diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1fdaeae --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +on: + push: + branches: + - main + - "feature/**" + pull_request: + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Syntax, Slurm & End-to-End Pytest Suite (Python ${{ matrix.python-version }}) + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + python-version: ["3.11"] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install uv and Python + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" + python-version: ${{ matrix.python-version }} + + - name: Install CPU PyTorch and PorousDiff test dependencies + run: | + uv venv --clear .venv + uv pip install --python .venv/bin/python torch --index-url https://download.pytorch.org/whl/cpu + uv pip install --python .venv/bin/python -e ".[dev]" + + - name: Validate Python syntax across package, scripts, and tests + run: .venv/bin/python -m compileall -q porousdiff scripts tests + + - name: Run Ruff static lint checks + run: .venv/bin/ruff check porousdiff scripts tests --select E9,F63,F7,F82,F401 + + - name: Validate Slurm batch script syntax + run: bash -n slurm/*.slurm + + - name: Run unit, conditioning convergence, and 5-stage end-to-end CLI pipeline tests + run: .venv/bin/pytest -v diff --git a/.gitignore b/.gitignore index 787de4d..771a793 100644 --- a/.gitignore +++ b/.gitignore @@ -210,6 +210,7 @@ __marimo__/ data/ datasets/ checkpoints/ +results/ *.pth *.pt *.ckpt diff --git a/README.md b/README.md index 88948eb..99198da 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,169 @@ # PorousDiff -**Generative modelling of three-dimensional porous electrode microstructures using diffusion models.** +**Continuous compression-conditioned 3D diffusion modelling of porous electrode microstructures from synchrotron X-ray computed tomography (XCT).** -Developed by [BASE Laboratory](https://github.com/BASE-Laboratory) using NVIDIA GH200 accelerators on Isambard-AI. +Developed by [BASE Laboratory](https://github.com/BASE-Laboratory) using NVIDIA GH200 Grace Hopper accelerators on Isambard-AI. -**Research status:** Active development — unconditional 3D diffusion benchmarks completed; compression-conditioned generation planned. +--- ## Overview -PorousDiff investigates the use of generative artificial intelligence to produce synthetic three-dimensional electrode microstructures from X-ray computed tomography (XCT) data. +PorousDiff provides a 3D denoising diffusion surrogate framework for multi-resolution synchrotron XCT compression scans of porous battery electrodes—spanning **Virgin Carbon Felt**, **$\text{TiO}_2$-coated Carbon Felt Anodes**, and **Copper Hexacyanoferrate (CuHCF) Coated Cathodes** (plus **Diamond K11 DIAD** co-registered XCT/XRD scans). -The long-term objective is to develop a physics-validated generative framework capable of modelling microstructural changes under mechanical compression. +PorousDiff is designed to evaluate two core hypotheses on held-out synchrotron compression states: +1. **Continuous Mechanical Compression Conditioning:** Given a compressive stress (`kPa`) or load (`mV/V`) within the calibrated training range, generates 3D multi-phase microstructures (`0 = pore`, `1 = carbon fiber`, `2 = active TiO2/CuHCF coating`) to model fiber compaction, `XY/Z` chord anisotropy, active-coating occlusion ($\text{SSA}_{\text{pore–coating}}$), triple-phase boundaries ($L_{\text{TPB}}$), electrical wiring, and 26-connected Euclidean geodesic tortuosity relative to affine Z-squashing and vertical pillar pore-deletion baselines. +2. **Few-Shot Cross-Electrode Transfer Benchmark:** Combines a densely scanned mechanical reference felt (Virgin Carbon Felt) with a sparse subset (`2–3` loads) of functionalized coated-electrode scans ($\text{TiO}_2$ or CuHCF) to test how accurately intermediate held-out coated-electrode states can be interpolated compared with dense single-material training. -Current experiments focus on TiO₂ electrode microstructures and evaluate whether 3D denoising diffusion probabilistic models (DDPMs) can reproduce realistic pore and solid structures. +--- -## Research Objectives +### Key Capabilities (`v0.4` Multi-Electrode & OpenImpala Pipeline) -1. Develop reproducible preprocessing pipelines for XCT datasets. -2. Train and benchmark 3D autoencoder and diffusion models. -3. Evaluate generated microstructures using physically meaningful metrics. -4. Investigate how preprocessing, patch size, and architecture affect generative fidelity. -5. Extend the framework to compression-conditioned generation using CuCHF datasets. +1. **Calibrated 2-Phase & 3-Phase XCT Segmentation, Partial-Volume Halo Cleanup & Mirror-Free Baseline Slabs** (`porousdiff/preprocessing.py`, `scripts/preprocessing/prepare_local_drive_bundle.py`) + - Masks exterior air outside the circular electrode punch (`--roi-radius-fraction 0.70`), subtracts pore-mode-clamped background, applies shared per-`(material, beamline)` lowest-load thresholds (`delta_carbon`, `delta_coating`), and strips Paganin/PSF partial-volume "carbon rind" blur shells at `0–2` pore–coating boundaries (`--strip-halo-px 1`) while preserving $\ge 3\text{ px}$ carbon fibres and `0–1–2` triple-phase junctions. + - Excludes bullseye ring seams (`compute_radial_ring_score`) and compression plungers (`valid_slices.json`), supports volume-fraction-matched `--spatial-bin 2` coarse-graining (with optional `audit_binning_topology_preservation` checks), extracts full-height `val/tall_patches.npy` columns so Z-compression baselines require zero mirror-tiling reflection, and falls back from Z-slab splitting to buffered XY-sector `train`/`val` splitting (`--split-mode auto`) on thin scans (`< 3 × raw_window`). -## Implemented Features +2. **Physics-Regularized Hierarchical Multi-Phase Diffusion (`v`-prediction + Zero-SNR + $\hat{x}_0$ Penalties)** (`porousdiff/models.py`, `porousdiff/losses.py`) + - **Hierarchical 2-channel Zero-SNR representation (`c_solid`, `c_coating`):** Aligns 2-phase Virgin Felt and 3-phase $\text{TiO}_2$/CuHCF electrodes via Channel 0 (`total solid`) and Channel 1 (`active coating`), conditioned on stress/load, categorical electrode chemistry (`nn.Embedding`), and `log(dx / dx_ref)` voxel size. Enforces a single load-unit source of truth across training, `--resume`, sampling, and evaluation so `mV/V` and `kPa` can never be mixed. + - **Optional $\hat{x}_0$ auxiliary penalties (`--physics-loss`):** Combines Min-SNR-$\gamma$ timestep weighting with 3D finite-difference interfacial sharpness, hierarchical no-floating-coating support penalties, and catalyst-to-scaffold mass-loading conservation, alongside high-load coating-retention diagnostics (`coating_retention_diagnostics.json`). -- XCT TIFF inspection and preprocessing. -- Background correction and binary segmentation. -- 3D patch extraction with training and validation splits. -- MONAI-based 3D DDPM training. -- Checkpointing and training resumption. -- Slurm-based GPU training on Isambard-AI. -- Generation of synthetic 3D volumes. -- Orthogonal XY, XZ, and YZ visualization. -- Preliminary solid-fraction and morphological evaluation. +3. **Conditional DDIM Microstructure Transfer & Seamless $>96^3$ MultiDiffusion Tiling** (`porousdiff/virtual_compression.py`, `scripts/evaluation/virtual_compress_scan.py`) + - **Conditional DDIM Transfer (`invert_strength = 0.60`):** Partially inverts a real low-load segmented scan along the conditional DDIM trajectory and denoises it at higher in-range target stresses $\sigma$. Automatically audits `check_load_extrapolation`, roundtrip `same_load_solid_dice`, and `paired_xy_column_corr` vs. unpaired scans to verify specimen-specific structural retention rather than unconditional resampling (note: fixed Eulerian $96^3$ windows model mesoscale morphology rather than Lagrangian material-point tracking). + - **Seamless $>96^3$ 3D MultiDiffusion (`--tiled-shape`):** Synthesizes larger 3D domains (e.g. `96 × 96 × 192` or `160³`) via symmetrically spaced 3D raised-cosine (Hann) overlapped denoising so domain-size REV convergence can be tested explicitly in OpenImpala. -## Experimental Progress +4. **Optimal Clamping-Pressure ($\sigma^*$) Analysis via 1D Newman ASR ($\Omega\cdot\text{cm}^2$), Weight Sensitivity & DIAD XRD Partial Correlation** (`porousdiff/optimization.py`, `scripts/evaluation/optimize_clamping_pressure.py`, `porousdiff/transport_and_xrd.py`) + - Couples uniform OpenImpala continuum transport (`openimpala_transport_parity.csv`, refusing mixed partial OpenImpala + surrogate curves) into both a normalized harmonic Figure of Merit (with an 11-point simplex `weight_sensitivity` sweep) and a **dimensional 1D Newman & Tobias porous-electrode Area-Specific Resistance $R_{\text{ASR}}(\sigma)$ ($\Omega\cdot\text{cm}^2$)** accounting for compressed thickness $L(\sigma) = L_0(1 - \epsilon_Z)$, Nernst mass-transfer resistance, clamping contact resistance, Darcy pumping penalty, and 3-regime sensitivity (`kinetics_limited`, `balanced`, `mass_transport_limited`). + - Quantifies 3D compressive contact metrics (`specific_z_pinch_excess`, `coating_pinch_fraction`, `mean_coating_coordination`) correlated against co-registered DLS K11 DIAD XRD microstrain with OLS partial correlation (`partial_pearson_r_given_sf`) controlling for solid fraction $\phi_s$. -### 3D Autoencoder Baselines - -Initial experiments investigated convolutional 3D autoencoders using mean squared error (MSE) and slice-wise learned perceptual image patch similarity (LPIPS) losses. - -Perceptual-loss experiments improved qualitative reconstruction detail compared with MSE-only baselines, although reconstruction smoothing remained. - -### Grayscale 3D Diffusion Models - -MONAI-based DDPMs were trained on 64³ XCT patches from TiO₂ datasets, including Op001mVV and Op026mVV. - -Training was completed for up to 100 epochs. - -Generated volumes developed porous textures but exhibited limitations, including horizontal banding, blurred boundaries, and incomplete reproduction of fine-scale morphology. - -Increasing the number of residual blocks did not eliminate these artifacts. - -### Binary 3D Diffusion Models - -Binary segmentation was introduced to support more direct evaluation of pore and solid structures. - -Two Op001mVV experiments were completed: - -| Configuration | 64³ DDPM | 96³ DDPM | -|---|---:|---:| -| Training patches | 960 | 960 | -| Validation patches | 240 | 240 | -| Training epochs | 100 | 100 | -| Final training loss | 0.004893 | 0.003370 | -| Mean real solid fraction | 0.1318 | 0.0984 | -| Mean generated solid fraction | 0.3948 | 0.2806 | -| Generated samples evaluated | 10 | 10 | - -**Main finding:** Successful DDPM training and decreasing noise-prediction loss did not guarantee physically representative synthetic microstructures. - -The generated volumes showed substantial differences in solid fraction and morphology compared with real XCT patches. - -The 64³ and 96³ datasets also had different solid-fraction distributions, so the results do not establish a controlled patch-size comparison. - -These findings are preliminary. Comprehensive connectivity, spatial-correlation, and transport-property validation remains incomplete. - -Detailed experiment records are available in [experiments.md](experiments.md). - -## Technical Stack - -- **Language:** Python -- **Deep learning:** PyTorch -- **Generative modelling:** MONAI Generative Models -- **Scientific computing:** NumPy and SciPy -- **Image processing:** tifffile and scikit-image -- **Visualization:** Matplotlib -- **HPC:** Slurm and NVIDIA GH200 GPUs +--- ## Repository Structure ```text PorousDiff/ -├── configs/ -├── docs/ +├── porousdiff/ +│ ├── __init__.py +│ ├── preprocessing.py # Circular ROI, 2/3-phase segmentation, halo cleanup, 2x binning + topology audit, kPa calibration +│ ├── baselines.py # Literature heuristics: affine Z-rescaling, pillar pore-deletion & mirror-tiling audit +│ ├── data.py # Hierarchical {0,1,2} <-> [-1,1]^2 encoding, load-unit source-of-truth helpers, symmetries +│ ├── models.py # Zero-SNR schedule, MultiModalConditionEmbedder (stress + chemistry + voxel size), 3D UNet, EMA +│ ├── losses.py # Auxiliary x0_hat loss (Min-SNR, 3D gradients, support, loading) & coating-retention check +│ ├── virtual_compression.py # Conditional DDIM microstructure transfer, fidelity diagnostics & >96^3 Hann MultiDiffusion +│ ├── optimization.py # 1D Newman ASR (Ohm*cm^2), simplex weight-sensitivity & bootstrap sigma* analysis +│ ├── metrics.py # Porosity, EASA, TPB, wired coating, pinch excess, S2(r), 26-neighbour Euclidean tortuosity +│ └── transport_and_xrd.py # OpenImpala INI/batch suite generator & parser + partial-correlation K11 DIAD XRD analysis ├── scripts/ │ ├── preprocessing/ +│ │ ├── prepare_local_drive_bundle.py # Spyder/CLI multi-electrode stream-bundler + tall baseline slabs + --preview-only +│ │ ├── segment_carbon_xct.py +│ │ └── extract_carbon_3d_patches.py │ ├── training/ +│ │ └── train_carbon_conditional_96.py # Multi-electrode 3-phase training + --resume config guard + --physics-loss │ └── evaluation/ -├── slurm/ -├── experiments.md +│ ├── sample_carbon_conditional_96.py # Standard & >96^3 (--tiled-shape) 3D sampling + OpenImpala TIFFs +│ ├── virtual_compress_scan.py # Conditional DDIM transfer across loads + same-load Dice & XY fidelity audit +│ ├── evaluate_carbon_physics.py +│ ├── evaluate_beamtime_multiplier.py # Real Held-Out XCT vs. Mirror-Free Baselines vs. PorousDiff +│ ├── parse_openimpala_suite.py # Ingest OpenImpala solver outputs into openimpala_transport_parity.csv +│ └── optimize_clamping_pressure.py # Newman ASR & FoM clamping-pressure sigma* + weight-sensitivity analysis +├── legacy/scripts/ # Archived pre-conditional v0.1-v0.2 prototype scripts (Exps 1-8) +├── slurm/ # Isambard-AI GH200 Slurm batch scripts (configurable via POROUSDIFF_ROOT) +├── tests/ # Automated pytest unit, trained-model DDIM fidelity, and end-to-end CLI verification +├── experiments.md # Chronological experimental log +├── pyproject.toml ├── environment.yml -├── requirements-isambard.txt -└── README.md +└── requirements-isambard.txt ``` -## Getting Started +--- -Clone the repository: +## Quickstart & Handover Guide + +### 1. Environment Setup (Isambard-AI GH200 or Linux Workstation) ```bash -git clone https://github.com/BASE-Laboratory/PorousDiff.git -cd PorousDiff +export POROUSDIFF_ROOT="/path/to/porousdiff_workspace" # e.g. /projects// on Isambard-AI +cd "$POROUSDIFF_ROOT/code/PorousDiff" +git fetch origin && git checkout feature/paper-ready-conditional && git pull +source "$POROUSDIFF_ROOT/.venv/bin/activate" +pip install --no-build-isolation --no-deps -e . +pytest -v ``` -The repository includes environment specifications and experimental scripts. Some Slurm scripts contain cluster-specific paths and settings that must be adapted before use on another HPC system. - -Research datasets and trained checkpoints are not distributed through this repository. - -## Planned Work: Compression-Conditioned Generation +### 2. Local Drive Bundling (Windows Lab PC / Spyder) + +1. **10-Second 3-Phase Threshold Preview** (verify `pore = black`, `carbon = cyan`, `TiO2/CuHCF = gold` on 1 scan per material first): + ```bash + python scripts/preprocessing/prepare_local_drive_bundle.py \ + --drive-root "D:/dry_samples" \ + --output-dir "D:/porousdiff_upload_bundle" \ + --datasets tio2_sample_folder cuhcf_sample_folder \ + --auto-multiphase --preview-only + ``` +2. **Full Multi-Electrode Bundle** (recursively discovers nested `Virgin/`, `TiO2/Op001mVV`, `CuHCF/` folders, shares lowest-load thresholds per electrode, saves mirror-free `val/tall_patches.npy` baseline columns, supports `--resume`, and writes a `.zip` upload archive): + ```bash + python scripts/preprocessing/prepare_local_drive_bundle.py \ + --drive-root "D:/dry_samples" \ + --output-dir "D:/porousdiff_upload_bundle" \ + --auto-multiphase --spatial-bin 2 \ + --voxel-size-um 1.6 --kpa-per-mvv 8500 \ + --beamline "DLS_I12" --resume --zip + + # Optional: merge a second drive/beamline (e.g. DLS K11 DIAD) into the SAME manifest without overwriting: + python scripts/preprocessing/prepare_local_drive_bundle.py \ + --drive-root "E:/K11_DIAD_samples" \ + --output-dir "D:/porousdiff_upload_bundle" \ + --auto-multiphase --auto-calibrate-carbon \ + --voxel-size-um 0.54 --punch-diameter-mm 10.0 \ + --beamline "DLS_K11_DIAD" --append --resume --zip + ``` + +### 3. Training, OpenImpala Transport, Conditional DDIM Transfer & Clamping $\sigma^*$ Analysis -The next research phase will investigate CuCHF electrode microstructures acquired under three experimental load-cell conditions. - -The proposed conditional diffusion model will use an experimental condition as an input to generate synthetic microstructures corresponding to that condition. - -Planned developments include: - -- Conditioning on measured load-cell signals. -- Generation across three observed experimental states. -- Investigation of larger 3D patches. -- Physical validation of generated structures. -- Portable inference from trained model checkpoints. - -These capabilities are research objectives and are not yet presented as completed implementations. +```bash +# Core Paper Benchmark: Multi-electrode training (auto-detecting calibrated stress_kpa), +# sampling, mirror-free baseline benchmark, OpenImpala suite generation, and preliminary sigma* sweep: +sbatch --export=ALL,POROUSDIFF_ROOT="$POROUSDIFF_ROOT",EXPERIMENT="beamtime_fewshot_96",HELDOUT_STATES="tio2_0p018mVV tio2_0p033mVV cuhcf_0p018mVV cuhcf_0p033mVV" \ + slurm/run_beamtime_multiplier.slurm + +# Run generated OpenImpala continuum diffusion/conduction jobs & re-evaluate Newman ASR + sigma*: +bash "$POROUSDIFF_ROOT/results/beamtime_fewshot_96/beamtime_benchmark/openimpala_suite/run_openimpala_batch.sh" +python scripts/evaluation/parse_openimpala_suite.py \ + --suite-dir "$POROUSDIFF_ROOT/results/beamtime_fewshot_96/beamtime_benchmark/openimpala_suite" +python scripts/evaluation/optimize_clamping_pressure.py \ + --experiments porousdiff=beamtime_fewshot_96 \ + --data-root "$POROUSDIFF_ROOT/data/carbon_virgin" \ + --manifest-json "$POROUSDIFF_ROOT/data/carbon_virgin/bundle_manifest.json" + +# MultiDiffusion >96^3 synthesis (e.g. 96x96x192 or 160^3) to test OpenImpala REV domain-size convergence: +python scripts/evaluation/sample_carbon_conditional_96.py \ + --experiment beamtime_fewshot_96 \ + --manifest-json "$POROUSDIFF_ROOT/data/carbon_virgin/bundle_manifest.json" \ + --tiled-shape 96,96,192 --num-samples 5 --export-openimpala + +# Conditional DDIM transfer of a real low-load (in-range, e.g. 68 kPa) 3D scan to higher target stresses: +python scripts/evaluation/virtual_compress_scan.py \ + --input "$POROUSDIFF_ROOT/data/low_load_68kpa.tif" \ + --experiment beamtime_fewshot_96 \ + --source-load 68.0 --target-loads "150,300,450" \ + --material tio2_anode --export-openimpala +``` -## Limitations +--- -Current generated microstructures have not consistently reproduced the solid-fraction distributions or spatial morphology of the XCT training data. +## Experimental & Pipeline Progression -Additional validation is required before using generated volumes as physically representative electrode digital twins. +Earlier unconditional baselines (`TiO₂`, `Op001mVV` / `Op026mVV`) established the motivation for the calibrated segmentation and Zero-SNR `v`-prediction formulation (full log in [`experiments.md`](experiments.md)): -## Acknowledgements +| Stage | Formulation | Patch Size | Real $\phi_s$ | Generated $\phi_s$ | Status & Diagnostic Takeaway | +|---|---|---:|---:|---:|---| +| Exp 1–5 | 3D Conv Autoencoder (MSE + LPIPS) | `64³` | — | — | Completed — reconstructs global envelope; smooths fine pores | +| Exp 6–11 | Grayscale 3D DDPM (`epsilon`, std linear) | `64³` | — | — | Completed — blurry boundaries & horizontal banding | +| Exp 12 | Unconditional Binary DDPM (`epsilon`, std schedule) | `64³` | `0.1318` | `0.3948` | Completed — non-zero terminal SNR shifts sparse binary mean ~3× toward `0.50` | +| Exp 13 | Unconditional Binary DDPM (`epsilon`, std schedule) | `96³` | `0.0984` | `0.2806` | Completed — `96³` feasible on GH200, motivating Zero-SNR & conditioning fixes | +| **Exp 14–16 (`v0.4`)** | **Multi-Electrode 3-Phase Load-Conditioned DDPM (`v`-pred + Zero-SNR + OpenImpala ASR)** | **`96³` (`2×` binned)** | **Virgin + $\text{TiO}_2$ + CuHCF + DIAD** | **Pending full GH200 benchmark** | **Software pipeline & unit/methodology guards verified; ready for held-out synchrotron evaluation** | -Developed within BASE Laboratory, with experiments performed using Isambard-AI computing resources. +--- -## License +## Acknowledgements & License -See [LICENSE](LICENSE) for licensing information. \ No newline at end of file +Developed within [BASE Laboratory](https://github.com/BASE-Laboratory), with experiments performed on Isambard-AI NVIDIA GH200 computing resources (`brics.u6uc`). See [`LICENSE`](LICENSE) for licensing details. \ No newline at end of file diff --git a/environment.yml b/environment.yml index 9dacfd9..58bbf4c 100644 --- a/environment.yml +++ b/environment.yml @@ -4,14 +4,19 @@ channels: - nvidia - conda-forge dependencies: - - python=3.10 - - pytorch=2.1 - - torchvision - - pytorch-cuda=12.1 + - python=3.11 - pip - pip: - - monai-generative - - hydra-core - - wandb - - porespy - - zarr + - torch>=2.4 + - torchvision + - monai>=1.4 + - einops>=0.7 + - numpy>=1.26 + - scipy>=1.11 + - scikit-image>=0.22 + - tifffile>=2024.2 + - matplotlib>=3.8 + - tqdm>=4.66 + - porespy>=2.4 + - pytest>=8.0 + - -e . diff --git a/experiments.md b/experiments.md index 32f0eb3..a9d5a96 100644 --- a/experiments.md +++ b/experiments.md @@ -428,3 +428,67 @@ Investigate whether increasing patch size from 64³ to 96³ improves the fidelit Increasing patch size to 96³ was computationally feasible on the GH200 but did not consistently reproduce the physical characteristics of the training microstructures. Future work will investigate compression-conditioned generation using CuCHF datasets with three experimentally measured load-cell conditions. + + +## Experiment 14: Synchrotron XCT Diagnostic Audit & Continuous Load-Conditioned 3D DDPM (`v0.2`) + +**Status:** Implemented and verified end-to-end (`feature/paper-ready-conditional`) — ready for full Isambard-AI GH200 paper runs + +### Diagnostic Findings on Raw Carbon-Felt XCT Volumes (`2160 × 2560 × 2560` `uint16` Paganin) +Direct inspection of raw synchrotron compression scans (`crbon_virgin_1_0p008mVV` through `0p053mVV`) identified five root causes behind earlier binary/conditional instabilities: +1. **Compression axis orientation:** `Z` (slice index `0..2159`) is the vertical mechanical compression plunger axis; `XY` is the carbon-felt sheet plane. +2. **Circular punch geometry:** The cylindrical carbon-felt punch (`R ≈ 950 px` centered at `(1280, 1280)`) sits inside square `2560 × 2560` reconstructions surrounded by gray exterior air (`~30,000` counts). Unmasked square grids extracted exterior air corners. +3. **Plunger intrusion & mid-stack bullseye ring seams:** Under higher loads, the bottom compression plunger rises into the stack (`Z ≈ 1850+` at `0.028 mV/V`; `Z ≈ 1280+` at `0.053 mV/V`), and non-physical concentric ring / stitch interference bands appear mid-stack (including `Z ≈ 520–620` inside the first `800` slices at `0.053 mV/V`). +4. **Per-slice Otsu inversion on sparse low-load slices:** Per-slice `[1, 99]` percentile clipping + `sigma = 50` Gaussian subtraction + per-slice Otsu inverted empty pore noise into ~80% false solid on sparse `0.008 mV/V` slices and erased true monotonic compression trends. +5. **1-token cross-attention degeneracy & non-zero terminal SNR:** Passing a 1-token context `[B, 1, 32]` into cross-attention forced `softmax([s]) = 1.0` identically (zero gradient to `W_Q, W_K`), while standard linear/scaled-linear schedules left residual signal at `t = T - 1` that biased sparse `{−1, +1}` microstructures upward toward `50%` solid fraction. + +### Architectural & Pipeline Fixes (`porousdiff/`) +- **Calibrated segmentation (`segment_carbon_xct.py`):** Circular ROI mask (`radius_fraction = 0.70`), pore-mode-clamped background subtraction (`sigma = 32`), shared `delta_threshold = 5500` across all loads (recovering clean monotonic solid fractions `~5.4%` at `0.008 mV/V` → `~23.5%` at `0.028 mV/V` → `~33.8%` at `0.053 mV/V`), and automatic Z-interval gating (`compute_radial_ring_score` + slice-jump + pore-mode drift). +- **Z-stratified patch extraction (`extract_carbon_3d_patches.py`):** Round-robin sampling across all valid Z-windows, `>= patch_size` buffer between `train` and `val` Z-slabs, and stacked `patches.npy` archives. +- **Multi-scale continuous conditioning (`ContinuousLoadConditionedUNet`):** Smooth Fourier scalar load embedder (`ContinuousScalarLoadEmbedder`) injected into `temb` at every ResBlock across all resolution levels plus 3-channel spatial input `[x_t, cond_norm * mask, mask]`, Zero-Terminal-SNR schedule (`zero_snr_scaled_linear`), `v_prediction`, EMA (`0.9995`), compression-safe `D_4(XY) × Z_2(Z)` augmentation, and `bf16` GH200 mixed precision. +- **Physics & transport evaluation (`evaluate_carbon_physics.py`):** Automated comparison of solid fraction, specific surface area, directional chord lengths, `XY/Z` compression anisotropy, directional $S_2(r)$, 3D percolation, geodesic pore tortuosity, and OpenImpala multi-page TIFF export across both training and held-out interpolated loads. + + +## Experiment 15: Multi-Electrode, 3-Phase Few-Shot Transfer Architecture (`v0.3`) + +**Status:** Software pipeline implemented and verified on synthetic test suites (`feature/paper-ready-conditional`) — awaiting full GH200 evaluation on Virgin Carbon, $\text{TiO}_2$ Anode, CuHCF Cathode, and DLS K11 DIAD datasets + +### Scientific Motivation & Literature Gap +Existing 3D diffusion papers for porous electrodes generate static microstructures or condition on global porosity, while recent compression-transport studies (*ACS Appl. Energy Mater.* 2025) rely on heuristic numerical Z-squashing or column-wise pore-voxel deletion. Across our multi-electrode synchrotron compression suite—**Virgin Carbon Felt** (`virgin_carbon`), **$\text{TiO}_2$-coated Carbon Felt Anode** (`tio2_anode`), and **Copper Hexacyanoferrate Coated Cathode** (`cuhcf_cathode`)—plus an independent cross-resolution repeat at **Diamond Light Source K11 DIAD**, `PorousDiff` implements a held-out cross-electrode benchmark. + +### Key `v0.3` Upgrades (`porousdiff/`) +1. **3-Phase Solid-Masked Calibration & Coarse-Grained Binning (`preprocessing.py`):** + - Two-stage Otsu calibration (`calibrate_multiphase_thresholds` + `segment_slice_multiphase`) separates `0 = pore`, `1 = carbon_fiber`, and `2 = active_coating` ($\text{TiO}_2$ / CuHCF) without splitting the dominant 80–90% pore background peak. + - Volume-fraction-matched `downsample_volume_rev` (`spatial_bin = 2` or `3`, paired with `audit_binning_topology_preservation`) enlarges $96^3$ patches to $2\times$–$3\times$ physical field-of-view (`~20–30` fiber diameters per patch edge). + - `CalibrationMetadata` records physical `voxel_size_um`, `kpa_per_mvv` mechanical stress conversion, `beamline`, and `is_repeat` in `bundle_manifest.json`, plus a ~10-second `--preview-only` Spyder mode (`qa/multiphase_overlay_diagnostic.png`). +2. **Literature Numerical Z-Compression Baselines (`baselines.py`):** + - `affine_z_compress_to_target` (affine Z-resampling + occupancy-quantile projection matching target compressed porosity) and `pillar_pore_deletion_compress` (stochastic XY-column pore-layer removal). +3. **Hierarchical 2-Channel 3-Phase Representation & Multi-Modal Conditioning (`data.py`, `models.py`):** + - Hierarchical encoding `[c_solid, c_coating]` in `{-1, +1}` aligns Channel 0 (`total solid vs. pore`) across 2-phase Virgin Felt and 3-phase coated electrodes while `decode_channels_to_phases` prevents floating unattached coating artifacts. + - `MultiModalConditionEmbedder` embeds normalized stress/load + categorical `material_id` (`nn.Embedding`) + continuous `log(dx / dx_ref)` voxel size into `temb` and spatial conditioning channels. +4. **3-Phase Electrochemical Interfacial Metrics & Held-Out Benchmark (`metrics.py`, `evaluate_beamtime_multiplier.py`):** + - Computes Electrochemically Active Surface Area (`ssa_pore_coating`), Triple-Phase Boundary line density (`tpb_density` in $\mu\text{m}^{-2}$), electronically wired coating fraction (`wired_coating_fraction`), 26-connected Euclidean geodesic tortuosity, and multi-phase OpenImpala TIFF export (`{0, 1, 2}`). + + +## Experiment 16: Conditional DDIM Transfer, MultiDiffusion Tiling, $\hat{x}_0$ Regularization & OpenImpala Clamping-Pressure ($\sigma^*$) Pipeline (`v0.4`) + +**Status:** Software pipeline & peer-review audit fixes verified locally and in CI (`feature/paper-ready-conditional`) + +### Key `v0.4` Capabilities & Methodological Safeguards (`porousdiff/`) +1. **Conditional DDIM Microstructure Transfer & Fidelity Audit (`porousdiff/virtual_compression.py`, `scripts/evaluation/virtual_compress_scan.py`):** + - Partially inverts a real low-load segmented 3D scan (`invert_strength = 0.60`, $\bar{\alpha} \approx 0.16$) along the conditional DDIM trajectory and denoises to higher in-range target stresses $\sigma$. Automatically evaluates `check_load_extrapolation`, roundtrip `same_load_solid_dice`, and `paired_xy_column_corr` vs. unpaired scans (`xy_correlation_lift`) to distinguish specimen-specific structural retention from unconditional resampling. +2. **Seamless $>96^3$ 3D Volume Stitching (`generate_tiled_3d`, `--tiled-shape`):** + - Symmetrically spaced 3D raised-cosine (Hann) MultiDiffusion synthesizes larger domains (e.g. $96 \times 96 \times 192$ or $160^3$) so REV domain-size convergence can be tested explicitly in OpenImpala. +3. **Auxiliary $\hat{x}_0$ Training Penalties & Coating-Retention Audit (`porousdiff/losses.py`, `--physics-loss`):** + - Closed-form single-step $\hat{x}_0$ reconstruction paired with Min-SNR-$\gamma$ weighting, 3D finite-difference interfacial sharpness, hierarchical no-floating-coating support penalty $\text{ReLU}(\hat{c}_{\text{coating}} - \hat{c}_{\text{solid}})$, and solid-phase catalyst-to-scaffold ratio conservation, plus `val_x0_sf_mae` validation logging and `coating_retention_diagnostics.json`. +4. **Clamping-Pressure $\sigma^*$ Analysis via 1D Newman ASR & Weight Sensitivity (`porousdiff/optimization.py`, `scripts/evaluation/optimize_clamping_pressure.py`):** + - Computes both a normalized harmonic Figure of Merit (accompanied by an 11-point simplex `weight_sensitivity` sweep) and a dimensional **1D Newman & Tobias porous-electrode Area-Specific Resistance $R_{\text{ASR}}(\sigma)$ ($\Omega\cdot\text{cm}^2$)** incorporating compressed thickness $L(\sigma) = L_0(1 - \epsilon_Z)$, Nernst diffusive mass-transfer resistance, clamping contact resistance, Darcy hydraulic penalty, and 3-regime sensitivity (`kinetics_limited`, `balanced`, `mass_transport_limited`). Refuses to mix partial OpenImpala + surrogate estimates on the same curve. +5. **Partial-Correlation DIAD XRD & Pinch-Point Analysis (`porousdiff/metrics.py`, `porousdiff/transport_and_xrd.py`):** + - Quantifies `specific_z_pinch_excess` (`P(solid[z] & solid[z-1]) / phi_s^2`), `coating_pinch_fraction`, and `mean_coating_coordination`, reporting both raw Pearson $r$ and OLS partial correlation `partial_pearson_r_given_sf` (controlling for solid fraction $\phi_s$) against co-registered DLS K11 DIAD XRD microstrain. +6. **Peer-Review & Red-Team Engineering Hardening (`v0.4`):** + - **Single Load-Unit Source of Truth:** `sample_carbon_conditional_96.py`, `evaluate_beamtime_multiplier.py`, and `optimize_clamping_pressure.py` read `use_stress_kpa` directly from checkpoint config (`resolve_use_stress_kpa_from_checkpoints`) and resolve loads via `resolve_scan_load_value` (preserving `0.0 kPa` and raising on missing units rather than falling back to `mV/V`). + - **Resume Data-Leakage Guard (`validate_resume_config`):** `train_carbon_conditional_96.py --resume` verifies `heldout_states`, `include_states`, `use_stress_kpa`, `states`, and `scan_specs` match the saved checkpoint before loading weights. + - **Mirror-Free Tall Baseline Sub-Volumes (`val/tall_patches.npy`):** `prepare_local_drive_bundle.py` extracts full-clean-height validation columns so `affine_z_compress_to_target` and `pillar_pore_deletion_compress` operate with `baseline_mirror_tiled_fraction == 0.0` whenever clean scan height permits. + - **26-Connected Euclidean Geodesic Tortuosity:** Upgraded `geodesic_pore_tortuosity` from 6-neighbour Manhattan BFS to 26-neighbour Euclidean Dijkstra (`1, sqrt(2), sqrt(3)` step weights, recovering $\tau = \sqrt{2}$ on $45^\circ$ diagonal channels). + - **3-Phase Partial-Volume Halo Cleanup & Thin-Scan XY Split:** Strips 1–2 px Paganin/PSF phase-`1` blur rinds at `0–2` pore boundaries (`strip_partial_volume_carbon_halo`) and falls back to buffered XY-sector splitting (`split_xy_positions_with_buffer`) on thin compressed stacks. + diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..c70e05b --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,11 @@ +# Legacy Pre-Conditional Scripts (v0.1 – v0.2) + +This directory archives the original single-state unconditional $64^3$ / early prototype scripts (`make_patches64*.py`, `train_diffusion*.py`, `sample_diffusion*.py`, `train_autoencoder.py`, `check_tiffs.py`, `create_volume.py`) from the initial PorousDiff exploratory runs (`Exps 1–8`). + +They are preserved for historical reproducibility of `main`, while all current conditional $96^3$ 2-phase and 3-phase multi-electrode workflows live under: +- `scripts/preprocessing/prepare_local_drive_bundle.py` +- `scripts/training/train_carbon_conditional_96.py` +- `scripts/evaluation/sample_carbon_conditional_96.py` +- `scripts/evaluation/evaluate_beamtime_multiplier.py` +- `scripts/evaluation/optimize_clamping_pressure.py` +- `scripts/evaluation/virtual_compress_scan.py` diff --git a/scripts/check_tiffs.py b/legacy/scripts/check_tiffs.py similarity index 100% rename from scripts/check_tiffs.py rename to legacy/scripts/check_tiffs.py diff --git a/scripts/create_volume.py b/legacy/scripts/create_volume.py similarity index 100% rename from scripts/create_volume.py rename to legacy/scripts/create_volume.py diff --git a/scripts/make_patches64.py b/legacy/scripts/make_patches64.py similarity index 100% rename from scripts/make_patches64.py rename to legacy/scripts/make_patches64.py diff --git a/scripts/make_patches64_op026.py b/legacy/scripts/make_patches64_op026.py similarity index 100% rename from scripts/make_patches64_op026.py rename to legacy/scripts/make_patches64_op026.py diff --git a/scripts/sample_diffusion.py b/legacy/scripts/sample_diffusion.py similarity index 100% rename from scripts/sample_diffusion.py rename to legacy/scripts/sample_diffusion.py diff --git a/scripts/sample_diffusion_deeper.py b/legacy/scripts/sample_diffusion_deeper.py similarity index 100% rename from scripts/sample_diffusion_deeper.py rename to legacy/scripts/sample_diffusion_deeper.py diff --git a/scripts/sample_diffusion_op026.py b/legacy/scripts/sample_diffusion_op026.py similarity index 100% rename from scripts/sample_diffusion_op026.py rename to legacy/scripts/sample_diffusion_op026.py diff --git a/scripts/train_autoencoder.py b/legacy/scripts/train_autoencoder.py similarity index 100% rename from scripts/train_autoencoder.py rename to legacy/scripts/train_autoencoder.py diff --git a/scripts/train_diffusion.py b/legacy/scripts/train_diffusion.py similarity index 100% rename from scripts/train_diffusion.py rename to legacy/scripts/train_diffusion.py diff --git a/scripts/train_diffusion_deeper.py b/legacy/scripts/train_diffusion_deeper.py similarity index 100% rename from scripts/train_diffusion_deeper.py rename to legacy/scripts/train_diffusion_deeper.py diff --git a/scripts/train_diffusion_op026.py b/legacy/scripts/train_diffusion_op026.py similarity index 100% rename from scripts/train_diffusion_op026.py rename to legacy/scripts/train_diffusion_op026.py diff --git a/porousdiff/__init__.py b/porousdiff/__init__.py new file mode 100644 index 0000000..6caf5eb --- /dev/null +++ b/porousdiff/__init__.py @@ -0,0 +1,3 @@ +"""PorousDiff: Multi-electrode, multi-phase conditional 3D diffusion for compressed porous electrodes.""" + +__version__ = "0.4.0" diff --git a/porousdiff/baselines.py b/porousdiff/baselines.py new file mode 100644 index 0000000..016752a --- /dev/null +++ b/porousdiff/baselines.py @@ -0,0 +1,201 @@ +"""Geometry-based numerical uniaxial Z-compression baselines from porous electrode literature. + +Implements the two standard heuristics currently used in fuel-cell GDL and redox-flow +felt modelling when intermediate synchrotron XCT compression scans are unavailable: + 1. `affine_z_compress_to_target`: Uniaxial Z-coordinate rescaling with mass-conserving + occupancy projection (affinely squashes Z without non-affine fiber bending/slip). + 2. `pillar_pore_deletion_compress`: Vertical 1-voxel pillar pore-deletion algorithm + (e.g., ACS Appl. Energy Mater. 2025; GeoDict uniaxial geometrical compression). +""" + +from __future__ import annotations + +import math + +import numpy as np +from scipy.ndimage import gaussian_filter + + +def _tile_slab_along_z(volume: np.ndarray, required_depth: int) -> np.ndarray: + """Extend a 3D volume `(Z, Y, X)` to `required_depth` via mirror-bounce tiling along Z.""" + d = volume.shape[0] + if d >= required_depth: + return volume[:required_depth] + if d == 1: + return np.repeat(volume, required_depth, axis=0) + + period = np.concatenate([volume, volume[-2:0:-1]], axis=0) + reps = math.ceil(required_depth / period.shape[0]) + return np.tile(period, (reps, 1, 1))[:required_depth] + + +def resolve_compression_ratio( + initial_solid_fraction: float, + target_solid_fraction: float | None = None, + compression_ratio: float | None = None, +) -> float: + """Resolve compressed-to-initial thickness ratio `H_comp / H_0` in `(0, 1]`.""" + if compression_ratio is not None: + if not 0.05 <= compression_ratio <= 1.0: + raise ValueError("compression_ratio (H_comp / H_0) must be in [0.05, 1.0]") + return float(compression_ratio) + if target_solid_fraction is None: + raise ValueError("Provide either target_solid_fraction or compression_ratio") + if not 0.0 < target_solid_fraction < 1.0: + raise ValueError("target_solid_fraction must be in (0, 1)") + if initial_solid_fraction <= 0.0: + return 1.0 + # Mass conservation of incompressible solid fibers under uniaxial Z compression: + # phi_target * H_comp = phi_0 * H_0 => H_comp / H_0 = phi_0 / phi_target + return float(np.clip(initial_solid_fraction / target_solid_fraction, 0.08, 1.0)) + + +def compute_mirror_tiled_fraction( + source_depth: int, + output_depth: int, + initial_solid_fraction: float, + target_solid_fraction: float | None = None, + compression_ratio: float | None = None, +) -> float: + """Return the fraction in `[0.0, 1.0)` of required uncompressed Z slices extended via mirror tiling. + + Equals `0.0` whenever `source_depth >= round(output_depth / ratio)` (i.e. when a tall + real uncompressed sub-volume supplies the full required Z height without reflection). + """ + ratio = resolve_compression_ratio(initial_solid_fraction, target_solid_fraction, compression_ratio) + req_src_d = max(int(output_depth), int(round(output_depth / ratio))) + if source_depth >= req_src_d or req_src_d <= 0: + return 0.0 + return float(req_src_d - source_depth) / float(req_src_d) + + +def affine_z_compress_to_target( + uncompressed_vol: np.ndarray, + target_solid_fraction: float | None = None, + compression_ratio: float | None = None, + output_depth: int | None = None, +) -> np.ndarray: + """Affinely rescale `Z` (axis 0) to simulate purely geometric uniaxial compression. + + Preserves both `{0, 1}` binary and `{0, 1, 2}` 3-phase labels while conserving + solid/coating mass up to target solid fraction. + """ + vol = np.asarray(uncompressed_vol, dtype=np.uint8) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got {vol.shape}") + + phi_0 = float((vol >= 1).mean()) + ratio = resolve_compression_ratio(phi_0, target_solid_fraction, compression_ratio) + out_d = int(output_depth) if output_depth is not None else max(2, int(round(vol.shape[0] * ratio))) + src_d = max(out_d, int(round(out_d / ratio))) + slab = _tile_slab_along_z(vol, src_d) + + _, h, w = slab.shape + edges = np.linspace(0.0, float(src_d), out_d + 1) + occ_solid = np.zeros((out_d, h, w), dtype=np.float32) + occ_coat = np.zeros((out_d, h, w), dtype=np.float32) + has_coat = bool(np.any(slab == 2)) + + for z_out in range(out_d): + z0 = edges[z_out] + z1 = edges[z_out + 1] + i0 = int(math.floor(z0)) + i1 = min(src_d, int(math.ceil(z1))) + span = max(z1 - z0, 1e-6) + acc_s = np.zeros((h, w), dtype=np.float32) + acc_c = np.zeros((h, w), dtype=np.float32) + for iz in range(i0, i1): + wt = float(min(z1, iz + 1.0) - max(z0, float(iz))) / span + if wt > 0.0: + acc_s += wt * (slab[iz] >= 1).astype(np.float32) + if has_coat: + acc_c += wt * (slab[iz] == 2).astype(np.float32) + occ_solid[z_out] = acc_s + if has_coat: + occ_coat[z_out] = acc_c + + expected_phi = ( + float(target_solid_fraction) + if target_solid_fraction is not None + else min(0.95, float((slab >= 1).mean()) / ratio) + ) + n_vox = out_d * h * w + target_s_count = int(round(expected_phi * n_vox)) + out = np.zeros((out_d, h, w), dtype=np.uint8) + if target_s_count == 0: + return out + + score_s = np.where( + occ_solid > 0.0, + occ_solid + 1e-2 * gaussian_filter(occ_solid, sigma=(0.6, 0.4, 0.4)), + 0.0, + ) + k_s = min(target_s_count, int(np.count_nonzero(score_s > 0.0))) + if k_s > 0: + idx_s = np.argpartition(score_s.ravel(), -k_s)[-k_s:] + out.ravel()[idx_s] = 1 + + if has_coat: + coat_ratio_0 = float((slab == 2).mean()) / max(float((slab >= 1).mean()), 1e-6) + target_c_count = int(round(expected_phi * coat_ratio_0 * n_vox)) + score_c = np.where( + (out == 1) & (occ_coat > 0.0), + occ_coat + 1e-2 * gaussian_filter(occ_coat, sigma=(0.6, 0.4, 0.4)), + 0.0, + ) + k_c = min(target_c_count, int(np.count_nonzero(score_c > 0.0))) + if k_c > 0: + idx_c = np.argpartition(score_c.ravel(), -k_c)[-k_c:] + out.ravel()[idx_c] = 2 + + return out + + +def pillar_pore_deletion_compress( + uncompressed_vol: np.ndarray, + target_solid_fraction: float | None = None, + compression_ratio: float | None = None, + output_depth: int | None = None, + rng: np.random.Generator | None = None, +) -> np.ndarray: + """Compress 3D electrode along Z by deleting pore voxels inside vertical 1-voxel pillars. + + Mirrors the standard geometrical compression heuristic in GDL/felt literature: + each `(y, x)` column independently removes `(H_0 - H_comp)` pore voxels (`phase == 0`), + preserving solid/coating voxel order along Z until column pores are exhausted. + """ + vol = np.asarray(uncompressed_vol, dtype=np.uint8) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got {vol.shape}") + + gen = np.random.default_rng(42) if rng is None else rng + phi_0 = float((vol >= 1).mean()) + ratio = resolve_compression_ratio(phi_0, target_solid_fraction, compression_ratio) + out_d = int(output_depth) if output_depth is not None else max(2, int(round(vol.shape[0] * ratio))) + src_d = max(out_d, int(round(out_d / ratio))) + slab = _tile_slab_along_z(vol, src_d) + n_delete = src_d - out_d + if n_delete == 0: + return slab.copy() + + _, h, w = slab.shape + cols = slab.reshape(src_d, h * w) + out_cols = np.empty((out_d, h * w), dtype=np.uint8) + fallback_idx = np.linspace(0, src_d - 1, out_d).round().astype(np.int32) + + for c_idx in range(h * w): + col = cols[:, c_idx] + pore_pos = np.flatnonzero(col == 0) + if pore_pos.size >= n_delete: + drop = gen.choice(pore_pos, size=n_delete, replace=False) + keep_mask = np.ones(src_d, dtype=bool) + keep_mask[drop] = False + out_cols[:, c_idx] = col[keep_mask] + elif pore_pos.size > 0: + solid_only = col[col > 0] + sub_idx = np.linspace(0, solid_only.size - 1, out_d).round().astype(np.int32) + out_cols[:, c_idx] = solid_only[sub_idx] + else: + out_cols[:, c_idx] = col[fallback_idx] + + return out_cols.reshape(out_d, h, w) diff --git a/porousdiff/data.py b/porousdiff/data.py new file mode 100644 index 0000000..893cde6 --- /dev/null +++ b/porousdiff/data.py @@ -0,0 +1,454 @@ +"""RAM-cached datasets, multi-phase hierarchical encoding, symmetries, and RNG helpers.""" + +from __future__ import annotations + +import os +import random +from pathlib import Path +from typing import Mapping, Sequence + +import numpy as np +import torch +from torch.utils.data import Dataset + +MATERIAL_TO_ID: dict[str, int] = { + "virgin_carbon": 0, + "tio2_anode": 1, + "cuhcf_cathode": 2, +} +ID_TO_MATERIAL: dict[int, str] = {v: k for k, v in MATERIAL_TO_ID.items()} + + +def resolve_material_id(material: str | int) -> int: + """Map canonical material name or integer index to `[0, 1, 2]`.""" + if isinstance(material, int): + if material not in ID_TO_MATERIAL: + raise ValueError(f"Unknown material_id {material}; valid={list(ID_TO_MATERIAL)}") + return material + key = material.strip().lower() + if key in MATERIAL_TO_ID: + return MATERIAL_TO_ID[key] + raise ValueError(f"Unknown material {material!r}; choose from {list(MATERIAL_TO_ID)}") + + +def default_project_root() -> Path: + """Resolve data/checkpoint root via `POROUSDIFF_ROOT` environment variable or CWD.""" + env = os.environ.get("POROUSDIFF_ROOT") + if env: + return Path(env).expanduser() + return Path.cwd() + + +def resolve_scan_load_value(scan: Mapping[str, object], use_stress_kpa: bool) -> float: + """Extract physical load or calibrated compressive stress (`kPa`) without unit mixing. + + Preserves `0.0 kPa` accurately and raises `ValueError` if the requested unit field + is missing (`None`) on `scan`. + """ + st = str(scan.get("state", "")) + if use_stress_kpa: + val = scan.get("stress_kpa") + if val is None: + raise ValueError( + f"use_stress_kpa=True, but scan {st!r} has stress_kpa=None. " + "Ensure all manifest scans carry calibrated stress_kpa." + ) + return float(val) # type: ignore[arg-type] + + val_mv = scan.get("load_mv_per_v") + if val_mv is None: + raise ValueError(f"use_stress_kpa=False, but scan {st!r} has load_mv_per_v=None.") + return float(val_mv) # type: ignore[arg-type] + + +def resolve_checkpoint_path( + output_root: Path, + experiment: str, + explicit_checkpoint: Path | None = None, +) -> Path | None: + """Locate `explicit_checkpoint`, `best_val.pth`, or `latest.pth` if present.""" + if explicit_checkpoint is not None: + return explicit_checkpoint if explicit_checkpoint.is_file() else None + ckpt_dir = Path(output_root) / "checkpoints" / experiment + for name in ("best_val.pth", "latest.pth"): + cand = ckpt_dir / name + if cand.is_file(): + return cand + return None + + +def resolve_use_stress_kpa_from_checkpoints( + output_root: Path, + experiments: Sequence[str], + explicit_checkpoint: Path | None = None, + default: bool = False, +) -> tuple[bool, bool]: + """Read `use_stress_kpa` directly from saved checkpoint config(s) as single source of truth. + + Returns `(use_stress_kpa, found_any_checkpoint)`. Raises `ValueError` if multiple + compared checkpoints were trained with incompatible load units (`kPa` vs `mV/V`). + """ + units_by_exp: dict[str, bool] = {} + for idx, exp in enumerate(experiments): + ckpt_p = resolve_checkpoint_path( + output_root=output_root, + experiment=exp, + explicit_checkpoint=explicit_checkpoint if idx == 0 else None, + ) + if ckpt_p is not None: + saved = torch.load(ckpt_p, map_location="cpu", weights_only=False) + cfg = saved.get("config", {}) + units_by_exp[exp] = bool(cfg.get("use_stress_kpa", False)) + + if not units_by_exp: + return bool(default), False + + unique_units = set(units_by_exp.values()) + if len(unique_units) > 1: + raise ValueError( + f"Cannot compare checkpoints trained with incompatible load units (use_stress_kpa={units_by_exp})." + ) + return next(iter(unique_units)), True + + +def apply_compression_safe_symmetry( + patch: np.ndarray, + sym_id: int, + allow_z_flip: bool = True, +) -> np.ndarray: + """Apply symmetries of `D_4(XY) x Z_2(Z)` (16) or `D_4(XY)` (8) preserving compression axis Z. + + Axis 0 (`Z`) is the vertical compression plunger axis; axes 1 and 2 (`Y, X`) + span the isotropic carbon-felt sheet plane. Never swaps `Z` with `X` or `Y`. + Set `allow_z_flip=False` to preserve directional plunger-vs-base Z compaction gradients. + """ + code = int(sym_id) % (16 if allow_z_flip else 8) + rot_k = code & 0b11 + flip_y = bool((code >> 2) & 0b1) + flip_z = bool((code >> 3) & 0b1) if allow_z_flip else False + + out = patch + if rot_k: + out = np.rot90(out, k=rot_k, axes=(1, 2)) + if flip_y: + out = np.flip(out, axis=1) + if flip_z: + out = np.flip(out, axis=0) + return np.ascontiguousarray(out) + + +def encode_phases_to_channels(patch: np.ndarray, num_phases: int = 2) -> torch.Tensor: + """Encode `{0, 1}` or `{0, 1, 2}` `uint8` 3D volume into `{-1.0, +1.0}` float32 channels. + + For `num_phases == 3`, uses hierarchical 2-channel parameterization: + - Channel 0 (`c_solid`): `+1` if `patch >= 1` (`carbon` OR `coating`), `-1` if `pore` (`0`). + - Channel 1 (`c_coating`): `+1` if `patch == 2` (`active_coating`), `-1` otherwise. + """ + if num_phases == 2: + return ( + torch.from_numpy((patch > 0).astype(np.float32)) + .unsqueeze(0) + .mul_(2.0) + .sub_(1.0) + ) + if num_phases == 3: + solid = (patch >= 1).astype(np.float32) * 2.0 - 1.0 + coating = (patch == 2).astype(np.float32) * 2.0 - 1.0 + return torch.from_numpy(np.stack([solid, coating], axis=0)) + raise ValueError(f"Unsupported num_phases={num_phases}; expected 2 or 3") + + +def decode_channels_to_phases( + tensor: torch.Tensor | np.ndarray, + enforce_solid_support: bool = True, + threshold: float = 0.0, +) -> np.ndarray: + """Decode continuous diffusion output `[C, D, H, W]` or `[B, C, D, H, W]` to `uint8` phases.""" + arr = tensor.detach().cpu().numpy() if isinstance(tensor, torch.Tensor) else np.asarray(tensor) + if arr.ndim not in (4, 5): + raise ValueError(f"Expected 4D [C,D,H,W] or 5D [B,C,D,H,W] array, got {arr.shape}") + + c_axis = 0 if arr.ndim == 4 else 1 + n_ch = arr.shape[c_axis] + if n_ch == 1: + slc = arr[0] if arr.ndim == 4 else arr[:, 0] + return (slc > threshold).astype(np.uint8) + if n_ch == 2: + s_raw = arr[0] if arr.ndim == 4 else arr[:, 0] + c_raw = arr[1] if arr.ndim == 4 else arr[:, 1] + solid = s_raw > threshold + coat = c_raw > threshold + if enforce_solid_support: + coat = coat & solid + return (solid.astype(np.uint8) + coat.astype(np.uint8)).astype(np.uint8) + raise ValueError(f"Unsupported channel count {n_ch}; expected 1 or 2") + + +def resolve_patch_split_dir( + data_root: Path, + state_name: str, + patch_size: int, + version: str, + split: str, + patch_dir_name: str | None = None, +) -> Path: + """Resolve `_{binary,multiphase}_patches

_/` directory.""" + candidates: list[Path] = [] + if patch_dir_name: + candidates.append(data_root / patch_dir_name / split) + candidates.extend( + [ + data_root / f"{state_name}_binary_patches{patch_size}_{version}" / split, + data_root / f"{state_name}_multiphase_patches{patch_size}_{version}" / split, + ] + ) + for cand in candidates: + if cand.is_dir(): + return cand + return candidates[0] + + +def load_split_patches( + split_dir: Path, + expected_shape: tuple[int, int, int] | None = None, +) -> np.ndarray: + """Load `[N, D, H, W]` `uint8` patches from stacked `patches.npy` or individual `.npy` files.""" + stacked_path = split_dir / "patches.npy" + if stacked_path.is_file(): + arr = np.load(stacked_path) + if arr.ndim != 4 or arr.dtype != np.uint8: + raise ValueError(f"Expected 4D uint8 array in {stacked_path}, got {arr.shape} {arr.dtype}") + if expected_shape is not None and arr.shape[1:] != expected_shape: + raise ValueError(f"Expected patch shape {expected_shape} in {stacked_path}, got {arr.shape[1:]}") + return arr + + files = sorted( + p for p in split_dir.glob("*.npy") if p.name not in ("patches.npy", "tall_patches.npy") + ) + if not files: + raise RuntimeError(f"No patches found in {split_dir}") + + patches = [] + for path in files: + p = np.load(path) + if p.dtype != np.uint8: + raise ValueError(f"Expected uint8 patch in {path}, got {p.dtype}") + if expected_shape is not None and p.shape != expected_shape: + raise ValueError(f"Expected patch shape {expected_shape} in {path}, got {p.shape}") + patches.append(p) + return np.stack(patches, axis=0) + + +def load_tall_or_split_patches( + split_dir: Path, + fallback_patches: np.ndarray, +) -> np.ndarray: + """Load full-clean-height `[N, D_tall, H, W]` `tall_patches.npy` if available for mirror-free baselines.""" + tall_path = Path(split_dir) / "tall_patches.npy" + if tall_path.is_file(): + arr = np.load(tall_path) + if ( + arr.ndim == 4 + and arr.dtype == np.uint8 + and arr.shape[0] >= 1 + and arr.shape[1] >= fallback_patches.shape[1] + and arr.shape[2:] == fallback_patches.shape[2:] + ): + return arr + return fallback_patches + + +class LoadConditionedPatchDataset(Dataset): + """RAM-cached 3D microstructure dataset paired with continuous load values.""" + + def __init__( + self, + data_root: Path, + states: Mapping[str, float], + split: str, + patch_size: int = 96, + version: str = "carbon_calibrated_v1", + augment: bool = False, + num_phases: int = 2, + allow_z_flip: bool = True, + ) -> None: + if split not in ("train", "val"): + raise ValueError(f"Invalid split {split!r}; expected 'train' or 'val'") + if not states: + raise ValueError("states mapping must not be empty") + + self.data_root = Path(data_root) + self.states = {str(k): float(v) for k, v in states.items()} + self.split = split + self.patch_size = patch_size + self.version = version + self.augment = augment + self.num_phases = num_phases + self.allow_z_flip = allow_z_flip + + expected_shape = (patch_size, patch_size, patch_size) + volume_blocks: list[np.ndarray] = [] + load_blocks: list[np.ndarray] = [] + self.state_counts: dict[str, int] = {} + + for state_name, load_val in self.states.items(): + split_dir = resolve_patch_split_dir( + self.data_root, state_name, patch_size, version, split + ) + arr = load_split_patches(split_dir, expected_shape=expected_shape) + volume_blocks.append(arr) + load_blocks.append(np.full(arr.shape[0], load_val, dtype=np.float32)) + self.state_counts[state_name] = int(arr.shape[0]) + + self.patches = np.concatenate(volume_blocks, axis=0) + self.loads = np.concatenate(load_blocks, axis=0) + + def __len__(self) -> int: + return int(self.patches.shape[0]) + + def __getitem__(self, index: int) -> tuple[torch.Tensor, torch.Tensor]: + patch = self.patches[index] + if self.augment: + sym_id = random.randrange(16 if self.allow_z_flip else 8) + patch = apply_compression_safe_symmetry(patch, sym_id, allow_z_flip=self.allow_z_flip) + + image = encode_phases_to_channels(patch, num_phases=self.num_phases) + load_val = torch.tensor(float(self.loads[index]), dtype=torch.float32) + return image, load_val + + +class MultiElectrodePatchDataset(Dataset): + """Unified RAM-cached dataset across Virgin, TiO2-Anode, and CuHCF-Cathode scans. + + Each entry in `scan_specs` specifies: + - `state`: dataset state identifier + - `load_value`: scalar load or calibrated compressive stress (`kPa` or `mV/V`) + - `material`: `'virgin_carbon'`, `'tio2_anode'`, `'cuhcf_cathode'` (or int `0..2`) + - `voxel_size_um`: effective voxel size in micrometers (default `1.0`) + - `patch_dir` (optional): explicit patch directory override + """ + + def __init__( + self, + data_root: Path, + scan_specs: Sequence[Mapping[str, object]], + split: str, + patch_size: int = 96, + version: str = "carbon_calibrated_v1", + num_phases: int = 3, + augment: bool = False, + allow_z_flip: bool = True, + ) -> None: + if split not in ("train", "val"): + raise ValueError(f"Invalid split {split!r}; expected 'train' or 'val'") + if not scan_specs: + raise ValueError("scan_specs must not be empty") + + self.data_root = Path(data_root) + self.split = split + self.patch_size = patch_size + self.version = version + self.num_phases = num_phases + self.augment = augment + self.allow_z_flip = allow_z_flip + + expected_shape = (patch_size, patch_size, patch_size) + vol_blocks: list[np.ndarray] = [] + load_blocks: list[np.ndarray] = [] + mat_blocks: list[np.ndarray] = [] + vox_blocks: list[np.ndarray] = [] + group_ids: list[np.ndarray] = [] + self.state_counts: dict[str, int] = {} + self.material_counts: dict[str, int] = {} + + for group_idx, spec in enumerate(scan_specs): + state_name = str(spec["state"]) + load_val = float(spec["load_value"]) # type: ignore[arg-type] + mat_id = resolve_material_id(spec.get("material", "virgin_carbon")) # type: ignore[arg-type] + mat_name = ID_TO_MATERIAL[mat_id] + vox_um = float(spec.get("voxel_size_um", 1.0)) # type: ignore[arg-type] + + if "patch_dir" in spec and spec["patch_dir"] is not None: + split_dir = Path(str(spec["patch_dir"])) / split + else: + split_dir = resolve_patch_split_dir( + self.data_root, state_name, patch_size, version, split + ) + + arr = load_split_patches(split_dir, expected_shape=expected_shape) + n = int(arr.shape[0]) + vol_blocks.append(arr) + load_blocks.append(np.full(n, load_val, dtype=np.float32)) + mat_blocks.append(np.full(n, mat_id, dtype=np.int64)) + vox_blocks.append(np.full(n, vox_um, dtype=np.float32)) + group_ids.append(np.full(n, group_idx, dtype=np.int64)) + + self.state_counts[state_name] = n + self.material_counts[mat_name] = self.material_counts.get(mat_name, 0) + n + + self.patches = np.concatenate(vol_blocks, axis=0) + self.loads = np.concatenate(load_blocks, axis=0) + self.materials = np.concatenate(mat_blocks, axis=0) + self.voxel_sizes = np.concatenate(vox_blocks, axis=0) + self._group_ids = np.concatenate(group_ids, axis=0) + + def __len__(self) -> int: + return int(self.patches.shape[0]) + + def build_balanced_weights(self) -> torch.Tensor: + """Compute per-sample weights giving equal gradient share to each material class + and each state within a material (prevents 12-load Virgin from drowning 2-load Few-Shot).""" + weights = np.zeros(len(self), dtype=np.float64) + unique_mats = np.unique(self.materials) + for mat_id in unique_mats: + mat_mask = self.materials == mat_id + mat_groups = np.unique(self._group_ids[mat_mask]) + for g_id in mat_groups: + g_mask = mat_mask & (self._group_ids == g_id) + count = int(np.count_nonzero(g_mask)) + if count > 0: + weights[g_mask] = 1.0 / (len(unique_mats) * len(mat_groups) * count) + return torch.from_numpy(weights) + + def __getitem__( + self, index: int + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + patch = self.patches[index] + if self.augment: + sym_id = random.randrange(16 if self.allow_z_flip else 8) + patch = apply_compression_safe_symmetry(patch, sym_id, allow_z_flip=self.allow_z_flip) + + image = encode_phases_to_channels(patch, num_phases=self.num_phases) + load_val = torch.tensor(float(self.loads[index]), dtype=torch.float32) + mat_id = torch.tensor(int(self.materials[index]), dtype=torch.long) + vox_um = torch.tensor(float(self.voxel_sizes[index]), dtype=torch.float32) + return image, load_val, mat_id, vox_um + + +def capture_rng_states() -> dict[str, object]: + """Capture Python, NumPy, CPU Torch, and all CUDA Torch RNG states.""" + state: dict[str, object] = { + "rng_python": random.getstate(), + "rng_numpy": np.random.get_state(), + "rng_torch_cpu": torch.get_rng_state(), + } + if torch.cuda.is_available(): + state["rng_torch_cuda"] = [s.cpu() for s in torch.cuda.get_rng_state_all()] + return state + + +def restore_rng_states(state: Mapping[str, object]) -> None: + """Restore RNG states saved by `capture_rng_states` (cross-device tolerant).""" + if "rng_python" in state: + random.setstate(state["rng_python"]) # type: ignore[arg-type] + if "rng_numpy" in state: + np.random.set_state(state["rng_numpy"]) # type: ignore[arg-type] + if "rng_torch_cpu" in state: + torch.set_rng_state(state["rng_torch_cpu"].cpu()) # type: ignore[union-attr] + elif "rng_torch" in state: + torch.set_rng_state(state["rng_torch"].cpu()) # type: ignore[union-attr] + + if torch.cuda.is_available() and "rng_torch_cuda" in state: + saved_cuda = state["rng_torch_cuda"] + if isinstance(saved_cuda, list) and len(saved_cuda) == torch.cuda.device_count(): + torch.cuda.set_rng_state_all([s.cpu() for s in saved_cuda]) diff --git a/porousdiff/losses.py b/porousdiff/losses.py new file mode 100644 index 0000000..c5a1503 --- /dev/null +++ b/porousdiff/losses.py @@ -0,0 +1,279 @@ +"""Physics-informed 3D diffusion training objective & coating delamination diagnostics. + +Provides: + 1. Closed-form single-step clean phase reconstruction `x0_hat` from `v_prediction` or `epsilon` + on Zero-Terminal-SNR schedules without numerical blow-up at `t = T - 1`. + 2. `PhysicsInformedDiffusionLoss`: combines Min-SNR-gamma timestep-weighted diffusion MSE with + signal-weighted 3D interfacial gradient sharpness, hierarchical no-floating-coating support + enforcement (`ReLU(c_coating - c_solid)`), and solid-phase catalyst-to-scaffold loading + ratio conservation (`phi_coating / phi_solid`). + 3. `compute_coating_retention_diagnostics`: checks experimental XCT states across increasing + compressive stress to quantify catalyst retention vs. mechanical spallation/delamination. +""" + +from __future__ import annotations + +from typing import Mapping + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + + +def reconstruct_x0( + x_t: torch.Tensor, + model_pred: torch.Tensor, + timesteps: torch.Tensor, + alphas_cumprod: torch.Tensor, + prediction_type: str = "v_prediction", + clamp_range: tuple[float, float] | None = (-1.5, 1.5), +) -> torch.Tensor: + """Closed-form clean sample estimate `x0_hat` from noisy `x_t` and network prediction.""" + ab = alphas_cumprod.to(device=x_t.device, dtype=torch.float32)[timesteps] + while ab.ndim < x_t.ndim: + ab = ab.unsqueeze(-1) + + sqrt_ab = ab.clamp(min=0.0).sqrt() + sqrt_one_minus_ab = (1.0 - ab).clamp(min=0.0).sqrt() + + x_f = x_t.float() + p_f = model_pred.float() + + if prediction_type == "v_prediction": + x0_hat = sqrt_ab * x_f - sqrt_one_minus_ab * p_f + elif prediction_type == "epsilon": + x0_hat = (x_f - sqrt_one_minus_ab * p_f) / sqrt_ab.clamp(min=1e-3) + else: + raise ValueError(f"Unsupported prediction_type={prediction_type!r}") + + if clamp_range is not None: + x0_hat = x0_hat.clamp(clamp_range[0], clamp_range[1]) + return x0_hat + + +def min_snr_weights( + alphas_cumprod: torch.Tensor, + timesteps: torch.Tensor, + snr_gamma: float = 5.0, + prediction_type: str = "v_prediction", + zero_snr_floor: float = 0.05, +) -> torch.Tensor: + """Min-SNR-gamma timestep weights (Hang et al., ICCV 2023) adapted for Zero-Terminal-SNR.""" + if snr_gamma <= 0.0: + return torch.ones_like(timesteps, dtype=torch.float32) + + ab = alphas_cumprod.to(device=timesteps.device, dtype=torch.float32)[timesteps] + snr = ab / (1.0 - ab).clamp(min=1e-6) + clipped = torch.clamp(snr, max=float(snr_gamma)) + + if prediction_type == "v_prediction": + weights = clipped / (snr + 1.0) + return weights.clamp(min=float(zero_snr_floor)) + if prediction_type == "epsilon": + return (clipped / snr.clamp(min=1e-4)).clamp(min=float(zero_snr_floor)) + raise ValueError(f"Unsupported prediction_type={prediction_type!r}") + + +def spatial_gradient_l1( + x0_hat: torch.Tensor, + x0_true: torch.Tensor, + sample_weights: torch.Tensor, +) -> torch.Tensor: + """3D finite-difference interfacial sharpness loss along `Z`, `Y`, and `X`.""" + dz = ( + (x0_hat[:, :, 1:, :, :] - x0_hat[:, :, :-1, :, :]) + - (x0_true[:, :, 1:, :, :] - x0_true[:, :, :-1, :, :]) + ).abs().mean(dim=(1, 2, 3, 4)) + dy = ( + (x0_hat[:, :, :, 1:, :] - x0_hat[:, :, :, :-1, :]) + - (x0_true[:, :, :, 1:, :] - x0_true[:, :, :, :-1, :]) + ).abs().mean(dim=(1, 2, 3, 4)) + dx = ( + (x0_hat[:, :, :, :, 1:] - x0_hat[:, :, :, :, :-1]) + - (x0_true[:, :, :, :, 1:] - x0_true[:, :, :, :, :-1]) + ).abs().mean(dim=(1, 2, 3, 4)) + grad_err = (dz + dy + dx) / 3.0 + return (sample_weights * grad_err).mean() + + +def hierarchical_support_penalty( + x0_hat: torch.Tensor, + sample_weights: torch.Tensor, +) -> torch.Tensor: + """Penalize physically impossible floating catalyst coating (`c_coating > c_solid`).""" + if x0_hat.shape[1] < 2: + return torch.zeros((), device=x0_hat.device, dtype=torch.float32) + violation = F.relu(x0_hat[:, 1] - x0_hat[:, 0]).mean(dim=(1, 2, 3)) + return (sample_weights * violation).mean() + + +def catalyst_loading_conservation_loss( + x0_hat: torch.Tensor, + x0_true: torch.Tensor, + sample_weights: torch.Tensor, +) -> torch.Tensor: + """Conserve catalyst-to-total-solid volume ratio (`phi_coating / phi_solid`) per patch.""" + if x0_hat.shape[1] < 2: + return torch.zeros((), device=x0_hat.device, dtype=torch.float32) + + solid_hat = ((x0_hat[:, 0].clamp(-1.0, 1.0) + 1.0) * 0.5).mean(dim=(1, 2, 3)) + coat_hat = ((x0_hat[:, 1].clamp(-1.0, 1.0) + 1.0) * 0.5).mean(dim=(1, 2, 3)) + solid_true = ((x0_true[:, 0].clamp(-1.0, 1.0) + 1.0) * 0.5).mean(dim=(1, 2, 3)) + coat_true = ((x0_true[:, 1].clamp(-1.0, 1.0) + 1.0) * 0.5).mean(dim=(1, 2, 3)) + + ratio_hat = coat_hat / (solid_hat + 1e-3) + ratio_true = coat_true / (solid_true + 1e-3) + return (sample_weights * (ratio_hat - ratio_true).pow(2)).mean() + + +class PhysicsInformedDiffusionLoss(nn.Module): + """Composite physics-informed 3D porous-electrode diffusion training criterion.""" + + def __init__( + self, + alphas_cumprod: torch.Tensor, + prediction_type: str = "v_prediction", + snr_gamma: float = 5.0, + interface_weight: float = 0.05, + support_weight: float = 0.05, + loading_weight: float = 0.05, + ) -> None: + super().__init__() + self.register_buffer( + "alphas_cumprod", + alphas_cumprod.detach().clone().to(dtype=torch.float32), + persistent=False, + ) + self.prediction_type = prediction_type + self.snr_gamma = float(snr_gamma) + self.interface_weight = float(interface_weight) + self.support_weight = float(support_weight) + self.loading_weight = float(loading_weight) + + def forward( + self, + model_pred: torch.Tensor, + target: torch.Tensor, + x_t: torch.Tensor, + x_0: torch.Tensor, + timesteps: torch.Tensor, + ) -> tuple[torch.Tensor, dict[str, float]]: + """Compute total physics-informed diffusion loss and detached diagnostic components.""" + pred_f = model_pred.float() + targ_f = target.float() + x0_f = x_0.float() + + per_sample_mse = (pred_f - targ_f).pow(2).mean(dim=(1, 2, 3, 4)) + w_snr = min_snr_weights( + self.alphas_cumprod, + timesteps, + snr_gamma=self.snr_gamma, + prediction_type=self.prediction_type, + ) + loss_diff = (w_snr * per_sample_mse).mean() + + ab_weights = self.alphas_cumprod.to(device=x_t.device)[timesteps].clamp(min=0.05) + x0_hat = reconstruct_x0( + x_t=x_t, + model_pred=pred_f, + timesteps=timesteps, + alphas_cumprod=self.alphas_cumprod, + prediction_type=self.prediction_type, + ) + + loss_iface = ( + spatial_gradient_l1(x0_hat, x0_f, ab_weights) + if self.interface_weight > 0.0 + else torch.zeros_like(loss_diff) + ) + loss_support = ( + hierarchical_support_penalty(x0_hat, ab_weights) + if self.support_weight > 0.0 + else torch.zeros_like(loss_diff) + ) + loss_loading = ( + catalyst_loading_conservation_loss(x0_hat, x0_f, ab_weights) + if self.loading_weight > 0.0 + else torch.zeros_like(loss_diff) + ) + + total = ( + loss_diff + + self.interface_weight * loss_iface + + self.support_weight * loss_support + + self.loading_weight * loss_loading + ) + stats = { + "loss_total": float(total.detach().item()), + "loss_diffusion": float(loss_diff.detach().item()), + "loss_interface": float(loss_iface.detach().item()), + "loss_support": float(loss_support.detach().item()), + "loss_loading": float(loss_loading.detach().item()), + } + return total, stats + + +def compute_coating_retention_diagnostics( + benchmark_report: Mapping[str, object], + spallation_drop_threshold: float = 0.15, +) -> dict[str, dict[str, object]]: + """Quantify experimental catalyst-to-carbon mass retention vs. delamination across stress. + + Groups real XCT states by `(material, beamline)`, normalizes each state's + `coating_to_carbon_ratio` by the lowest-load reference scan, and flags potential + high-load coating spallation if the ratio drops by more than `spallation_drop_threshold`. + """ + grouped: dict[tuple[str, str], list[tuple[float, str, float]]] = {} + for st, raw_entry in benchmark_report.items(): + entry = raw_entry # type: ignore[assignment] + mat = str(entry.get("material", "virgin_carbon")) # type: ignore[union-attr] + beam = str(entry.get("beamline", "primary")) # type: ignore[union-attr] + lv = float(entry.get("load_value", 0.0)) # type: ignore[union-attr] + real_metrics = entry.get("real", {}).get("metrics", {}) # type: ignore[union-attr] + if "coating_to_carbon_ratio" not in real_metrics: + continue + ratio = float(real_metrics["coating_to_carbon_ratio"]["mean"]) + grouped.setdefault((mat, beam), []).append((lv, str(st), ratio)) + + diagnostics: dict[str, dict[str, object]] = {} + for (mat, beam), items in grouped.items(): + items.sort(key=lambda t: t[0]) + ref_lv, ref_st, ref_ratio = items[0] + group_key = f"{mat}__{beam}" + if ref_ratio <= 1e-6: + diagnostics[group_key] = { + "material": mat, + "beamline": beam, + "reference_state": ref_st, + "reference_load": ref_lv, + "is_coated": False, + "states": {}, + } + continue + + state_rows: dict[str, object] = {} + min_ret = 1.0 + for lv, st, ratio in items: + ret = float(ratio / ref_ratio) if np.isfinite(ratio) else float("nan") + if np.isfinite(ret): + min_ret = min(min_ret, ret) + state_rows[st] = { + "load_value": lv, + "coating_to_carbon_ratio": round(ratio, 6), + "retention_vs_uncompressed": round(ret, 6) if np.isfinite(ret) else float("nan"), + "possible_spallation": bool(np.isfinite(ret) and ret < (1.0 - spallation_drop_threshold)), + } + + diagnostics[group_key] = { + "material": mat, + "beamline": beam, + "reference_state": ref_st, + "reference_load": ref_lv, + "reference_coating_to_carbon_ratio": round(ref_ratio, 6), + "min_retention_across_loads": round(min_ret, 6), + "any_spallation_detected": bool(min_ret < (1.0 - spallation_drop_threshold)), + "is_coated": True, + "states": state_rows, + } + return diagnostics diff --git a/porousdiff/metrics.py b/porousdiff/metrics.py new file mode 100644 index 0000000..a11aae6 --- /dev/null +++ b/porousdiff/metrics.py @@ -0,0 +1,437 @@ +"""3D physical and morphological validation metrics for porous electrode volumes.""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Sequence + +import numpy as np +import tifffile +from scipy.ndimage import label +from scipy.sparse import coo_matrix +from scipy.sparse.csgraph import dijkstra + + +def validate_binary_volume(volume: np.ndarray) -> np.ndarray: + """Validate and cast a 3D `{0, 1}` microstructure volume `(Z, Y, X)`.""" + vol = np.asarray(volume) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got shape {vol.shape}") + return (vol > 0).astype(np.uint8) + + +def solid_fraction(volume: np.ndarray) -> float: + vol = validate_binary_volume(volume) + return float(vol.mean()) + + +def specific_surface_area(volume: np.ndarray, voxel_size: float = 1.0) -> dict[str, float]: + """Compute exposed pore-solid interfacial area per unit volume (`1 / voxel_size`).""" + if voxel_size <= 0: + raise ValueError("voxel_size must be positive") + vol = validate_binary_volume(volume) + n_voxels = float(vol.size) + + faces_z = float(np.count_nonzero(vol[1:, :, :] != vol[:-1, :, :])) + faces_y = float(np.count_nonzero(vol[:, 1:, :] != vol[:, :-1, :])) + faces_x = float(np.count_nonzero(vol[:, :, 1:] != vol[:, :, :-1])) + + total_faces = faces_z + faces_y + faces_x + scale = n_voxels * voxel_size + return { + "ssa_total": total_faces / scale, + "ssa_z": faces_z / scale, + "ssa_y": faces_y / scale, + "ssa_x": faces_x / scale, + } + + +def _chord_lengths_1d(lines_2d: np.ndarray, phase_val: int) -> np.ndarray: + """Vectorized run-length chord extraction along the last axis of a 2D array `(N_lines, L)`.""" + mask = (lines_2d == phase_val).astype(np.int8) + n_lines, length = mask.shape + if length == 0: + return np.empty(0, dtype=np.float32) + + padded = np.zeros((n_lines, length + 2), dtype=np.int8) + padded[:, 1:-1] = mask + diffs = np.diff(padded, axis=1) + + starts_r, starts_c = np.nonzero(diffs == 1) + ends_r, ends_c = np.nonzero(diffs == -1) + if starts_c.size == 0: + return np.empty(0, dtype=np.float32) + + # Because rows are sorted identically in `np.nonzero`, starts and ends pair 1-to-1 + assert np.array_equal(starts_r, ends_r) + chords = (ends_c - starts_c).astype(np.float32) + + # Prefer strictly interior (uncensored) chords if enough exist, else keep all chords + interior = (starts_c > 0) & (ends_c < length) + if np.count_nonzero(interior) >= 16: + return chords[interior] + return chords + + +def directional_chord_metrics(volume: np.ndarray, phase: int = 0) -> dict[str, float]: + """Mean chord lengths along Z (through-plane) and X, Y (in-plane) + anisotropy.""" + vol = validate_binary_volume(volume) + if phase not in (0, 1): + raise ValueError("phase must be 0 (pore) or 1 (solid)") + + # Transpose target axis to last dimension and flatten remaining 2 dims + z_lines = np.moveaxis(vol, 0, -1).reshape(-1, vol.shape[0]) + y_lines = np.moveaxis(vol, 1, -1).reshape(-1, vol.shape[1]) + x_lines = vol.reshape(-1, vol.shape[2]) + + cz = _chord_lengths_1d(z_lines, phase) + cy = _chord_lengths_1d(y_lines, phase) + cx = _chord_lengths_1d(x_lines, phase) + + mean_z = float(cz.mean()) if cz.size else 0.0 + mean_y = float(cy.mean()) if cy.size else 0.0 + mean_x = float(cx.mean()) if cx.size else 0.0 + inplane = 0.5 * (mean_x + mean_y) + anisotropy = float(inplane / mean_z) if mean_z > 1e-6 else 0.0 + + prefix = "pore" if phase == 0 else "solid" + return { + f"{prefix}_chord_z": mean_z, + f"{prefix}_chord_y": mean_y, + f"{prefix}_chord_x": mean_x, + f"{prefix}_anisotropy_xy_over_z": anisotropy, + } + + +def directional_two_point_correlation( + volume: np.ndarray, + max_lag: int | None = None, +) -> dict[str, np.ndarray]: + """Exact non-periodic two-point correlation `S_2(r)` along `Z`, `Y`, and `X`.""" + vol = validate_binary_volume(volume).astype(np.float64) + d, h, w = vol.shape + limit = min(d, h, w) // 2 if max_lag is None else min(max_lag, d - 1, h - 1, w - 1) + + curves: dict[str, np.ndarray] = {"lags": np.arange(limit + 1, dtype=np.int32)} + for axis, name in ((0, "s2_z"), (1, "s2_y"), (2, "s2_x")): + n = vol.shape[axis] + pad_len = 1 << (2 * n - 1).bit_length() + spec = np.fft.rfft(vol, n=pad_len, axis=axis) + ac = np.fft.irfft(spec * np.conj(spec), n=pad_len, axis=axis) + + slc = [slice(None)] * 3 + slc[axis] = slice(0, limit + 1) + summed = ac[tuple(slc)].sum(axis=tuple(i for i in range(3) if i != axis)) + perp_count = float(vol.size // n) + valid_pairs = (n - np.arange(limit + 1, dtype=np.float64)) * perp_count + curves[name] = (summed / np.maximum(valid_pairs, 1.0)).astype(np.float32) + + return curves + + +def connectivity_and_percolation(volume: np.ndarray) -> dict[str, float]: + """3D 6-connected solid network integrity and directional pore percolation.""" + vol = validate_binary_volume(volume) + struct6 = np.zeros((3, 3, 3), dtype=np.uint8) + struct6[1, 1, :] = 1 + struct6[1, :, 1] = 1 + struct6[:, 1, 1] = 1 + + # Solid network connectivity + total_solid = int(vol.sum()) + if total_solid == 0: + largest_solid_frac = 0.0 + else: + solid_labels, n_solid = label(vol == 1, structure=struct6) + if n_solid == 0: + largest_solid_frac = 0.0 + else: + counts = np.bincount(solid_labels.ravel())[1:] + largest_solid_frac = float(counts.max() / total_solid) + + # Pore percolation across opposite faces + pore_labels, _ = label(vol == 0, structure=struct6) + + def _spans(lbl_a: np.ndarray, lbl_b: np.ndarray) -> float: + a = set(np.unique(lbl_a)) - {0} + b = set(np.unique(lbl_b)) - {0} + return 1.0 if (a & b) else 0.0 + + return { + "solid_connected_fraction": largest_solid_frac, + "pore_percolates_z": _spans(pore_labels[0, :, :], pore_labels[-1, :, :]), + "pore_percolates_y": _spans(pore_labels[:, 0, :], pore_labels[:, -1, :]), + "pore_percolates_x": _spans(pore_labels[:, :, 0], pore_labels[:, :, -1]), + } + + +def _slice_pair(length: int, delta: int) -> tuple[slice, slice]: + if delta > 0: + return slice(0, length - delta), slice(delta, length) + if delta < 0: + return slice(-delta, length), slice(0, length + delta) + return slice(0, length), slice(0, length) + + +def geodesic_pore_tortuosity( + volume: np.ndarray, + axis: int = 0, + connectivity: int = 26, +) -> float: + """26-connected Euclidean shortest-path geometric tortuosity across `axis`. + + Uses exact `1.0`, `sqrt(2)`, and `sqrt(3)` voxel-center Euclidean step weights + via compiled Cython `scipy.sparse.csgraph.dijkstra` (eliminating the `~40%` Manhattan + staircase bias on diagonal `45 deg` channels while returning `nan` when non-percolating). + """ + if connectivity not in (6, 26): + raise ValueError(f"connectivity must be 6 or 26, got {connectivity}") + vol = validate_binary_volume(volume) + pore = np.moveaxis(vol == 0, axis, 0) + depth, height, width = pore.shape + if depth < 2: + return 1.0 + if not (pore[0].any() and pore[-1].any()): + return float("nan") + + idx_map = np.full(pore.shape, -1, dtype=np.int32) + n_pore = int(np.count_nonzero(pore)) + idx_map[pore] = np.arange(n_pore, dtype=np.int32) + + inlet_ids = idx_map[0][idx_map[0] >= 0] + outlet_ids = idx_map[-1][idx_map[-1] >= 0] + if inlet_ids.size == 0 or outlet_ids.size == 0: + return float("nan") + + forward_offsets: list[tuple[int, int, int, float]] = [] + for dz in (-1, 0, 1): + for dy in (-1, 0, 1): + for dx in (-1, 0, 1): + if (dz, dy, dx) == (0, 0, 0): + continue + if dz < 0 or (dz == 0 and dy < 0) or (dz == 0 and dy == 0 and dx < 0): + continue + norm_sq = dz * dz + dy * dy + dx * dx + if connectivity == 6 and norm_sq > 1: + continue + forward_offsets.append((dz, dy, dx, math.sqrt(float(norm_sq)))) + + rows_list: list[np.ndarray] = [np.full(inlet_ids.shape, n_pore, dtype=np.int32)] + cols_list: list[np.ndarray] = [inlet_ids] + weights_list: list[np.ndarray] = [np.zeros(inlet_ids.shape, dtype=np.float32)] + + for dz, dy, dx, step_w in forward_offsets: + sz0, sz1 = _slice_pair(depth, dz) + sy0, sy1 = _slice_pair(height, dy) + sx0, sx1 = _slice_pair(width, dx) + u = idx_map[sz0, sy0, sx0] + v = idx_map[sz1, sy1, sx1] + valid = (u >= 0) & (v >= 0) + if np.any(valid): + u_v = u[valid] + v_v = v[valid] + rows_list.append(u_v) + cols_list.append(v_v) + weights_list.append(np.full(u_v.shape, step_w, dtype=np.float32)) + + rows = np.concatenate(rows_list) + cols = np.concatenate(cols_list) + wts = np.concatenate(weights_list) + graph = coo_matrix((wts, (rows, cols)), shape=(n_pore + 1, n_pore + 1)).tocsr() + + dists = dijkstra(csgraph=graph, directed=False, indices=n_pore) + outlet_dists = dists[outlet_ids] + reached = outlet_dists[np.isfinite(outlet_dists)] + if reached.size == 0: + return float("nan") + return float(reached.mean() / float(depth - 1)) + + +def _count_tpb_edges_along_axis(vol: np.ndarray, axis: int) -> int: + """Count voxel grid edges along `axis` whose 2x2 incident voxels contain {0, 1, 2}.""" + v = np.moveaxis(vol, axis, 0) + # Incident 2x2 voxels around each interior grid line parallel to axis 0 + c00 = v[:, :-1, :-1] + c01 = v[:, :-1, 1:] + c10 = v[:, 1:, :-1] + c11 = v[:, 1:, 1:] + + has0 = (c00 == 0) | (c01 == 0) | (c10 == 0) | (c11 == 0) + has1 = (c00 == 1) | (c01 == 1) | (c10 == 1) | (c11 == 1) + has2 = (c00 == 2) | (c01 == 2) | (c10 == 2) | (c11 == 2) + return int(np.count_nonzero(has0 & has1 & has2)) + + +def multiphase_interfacial_metrics(volume: np.ndarray, voxel_size: float = 1.0) -> dict[str, float]: + """Compute 3-phase coated-electrode electrochemical descriptors (`0=pore, 1=carbon, 2=coating`). + + Returns: + - `carbon_fraction`, `coating_fraction`, `coating_to_carbon_ratio` + - `ssa_pore_coating`: Electrochemically Active Surface Area (EASA, 1/voxel_size) + - `ssa_pore_carbon`: Bare carbon-electrolyte surface area (1/voxel_size) + - `ssa_carbon_coating`: Electronic current-collector/coating contact area (1/voxel_size) + - `tpb_density`: Triple-Phase Boundary length per unit volume (1/voxel_size^2) + - `wired_coating_fraction`: Fraction of active coating electrically wired to Z boundaries + - `specific_z_pinch_excess`: Z-contact probability divided by `solid_fraction^2` + """ + if voxel_size <= 0: + raise ValueError("voxel_size must be positive") + vol = np.asarray(volume, dtype=np.uint8) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got shape {vol.shape}") + + n_voxels = float(vol.size) + area_norm = n_voxels * voxel_size + carbon_frac = float((vol == 1).mean()) + coat_frac = float((vol == 2).mean()) + solid_frac = carbon_frac + coat_frac + + def _pair_faces(a: int, b: int) -> float: + total = 0 + for sl1, sl2 in ( + (vol[1:, :, :], vol[:-1, :, :]), + (vol[:, 1:, :], vol[:, :-1, :]), + (vol[:, :, 1:], vol[:, :, :-1]), + ): + total += int(np.count_nonzero(((sl1 == a) & (sl2 == b)) | ((sl1 == b) & (sl2 == a)))) + return float(total) / area_norm + + tpb_edges = ( + _count_tpb_edges_along_axis(vol, 0) + + _count_tpb_edges_along_axis(vol, 1) + + _count_tpb_edges_along_axis(vol, 2) + ) + tpb_density = float(tpb_edges) / (n_voxels * (voxel_size**2)) + + # Electrical percolation of active coating via 6-connected solid backbone (phase >= 1) to Z=0 or Z=D-1 + total_coat = int(np.count_nonzero(vol == 2)) + if total_coat == 0: + wired_frac = 0.0 + else: + struct6 = np.zeros((3, 3, 3), dtype=np.uint8) + struct6[1, 1, :] = 1 + struct6[1, :, 1] = 1 + struct6[:, 1, 1] = 1 + solid_lbl, _ = label(vol >= 1, structure=struct6) + boundary_ids = (set(np.unique(solid_lbl[0, :, :])) | set(np.unique(solid_lbl[-1, :, :]))) - {0} + if not boundary_ids: + wired_frac = 0.0 + else: + wired_mask = np.isin(solid_lbl, list(boundary_ids)) & (vol == 2) + wired_frac = float(np.count_nonzero(wired_mask) / total_coat) + + # 3D compressive pinch-point & solid contact coordination (links to K11 DIAD XRD microstrain) + solid_mask = vol >= 1 + z_solid_faces = int(np.count_nonzero(solid_mask[1:, :, :] & solid_mask[:-1, :, :])) + z_pinch_density = float(z_solid_faces) / area_norm + n_z_pairs = max(1, (vol.shape[0] - 1) * vol.shape[1] * vol.shape[2]) + z_pair_prob = float(z_solid_faces) / float(n_z_pairs) + specific_z_pinch_excess = ( + z_pair_prob / max(solid_frac**2, 1e-6) if solid_frac > 1e-6 else 0.0 + ) + + if total_coat == 0 or vol.shape[0] < 3: + coat_pinch_frac = 0.0 + else: + sandwiched_coat = solid_mask[:-2, :, :] & (vol[1:-1, :, :] == 2) & solid_mask[2:, :, :] + coat_pinch_frac = float(np.count_nonzero(sandwiched_coat) / total_coat) + + if total_coat == 0: + mean_coat_coord = 0.0 + else: + padded_solid = np.pad(solid_mask.astype(np.uint8), 1, mode="constant", constant_values=0) + coord6 = ( + padded_solid[2:, 1:-1, 1:-1] + + padded_solid[:-2, 1:-1, 1:-1] + + padded_solid[1:-1, 2:, 1:-1] + + padded_solid[1:-1, :-2, 1:-1] + + padded_solid[1:-1, 1:-1, 2:] + + padded_solid[1:-1, 1:-1, :-2] + ) + mean_coat_coord = float(coord6[vol == 2].mean()) + + return { + "carbon_fraction": carbon_frac, + "coating_fraction": coat_frac, + "coating_to_carbon_ratio": coat_frac / max(carbon_frac, 1e-8) if coat_frac > 0 else 0.0, + "ssa_pore_coating": _pair_faces(0, 2), + "ssa_pore_carbon": _pair_faces(0, 1), + "ssa_carbon_coating": _pair_faces(1, 2), + "tpb_density": tpb_density, + "wired_coating_fraction": wired_frac, + "z_pinch_contact_density": z_pinch_density, + "specific_z_pinch_excess": specific_z_pinch_excess, + "coating_pinch_fraction": coat_pinch_frac, + "mean_coating_coordination": mean_coat_coord, + } + + +def evaluate_volume_physics(volume: np.ndarray, voxel_size: float = 1.0) -> dict[str, float]: + """Compute the full scalar physics validation suite on a single 3D binary volume.""" + vol = validate_binary_volume(volume) + sf = solid_fraction(vol) + metrics: dict[str, float] = { + "solid_fraction": sf, + "porosity": 1.0 - sf, + } + metrics.update(specific_surface_area(vol, voxel_size=voxel_size)) + metrics.update(directional_chord_metrics(vol, phase=0)) + metrics.update(directional_chord_metrics(vol, phase=1)) + metrics.update(connectivity_and_percolation(vol)) + metrics["geodesic_tortuosity_z"] = geodesic_pore_tortuosity(vol, axis=0) + metrics["geodesic_tortuosity_y"] = geodesic_pore_tortuosity(vol, axis=1) + metrics["geodesic_tortuosity_x"] = geodesic_pore_tortuosity(vol, axis=2) + metrics["geodesic_solid_tortuosity_z"] = geodesic_pore_tortuosity(1 - vol, axis=0) + return metrics + + +def evaluate_multiphase_physics(volume: np.ndarray, voxel_size: float = 1.0) -> dict[str, float]: + """Compute combined pore-transport + 3-phase electrochemical metrics on `{0, 1, 2}` volumes.""" + vol = np.asarray(volume, dtype=np.uint8) + metrics = evaluate_volume_physics(vol >= 1, voxel_size=voxel_size) + metrics.update(multiphase_interfacial_metrics(vol, voxel_size=voxel_size)) + return metrics + + +def summarize_metric_records(records: Sequence[dict[str, float]]) -> dict[str, dict[str, float]]: + """Aggregate `{mean, std, min, max, valid_n}` across multiple 3D realizations.""" + if not records: + return {} + keys = list(records[0].keys()) + summary: dict[str, dict[str, float]] = {} + for k in keys: + vals = np.array([r[k] for r in records], dtype=np.float64) + finite = vals[np.isfinite(vals)] + if finite.size == 0: + summary[k] = {"mean": float("nan"), "std": float("nan"), "min": float("nan"), "max": float("nan"), "valid_n": 0.0} + else: + summary[k] = { + "mean": float(finite.mean()), + "std": float(finite.std()), + "min": float(finite.min()), + "max": float(finite.max()), + "valid_n": float(finite.size), + } + return summary + + +def export_openimpala_tiff( + volume: np.ndarray, + output_path: Path, + pore_label: int = 0, + solid_label: int = 1, + preserve_multiphase: bool = False, +) -> Path: + """Export a 3D multi-page `uint8` TIFF stack formatted for OpenImpala transport solvers.""" + vol = np.asarray(volume) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got shape {vol.shape}") + if preserve_multiphase: + mapped = np.clip(vol, 0, 255).astype(np.uint8) + else: + mapped = np.where(vol > 0, solid_label, pore_label).astype(np.uint8) + output_path = Path(output_path) + output_path.parent.mkdir(parents=True, exist_ok=True) + tifffile.imwrite(output_path, mapped) + return output_path diff --git a/porousdiff/models.py b/porousdiff/models.py new file mode 100644 index 0000000..0d22755 --- /dev/null +++ b/porousdiff/models.py @@ -0,0 +1,491 @@ +"""Multi-electrode, multi-phase, continuous stress & resolution conditioned 3D Diffusion UNet.""" + +from __future__ import annotations + +import copy +import inspect +import math +from typing import Mapping, Sequence + +import torch +import torch.nn as nn + +try: + from monai.networks.nets import DiffusionModelUNet + from monai.networks.schedulers import DDIMScheduler, DDPMScheduler + from monai.networks.schedulers.scheduler import NoiseSchedules +except ImportError: # pragma: no cover - legacy fallback for monai-generative<=0.2.3 + from generative.networks.nets import DiffusionModelUNet + from generative.networks.schedulers import DDIMScheduler, DDPMScheduler + from generative.networks.schedulers.scheduler import NoiseSchedules + + +def _enforce_zero_terminal_snr(betas: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Rescale beta schedule to have exact zero terminal SNR (Lin et al., WACV 2024). + + Eliminates residual signal leakage at `t = T - 1` so sampling from zero-mean + Gaussian noise does not bias strongly porous microstructures (`~5-35%` solid) + upward toward `50%` solid fraction. + """ + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + alphas_bar_sqrt = alphas_bar_sqrt - alphas_bar_sqrt_T + alphas_bar_sqrt = alphas_bar_sqrt * ( + alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + ) + + alphas_bar = alphas_bar_sqrt**2 + alphas_new = torch.empty_like(alphas_bar) + alphas_new[0] = alphas_bar[0] + alphas_new[1:] = alphas_bar[1:] / alphas_bar[:-1].clamp(min=1e-12) + betas_new = (1.0 - alphas_new).clamp(0.0, 1.0) + return betas_new, alphas_new, alphas_bar + + +if "zero_snr_scaled_linear" not in NoiseSchedules: + + @NoiseSchedules.add_def( + "zero_snr_scaled_linear", + "Zero-terminal-SNR scaled linear schedule for unbiased porous media generation", + ) + def _zero_snr_scaled_linear( + num_train_timesteps: int, + beta_start: float = 0.00085, + beta_end: float = 0.012, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + base_betas = ( + torch.linspace( + beta_start**0.5, + beta_end**0.5, + num_train_timesteps, + dtype=torch.float32, + ) + ** 2 + ) + return _enforce_zero_terminal_snr(base_betas) + + +def build_noise_scheduler( + sampler: str = "ddpm", + num_train_timesteps: int = 1000, + schedule: str = "zero_snr_scaled_linear", + prediction_type: str = "v_prediction", +) -> DDPMScheduler | DDIMScheduler: + """Construct a MONAI diffusion scheduler configured for unbiased porous volumes.""" + kind = sampler.lower() + if kind == "ddpm": + return DDPMScheduler( + num_train_timesteps=num_train_timesteps, + schedule=schedule, + prediction_type=prediction_type, + clip_sample=True, + clip_sample_min=-1.0, + clip_sample_max=1.0, + ) + if kind == "ddim": + return DDIMScheduler( + num_train_timesteps=num_train_timesteps, + schedule=schedule, + prediction_type=prediction_type, + clip_sample=True, + clip_sample_min=-1.0, + clip_sample_max=1.0, + set_alpha_to_one=False, + ) + raise ValueError(f"Unknown sampler {sampler!r}; choose 'ddpm' or 'ddim'") + + +class ContinuousScalarLoadEmbedder(nn.Module): + """Smooth Fourier/sinusoidal scalar conditioner with a learned null token for CFG.""" + + def __init__(self, freq_dim: int, embed_dim: int, max_period: float = 16.0) -> None: + super().__init__() + if freq_dim % 2 != 0: + raise ValueError("freq_dim must be even") + self.freq_dim = freq_dim + self.embed_dim = embed_dim + + half = freq_dim // 2 + freqs = torch.exp( + torch.linspace(math.log(0.5 * math.pi), math.log(max_period * math.pi), half) + ) + self.register_buffer("freqs", freqs, persistent=False) + + self.mlp = nn.Sequential( + nn.Linear(freq_dim + 1, embed_dim), + nn.SiLU(), + nn.Linear(embed_dim, embed_dim), + ) + self.null_embedding = nn.Parameter(torch.zeros(1, embed_dim)) + + def forward(self, packed: torch.Tensor) -> torch.Tensor: + """Accepts `packed` of shape `(B, 2)` containing `[cond_norm, cond_mask]`.""" + cond_norm = packed[:, 0:1].float() + cond_mask = packed[:, 1:2].float() + + angles = cond_norm * self.freqs.unsqueeze(0) + fourier = torch.cat([cond_norm, torch.sin(angles), torch.cos(angles)], dim=-1) + cond_emb = self.mlp(fourier.to(self.null_embedding.dtype)) + null_emb = self.null_embedding.expand(cond_emb.shape[0], -1) + mask = cond_mask.to(cond_emb.dtype) + return mask * cond_emb + (1.0 - mask) * null_emb + + +class MultiModalConditionEmbedder(nn.Module): + """Joint embedder for continuous compressive stress/load, electrode chemistry ID, + and physical voxel size (`log(dx / dx_ref)`), with learned CFG null embedding.""" + + def __init__( + self, + num_materials: int, + freq_dim: int, + embed_dim: int, + max_period: float = 16.0, + ) -> None: + super().__init__() + if freq_dim % 2 != 0: + raise ValueError("freq_dim must be even") + if num_materials < 1: + raise ValueError("num_materials must be >= 1") + + self.num_materials = num_materials + self.freq_dim = freq_dim + self.embed_dim = embed_dim + + half = freq_dim // 2 + freqs = torch.exp( + torch.linspace(math.log(0.5 * math.pi), math.log(max_period * math.pi), half) + ) + self.register_buffer("freqs", freqs, persistent=False) + + self.material_embedding = nn.Embedding(num_materials, embed_dim) + nn.init.normal_(self.material_embedding.weight, std=0.02) + + # Concatenate Fourier(load) [freq_dim + 1] + Fourier(log_voxel) [freq_dim + 1] + self.scalar_mlp = nn.Sequential( + nn.Linear(2 * (freq_dim + 1), embed_dim), + nn.SiLU(), + nn.Linear(embed_dim, embed_dim), + ) + self.fusion_mlp = nn.Sequential( + nn.SiLU(), + nn.Linear(embed_dim, embed_dim), + ) + self.null_embedding = nn.Parameter(torch.zeros(1, embed_dim)) + + def _fourier_encode(self, val: torch.Tensor) -> torch.Tensor: + angles = val * self.freqs.unsqueeze(0) + return torch.cat([val, torch.sin(angles), torch.cos(angles)], dim=-1) + + def forward(self, packed: torch.Tensor) -> torch.Tensor: + """Accepts `packed` of shape `(B, 4)`: `[load_norm, mat_id, log_vox, cond_mask]`.""" + load_norm = packed[:, 0:1].float() + mat_ids = packed[:, 1].round().long().clamp(0, self.num_materials - 1) + log_vox = packed[:, 2:3].float() + cond_mask = packed[:, 3:4].float() + + dtype = self.null_embedding.dtype + scalar_feat = torch.cat( + [self._fourier_encode(load_norm), self._fourier_encode(log_vox)], dim=-1 + ).to(dtype) + fused = self.fusion_mlp(self.scalar_mlp(scalar_feat) + self.material_embedding(mat_ids)) + null_emb = self.null_embedding.expand(fused.shape[0], -1) + mask = cond_mask.to(dtype) + return mask * fused + (1.0 - mask) * null_emb + + +class ContinuousLoadConditionedUNet(nn.Module): + """3D Diffusion UNet conditioned on continuous compression load, electrode material, + and physical voxel resolution across 2-phase (`1 ch`) or 3-phase (`2 ch`) microstructures. + + Defaults (`num_phases=2, num_materials=1`) preserve exact backward compatibility with + single-material binary checkpoints while `num_phases=3` or `num_materials>1` activates + hierarchical multi-phase + multi-electrode conditioning. + """ + + def __init__( + self, + channels: Sequence[int] = (32, 64, 128), + attention_levels: Sequence[bool] = (False, False, True), + num_res_blocks: int = 2, + norm_num_groups: int = 16, + load_min: float = 0.008, + load_max: float = 0.053, + use_flash_attention: bool = False, + conditioning_mode: str = "multiscale_fourier", + num_phases: int = 2, + num_materials: int = 1, + ref_voxel_size_um: float = 1.0, + use_ordinal_material_channel: bool = False, + ) -> None: + super().__init__() + if load_max <= load_min: + raise ValueError("load_max must be strictly greater than load_min") + if conditioning_mode not in ("multiscale_fourier", "legacy_crossattn"): + raise ValueError( + f"Unknown conditioning_mode {conditioning_mode!r}; " + "choose 'multiscale_fourier' or 'legacy_crossattn'" + ) + if num_phases not in (2, 3): + raise ValueError(f"num_phases must be 2 or 3, got {num_phases}") + if num_materials < 1: + raise ValueError(f"num_materials must be >= 1, got {num_materials}") + if ref_voxel_size_um <= 0: + raise ValueError("ref_voxel_size_um must be positive") + + self.conditioning_mode = conditioning_mode + self.num_phases = num_phases + self.num_materials = num_materials + self.data_channels = 1 if num_phases == 2 else 2 + self.is_multimodal = (num_phases == 3) or (num_materials > 1) + self.use_ordinal_material_channel = bool(use_ordinal_material_channel) + + unet_params = inspect.signature(DiffusionModelUNet.__init__).parameters + if conditioning_mode == "multiscale_fourier": + aux_channels = (4 if self.use_ordinal_material_channel else 3) if self.is_multimodal else 2 + unet_kwargs: dict[str, object] = { + "spatial_dims": 3, + "in_channels": self.data_channels + aux_channels, + "out_channels": self.data_channels, + "attention_levels": tuple(attention_levels), + "num_res_blocks": num_res_blocks, + "norm_num_groups": norm_num_groups, + "with_conditioning": False, + "num_class_embeds": 1, + } + else: + unet_kwargs = { + "spatial_dims": 3, + "in_channels": self.data_channels, + "out_channels": self.data_channels, + "attention_levels": tuple(attention_levels), + "num_res_blocks": num_res_blocks, + "norm_num_groups": norm_num_groups, + "with_conditioning": True, + "cross_attention_dim": 32, + } + + if "channels" in unet_params: + unet_kwargs["channels"] = tuple(channels) + else: + unet_kwargs["num_channels"] = tuple(channels) + + if use_flash_attention and "use_flash_attention" in unet_params: + unet_kwargs["use_flash_attention"] = True + + self.unet = DiffusionModelUNet(**unet_kwargs) + base_ch = int(channels[0]) + time_embed_dim = base_ch * 4 + if conditioning_mode == "multiscale_fourier": + if self.is_multimodal: + self.unet.class_embedding = MultiModalConditionEmbedder( + num_materials=num_materials, + freq_dim=max(32, base_ch), + embed_dim=time_embed_dim, + ) + else: + self.unet.class_embedding = ContinuousScalarLoadEmbedder( + freq_dim=max(32, base_ch), + embed_dim=time_embed_dim, + ) + self.legacy_embedder: nn.Module | None = None + else: + legacy_in_dim = 3 if self.is_multimodal else 1 + self.legacy_embedder = nn.Sequential( + nn.Linear(legacy_in_dim, 32), + nn.SiLU(), + nn.Linear(32, 32), + ) + + self.register_buffer("load_min", torch.tensor(float(load_min), dtype=torch.float32)) + self.register_buffer("load_max", torch.tensor(float(load_max), dtype=torch.float32)) + self.register_buffer( + "ref_voxel_size_um", torch.tensor(float(ref_voxel_size_um), dtype=torch.float32) + ) + + def normalize_load(self, load_values: torch.Tensor) -> torch.Tensor: + """Linearly map physical load/stress values into `[-1, 1]`.""" + span = (self.load_max - self.load_min).clamp(min=1e-8) + return 2.0 * (load_values.float() - self.load_min) / span - 1.0 + + def normalize_voxel_size(self, voxel_sizes: torch.Tensor) -> torch.Tensor: + """Map physical voxel size into log-ratio `log(dx / dx_ref)`.""" + safe = voxel_sizes.float().clamp(min=1e-4) + return torch.log(safe / self.ref_voxel_size_um.clamp(min=1e-4)) + + def forward( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + load_values: torch.Tensor, + cond_mask: torch.Tensor | None = None, + material_ids: torch.Tensor | None = None, + voxel_sizes: torch.Tensor | None = None, + ) -> torch.Tensor: + """Predict velocity `v` (or `epsilon`) for noisy 3D volume `x` `[B, C, D, H, W]`.""" + b, _, d, h, w = x.shape + loads = load_values.view(b).to(device=x.device, dtype=torch.float32) + cond_norm = self.normalize_load(loads) + + if cond_mask is None: + mask = torch.ones(b, device=x.device, dtype=torch.float32) + else: + mask = cond_mask.view(b).to(device=x.device, dtype=torch.float32) + + if not self.is_multimodal: + if self.conditioning_mode == "legacy_crossattn": + assert self.legacy_embedder is not None + masked_scalar = (cond_norm * mask).unsqueeze(-1) + ctx = self.legacy_embedder(masked_scalar).unsqueeze(1).to(dtype=x.dtype) + return self.unet(x=x, timesteps=timesteps, context=ctx) + + masked_cond = (cond_norm * mask).to(dtype=x.dtype) + mask_ch = mask.to(dtype=x.dtype) + cond_vol = masked_cond.view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + mask_vol = mask_ch.view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + unet_in = torch.cat([x, cond_vol, mask_vol], dim=1) + packed_labels = torch.stack([cond_norm, mask], dim=-1) + return self.unet(x=unet_in, timesteps=timesteps, class_labels=packed_labels) + + # Multi-modal / 3-phase forward path: material chemistry conditioned purely via + # learned categorical embedding in MultiModalConditionEmbedder unless legacy flag set. + if material_ids is None: + mats = torch.zeros(b, device=x.device, dtype=torch.long) + else: + mats = material_ids.view(b).to(device=x.device, dtype=torch.long) + + if voxel_sizes is None: + vox = self.ref_voxel_size_um.expand(b).to(device=x.device, dtype=torch.float32) + else: + vox = voxel_sizes.view(b).to(device=x.device, dtype=torch.float32) + log_vox = self.normalize_voxel_size(vox) + + mat_span = max(1, self.num_materials - 1) + mat_norm = 2.0 * (mats.float() / float(mat_span)) - 1.0 if self.num_materials > 1 else torch.zeros_like(cond_norm) + + if self.conditioning_mode == "legacy_crossattn": + assert self.legacy_embedder is not None + vec = torch.stack([cond_norm * mask, mat_norm * mask, log_vox * mask], dim=-1) + ctx = self.legacy_embedder(vec).unsqueeze(1).to(dtype=x.dtype) + return self.unet(x=x, timesteps=timesteps, context=ctx) + + c_load = (cond_norm * mask).to(dtype=x.dtype).view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + c_vox = (log_vox * mask).to(dtype=x.dtype).view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + c_msk = mask.to(dtype=x.dtype).view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + if self.use_ordinal_material_channel: + c_mat = (mat_norm * mask).to(dtype=x.dtype).view(b, 1, 1, 1, 1).expand(b, 1, d, h, w) + unet_in = torch.cat([x, c_load, c_mat, c_vox, c_msk], dim=1) + else: + unet_in = torch.cat([x, c_load, c_vox, c_msk], dim=1) + + packed_mm = torch.stack([cond_norm, mats.float(), log_vox, mask], dim=-1) + return self.unet(x=unet_in, timesteps=timesteps, class_labels=packed_mm) + + def forward_cfg( + self, + x: torch.Tensor, + timesteps: torch.Tensor, + load_values: torch.Tensor, + guidance_scale: float = 1.0, + material_ids: torch.Tensor | None = None, + voxel_sizes: torch.Tensor | None = None, + ) -> torch.Tensor: + """Classifier-free guided prediction combining conditional and unconditional passes.""" + b = x.shape[0] + if abs(guidance_scale - 1.0) < 1e-6: + ones = torch.ones(b, device=x.device, dtype=torch.float32) + return self.forward( + x, + timesteps, + load_values, + cond_mask=ones, + material_ids=material_ids, + voxel_sizes=voxel_sizes, + ) + + x_cat = torch.cat([x, x], dim=0) + t_cat = torch.cat([timesteps, timesteps], dim=0) + loads_cat = torch.cat([load_values.view(b), load_values.view(b)], dim=0) + mats_cat = ( + torch.cat([material_ids.view(b), material_ids.view(b)], dim=0) + if material_ids is not None + else None + ) + vox_cat = ( + torch.cat([voxel_sizes.view(b), voxel_sizes.view(b)], dim=0) + if voxel_sizes is not None + else None + ) + mask_cat = torch.cat( + [ + torch.ones(b, device=x.device, dtype=torch.float32), + torch.zeros(b, device=x.device, dtype=torch.float32), + ], + dim=0, + ) + pred_both = self.forward( + x_cat, + t_cat, + loads_cat, + cond_mask=mask_cat, + material_ids=mats_cat, + voxel_sizes=vox_cat, + ) + pred_cond, pred_uncond = pred_both.chunk(2, dim=0) + return pred_uncond + guidance_scale * (pred_cond - pred_uncond) + + +class EMAModel: + """Exponential Moving Average of model parameters with early-step warmup.""" + + def __init__(self, model: nn.Module, decay: float = 0.9995) -> None: + if not 0.0 < decay < 1.0: + raise ValueError("EMA decay must be in (0, 1)") + self.decay = decay + self.num_updates = 0 + self.shadow_model = copy.deepcopy(model).eval() + for p in self.shadow_model.parameters(): + p.requires_grad_(False) + + @torch.no_grad() + def update(self, model: nn.Module) -> None: + self.num_updates += 1 + eff_decay = min(self.decay, (1.0 + self.num_updates) / (10.0 + self.num_updates)) + for s_param, param in zip(self.shadow_model.parameters(), model.parameters()): + s_param.lerp_(param.detach().to(s_param.dtype), 1.0 - eff_decay) + for s_buf, buf in zip(self.shadow_model.buffers(), model.buffers()): + s_buf.copy_(buf.detach()) + + def state_dict(self) -> dict[str, object]: + return { + "decay": self.decay, + "num_updates": self.num_updates, + "shadow_state_dict": self.shadow_model.state_dict(), + } + + def load_state_dict(self, state: dict[str, object]) -> None: + self.decay = float(state["decay"]) # type: ignore[arg-type] + self.num_updates = int(state["num_updates"]) # type: ignore[arg-type] + self.shadow_model.load_state_dict(state["shadow_state_dict"]) # type: ignore[arg-type] + + +def infer_use_ordinal_material_channel(saved_payload: Mapping[str, object]) -> bool: + """Resolve `use_ordinal_material_channel` from checkpoint config or state_dict input channels.""" + cfg = saved_payload.get("config", {}) + if isinstance(cfg, Mapping) and "use_ordinal_material_channel" in cfg: + return bool(cfg["use_ordinal_material_channel"]) + + num_phases = int(cfg.get("num_phases", 2)) if isinstance(cfg, Mapping) else 2 # type: ignore[arg-type] + data_ch = 1 if num_phases == 2 else 2 + model_sd = saved_payload.get("model", {}) + if isinstance(model_sd, Mapping): + for k, v in model_sd.items(): + if str(k).endswith("conv_in.conv.weight") and isinstance(v, torch.Tensor) and v.ndim == 5: + return int(v.shape[1]) == data_ch + 4 + return False + diff --git a/porousdiff/optimization.py b/porousdiff/optimization.py new file mode 100644 index 0000000..6a5eedc --- /dev/null +++ b/porousdiff/optimization.py @@ -0,0 +1,628 @@ +"""Optimal clamping-pressure (`sigma*`) discovery & OpenImpala reaction-transport coupling. + +Unifies PorousDiff continuous stress conditioning with OpenImpala continuum transport solvers +(and exact 3D geodesic / Carman-Kozeny microstructural transport surrogates) to identify the +electrochemical cell clamping sweet spot `sigma*` balancing: + 1. Ionic pore-phase transport (`D_eff,Z / D_0` and hydraulic permeability `K_Z`), + 2. Electronic solid-backbone percolation & wiring (`sigma_eff,Z / sigma_0` and wired coating), + 3. Electrochemically active wired catalyst interface (`ssa_pore_coating * wired_fraction` & TPB). + +Models total cell conductance via the series resistance law: + `R_cell(sigma) = w_ion / D_tilde(sigma) + w_elec / sigma_tilde(sigma) + w_kin / A_tilde(sigma)` +so `FoM(sigma) = 1 / R_cell(sigma)` vanishes if either pores pinch closed at over-compression OR +the fibre/coating network loses electrical percolation at under-compression. +""" + +from __future__ import annotations + +import csv +from pathlib import Path +from typing import Mapping, Sequence + +import numpy as np +from scipy.interpolate import PchipInterpolator + + +def load_openimpala_lookup( + csv_path: Path, +) -> dict[tuple[str, str], dict[str, float]]: + """Group `openimpala_transport_parity.csv` rows by `(method, state)` into mean transport dicts.""" + if not csv_path.is_file(): + return {} + buckets: dict[tuple[str, str], dict[str, list[float]]] = {} + with csv_path.open("r", newline="") as f: + for row in csv.DictReader(f): + method = str(row.get("method", "")).strip() + state = str(row.get("state", "")).strip() + phase = str(row.get("transport_phase", "")).strip().lower() + raw_deff = row.get("d_eff_z") + if not method or not state or raw_deff is None or not str(raw_deff).strip(): + continue + try: + val = float(str(raw_deff).strip()) + except ValueError: + continue + if not np.isfinite(val): + continue + key = (method, state) + entry = buckets.setdefault( + key, + {"pore_d_eff_z": [], "solid_sigma_eff_z": [], "solid_hetero_sigma_eff_z": []}, + ) + if "pore" in phase or "ion" in phase: + entry["pore_d_eff_z"].append(val) + elif "hetero" in phase: + entry["solid_hetero_sigma_eff_z"].append(val) + elif "carbon" in phase or "solid" in phase or "elec" in phase: + entry["solid_sigma_eff_z"].append(val) + + lookup: dict[tuple[str, str], dict[str, float]] = {} + for key, vals in buckets.items(): + agg: dict[str, float] = {} + if vals["pore_d_eff_z"]: + agg["pore_d_eff_z"] = float(np.mean(vals["pore_d_eff_z"])) + if vals["solid_hetero_sigma_eff_z"]: + agg["solid_hetero_sigma_eff_z"] = float(np.mean(vals["solid_hetero_sigma_eff_z"])) + if vals["solid_sigma_eff_z"]: + agg["solid_sigma_eff_z"] = float(np.mean(vals["solid_sigma_eff_z"])) + if agg: + lookup[key] = agg + return lookup + + +def compute_transport_and_fom_record( + phys_metrics: Mapping[str, float], + openimpala_metrics: Mapping[str, float] | None = None, + voxel_size_um: float = 1.0, + strain_z: float | None = None, +) -> dict[str, float]: + """Compute ionic, electronic, hydraulic, and kinetic observables for a single 3D realization.""" + sf = float(phys_metrics.get("solid_fraction", 0.0)) + porosity = float(phys_metrics.get("porosity", max(0.0, 1.0 - sf))) + coat_frac = float(phys_metrics.get("coating_fraction", 0.0)) + tau_pore_z = float(phys_metrics.get("geodesic_tortuosity_z", float("nan"))) + tau_solid_z = float(phys_metrics.get("geodesic_solid_tortuosity_z", float("nan"))) + + # 1. Through-plane ionic effective diffusivity D_eff,Z / D_0 + if openimpala_metrics and "pore_d_eff_z" in openimpala_metrics and np.isfinite(openimpala_metrics["pore_d_eff_z"]): + d_eff_pore_z = float(max(0.0, openimpala_metrics["pore_d_eff_z"])) + used_openimpala = 1.0 + elif np.isfinite(tau_pore_z) and tau_pore_z >= 1.0: + d_eff_pore_z = float(porosity / (tau_pore_z**2)) + used_openimpala = 0.0 + else: + d_eff_pore_z = 0.0 + used_openimpala = 0.0 + + # 2. Through-plane electronic effective conductivity sigma_eff,Z / sigma_0 + is_coated = coat_frac > 1e-6 or float(phys_metrics.get("ssa_pore_coating", 0.0)) > 1e-6 + wired_frac = float( + phys_metrics.get("wired_coating_fraction", phys_metrics.get("solid_connected_fraction", 1.0)) + ) + solid_conn = float(phys_metrics.get("solid_connected_fraction", 1.0)) + elec_factor = wired_frac if is_coated else solid_conn + + if ( + openimpala_metrics + and "solid_hetero_sigma_eff_z" in openimpala_metrics + and np.isfinite(openimpala_metrics["solid_hetero_sigma_eff_z"]) + ): + sigma_eff_solid_z = float(max(0.0, openimpala_metrics["solid_hetero_sigma_eff_z"])) + elif ( + openimpala_metrics + and "solid_sigma_eff_z" in openimpala_metrics + and np.isfinite(openimpala_metrics["solid_sigma_eff_z"]) + ): + sigma_eff_solid_z = float(max(0.0, openimpala_metrics["solid_sigma_eff_z"])) + elif np.isfinite(tau_solid_z) and tau_solid_z >= 1.0: + sigma_eff_solid_z = float(sf * elec_factor / (tau_solid_z**2)) + else: + # Fallback Bruggeman-percolation estimate when thin patches barely miss Z-face contact + sigma_eff_solid_z = float((sf**1.8) * elec_factor * 0.25) + + # 3. Carman-Kozeny hydraulic permeability proxy (um^2) + pore_chord_um = float(phys_metrics.get("pore_chord_z", 0.0)) * max(voxel_size_um, 1e-4) + if np.isfinite(tau_pore_z) and tau_pore_z >= 1.0 and porosity > 1e-4: + denom = 180.0 * max((1.0 - porosity) ** 2, 1e-5) * (tau_pore_z**2) + perm_ck_um2 = float((porosity**3) * (pore_chord_um**2) / denom) + else: + perm_ck_um2 = 0.0 + + # 4. Electrochemically active wired interfacial density (1/um) + ssa_coat = float(phys_metrics.get("ssa_pore_coating", 0.0)) + tpb = float(phys_metrics.get("tpb_density", 0.0)) + ssa_total = float(phys_metrics.get("ssa_total", 0.0)) + if is_coated: + active_interface = float(ssa_coat * wired_frac + 0.05 * voxel_size_um * tpb * wired_frac) + else: + active_interface = float(ssa_total * solid_conn) + + raw_sz = strain_z if strain_z is not None else phys_metrics.get("strain_z", float("nan")) + strain_val = float(raw_sz) if (raw_sz is not None and np.isfinite(raw_sz)) else float("nan") + + return { + "solid_fraction": sf, + "porosity": porosity, + "strain_z": strain_val, + "d_eff_pore_z": d_eff_pore_z, + "sigma_eff_solid_z": sigma_eff_solid_z, + "permeability_ck_um2": perm_ck_um2, + "active_wired_interface": active_interface, + "wired_coating_fraction": wired_frac if is_coated else solid_conn, + "tpb_density": tpb, + "z_pinch_contact_density": float(phys_metrics.get("z_pinch_contact_density", 0.0)), + "coating_pinch_fraction": float(phys_metrics.get("coating_pinch_fraction", 0.0)), + "used_openimpala": used_openimpala, + } + + +def _evaluate_harmonic_fom_curve( + loads: np.ndarray, + d_eff: np.ndarray, + sigma_eff: np.ndarray, + active_if: np.ndarray, + grid_loads: np.ndarray, + weights: tuple[float, float, float], + norm_scales: tuple[float, float, float] | None = None, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, tuple[float, float, float]]: + """Interpolate transport pillars via PCHIP and evaluate series-conductance FoM on `grid_loads`.""" + w_ion, w_elec, w_kin = weights + w_sum = max(w_ion + w_elec + w_kin, 1e-8) + w_ion, w_elec, w_kin = w_ion / w_sum, w_elec / w_sum, w_kin / w_sum + + if norm_scales is None: + s_ion = float(max(np.max(d_eff), 1e-8)) + s_elec = float(max(np.max(sigma_eff), 1e-8)) + s_kin = float(max(np.max(active_if), 1e-8)) + scales = (s_ion, s_elec, s_kin) + else: + scales = norm_scales + + ion_n = np.clip(d_eff / scales[0], 0.0, None) + elec_n = np.clip(sigma_eff / scales[1], 0.0, None) + kin_n = np.clip(active_if / scales[2], 0.0, None) + + if loads.size >= 2: + ion_g = np.clip(PchipInterpolator(loads, ion_n)(grid_loads), 0.0, None) + elec_g = np.clip(PchipInterpolator(loads, elec_n)(grid_loads), 0.0, None) + kin_g = np.clip(PchipInterpolator(loads, kin_n)(grid_loads), 0.0, None) + else: + ion_g = np.full_like(grid_loads, ion_n[0]) + elec_g = np.full_like(grid_loads, elec_n[0]) + kin_g = np.full_like(grid_loads, kin_n[0]) + + eps = 1e-4 + inv_conductance = ( + w_ion / (ion_g + eps) + + w_elec / (elec_g + eps) + + w_kin / (kin_g + eps) + ) + fom_grid = 1.0 / inv_conductance + return fom_grid, ion_g, elec_g, kin_g, scales + + +def evaluate_newman_asr_curve( + loads: np.ndarray, + d_eff: np.ndarray, + sigma_eff: np.ndarray, + active_if_um: np.ndarray, + solid_frac: np.ndarray, + perm_um2: np.ndarray, + z_pinch: np.ndarray, + strain_z: np.ndarray, + grid_loads: np.ndarray, + uncompressed_thickness_cm: float = 0.30, + kappa_bulk_s_cm: float = 0.05, + sigma_bulk_s_cm: float = 2.0, + r_ct_ohm_cm2: float = 2.5, + r_mt_ohm_cm2: float = 0.05, + r_contact_base_ohm_cm2: float = 0.04, + pump_penalty_ohm_cm2_um2_cm: float = 1.5e-3, +) -> tuple[np.ndarray, np.ndarray]: + """Evaluate dimensional 1D Newman & Tobias porous-electrode ASR (`Ohm * cm^2`) on `grid_loads`. + + Incorporates macroscopic compressed thickness shrinkage `L(sigma) = L_0 * (1 - strain_z)` + (or uniaxial solid-conservation `L_0 * phi_s,min / phi_s(sigma)` when `strain_z` is absent), + analytical 1D porous-electrode transmission-line polarization `R_tl(sigma)`, Nernst diffusive + mass-transport concentration resistance `R_mt(sigma)`, clamping contact resistance `R_contact(sigma)`, + and flow-through hydraulic pumping penalty `R_pump(sigma)`. + """ + l0_cm = max(float(uncompressed_thickness_cm), 1e-3) + ref_sf = float(max(np.min(solid_frac), 1e-3)) + + thick_discrete = np.empty_like(loads, dtype=np.float64) + for i in range(loads.size): + sz = float(strain_z[i]) + if np.isfinite(sz) and 0.0 <= sz < 0.90: + thick_discrete[i] = l0_cm * (1.0 - sz) + else: + ratio = np.clip(ref_sf / max(float(solid_frac[i]), 1e-3), 0.20, 1.0) + thick_discrete[i] = l0_cm * ratio + + def _interp(vals: np.ndarray, floor: float) -> np.ndarray: + if loads.size >= 2: + return np.clip(PchipInterpolator(loads, vals)(grid_loads), floor, None) + return np.full_like(grid_loads, max(float(vals[0]), floor), dtype=np.float64) + + d_g = _interp(d_eff, 1e-6) + s_g = _interp(sigma_eff, 1e-6) + a_cm_inv = _interp(active_if_um * 1e4, 1e-2) # 1/um -> 1/cm + sf_g = _interp(solid_frac, 1e-3) + k_g = _interp(perm_um2, 1e-5) + zp_g = _interp(z_pinch, 0.0) + l_cm = np.clip(_interp(thick_discrete, 0.05 * l0_cm), 0.05 * l0_cm, l0_cm) + + r_ion = 1.0 / np.maximum(kappa_bulk_s_cm * d_g, 1e-6) # Ohm * cm + r_elec = 1.0 / np.maximum(sigma_bulk_s_cm * s_g, 1e-6) # Ohm * cm + r_kin = r_ct_ohm_cm2 / np.maximum(a_cm_inv, 1e-2) # Ohm * cm^3 + + r_sum = r_ion + r_elec + nu = l_cm * np.sqrt(r_sum / np.maximum(r_kin, 1e-12)) + + # Overflow-free evaluation of Newman & Tobias (1962) 1D transmission-line impedance + r_tl = np.empty_like(grid_loads, dtype=np.float64) + small = nu < 1e-3 + if np.any(small): + r_tl[small] = r_kin[small] / np.maximum(l_cm[small], 1e-6) + l_cm[small] * r_sum[small] / 3.0 + if np.any(~small): + nu_b = nu[~small] + ri_b = r_ion[~small] + re_b = r_elec[~small] + rk_b = r_kin[~small] + rs_b = r_sum[~small] + l_b = l_cm[~small] + e1 = np.exp(-nu_b) + e2 = e1 * e1 + denom = np.maximum(1.0 - e2, 1e-15) + coth_nu = (1.0 + e2) / denom + csch_nu = 2.0 * e1 / denom + r_tl[~small] = (l_b * ri_b * re_b) / rs_b + ( + np.sqrt(rk_b) * ((ri_b**2 + re_b**2) * coth_nu + 2.0 * ri_b * re_b * csch_nu) + ) / (rs_b**1.5) + + r_mt = r_mt_ohm_cm2 / np.maximum(d_g, 1e-6) + r_contact = r_contact_base_ohm_cm2 / np.maximum(sf_g + 2.0 * zp_g, 0.02) + r_pump = pump_penalty_ohm_cm2_um2_cm / np.maximum(k_g * l_cm, 1e-4) + asr_grid = r_tl + r_mt + r_contact + r_pump + return asr_grid, l_cm + + +NEWMAN_REGIMES: dict[str, dict[str, float]] = { + "kinetics_limited": { + "kappa_bulk_s_cm": 0.08, + "sigma_bulk_s_cm": 2.0, + "r_ct_ohm_cm2": 6.0, + "r_mt_ohm_cm2": 0.01, + "pump_penalty_ohm_cm2_um2_cm": 5e-4, + }, + "balanced": { + "kappa_bulk_s_cm": 0.05, + "sigma_bulk_s_cm": 2.0, + "r_ct_ohm_cm2": 2.5, + "r_mt_ohm_cm2": 0.05, + "pump_penalty_ohm_cm2_um2_cm": 1.5e-3, + }, + "mass_transport_limited": { + "kappa_bulk_s_cm": 0.025, + "sigma_bulk_s_cm": 2.0, + "r_ct_ohm_cm2": 0.8, + "r_mt_ohm_cm2": 0.22, + "pump_penalty_ohm_cm2_um2_cm": 6e-3, + }, +} + + +SIMPLEX_WEIGHT_GRID: tuple[tuple[float, float, float], ...] = ( + (1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0), + (0.35, 0.35, 0.30), + (0.60, 0.20, 0.20), + (0.20, 0.60, 0.20), + (0.20, 0.20, 0.60), + (0.50, 0.35, 0.15), + (0.50, 0.15, 0.35), + (0.35, 0.50, 0.15), + (0.15, 0.50, 0.35), + (0.35, 0.15, 0.50), + (0.15, 0.35, 0.50), +) + + +def find_optimal_clamping_pressure( + records_by_load: Mapping[float, Sequence[Mapping[str, float]]], + weights: tuple[float, float, float] = (0.35, 0.35, 0.30), + grid_points: int = 201, + n_bootstrap: int = 200, + seed: int = 42, + norm_scales: tuple[float, float, float] | None = None, + uncompressed_thickness_cm: float = 0.30, + allow_mixed_estimators: bool = False, +) -> dict[str, object]: + """Identify optimal clamping stress `sigma*` via dimensional 1D Newman ASR and harmonic FoM.""" + sorted_loads = sorted(float(k) for k in records_by_load.keys() if records_by_load[k]) + if not sorted_loads: + raise ValueError("records_by_load must contain at least one non-empty load level") + + oi_per_load = [ + float(np.mean([float(r.get("used_openimpala", 0.0)) for r in records_by_load[lv]])) + for lv in sorted_loads + ] + n_oi_loads = sum(1 for v in oi_per_load if v > 0.5) + if 0 < n_oi_loads < len(sorted_loads) and not allow_mixed_estimators: + raise ValueError( + f"Refusing to mix OpenImpala ({n_oi_loads}/{len(sorted_loads)} loads) and geodesic surrogate " + "transport estimates on the same clamping-pressure curve. Run OpenImpala across all loads " + "or pass allow_mixed_estimators=True." + ) + estimator_mode = ( + "openimpala" + if n_oi_loads == len(sorted_loads) + else ("mixed" if n_oi_loads > 0 else "geodesic_surrogate") + ) + + loads_arr = np.asarray(sorted_loads, dtype=np.float64) + mean_d_eff = np.asarray( + [np.mean([float(r["d_eff_pore_z"]) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_sigma = np.asarray( + [np.mean([float(r["sigma_eff_solid_z"]) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_active = np.asarray( + [np.mean([float(r["active_wired_interface"]) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_perm = np.asarray( + [np.mean([float(r["permeability_ck_um2"]) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_sf = np.asarray( + [np.mean([float(r.get("solid_fraction", 0.15)) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_zpinch = np.asarray( + [np.mean([float(r.get("z_pinch_contact_density", 0.0)) for r in records_by_load[lv]]) for lv in sorted_loads], + dtype=np.float64, + ) + mean_strain = np.empty_like(loads_arr) + for i, lv in enumerate(sorted_loads): + vals = [float(r.get("strain_z", float("nan"))) for r in records_by_load[lv]] + fin = [v for v in vals if np.isfinite(v)] + mean_strain[i] = float(np.mean(fin)) if fin else float("nan") + + if len(sorted_loads) >= 2 and sorted_loads[-1] > sorted_loads[0]: + grid_loads = np.linspace(sorted_loads[0], sorted_loads[-1], max(11, int(grid_points))) + else: + grid_loads = loads_arr.copy() + + fom_grid, ion_g, elec_g, kin_g, scales = _evaluate_harmonic_fom_curve( + loads_arr, + mean_d_eff, + mean_sigma, + mean_active, + grid_loads, + weights=weights, + norm_scales=norm_scales, + ) + best_idx = int(np.argmax(fom_grid)) + opt_load = float(grid_loads[best_idx]) + opt_fom = float(fom_grid[best_idx]) + + weight_sweep_rows: list[dict[str, float]] = [] + weight_opt_loads: list[float] = [] + for w_cand in SIMPLEX_WEIGHT_GRID: + w_fom_g, _, _, _, _ = _evaluate_harmonic_fom_curve( + loads_arr, + mean_d_eff, + mean_sigma, + mean_active, + grid_loads, + weights=w_cand, + norm_scales=scales, + ) + w_idx = int(np.argmax(w_fom_g)) + w_opt = float(grid_loads[w_idx]) + weight_opt_loads.append(w_opt) + weight_sweep_rows.append( + { + "w_ionic": round(w_cand[0], 4), + "w_electronic": round(w_cand[1], 4), + "w_kinetic": round(w_cand[2], 4), + "optimal_load": round(w_opt, 6), + "optimal_fom": round(float(w_fom_g[w_idx]), 6), + } + ) + + asr_grid, thick_grid_cm = evaluate_newman_asr_curve( + loads_arr, + mean_d_eff, + mean_sigma, + mean_active, + mean_sf, + mean_perm, + mean_zpinch, + mean_strain, + grid_loads, + uncompressed_thickness_cm=uncompressed_thickness_cm, + ) + best_asr_idx = int(np.argmin(asr_grid)) + opt_load_asr = float(grid_loads[best_asr_idx]) + min_asr = float(asr_grid[best_asr_idx]) + + regime_sensitivity: dict[str, dict[str, float]] = {} + for reg_name, reg_params in NEWMAN_REGIMES.items(): + r_asr_g, _ = evaluate_newman_asr_curve( + loads_arr, + mean_d_eff, + mean_sigma, + mean_active, + mean_sf, + mean_perm, + mean_zpinch, + mean_strain, + grid_loads, + uncompressed_thickness_cm=uncompressed_thickness_cm, + **reg_params, + ) + r_idx = int(np.argmin(r_asr_g)) + regime_sensitivity[reg_name] = { + "optimal_load_asr": round(float(grid_loads[r_idx]), 6), + "min_asr_ohm_cm2": round(float(r_asr_g[r_idx]), 6), + **reg_params, + } + + # Non-parametric bootstrap across realizations at each discrete load + rng = np.random.default_rng(seed) + boot_opt_loads: list[float] = [] + boot_curves: list[np.ndarray] = [] + boot_opt_loads_asr: list[float] = [] + boot_asr_curves: list[np.ndarray] = [] + can_bootstrap = any(len(records_by_load[lv]) > 1 for lv in sorted_loads) and n_bootstrap > 0 + + if can_bootstrap: + for _ in range(int(n_bootstrap)): + b_deff = np.empty_like(loads_arr) + b_sig = np.empty_like(loads_arr) + b_act = np.empty_like(loads_arr) + b_sf = np.empty_like(loads_arr) + b_perm = np.empty_like(loads_arr) + b_zp = np.empty_like(loads_arr) + for i, lv in enumerate(sorted_loads): + recs = records_by_load[lv] + n_r = len(recs) + pick = rng.integers(0, n_r, size=n_r) + b_deff[i] = np.mean([float(recs[j]["d_eff_pore_z"]) for j in pick]) + b_sig[i] = np.mean([float(recs[j]["sigma_eff_solid_z"]) for j in pick]) + b_act[i] = np.mean([float(recs[j]["active_wired_interface"]) for j in pick]) + b_sf[i] = np.mean([float(recs[j].get("solid_fraction", 0.15)) for j in pick]) + b_perm[i] = np.mean([float(recs[j].get("permeability_ck_um2", 1.0)) for j in pick]) + b_zp[i] = np.mean([float(recs[j].get("z_pinch_contact_density", 0.0)) for j in pick]) + b_fom, _, _, _, _ = _evaluate_harmonic_fom_curve( + loads_arr, b_deff, b_sig, b_act, grid_loads, weights=weights, norm_scales=scales + ) + boot_opt_loads.append(float(grid_loads[int(np.argmax(b_fom))])) + boot_curves.append(b_fom) + + b_asr, _ = evaluate_newman_asr_curve( + loads_arr, + b_deff, + b_sig, + b_act, + b_sf, + b_perm, + b_zp, + mean_strain, + grid_loads, + uncompressed_thickness_cm=uncompressed_thickness_cm, + ) + boot_opt_loads_asr.append(float(grid_loads[int(np.argmin(b_asr))])) + boot_asr_curves.append(b_asr) + + ci_low, ci_high = ( + float(np.percentile(boot_opt_loads, 2.5)), + float(np.percentile(boot_opt_loads, 97.5)), + ) + curve_stack = np.stack(boot_curves, axis=0) + fom_ci_low = np.percentile(curve_stack, 2.5, axis=0) + fom_ci_high = np.percentile(curve_stack, 97.5, axis=0) + + asr_opt_ci_low, asr_opt_ci_high = ( + float(np.percentile(boot_opt_loads_asr, 2.5)), + float(np.percentile(boot_opt_loads_asr, 97.5)), + ) + asr_stack = np.stack(boot_asr_curves, axis=0) + asr_ci_low = np.percentile(asr_stack, 2.5, axis=0) + asr_ci_high = np.percentile(asr_stack, 97.5, axis=0) + else: + ci_low, ci_high = opt_load, opt_load + fom_ci_low = fom_grid.copy() + fom_ci_high = fom_grid.copy() + asr_opt_ci_low, asr_opt_ci_high = opt_load_asr, opt_load_asr + asr_ci_low = asr_grid.copy() + asr_ci_high = asr_grid.copy() + + discrete_rows: list[dict[str, float]] = [] + discrete_fom, _, _, _, _ = _evaluate_harmonic_fom_curve( + loads_arr, mean_d_eff, mean_sigma, mean_active, loads_arr, weights=weights, norm_scales=scales + ) + discrete_asr, discrete_thick_cm = evaluate_newman_asr_curve( + loads_arr, + mean_d_eff, + mean_sigma, + mean_active, + mean_sf, + mean_perm, + mean_zpinch, + mean_strain, + loads_arr, + uncompressed_thickness_cm=uncompressed_thickness_cm, + ) + for i, lv in enumerate(sorted_loads): + recs = records_by_load[lv] + discrete_rows.append( + { + "load_value": lv, + "num_samples": float(len(recs)), + "solid_fraction": float(mean_sf[i]), + "porosity": float(np.mean([float(r.get("porosity", 1.0 - mean_sf[i])) for r in recs])), + "compressed_thickness_cm": float(discrete_thick_cm[i]), + "d_eff_pore_z": float(mean_d_eff[i]), + "sigma_eff_solid_z": float(mean_sigma[i]), + "permeability_ck_um2": float(mean_perm[i]), + "active_wired_interface": float(mean_active[i]), + "wired_coating_fraction": float( + np.mean([float(r.get("wired_coating_fraction", 1.0)) for r in recs]) + ), + "tpb_density": float(np.mean([float(r.get("tpb_density", 0.0)) for r in recs])), + "coating_pinch_fraction": float( + np.mean([float(r.get("coating_pinch_fraction", 0.0)) for r in recs]) + ), + "fom_harmonic": float(discrete_fom[i]), + "asr_ohm_cm2": float(discrete_asr[i]), + "used_openimpala": float(np.mean([float(r.get("used_openimpala", 0.0)) for r in recs])), + } + ) + + return { + "estimator_mode": estimator_mode, + "uncertainty_note": ( + "Bootstrap CIs quantify within-scan spatial patch variance; " + "model-form sensitivity is reported in weight_sensitivity and newman_asr.regime_sensitivity." + ), + "optimal_load": round(opt_load, 6), + "optimal_load_ci95": [round(ci_low, 6), round(ci_high, 6)], + "optimal_fom": round(opt_fom, 6), + "weights": {"ionic": weights[0], "electronic": weights[1], "kinetic": weights[2]}, + "weight_sensitivity": { + "sigma_star_min": round(float(min(weight_opt_loads)), 6), + "sigma_star_max": round(float(max(weight_opt_loads)), 6), + "sigma_star_std": round(float(np.std(weight_opt_loads)), 6), + "grid": weight_sweep_rows, + }, + "norm_scales": { + "d_eff_pore_z": scales[0], + "sigma_eff_solid_z": scales[1], + "active_wired_interface": scales[2], + }, + "newman_asr": { + "uncompressed_thickness_cm": float(uncompressed_thickness_cm), + "optimal_load_asr": round(opt_load_asr, 6), + "optimal_load_asr_ci95": [round(asr_opt_ci_low, 6), round(asr_opt_ci_high, 6)], + "min_asr_ohm_cm2": round(min_asr, 6), + "thickness_cm_grid": [float(x) for x in thick_grid_cm], + "asr_mean_ohm_cm2": [float(x) for x in asr_grid], + "asr_ci95_low_ohm_cm2": [float(x) for x in asr_ci_low], + "asr_ci95_high_ohm_cm2": [float(x) for x in asr_ci_high], + "regime_sensitivity": regime_sensitivity, + }, + "discrete_points": discrete_rows, + "continuous_curve": { + "grid_loads": [float(x) for x in grid_loads], + "fom_mean": [float(x) for x in fom_grid], + "fom_ci95_low": [float(x) for x in fom_ci_low], + "fom_ci95_high": [float(x) for x in fom_ci_high], + "ionic_norm": [float(x) for x in ion_g], + "electronic_norm": [float(x) for x in elec_g], + "kinetic_norm": [float(x) for x in kin_g], + }, + } diff --git a/porousdiff/preprocessing.py b/porousdiff/preprocessing.py new file mode 100644 index 0000000..f1bdf52 --- /dev/null +++ b/porousdiff/preprocessing.py @@ -0,0 +1,985 @@ +"""Physics-consistent XCT segmentation, artifact gating, and stratified 3D patch extraction.""" + +from __future__ import annotations + +import csv +from dataclasses import dataclass +import json +import re +from pathlib import Path +from typing import Iterable, Sequence + +import numpy as np +from scipy.ndimage import binary_closing, binary_dilation, binary_erosion, gaussian_filter + + +@dataclass(frozen=True) +class CalibrationMetadata: + """Physical calibration linking beamline XCT coordinates and load-cell readings.""" + + voxel_size_um: float = 1.0 + spatial_bin: int = 1 + kpa_per_mvv: float | None = None + punch_diameter_mm: float | None = None + zero_offset_mvv: float = 0.0 + reference_thickness_slices: int | None = None + beamline: str = "primary" + + @property + def effective_voxel_size_um(self) -> float: + return float(self.voxel_size_um * max(1, self.spatial_bin)) + + def load_to_stress_kpa(self, load_mv_v: float) -> float | None: + if self.kpa_per_mvv is None: + return None + return float(max(0.0, (load_mv_v - self.zero_offset_mvv) * self.kpa_per_mvv)) + + def resolve_stress_kpa(self, value: float, unit: str = "mv_per_v") -> float | None: + """Convert parsed load (`mv_per_v`, `kpa`, or `newton`) to mechanical stress in kPa.""" + u = unit.lower() + if u == "kpa": + return float(max(0.0, value)) + if u == "newton": + if self.punch_diameter_mm is None or self.punch_diameter_mm <= 0: + return None + radius_m = 0.5 * float(self.punch_diameter_mm) * 1e-3 + area_m2 = float(np.pi * (radius_m**2)) + return float(max(0.0, (value / area_m2) * 1e-3)) + return self.load_to_stress_kpa(value) + + def estimate_strain_z(self, clean_thickness_slices: int) -> float | None: + if self.reference_thickness_slices is None or self.reference_thickness_slices <= 0: + return None + ratio = float(clean_thickness_slices) / float(self.reference_thickness_slices) + return float(np.clip(1.0 - ratio, 0.0, 0.95)) + + +def has_explicit_material_keyword(text: str) -> bool: + """Return True if `text` explicitly names TiO2/anode, CuHCF/cathode, or virgin/carbon.""" + low = text.lower() + keywords = ( + "tio2", + "titania", + "anode", + "cuhcf", + "cuchf", + "hexacyano", + "prussian", + "cathode", + "virgin", + "crbon", + "carbon", + ) + return any(k in low for k in keywords) + + +def infer_material_name(name_or_path: str | Path) -> str: + """Infer canonical electrode material key from folder name or relative/full path.""" + low = str(name_or_path).replace("\\", "/").lower() + if any(k in low for k in ("tio2", "titania", "anode")): + return "tio2_anode" + if any(k in low for k in ("cuhcf", "cuchf", "hexacyano", "prussian", "cathode")): + return "cuhcf_cathode" + return "virgin_carbon" + + +def make_collision_safe_state_name( + scan_dir: Path | str, + material: str, + beamline: str = "primary", + existing_names: set[str] | None = None, +) -> str: + """Build a canonical state key avoiding collisions across generic leaf folders (e.g. `Op001mVV`).""" + leaf = Path(scan_dir).name.strip().rstrip("_") + parts: list[str] = [] + if beamline and beamline != "primary" and beamline.lower() not in leaf.lower(): + parts.append(beamline.strip()) + if material != "virgin_carbon" and not has_explicit_material_keyword(leaf): + parts.append(material) + parts.append(leaf) + candidate = "__".join(parts) + + if existing_names is not None and candidate in existing_names: + if material not in candidate: + candidate = f"{material}__{candidate}" + idx = 2 + base = candidate + while candidate in existing_names: + candidate = f"{base}_{idx}" + idx += 1 + return candidate + + +def _decode_p_number(token: str) -> float: + norm = re.sub(r"^[Oo](?=p\d)", "0", token.strip()) + return float(norm.replace("p", ".").replace("P", ".")) + + +def load_overrides_table(path: Path) -> dict[str, tuple[float, str]]: + """Load explicit folder->`(value, unit)` overrides from a `.csv` or `.json` file.""" + if path.suffix.lower() == ".json": + raw = json.loads(path.read_text()) + out: dict[str, tuple[float, str]] = {} + for k, v in raw.items(): + if isinstance(v, dict): + out[str(k)] = (float(v["value"]), str(v.get("unit", "mv_per_v"))) + else: + out[str(k)] = (float(v), "mv_per_v") + return out + + out = {} + with path.open("r", newline="") as f: + reader = csv.DictReader(f) + for row in reader: + key = ( + row.get("state") + or row.get("folder") + or row.get("dataset") + or row.get("scan") + or "" + ).strip() + val_str = ( + row.get("load") + or row.get("value") + or row.get("stress_kpa") + or row.get("load_mv_per_v") + or "" + ).strip() + if not key or not val_str: + continue + default_unit = "kpa" if ("stress_kpa" in row and row.get("stress_kpa")) else "mv_per_v" + unit = (row.get("unit") or default_unit).strip() + out[key] = (float(val_str), unit) + return out + + +def parse_load_with_unit( + name: str, + overrides: dict[str, tuple[float, str]] | None = None, +) -> tuple[float, str]: + """Parse mechanical load and physical unit (`mv_per_v`, `kpa`, or `newton`) from folder/path. + + Supports: + - `0p048mVV`, `Op026mVV`, `15mV` -> `("mv_per_v")` + - `200kPa`, `0p25MPa` (`250 kPa`), `2p5bar` (`250 kPa`) -> `("kpa")` + - `50N`, `12p5N` -> `("newton")` + - Explicit CSV/JSON override table entries + """ + clean = name.strip().rstrip("_") + leaf = Path(clean.replace("\\", "/")).name + if overrides: + for candidate in (clean, name, leaf): + if candidate in overrides: + return overrides[candidate] + + match_mvv = re.search(r"((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)mVV", clean) + if match_mvv: + return _decode_p_number(match_mvv.group(1)), "mv_per_v" + + match_mv = re.search(r"((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)\s*mV(?!V)", clean, flags=re.IGNORECASE) + if match_mv: + val = _decode_p_number(match_mv.group(1)) + return (val / 1000.0 if val >= 1.0 else val), "mv_per_v" + + match_mpa = re.search(r"((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)\s*MPa", clean, flags=re.IGNORECASE) + if match_mpa: + return _decode_p_number(match_mpa.group(1)) * 1000.0, "kpa" + + match_kpa = re.search(r"((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)\s*kPa", clean, flags=re.IGNORECASE) + if match_kpa: + return _decode_p_number(match_kpa.group(1)), "kpa" + + match_bar = re.search(r"((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)\s*bar", clean, flags=re.IGNORECASE) + if match_bar: + return _decode_p_number(match_bar.group(1)) * 100.0, "kpa" + + match_n = re.search(r"(?:^|[_\-\s])((?:0|[Oo]|\d+)p\d+|\d+(?:\.\d+)?)\s*N(?:$|[_\-\s])", clean) + if match_n: + return _decode_p_number(match_n.group(1)), "newton" + + raise ValueError( + f"Cannot parse load value from {name!r}; supply --loads-csv or an explicit states mapping." + ) + + +def parse_load_value( + name: str, + overrides: dict[str, tuple[float, str]] | None = None, +) -> float: + """Parse scalar load value from folder/state names (`mV/V`, `kPa`, `MPa`, `bar`, `N`).""" + val, _ = parse_load_with_unit(name, overrides=overrides) + return val + + +def resolve_tiff_directory(dataset_path: Path) -> Path: + """Locate `.tif` files directly in `dataset_path` or inside `rec_16bit_Paganin`.""" + direct = list(dataset_path.glob("*.tif")) + list(dataset_path.glob("*.tiff")) + if direct: + return dataset_path + paganin = dataset_path / "rec_16bit_Paganin" + if paganin.is_dir(): + sub = list(paganin.glob("*.tif")) + list(paganin.glob("*.tiff")) + if sub: + return paganin + raise FileNotFoundError(f"No TIFF slices found under {dataset_path}") + + +def make_circular_mask( + shape_hw: tuple[int, int], + radius_fraction: float = 0.70, + center: tuple[float, float] | None = None, +) -> np.ndarray: + """Boolean mask for the inscribed circular electrode punch inside a square XCT slice.""" + if not 0.0 < radius_fraction <= 1.0: + raise ValueError("radius_fraction must be in (0, 1]") + h, w = shape_hw + yc, xc = center if center is not None else (h / 2.0, w / 2.0) + radius = 0.5 * min(h, w) * radius_fraction + yy, xx = np.ogrid[:h, :w] + return ((yy - yc) ** 2 + (xx - xc) ** 2) <= (radius**2) + + +def estimate_pore_mode(values: np.ndarray) -> tuple[float, float]: + """Estimate the dominant pore/air intensity mode and robust spread (MAD). + + In porous electrodes (porosity > 50%), the lower/primary histogram mode inside + the sample ROI corresponds to the pore phase. + """ + flat = np.asarray(values, dtype=np.float32).ravel() + if flat.size == 0: + raise ValueError("Cannot estimate pore mode on empty array") + + lo, hi = np.percentile(flat, [0.5, 90.0]) + if hi <= lo: + med = float(np.median(flat)) + return med, 1.0 + + # Restrict search for the pore peak to below the 75th percentile so dense solid + # phases never hijack the mode even under heavy compression. + upper_search = float(np.percentile(flat, 75.0)) + sub = flat[(flat >= lo) & (flat <= max(upper_search, lo + 1.0))] + if sub.size < 32: + sub = flat + + hist, edges = np.histogram(sub, bins=128) + smooth_hist = np.convolve(hist, [0.25, 0.5, 0.25], mode="same") + peak_idx = int(np.argmax(smooth_hist)) + pore_mode = 0.5 * float(edges[peak_idx] + edges[peak_idx + 1]) + + pore_side = flat[flat <= pore_mode + (pore_mode - float(np.percentile(flat, 5.0)))] + if pore_side.size < 16: + pore_side = sub + mad = float(np.median(np.abs(pore_side - pore_mode))) * 1.4826 + return pore_mode, max(mad, 1.0) + + +def compute_pore_referenced_delta( + image: np.ndarray, + roi_mask: np.ndarray | None = None, + sigma: float = 32.0, +) -> tuple[np.ndarray, float, float]: + """Subtract gentle low-frequency pore background without hollowing out dense fibers. + + Clamping bright fiber pixels near `pore_mode + 2 * pore_std` before Gaussian + smoothing removes slice-to-slice synchrotron beam flicker and Paganin radial + cupping while keeping high fiber-packing regions intact under compression. + """ + img = np.asarray(image, dtype=np.float32) + roi_vals = img[roi_mask] if roi_mask is not None else img + pore_mode, pore_std = estimate_pore_mode(roi_vals) + + if sigma > 0: + clamped = np.minimum(img, pore_mode + 2.0 * pore_std) + if roi_mask is not None: + clamped = np.where(roi_mask, clamped, pore_mode) + background = gaussian_filter(clamped, sigma=sigma) + else: + background = np.full_like(img, pore_mode) + + delta = img - background + return delta, pore_mode, pore_std + + +def _plateau_isodata_threshold(vals: np.ndarray, nbins: int = 512) -> float: + """Compute a gap-centered Otsu threshold refined to the half-maximum class midpoint. + + Standard `skimage.filters.threshold_otsu` selects the *leftmost* histogram bin on + empty intensity gaps and shifts toward the dominant mode under 90%+ class imbalance. + Centering the optimal Otsu plateau and refining via 1D IsoData yields the unbiased + 50% partial-volume iso-contour `0.5 * (mu_low + mu_high)` for Paganin XCT interfaces. + """ + hist, bin_edges = np.histogram(vals, bins=nbins) + hist_f = hist.astype(np.float64) + centers = 0.5 * (bin_edges[:-1] + bin_edges[1:]) + w0 = np.cumsum(hist_f) + w1 = w0[-1] - w0 + valid = (w0 > 0) & (w1 > 0) + if not np.any(valid): + return float(np.median(vals)) + + csum = np.cumsum(hist_f * centers) + m0 = csum[valid] / w0[valid] + m1 = (csum[-1] - csum[valid]) / w1[valid] + var_between = w0[valid] * w1[valid] * (m0 - m1) ** 2 + best = var_between >= 0.999 * var_between.max() + best_centers = centers[valid][best] + thresh = float(0.5 * (best_centers[0] + best_centers[-1])) + + for _ in range(3): + lo = vals[vals <= thresh] + hi = vals[vals > thresh] + if lo.size == 0 or hi.size == 0: + break + thresh = float(0.5 * (lo.mean() + hi.mean())) + return thresh + + +def calibrate_delta_threshold( + reference_slices: Iterable[np.ndarray], + roi_mask: np.ndarray | None = None, + sigma: float = 32.0, +) -> float: + """Calibrate a single shared `delta_threshold` from reference XCT slices.""" + pooled = [] + pore_stds: list[float] = [] + rng = np.random.default_rng(0) + for slc in reference_slices: + delta, _, p_std = compute_pore_referenced_delta(slc, roi_mask=roi_mask, sigma=sigma) + pore_stds.append(p_std) + vals = delta[roi_mask] if roi_mask is not None else delta.ravel() + if vals.size > 50_000: + vals = rng.choice(vals, size=50_000, replace=False) + pooled.append(vals) + + if not pooled: + raise ValueError("No reference slices provided for threshold calibration") + + all_vals = np.concatenate(pooled) + med_pstd = max(float(np.median(pore_stds)), 1.0) + lo, hi = np.percentile(all_vals, [1.0, 99.8]) + clipped = all_vals[(all_vals >= lo) & (all_vals <= hi)] + return float(max(_plateau_isodata_threshold(clipped), 3.5 * med_pstd)) + + +def calibrate_multiphase_thresholds( + reference_slices: Iterable[np.ndarray], + roi_mask: np.ndarray | None = None, + sigma: float = 32.0, + fixed_delta_carbon: float | None = None, +) -> tuple[float, float]: + """Calibrate `(delta_carbon, delta_coating)` thresholds for 3-phase coated electrodes. + + Gating above `4 * pore_std` isolates solid voxels (`carbon_fiber + active_coating`) + from the 80–95% pore background so `delta_coating` cleanly splits carbon (`Z=6`) vs + high-Z coating (`TiO2`/`CuHCF`), and `delta_carbon` lands on the unbiased 50% + partial-volume midpoint between pore and carbon without high-Z skew. + """ + pooled: list[np.ndarray] = [] + pore_stds: list[float] = [] + rng = np.random.default_rng(0) + for slc in reference_slices: + delta, _, p_std = compute_pore_referenced_delta(slc, roi_mask=roi_mask, sigma=sigma) + pore_stds.append(p_std) + vals = delta[roi_mask] if roi_mask is not None else delta.ravel() + if vals.size > 80_000: + vals = rng.choice(vals, size=80_000, replace=False) + pooled.append(vals) + + if not pooled: + raise ValueError("No reference slices provided for multiphase calibration") + + all_vals = np.concatenate(pooled) + med_pstd = max(float(np.median(pore_stds)), 1.0) + solid_gate = 4.0 * med_pstd + + if fixed_delta_carbon is not None: + delta_carbon = float(fixed_delta_carbon) + solid_candidates = all_vals[all_vals > delta_carbon] + else: + solid_candidates = all_vals[all_vals > solid_gate] + + if solid_candidates.size < 32: + d_carb = float(fixed_delta_carbon) if fixed_delta_carbon is not None else max(solid_gate, 1000.0) + return d_carb, d_carb + max(4.0 * med_pstd, 2000.0) + + hi_solid = float(np.percentile(solid_candidates, 99.9)) + solid_clipped = solid_candidates[solid_candidates <= hi_solid] + if solid_clipped.size < 32 or float(solid_clipped.max() - solid_clipped.min()) < 1e-3: + d_carb = float(fixed_delta_carbon) if fixed_delta_carbon is not None else max(solid_gate, 1000.0) + return d_carb, d_carb + max(4.0 * med_pstd, 2000.0) + + delta_coating = _plateau_isodata_threshold(solid_clipped) + + if fixed_delta_carbon is None: + pore_vals = all_vals[all_vals <= solid_gate] + carbon_vals = solid_clipped[solid_clipped <= delta_coating] + if pore_vals.size > 0 and carbon_vals.size > 0: + delta_carbon = max( + 0.5 * (float(pore_vals.mean()) + float(carbon_vals.mean())), + 3.5 * med_pstd, + ) + else: + delta_carbon = max(_plateau_isodata_threshold(all_vals), 3.5 * med_pstd) + + delta_coating = max(float(delta_coating), float(delta_carbon) + max(2.0 * med_pstd, 500.0)) + return float(delta_carbon), float(delta_coating) + + +def segment_slice_calibrated( + image: np.ndarray, + delta_threshold: float = 5500.0, + roi_mask: np.ndarray | None = None, + sigma: float = 32.0, +) -> tuple[np.ndarray, dict[str, float]]: + """Segment a 2D XCT slice using a consistent pore-referenced physical threshold.""" + if delta_threshold <= 0: + raise ValueError("delta_threshold must be positive") + + delta, pore_mode, pore_std = compute_pore_referenced_delta( + image, roi_mask=roi_mask, sigma=sigma + ) + binary = (delta > delta_threshold).astype(np.uint8) + if roi_mask is not None: + binary = np.where(roi_mask, binary, 0).astype(np.uint8) + solid_fraction = float(binary[roi_mask].mean()) + else: + solid_fraction = float(binary.mean()) + + stats = { + "pore_mode": pore_mode, + "pore_std": pore_std, + "solid_fraction": solid_fraction, + } + return binary, stats + + +def strip_partial_volume_carbon_halo( + labels: np.ndarray, + delta: np.ndarray, + delta_carbon: float, + delta_coating: float, + halo_px: int = 1, +) -> np.ndarray: + """Remove thin partial-volume phase-`1` blur shells wrapping phase-`2` coating at pore boundaries. + + Paganin phase retrieval and optical PSF blur cause sharp `pore (0) <-> high-Z coating (2)` + interfaces to pass continuously through `(delta_carbon, delta_coating]`, creating a fake + 1-2 px phase-`1` ("carbon") rind that artificially zeroes direct `0-2` EASA. Eroding `labels == 1` + with a `3x3` structuring element isolates the spine of genuine carbon fibres (`>= 3 px` diameter) + so real fibres and their `0-1-2` triple-phase boundaries stay protected while fake partial-volume + rinds are reassigned across `0.5 * (delta_carbon + delta_coating)`. + """ + if halo_px < 1: + return labels + carb = labels == 1 + coat = labels == 2 + if not (np.any(carb) and np.any(coat)): + return labels + + pore = labels == 0 + struct3 = np.ones((3, 3), dtype=bool) + iters = int(halo_px) + + core_carbon = binary_erosion(carb, structure=struct3, iterations=1) + if np.any(core_carbon): + protected_carbon = binary_dilation(core_carbon, structure=struct3, iterations=iters + 1) & carb + else: + protected_carbon = np.zeros_like(carb) + + near_coat = binary_dilation(coat, structure=struct3, iterations=iters + 1) + near_pore = binary_dilation(pore, structure=struct3, iterations=iters + 1) + suspect_halo = carb & near_coat & near_pore & (~protected_carbon) + if not np.any(suspect_halo): + return labels + + cleaned = labels.copy() + mid_02 = 0.5 * (float(delta_carbon) + float(delta_coating)) + cleaned[suspect_halo & (delta >= mid_02)] = 2 + cleaned[suspect_halo & (delta < mid_02)] = 0 + return cleaned + + +def segment_slice_multiphase( + image: np.ndarray, + delta_carbon: float = 5500.0, + delta_coating: float | None = None, + roi_mask: np.ndarray | None = None, + sigma: float = 32.0, + strip_halo_px: int = 0, +) -> tuple[np.ndarray, dict[str, float]]: + """Segment a 2D XCT slice into `{0: pore, 1: carbon_fiber, 2: active_coating}`. + + When `delta_coating is None`, behaves identically to 2-phase `segment_slice_calibrated`. + """ + if delta_carbon <= 0: + raise ValueError("delta_carbon must be positive") + if delta_coating is not None and delta_coating <= delta_carbon: + raise ValueError("delta_coating must be strictly greater than delta_carbon") + + delta, pore_mode, pore_std = compute_pore_referenced_delta( + image, roi_mask=roi_mask, sigma=sigma + ) + labels = np.zeros(image.shape, dtype=np.uint8) + labels[delta > delta_carbon] = 1 + if delta_coating is not None: + labels[delta > delta_coating] = 2 + if strip_halo_px > 0: + labels = strip_partial_volume_carbon_halo( + labels, + delta, + delta_carbon=delta_carbon, + delta_coating=delta_coating, + halo_px=strip_halo_px, + ) + + if roi_mask is not None: + labels = np.where(roi_mask, labels, 0).astype(np.uint8) + region = labels[roi_mask] + else: + region = labels.ravel() + + solid_frac = float((region >= 1).mean()) + carbon_frac = float((region == 1).mean()) + coating_frac = float((region == 2).mean()) + stats = { + "pore_mode": pore_mode, + "pore_std": pore_std, + "solid_fraction": solid_frac, + "carbon_fraction": carbon_frac, + "coating_fraction": coating_frac, + } + return labels, stats + + +def downsample_volume_rev(volume: np.ndarray, factor: int = 2) -> np.ndarray: + """Volume-fraction-conserving, neighborhood-ranked 3D block binning for REV patches. + + Pools `(factor x factor x factor)` voxel blocks using block occupancy ranked with + Gaussian neighborhood tie-breaking so global solid and active-coating volume + fractions are conserved (`< 0.5%` drift) while favoring continuous fibre backbones. + Use `audit_binning_topology_preservation` to verify percolation and connectivity + retention on specific scans. + """ + if factor < 1: + raise ValueError("factor must be >= 1") + vol = np.asarray(volume, dtype=np.uint8) + if vol.ndim != 3: + raise ValueError(f"Expected 3D volume, got shape {vol.shape}") + if factor == 1: + return vol.copy() + + d, h, w = vol.shape + od, oh, ow = d // factor, h // factor, w // factor + if min(od, oh, ow) < 1: + raise ValueError(f"Volume shape {vol.shape} smaller than bin factor {factor}") + + cropped = vol[: od * factor, : oh * factor, : ow * factor] + blocks = cropped.reshape(od, factor, oh, factor, ow, factor) + + solid_mask = cropped >= 1 + target_solid_frac = float(solid_mask.mean()) + n_out = od * oh * ow + target_solid_count = int(round(target_solid_frac * n_out)) + + out = np.zeros((od, oh, ow), dtype=np.uint8) + if target_solid_count == 0: + return out + + occ_solid = (blocks >= 1).mean(axis=(1, 3, 5)).astype(np.float32) + # Break ties along continuous 3D fiber backbones while never activating empty blocks + score_solid = np.where( + occ_solid > 0.0, + occ_solid + 1e-2 * gaussian_filter(occ_solid, sigma=0.8), + 0.0, + ) + nonzero_solid = int(np.count_nonzero(score_solid > 0.0)) + k_solid = min(target_solid_count, nonzero_solid) + if k_solid > 0: + flat_scores = score_solid.ravel() + top_idx = np.argpartition(flat_scores, -k_solid)[-k_solid:] + out.ravel()[top_idx] = 1 + + if np.any(cropped == 2): + target_coat_frac = float((cropped == 2).mean()) + target_coat_count = int(round(target_coat_frac * n_out)) + if target_coat_count > 0: + occ_coat = (blocks == 2).mean(axis=(1, 3, 5)).astype(np.float32) + score_coat = np.where( + (out == 1) & (occ_coat > 0.0), + occ_coat + 1e-2 * gaussian_filter(occ_coat, sigma=0.8), + 0.0, + ) + nonzero_coat = int(np.count_nonzero(score_coat > 0.0)) + k_coat = min(target_coat_count, nonzero_coat) + if k_coat > 0: + flat_coat = score_coat.ravel() + coat_idx = np.argpartition(flat_coat, -k_coat)[-k_coat:] + out.ravel()[coat_idx] = 2 + + return out + + +def audit_binning_topology_preservation( + raw_volume: np.ndarray, + binned_volume: np.ndarray, +) -> dict[str, float]: + """Audit volume-fraction drift and percolation/backbone retention after 3D block binning.""" + from porousdiff.metrics import connectivity_and_percolation + + raw = np.asarray(raw_volume, dtype=np.uint8) + binned = np.asarray(binned_volume, dtype=np.uint8) + c_raw = connectivity_and_percolation(raw >= 1) + c_bin = connectivity_and_percolation(binned >= 1) + perc_agree = float( + int(c_raw["pore_percolates_z"] == c_bin["pore_percolates_z"]) + + int(c_raw["pore_percolates_y"] == c_bin["pore_percolates_y"]) + + int(c_raw["pore_percolates_x"] == c_bin["pore_percolates_x"]) + ) / 3.0 + return { + "solid_fraction_abs_err": abs(float((binned >= 1).mean()) - float((raw >= 1).mean())), + "coating_fraction_abs_err": abs(float((binned == 2).mean()) - float((raw == 2).mean())), + "solid_connected_fraction_raw": float(c_raw["solid_connected_fraction"]), + "solid_connected_fraction_binned": float(c_bin["solid_connected_fraction"]), + "solid_connected_abs_drift": abs( + float(c_bin["solid_connected_fraction"]) - float(c_raw["solid_connected_fraction"]) + ), + "pore_percolation_agreement": perc_agree, + } + + +def compute_radial_ring_score( + image: np.ndarray, + max_radius_fraction: float = 0.40, + center: tuple[float, float] | None = None, +) -> float: + """Scale-invariant concentric ring score via split-half detrended radial covariance. + + Splits the slice into disjoint upper and lower semicircles around `center`, + removes linear radial cupping, and measures their normalized radial covariance. + Uncorrelated fibers in opposite halves have zero expected covariance across both + small crops (`96x96`) and full `2560x2560` XCT slices (`0.00 - 0.08`), whereas + concentric bullseye ring seams and plunger interference bands spike (`> 0.15`). + """ + img = np.asarray(image, dtype=np.float32) + h, w = img.shape + yc, xc = center if center is not None else (h / 2.0, w / 2.0) + r_max = int(0.5 * min(h, w) * max_radius_fraction) + if r_max < 12: + return 0.0 + + # Subsample large slices by 2x for speed without losing ring structure + step = 2 if min(h, w) >= 512 else 1 + sub = img[::step, ::step] + sh, sw = sub.shape + syc, sxc = yc / step, xc / step + sr_max = max(12, r_max // step) + + yy, xx = np.indices((sh, sw)) + rr = np.sqrt((yy - syc) ** 2 + (xx - sxc) ** 2).astype(np.int32) + mask = (rr >= 4) & (rr < sr_max) + total_var = float(np.var(sub[mask])) + if total_var < 1e-6: + return 0.0 + + top = mask & (yy < syc - 1.5) + bot = mask & (yy > syc + 1.5) + s_top = np.bincount(rr[top], weights=sub[top], minlength=sr_max) + c_top = np.bincount(rr[top], minlength=sr_max) + s_bot = np.bincount(rr[bot], weights=sub[bot], minlength=sr_max) + c_bot = np.bincount(rr[bot], minlength=sr_max) + + valid = (c_top >= 6) & (c_bot >= 6) + if valid.sum() < 6: + return 0.0 + + r_val = np.nonzero(valid)[0].astype(np.float32) + r_val -= r_val.mean() + denom = max(float(np.dot(r_val, r_val)), 1e-6) + + m_top = s_top[valid] / c_top[valid] + m_bot = s_bot[valid] / c_bot[valid] + m_top = m_top - m_top.mean() - (float(np.dot(r_val, m_top)) / denom) * r_val + m_bot = m_bot - m_bot.mean() - (float(np.dot(r_val, m_bot)) / denom) * r_val + + cov = float(np.mean(m_top * m_bot)) + return float(np.clip(cov / total_var, 0.0, 1.0)) + + +def detect_valid_z_intervals( + ring_scores: Sequence[float], + pore_modes: Sequence[float], + slice_jumps: Sequence[float], + min_interval_length: int = 96, + max_ring_score: float = 0.15, + jump_factor: float = 3.5, + max_pore_mode_drift: float = 6500.0, + seam_margin: int = 12, +) -> tuple[np.ndarray, list[tuple[int, int]]]: + """Identify artifact-free Z intervals excluding bullseye seams and rig plungers. + + Returns: + valid_mask: boolean array of shape `(num_slices,)` + intervals: list of `(z_start, z_end)` half-open clean intervals with length >= `min_interval_length` + """ + rings = np.asarray(ring_scores, dtype=np.float32) + modes = np.asarray(pore_modes, dtype=np.float32) + jumps = np.asarray(slice_jumps, dtype=np.float32) + n = rings.size + if not (modes.size == n and jumps.size == n): + raise ValueError("Slice quality arrays must have equal length") + + clean_seed = rings <= max_ring_score + ref_modes = modes[clean_seed] if clean_seed.any() else modes + ref_jumps = jumps[clean_seed] if clean_seed.any() else jumps + + med_mode = float(np.median(ref_modes)) + med_jump = max(float(np.median(ref_jumps)), 1e-6) + + # Solid plungers/seams brighten the mode (> +max_pore_mode_drift), whereas Paganin + # dark halos around dense compressed fibers can dip locally without being artifacts. + bad = ( + (rings > max_ring_score) + | (jumps > jump_factor * med_jump) + | (modes > med_mode + max_pore_mode_drift) + | (modes < med_mode - 2.5 * max_pore_mode_drift) + ) + + # Close narrow gaps inside multi-slice seam/plunger zones, then pad by `seam_margin` + if bad.any(): + close_width = max(3, seam_margin) + bad = binary_closing(bad, structure=np.ones(close_width, dtype=bool)) + if seam_margin > 0: + pad_struct = np.ones(2 * seam_margin + 1, dtype=bool) + bad = binary_dilation(bad, structure=pad_struct) + + valid_mask = ~bad + intervals: list[tuple[int, int]] = [] + start: int | None = None + for z, ok in enumerate(valid_mask): + if ok and start is None: + start = z + elif not ok and start is not None: + if z - start >= min_interval_length: + intervals.append((start, z)) + start = None + if start is not None and n - start >= min_interval_length: + intervals.append((start, n)) + + # Zero out valid_mask in fragments shorter than min_interval_length + filtered_mask = np.zeros(n, dtype=bool) + for s, e in intervals: + filtered_mask[s:e] = True + + return filtered_mask, intervals + + +def split_z_intervals_with_buffer( + intervals: Sequence[tuple[int, int]], + patch_size: int, + val_fraction: float = 0.2, + buffer_slices: int | None = None, +) -> dict[str, list[tuple[int, int]]]: + """Split clean Z intervals into spatially disjoint `train` and `val` slabs. + + Inserts an unused buffer of at least `patch_size` slices wherever a contiguous + clean interval is partitioned between `train` and `val`, ensuring zero shared or + immediately adjacent voxels across splits. + """ + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be in (0, 1)") + buf = patch_size if buffer_slices is None else max(0, buffer_slices) + + usable = [(s, e) for s, e in intervals if (e - s) >= patch_size] + if not usable: + raise RuntimeError( + f"No clean Z intervals of length >= {patch_size} available" + ) + + total_clean = sum(e - s for s, e in usable) + target_val = max(patch_size, int(round(total_clean * val_fraction))) + + # Carve validation from the longest clean interval (or trailing clean interval if large enough) + train_intervals: list[tuple[int, int]] = [] + val_intervals: list[tuple[int, int]] = [] + + # Check if the last interval alone can serve as validation without splitting + if len(usable) >= 2 and usable[-1][1] - usable[-1][0] >= patch_size: + last_s, last_e = usable[-1] + last_len = last_e - last_s + if last_len <= max(target_val * 2, patch_size * 2): + train_intervals.extend(usable[:-1]) + val_intervals.append((last_s, last_e)) + return {"train": train_intervals, "val": val_intervals} + + # Otherwise split the longest interval with an explicit buffer gap + longest_idx = int(np.argmax([e - s for s, e in usable])) + for idx, (s, e) in enumerate(usable): + if idx != longest_idx: + train_intervals.append((s, e)) + continue + + span = e - s + val_len = max(patch_size, min(target_val, span - patch_size - buf)) + train_end = e - val_len - buf + val_start = e - val_len + if train_end - s < patch_size or e - val_start < patch_size: + # Fallback if interval is tight: shrink buffer only if unavoidable, else error + if span >= 2 * patch_size: + mid = s + (span - patch_size) // 2 + train_end = max(s + patch_size, e - patch_size - min(buf, span - 2 * patch_size)) + val_start = e - patch_size + else: + raise RuntimeError( + f"Clean Z span ({span} slices) cannot fit disjoint train and val patches " + f"of size {patch_size} (need >= {2 * patch_size + buf} slices)." + ) + train_intervals.append((s, train_end)) + val_intervals.append((val_start, e)) + + return {"train": train_intervals, "val": val_intervals} + + +def enumerate_cylinder_xy_positions( + shape_hw: tuple[int, int], + patch_size: int, + stride_xy: int, + radius_fraction: float = 0.70, + center: tuple[float, float] | None = None, +) -> list[tuple[int, int]]: + """Return `(y, x)` top-left patch coordinates whose entire square lies inside the circular ROI.""" + h, w = shape_hw + if h < patch_size or w < patch_size: + raise ValueError(f"Slice shape {shape_hw} smaller than patch_size={patch_size}") + + yc, xc = center if center is not None else (h / 2.0, w / 2.0) + radius_sq = (0.5 * min(h, w) * radius_fraction) ** 2 + + positions: list[tuple[int, int]] = [] + for y in range(0, h - patch_size + 1, stride_xy): + for x in range(0, w - patch_size + 1, stride_xy): + corners = ( + (y, x), + (y, x + patch_size - 1), + (y + patch_size - 1, x), + (y + patch_size - 1, x + patch_size - 1), + ) + if all((cy - yc) ** 2 + (cx - xc) ** 2 <= radius_sq for cy, cx in corners): + positions.append((y, x)) + + return positions + + +def split_xy_positions_with_buffer( + xy_positions: Sequence[tuple[int, int]], + patch_size: int, + val_fraction: float = 0.2, + buffer_px: int = 0, +) -> dict[str, list[tuple[int, int]]]: + """Partition `(y, x)` cylinder patch coordinates into strictly non-overlapping `train` and `val` sectors. + + Used automatically when a heavily compressed electrode scan is thinner than `3 * raw_window` + in Z (where Z-slab splitting would either fail or bias `val` toward platen compaction), + guaranteeing zero shared voxels across splits while sampling the full clean Z thickness. + """ + if not 0.0 < val_fraction < 1.0: + raise ValueError("val_fraction must be in (0, 1)") + pts = list(xy_positions) + if len(pts) < 2: + raise RuntimeError( + f"Need >= 2 XY patch positions to partition train/val in XY, got {len(pts)}" + ) + + ys = [p[0] for p in pts] + xs = [p[1] for p in pts] + axis = 1 if (max(xs) - min(xs)) >= (max(ys) - min(ys)) else 0 + + def _partition_with_gap(gap: int) -> tuple[list[tuple[int, int]], list[tuple[int, int]]] | None: + unique_coords = sorted({p[axis] for p in pts}) + best_pair: tuple[list[tuple[int, int]], list[tuple[int, int]]] | None = None + best_err = float("inf") + for cut in unique_coords[1:]: + val_pts = [p for p in pts if p[axis] >= cut] + train_pts = [p for p in pts if (p[axis] + patch_size + gap) <= cut] + if not train_pts or not val_pts: + continue + frac = len(val_pts) / float(len(train_pts) + len(val_pts)) + err = abs(frac - val_fraction) + if err < best_err: + best_err = err + best_pair = (train_pts, val_pts) + return best_pair + + res = _partition_with_gap(max(0, int(buffer_px))) + if res is None and buffer_px > 0: + res = _partition_with_gap(0) + if res is None: + raise RuntimeError( + f"XY cylinder ROI cannot accommodate two non-overlapping patches of size {patch_size} " + f"(positions={pts}); decrease --patch-size or increase --roi-radius-fraction." + ) + return {"train": res[0], "val": res[1]} + + +def build_stratified_patch_plan( + z_intervals: Sequence[tuple[int, int]], + xy_positions: Sequence[tuple[int, int]], + patch_size: int, + stride_z: int, + max_patches: int, + rng: np.random.Generator, +) -> list[tuple[int, list[tuple[int, int]]]]: + """Distribute patch budget uniformly across all valid Z-windows (round-robin). + + Prevents exhausting `max_patches` inside the first 2-3 Z-windows and guarantees + the full clean thickness of the electrode is represented. + """ + if not xy_positions: + raise ValueError("No valid XY positions inside circular ROI") + + z_windows: list[int] = [] + for s, e in z_intervals: + win = list(range(s, e - patch_size + 1, stride_z)) + # Ensure the trailing edge of the interval can also be sampled if missed by stride_z + last_valid = e - patch_size + if last_valid >= s and (not win or win[-1] != last_valid): + if not win or (last_valid - win[-1]) >= max(1, stride_z // 2): + win.append(last_valid) + z_windows.extend(win) + + if not z_windows: + raise ValueError("No valid Z windows available") + + z_order = list(z_windows) + rng.shuffle(z_order) + + per_z_queues: dict[int, list[tuple[int, int]]] = {} + for z in z_order: + xy_copy = list(xy_positions) + rng.shuffle(xy_copy) + per_z_queues[z] = xy_copy + + # Allocate extra candidate headroom (1.5x max_patches) in round-robin order + # so min/max solid-fraction filters still maintain uniform Z coverage. + target_candidates = min( + len(z_order) * len(xy_positions), + max(max_patches, int(max_patches * 1.5) + len(z_order)), + ) + + selected_per_z: dict[int, list[tuple[int, int]]] = {z: [] for z in z_order} + total_planned = 0 + cursor = 0 + while total_planned < target_candidates: + progressed = False + for z in z_order: + if cursor < len(per_z_queues[z]): + selected_per_z[z].append(per_z_queues[z][cursor]) + total_planned += 1 + progressed = True + if total_planned >= target_candidates: + break + if not progressed: + break + cursor += 1 + + # Sort Z ascending for sequential disk reads during extraction + return [(z, selected_per_z[z]) for z in sorted(selected_per_z) if selected_per_z[z]] diff --git a/porousdiff/transport_and_xrd.py b/porousdiff/transport_and_xrd.py new file mode 100644 index 0000000..149e848 --- /dev/null +++ b/porousdiff/transport_and_xrd.py @@ -0,0 +1,473 @@ +"""OpenImpala continuum-transport config/result integration & DLS K11 DIAD XRD correlation.""" + +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Iterable, Mapping + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +from scipy.stats import pearsonr, spearmanr + + +def write_openimpala_ini( + ini_path: Path, + tiff_path: Path, + output_prefix: Path, + transport_phase: str = "pore", + phase_value: int = 0, + directions: tuple[str, ...] = ("z", "x", "y"), + phase_conductivities: str | None = None, + template_ini: Path | None = None, +) -> Path: + """Write an OpenImpala `TortuosityFD` INI configuration file (or substitute a user template).""" + ini_path.parent.mkdir(parents=True, exist_ok=True) + output_prefix.parent.mkdir(parents=True, exist_ok=True) + dirs_str = ",".join(directions) + + if template_ini is not None: + raw_tpl = template_ini.read_text() + content = raw_tpl.format( + tiff_path=str(tiff_path.resolve()), + phase=int(phase_value), + transport_phase=transport_phase, + output_prefix=str(output_prefix.resolve()), + directions=dirs_str, + phase_conductivities=phase_conductivities or "", + ) + ini_path.write_text(content) + return ini_path + + geom_lines = [ + "# Auto-generated by PorousDiff v0.4 for OpenImpala TortuosityFD", + "[Geometry]", + f"filename = {tiff_path.resolve()}", + f"phase = {int(phase_value)}", + f"label = {transport_phase}", + ] + if phase_conductivities: + geom_lines.append(f"phase_conductivities = {phase_conductivities}") + + content = "\n".join( + [ + *geom_lines, + "", + "[Solver]", + "solver_type = DiffusionFD", + f"directions = {dirs_str}", + "tolerance = 1e-6", + "max_iterations = 20000", + "", + "[Output]", + f"output_prefix = {output_prefix.resolve()}", + "write_json = true", + "", + ] + ) + ini_path.write_text(content) + return ini_path + + +def generate_openimpala_job_suite( + tiff_entries: Iterable[Mapping[str, object]], + suite_dir: Path, + openimpala_cmd: str = "openimpala", + template_ini: Path | None = None, + include_hetero_solid: bool = False, + hetero_conductivities: str = "1:1.0,2:0.01", +) -> dict[str, object]: + """Generate INI configs, `openimpala_jobs.json`, and `run_openimpala_batch.sh` across methods. + + Each item in `tiff_entries` should contain: + - `method`: e.g. `"real"`, `"affine_z_squash"`, `"pillar_pore_deletion"`, `"porousdiff"` + - `state`: state identifier + - `material`: material key + - `beamline`: beamline identifier + - `load_value`: scalar load or stress + - `tiff_path`: `Path` to exported 3D multi-page `.tif` (`0=pore`, `1=carbon`, `2=coating`) + - `num_phases`: `2` or `3` + """ + suite_dir.mkdir(parents=True, exist_ok=True) + cfg_dir = suite_dir / "configs" + out_dir = suite_dir / "outputs" + cfg_dir.mkdir(parents=True, exist_ok=True) + out_dir.mkdir(parents=True, exist_ok=True) + + jobs: list[dict[str, object]] = [] + shell_lines = [ + "#!/bin/bash", + "set -euo pipefail", + f'OPENIMPALA_BIN="${{OPENIMPALA_BIN:-{openimpala_cmd}}}"', + "", + ] + + for entry in tiff_entries: + method = str(entry["method"]) + state = str(entry["state"]) + material = str(entry.get("material", "virgin_carbon")) + beamline = str(entry.get("beamline", "primary")) + load_val = float(entry.get("load_value", 0.0)) # type: ignore[arg-type] + tiff_path = Path(str(entry["tiff_path"])) + n_ph = int(entry.get("num_phases", 2)) # type: ignore[arg-type] + stem = tiff_path.stem + + # Configure ionic pore-phase diffusion (phase 0) and carbon backbone (phase 1); + # for 3-phase coated electrodes also configure active coating (phase 2) and optional + # heterogeneous composite solid conduction (`1:1.0,2:0.01`). + phase_targets: list[tuple[str, int, str | None]] = [ + ("pore_ionic", 0, None), + ("carbon_electronic", 1, None), + ] + if n_ph == 3: + phase_targets.append(("coating_active", 2, None)) + if include_hetero_solid: + phase_targets.append(("solid_hetero_electronic", 1, hetero_conductivities)) + + for transport_name, ph_val, ph_cond in phase_targets: + job_id = f"{method}__{state}__{stem}__{transport_name}" + ini_path = cfg_dir / f"{job_id}.ini" + out_prefix = out_dir / job_id + write_openimpala_ini( + ini_path=ini_path, + tiff_path=tiff_path, + output_prefix=out_prefix, + transport_phase=transport_name, + phase_value=ph_val, + phase_conductivities=ph_cond, + template_ini=template_ini, + ) + jobs.append( + { + "job_id": job_id, + "method": method, + "state": state, + "material": material, + "beamline": beamline, + "load_value": load_val, + "transport_phase": transport_name, + "phase_value": ph_val, + "phase_conductivities": ph_cond, + "tiff_path": str(tiff_path.resolve()), + "ini_path": str(ini_path.resolve()), + "output_json": str((out_dir / f"{job_id}.json").resolve()), + } + ) + shell_lines.append(f'"$OPENIMPALA_BIN" "{ini_path.resolve()}"') + + manifest = {"num_jobs": len(jobs), "jobs": jobs} + manifest_path = suite_dir / "openimpala_jobs.json" + manifest_path.write_text(json.dumps(manifest, indent=2)) + + script_path = suite_dir / "run_openimpala_batch.sh" + script_path.write_text("\n".join(shell_lines) + "\n") + script_path.chmod(0o755) + return { + "suite_dir": str(suite_dir), + "manifest_path": str(manifest_path), + "script_path": str(script_path), + "num_jobs": len(jobs), + } + + +def _parse_single_openimpala_file(path: Path) -> dict[str, float]: + """Extract `d_eff_x/y/z`, `tau_x/y/z`, and `volume_fraction` from JSON or key-value log.""" + if path.suffix.lower() == ".json": + raw = json.loads(path.read_text()) + out: dict[str, float] = {} + for k, v in raw.items(): + if isinstance(v, (int, float)): + out[k.lower()] = float(v) + elif isinstance(v, dict): + for sk, sv in v.items(): + if isinstance(sv, (int, float)): + out[f"{k.lower()}_{sk.lower()}"] = float(sv) + return out + + text = path.read_text(errors="replace") + parsed: dict[str, float] = {} + for m in re.finditer( + r"(d_eff_[xyz]|deff_[xyz]|tortuosity_[xyz]|tau_[xyz]|volume_fraction|porosity)\s*[:=]\s*([0-9.eE+-]+)", + text, + flags=re.IGNORECASE, + ): + key = m.group(1).lower().replace("deff_", "d_eff_").replace("tortuosity_", "tau_") + parsed[key] = float(m.group(2)) + return parsed + + +def parse_openimpala_results( + suite_dir: Path, + out_csv: Path | None = None, +) -> list[dict[str, object]]: + """Ingest completed OpenImpala outputs matched against `openimpala_jobs.json`.""" + jobs_file = suite_dir / "openimpala_jobs.json" + if not jobs_file.is_file(): + raise FileNotFoundError(f"Missing OpenImpala job manifest: {jobs_file}") + + jobs = json.loads(jobs_file.read_text()).get("jobs", []) + rows: list[dict[str, object]] = [] + outputs_dir = suite_dir / "outputs" + + for job in jobs: + job_id = str(job["job_id"]) + candidates = [ + Path(str(job["output_json"])), + outputs_dir / f"{job_id}.log", + outputs_dir / f"{job_id}.txt", + ] + found = next((p for p in candidates if p.is_file()), None) + if found is None: + continue + metrics = _parse_single_openimpala_file(found) + row: dict[str, object] = { + "job_id": job_id, + "method": job["method"], + "state": job["state"], + "material": job["material"], + "beamline": job["beamline"], + "load_value": job["load_value"], + "transport_phase": job["transport_phase"], + } + row.update(metrics) + rows.append(row) + + if out_csv is not None and rows: + out_csv.parent.mkdir(parents=True, exist_ok=True) + keys: list[str] = [] + for r in rows: + for k in r: + if k not in keys: + keys.append(k) + with out_csv.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=keys) + writer.writeheader() + writer.writerows(rows) + return rows + + +def correlate_xrd_with_microstructure( + xrd_csv_path: Path, + benchmark_report: Mapping[str, object], + out_dir: Path, + preferred_method: str | None = None, +) -> dict[str, object]: + """Correlate DLS K11 DIAD co-registered XRD observables with 3D XCT & PorousDiff metrics. + + Expects `xrd_csv_path` to contain a `state` column (matching manifest state names) OR + (`material` + `load_value`/`stress_kpa`/`load_mv_per_v`), plus one or more numeric XRD + columns (e.g. `d_spacing_A`, `lattice_strain_pct`, `fwhm_deg`, `coating_peak_ratio`). + """ + out_dir.mkdir(parents=True, exist_ok=True) + with xrd_csv_path.open("r", newline="") as f: + xrd_rows = list(csv.DictReader(f)) + if not xrd_rows: + raise ValueError(f"XRD CSV {xrd_csv_path} is empty") + + reserved = { + "state", + "folder", + "dataset", + "material", + "beamline", + "load", + "load_value", + "stress_kpa", + "load_mv_per_v", + } + xrd_cols = [ + c + for c in (xrd_rows[0].keys() if xrd_rows else []) + if c and c.strip().lower() not in reserved + ] + + micro_keys = [ + "solid_fraction", + "coating_fraction", + "ssa_pore_coating", + "tpb_density", + "wired_coating_fraction", + "geodesic_tortuosity_z", + "geodesic_solid_tortuosity_z", + "z_pinch_contact_density", + "specific_z_pinch_excess", + "coating_pinch_fraction", + "mean_coating_coordination", + ] + + merged_rows: list[dict[str, object]] = [] + for xrow in xrd_rows: + st_match: str | None = None + raw_st = (xrow.get("state") or xrow.get("folder") or xrow.get("dataset") or "").strip() + if raw_st and raw_st in benchmark_report: + st_match = raw_st + elif raw_st: + # Allow suffix match when collision-safe prefixes like `DLS_K11_DIAD__...` were added + for cand_st in benchmark_report: + if str(cand_st).endswith(raw_st): + st_match = str(cand_st) + break + + if st_match is None: + raw_lv = xrow.get("load_value") or xrow.get("stress_kpa") or xrow.get("load_mv_per_v") or xrow.get("load") + raw_mat = (xrow.get("material") or "").strip() + if raw_lv is not None and str(raw_lv).strip(): + target_lv = float(str(raw_lv).strip()) + best_st = None + best_diff = float("inf") + for cand_st, entry in benchmark_report.items(): + ent = entry # type: ignore[assignment] + if raw_mat and str(ent.get("material", "")) != raw_mat: # type: ignore[union-attr] + continue + diff = abs(float(ent.get("load_value", 0.0)) - target_lv) # type: ignore[union-attr] + if diff < best_diff and diff <= max(1e-4, 0.05 * max(abs(target_lv), 1e-3)): + best_diff = diff + best_st = str(cand_st) + st_match = best_st + + if st_match is None: + continue + + entry = benchmark_report[st_match] # type: ignore[index] + methods_dict = entry.get("methods", {}) # type: ignore[union-attr] + chosen_method = preferred_method + if chosen_method is None or chosen_method not in methods_dict: + non_base = [m for m in methods_dict if m not in ("affine_z_squash", "pillar_pore_deletion")] + chosen_method = non_base[0] if non_base else (next(iter(methods_dict), None)) + + merged: dict[str, object] = { + "state": st_match, + "material": entry.get("material", ""), # type: ignore[union-attr] + "load_value": float(entry.get("load_value", 0.0)), # type: ignore[union-attr] + "pred_method": chosen_method or "", + } + for xc in xrd_cols: + try: + merged[f"xrd_{xc}"] = float(str(xrow[xc]).strip()) + except (TypeError, ValueError): + merged[f"xrd_{xc}"] = float("nan") + + real_m = entry["real"]["metrics"] # type: ignore[index] + for mk in micro_keys: + if mk in real_m: + merged[f"real_{mk}"] = float(real_m[mk]["mean"]) + + if chosen_method and chosen_method in methods_dict: + pred_m = methods_dict[chosen_method]["metrics"] + for mk in micro_keys: + if mk in pred_m: + merged[f"pred_{mk}"] = float(pred_m[mk]["mean"]) + + merged_rows.append(merged) + + if not merged_rows: + raise ValueError("No rows in XRD CSV matched states or loads in benchmark_report") + + all_fieldnames: list[str] = [] + for row_d in merged_rows: + for k in row_d: + if k not in all_fieldnames: + all_fieldnames.append(k) + + merged_csv = out_dir / "xrd_microstructure_merged.csv" + with merged_csv.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=all_fieldnames) + writer.writeheader() + writer.writerows(merged_rows) + + def _ols_residual(vec: np.ndarray, cov: np.ndarray) -> np.ndarray: + if np.std(cov) < 1e-12: + return vec - np.mean(vec) + slope, inter = np.polyfit(cov, vec, 1) + return vec - (slope * cov + inter) + + correlations: dict[str, dict[str, dict[str, float]]] = {} + for xc in xrd_cols: + x_vals = np.asarray([float(r.get(f"xrd_{xc}", float("nan"))) for r in merged_rows], dtype=np.float64) + correlations[xc] = {} + for prefix in ("real", "pred"): + sf_col = f"{prefix}_solid_fraction" + sf_vals = ( + np.asarray([float(r.get(sf_col, float("nan"))) for r in merged_rows], dtype=np.float64) + if sf_col in all_fieldnames + else np.full_like(x_vals, float("nan")) + ) + for mk in micro_keys: + col_key = f"{prefix}_{mk}" + if col_key not in all_fieldnames: + continue + y_vals = np.asarray([float(r.get(col_key, float("nan"))) for r in merged_rows], dtype=np.float64) + mask = np.isfinite(x_vals) & np.isfinite(y_vals) + if int(mask.sum()) >= 3 and np.std(x_vals[mask]) > 1e-12 and np.std(y_vals[mask]) > 1e-12: + pr, pp = pearsonr(x_vals[mask], y_vals[mask]) + sr, sp = spearmanr(x_vals[mask], y_vals[mask]) + slope, intercept = np.polyfit(x_vals[mask], y_vals[mask], 1) + + part_r = float("nan") + part_p = float("nan") + pmask = mask & np.isfinite(sf_vals) + if mk != "solid_fraction" and int(pmask.sum()) >= 4 and np.std(sf_vals[pmask]) > 1e-12: + rx = _ols_residual(x_vals[pmask], sf_vals[pmask]) + ry = _ols_residual(y_vals[pmask], sf_vals[pmask]) + if np.std(rx) > 1e-12 and np.std(ry) > 1e-12: + pr_c, pp_c = pearsonr(rx, ry) + part_r, part_p = float(pr_c), float(pp_c) + else: + part_r, part_p = 0.0, 1.0 + + correlations[xc][col_key] = { + "n": int(mask.sum()), + "pearson_r": float(pr), + "pearson_p": float(pp), + "spearman_rho": float(sr), + "spearman_p": float(sp), + "partial_pearson_r_given_sf": part_r, + "partial_pearson_p_given_sf": part_p, + "linear_slope": float(slope), + "linear_intercept": float(intercept), + } + + summary_json = out_dir / "xrd_microstructure_correlation.json" + summary_payload = { + "num_matched_states": len(merged_rows), + "xrd_columns": xrd_cols, + "merged_csv": str(merged_csv), + "correlations": correlations, + } + summary_json.write_text(json.dumps(summary_payload, indent=2)) + + # Plot primary XRD observable against EASA, TPB density, tortuosity, and compressive pinch-points + if xrd_cols: + primary_xrd = xrd_cols[0] + candidate_targets = [ + ("ssa_pore_coating", "Active Coating EASA (1/µm)"), + ("tpb_density", "Triple-Phase Boundary (1/µm²)"), + ("geodesic_tortuosity_z", "Through-Plane Tortuosity τ_Z"), + ("coating_pinch_fraction", "Pinched Coating Fraction"), + ] + plot_targets = [ + (mk, lbl) for mk, lbl in candidate_targets if f"real_{mk}" in all_fieldnames + ] + if plot_targets: + fig, axes = plt.subplots(1, len(plot_targets), figsize=(4.2 * len(plot_targets), 4.0), squeeze=False) + xv = np.asarray([float(r.get(f"xrd_{primary_xrd}", float("nan"))) for r in merged_rows], dtype=np.float64) + for ax, (mk, label) in zip(axes[0], plot_targets): + ry = np.asarray([float(r.get(f"real_{mk}", float("nan"))) for r in merged_rows], dtype=np.float64) + ax.scatter(xv, ry, color="black", s=42, label="Real XCT") + if f"pred_{mk}" in all_fieldnames: + py = np.asarray([float(r.get(f"pred_{mk}", float("nan"))) for r in merged_rows], dtype=np.float64) + ax.scatter(xv, py, color="tab:blue", marker="^", s=46, label="PorousDiff") + ax.set_xlabel(f"DIAD XRD: {primary_xrd}") + ax.set_ylabel(label) + ax.grid(True, alpha=0.3) + ax.legend(fontsize=8) + fig.suptitle("DLS K11 DIAD Co-Registered XRD vs. 3D Microstructure & Pinch-Points", fontsize=12) + fig.tight_layout() + fig.savefig(out_dir / "diad_xrd_microstructure_correlation.png", dpi=220, bbox_inches="tight") + plt.close(fig) + + return summary_payload diff --git a/porousdiff/virtual_compression.py b/porousdiff/virtual_compression.py new file mode 100644 index 0000000..53ab98d --- /dev/null +++ b/porousdiff/virtual_compression.py @@ -0,0 +1,559 @@ +"""Virtual in-situ compression via DDIM ODE inversion & seamless >96^3 3D MultiDiffusion. + +Implements two zero-extra-beamtime inference capabilities on top of trained PorousDiff UNets: + 1. `virtual_compress_volumes`: Takes a real segmented 3D scan (e.g. benchtop XCT at `0 kPa` + or lowest synchrotron load `sigma_0`), deterministically inverts it along the conditional + DDIM probability-flow ODE at `source_load` (safely stopping before Zero-SNR `alpha_bar = 0`), + and denoises it back to `t = 0` conditioned on any `target_load` `sigma`. + 2. `generate_tiled_3d`: Synthesizes arbitrarily large, seam-free 3D electrode volumes + (e.g. `96 x 96 x 192` or `160^3`) for OpenImpala REV convergence using overlapped + 3D raised-cosine (Hann) weighted MultiDiffusion without increasing per-step GPU memory. +""" + +from __future__ import annotations + +import math + +import numpy as np +import torch +from scipy.ndimage import gaussian_filter + +from porousdiff.data import decode_channels_to_phases, encode_phases_to_channels +from porousdiff.models import ContinuousLoadConditionedUNet + + +def _pred_to_x0_and_eps( + x_t: torch.Tensor, + pred: torch.Tensor, + alpha_bar: float, + prediction_type: str, + clip_sample: bool = True, +) -> tuple[torch.Tensor, torch.Tensor]: + """Convert `(x_t, pred)` at scalar `alpha_bar` into consistent `(x0_hat, eps_eff)`.""" + ab = float(min(max(alpha_bar, 1e-6), 1.0 - 1e-6)) + sqrt_ab = math.sqrt(ab) + sqrt_omab = math.sqrt(1.0 - ab) + + x_f = x_t.float() + p_f = pred.float() + + if prediction_type == "v_prediction": + x0_hat = sqrt_ab * x_f - sqrt_omab * p_f + elif prediction_type == "epsilon": + x0_hat = (x_f - sqrt_omab * p_f) / sqrt_ab + else: + raise ValueError(f"Unsupported prediction_type={prediction_type!r}") + + if clip_sample: + x0_hat = x0_hat.clamp(-1.0, 1.0) + eps_eff = (x_f - sqrt_ab * x0_hat) / sqrt_omab + return x0_hat, eps_eff + + +def _build_safe_ddim_schedule( + alphas_cumprod: torch.Tensor, + num_steps: int, + invert_strength: float, + min_alpha_bar: float = 1e-4, +) -> list[int]: + """Select ascending DDIM timesteps `[t_0, ..., t_K]` strictly avoiding `alpha_bar == 0`.""" + if not 0.05 <= invert_strength <= 1.0: + raise ValueError(f"invert_strength must be in [0.05, 1.0], got {invert_strength}") + total_t = int(alphas_cumprod.shape[0]) + ab_np = alphas_cumprod.detach().cpu().numpy() + valid_indices = np.flatnonzero(ab_np >= min_alpha_bar) + if valid_indices.size < 2: + raise ValueError("Noise schedule has fewer than 2 timesteps with alpha_bar >= min_alpha_bar") + + max_safe_t = int(valid_indices[-1]) + raw_grid = np.linspace(0, max_safe_t, max(2, int(num_steps)), dtype=np.int64) + unique_asc = sorted({int(t) for t in raw_grid}) + if len(unique_asc) < 2: + unique_asc = [0, max_safe_t] + + stop_idx = max(1, min(len(unique_asc) - 1, int(round(invert_strength * (len(unique_asc) - 1))))) + return unique_asc[: stop_idx + 1] + + +@torch.inference_mode() +def ddim_invert_latent( + model: ContinuousLoadConditionedUNet, + x0: torch.Tensor, + alphas_cumprod: torch.Tensor, + source_load: float, + prediction_type: str = "v_prediction", + num_steps: int = 50, + invert_strength: float = 0.60, + guidance_scale_inv: float = 1.0, + material_id: int = 0, + voxel_size_um: float = 1.0, + use_amp: bool = False, + amp_dtype: torch.dtype = torch.float32, +) -> tuple[torch.Tensor, list[int]]: + """Deterministic forward DDIM ODE inversion mapping clean `x0` `[B, C, D, H, W]` -> `x_{t_K}`.""" + device = x0.device + bsz = x0.shape[0] + ab_cpu = alphas_cumprod.detach().cpu().float() + asc_timesteps = _build_safe_ddim_schedule( + ab_cpu, num_steps=num_steps, invert_strength=invert_strength + ) + + loads = torch.full((bsz,), float(source_load), device=device, dtype=torch.float32) + mats = ( + torch.full((bsz,), int(material_id), device=device, dtype=torch.long) + if model.is_multimodal + else None + ) + voxs = ( + torch.full((bsz,), float(voxel_size_um), device=device, dtype=torch.float32) + if model.is_multimodal + else None + ) + + # Lift clean x0 onto the first schedule timestep t_0 + t0 = asc_timesteps[0] + ab0 = float(ab_cpu[t0].item()) + cur = math.sqrt(ab0) * x0.float() + + for idx in range(len(asc_timesteps) - 1): + t_cur = asc_timesteps[idx] + t_next = asc_timesteps[idx + 1] + t_batch = torch.full((bsz,), t_cur, device=device, dtype=torch.long) + + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = model.forward_cfg( + cur, + t_batch, + loads, + guidance_scale=guidance_scale_inv, + material_ids=mats, + voxel_sizes=voxs, + ) + + # Do NOT clamp x0_hat during forward ODE inversion: on discrete {-1, +1} targets, + # clamping zeroes out eps_eff whenever |x0_hat| > 1 and collapses latent norm. + x0_hat, eps_eff = _pred_to_x0_and_eps( + cur, + pred, + float(ab_cpu[t_cur].item()), + prediction_type=prediction_type, + clip_sample=False, + ) + ab_next = float(ab_cpu[t_next].item()) + cur = math.sqrt(ab_next) * x0_hat + math.sqrt(max(1.0 - ab_next, 0.0)) * eps_eff + + return cur, asc_timesteps + + +@torch.inference_mode() +def ddim_denoise_from_inverted( + model: ContinuousLoadConditionedUNet, + x_inv: torch.Tensor, + asc_timesteps: list[int], + alphas_cumprod: torch.Tensor, + target_load: float, + prediction_type: str = "v_prediction", + guidance_scale: float = 1.5, + material_id: int = 0, + voxel_size_um: float = 1.0, + use_amp: bool = False, + amp_dtype: torch.dtype = torch.float32, +) -> tuple[np.ndarray, np.ndarray]: + """Deterministic reverse DDIM ODE denoising from inverted `x_inv` to `target_load`.""" + device = x_inv.device + bsz = x_inv.shape[0] + ab_cpu = alphas_cumprod.detach().cpu().float() + + loads = torch.full((bsz,), float(target_load), device=device, dtype=torch.float32) + mats = ( + torch.full((bsz,), int(material_id), device=device, dtype=torch.long) + if model.is_multimodal + else None + ) + voxs = ( + torch.full((bsz,), float(voxel_size_um), device=device, dtype=torch.float32) + if model.is_multimodal + else None + ) + + cur = x_inv.clone().float() + for idx in range(len(asc_timesteps) - 1, 0, -1): + t_cur = asc_timesteps[idx] + t_prev = asc_timesteps[idx - 1] + t_batch = torch.full((bsz,), t_cur, device=device, dtype=torch.long) + + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = model.forward_cfg( + cur, + t_batch, + loads, + guidance_scale=guidance_scale, + material_ids=mats, + voxel_sizes=voxs, + ) + + x0_hat, eps_eff = _pred_to_x0_and_eps( + cur, pred, float(ab_cpu[t_cur].item()), prediction_type=prediction_type + ) + ab_prev = float(ab_cpu[t_prev].item()) + cur = math.sqrt(ab_prev) * x0_hat + math.sqrt(max(1.0 - ab_prev, 0.0)) * eps_eff + + # Final projection at t_0 -> clean x0 + t0 = asc_timesteps[0] + t0_batch = torch.full((bsz,), t0, device=device, dtype=torch.long) + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred0 = model.forward_cfg( + cur, + t0_batch, + loads, + guidance_scale=guidance_scale, + material_ids=mats, + voxel_sizes=voxs, + ) + x0_final, _ = _pred_to_x0_and_eps( + cur, pred0, float(ab_cpu[t0].item()), prediction_type=prediction_type + ) + + raw_np = x0_final.cpu().numpy() + phases = decode_channels_to_phases(raw_np, enforce_solid_support=True, threshold=0.0) + probs = np.clip((raw_np[:, 0] + 1.0) * 0.5, 0.0, 1.0).astype(np.float32) + return phases, probs + + +@torch.inference_mode() +def virtual_compress_volumes( + model: ContinuousLoadConditionedUNet, + alphas_cumprod: torch.Tensor, + source_volumes: np.ndarray, + source_load: float, + target_loads: list[float] | tuple[float, ...], + prediction_type: str = "v_prediction", + num_steps: int = 50, + invert_strength: float = 0.60, + guidance_scale: float = 1.5, + material_id: int = 0, + voxel_size_um: float = 1.0, + batch_size: int = 4, + interface_smooth_sigma: float = 0.25, + use_amp: bool = False, + amp_dtype: torch.dtype = torch.float32, +) -> dict[float, np.ndarray]: + """Transfer real low-load 3D XCT patches `[N, D, H, W]` (or `[D, H, W]`) to `target_loads`. + + Inverts each batch of `source_volumes` ONCE at `source_load` (`invert_strength=0.60` + retains macroscopic in-plane XY fibre architecture while allowing Z pore collapse) and + branches the reverse DDIM trajectory across `target_loads`. Note that fixed-cube windows + represent statistical / mesoscale local microstructure conditioned on input latent structure, + not Lagrangian material-point tracking. + """ + vols = np.asarray(source_volumes, dtype=np.uint8) + single_input = vols.ndim == 3 + if single_input: + vols = vols[np.newaxis, ...] + if vols.ndim != 4: + raise ValueError(f"Expected 3D [D,H,W] or 4D [N,D,H,W] source_volumes, got {vols.shape}") + + device = next(model.parameters()).device + num_phases = model.num_phases + per_load_batches: dict[float, list[np.ndarray]] = {float(tl): [] for tl in target_loads} + + for start in range(0, vols.shape[0], max(1, batch_size)): + chunk = vols[start : start + max(1, batch_size)] + encoded_list: list[torch.Tensor] = [] + for v in chunk: + enc = encode_phases_to_channels(v, num_phases=num_phases) + if interface_smooth_sigma > 0.0: + enc_np = enc.numpy() + for ch_i in range(enc_np.shape[0]): + enc_np[ch_i] = np.clip( + gaussian_filter(enc_np[ch_i], sigma=interface_smooth_sigma), -1.0, 1.0 + ) + enc = torch.from_numpy(enc_np) + encoded_list.append(enc) + + x0_batch = torch.stack(encoded_list, dim=0).to(device=device, dtype=torch.float32) + x_inv, asc_steps = ddim_invert_latent( + model=model, + x0=x0_batch, + alphas_cumprod=alphas_cumprod, + source_load=source_load, + prediction_type=prediction_type, + num_steps=num_steps, + invert_strength=invert_strength, + guidance_scale_inv=1.0, + material_id=material_id, + voxel_size_um=voxel_size_um, + use_amp=use_amp, + amp_dtype=amp_dtype, + ) + + for tl in target_loads: + tl_f = float(tl) + eff_cfg = 1.0 if abs(tl_f - float(source_load)) < 1e-8 else guidance_scale + phases_out, _ = ddim_denoise_from_inverted( + model=model, + x_inv=x_inv, + asc_timesteps=asc_steps, + alphas_cumprod=alphas_cumprod, + target_load=tl_f, + prediction_type=prediction_type, + guidance_scale=eff_cfg, + material_id=material_id, + voxel_size_um=voxel_size_um, + use_amp=use_amp, + amp_dtype=amp_dtype, + ) + per_load_batches[tl_f].append(phases_out) + + results: dict[float, np.ndarray] = {} + for tl_f, chunks in per_load_batches.items(): + cat = np.concatenate(chunks, axis=0) + results[tl_f] = cat[0] if single_input else cat + return results + + +def check_load_extrapolation( + loads: list[float] | tuple[float, ...], + load_min: float, + load_max: float, + tol: float = 1e-6, +) -> dict[str, object]: + """Identify whether any requested source or target load extrapolates outside `[load_min, load_max]`.""" + out_of_range = [ + float(lv) + for lv in loads + if float(lv) < float(load_min) - tol or float(lv) > float(load_max) + tol + ] + return { + "load_min": float(load_min), + "load_max": float(load_max), + "is_extrapolated": len(out_of_range) > 0, + "extrapolated_loads": out_of_range, + } + + +def compute_virtual_compression_fidelity( + source_volumes: np.ndarray, + compressed_volumes: np.ndarray, + same_load_reconstructions: np.ndarray | None = None, +) -> dict[str, float]: + """Quantify structural retention vs random resampling in DDIM virtual compression. + + Computes: + - `same_load_solid_dice`: Solid-phase Dice overlap (`2|A&B| / (|A|+|B|)`) for `sigma_0 -> sigma_0` + roundtrip reconstruction (immune to high-porosity accuracy inflation). + - `paired_xy_column_corr`: Mean Pearson correlation between each input scan's Z-averaged + XY solid density `mean_Z(source_i >= 1)` and its own compressed output `mean_Z(comp_i >= 1)`. + - `unpaired_xy_column_corr`: Baseline correlation against unrelated shifted inputs `source_{(i+1)%N}`. + - `xy_correlation_lift`: `paired_xy_column_corr - unpaired_xy_column_corr` (`> 0` confirms + specimen-specific structural memory rather than unconditional resampling). + """ + src = np.asarray(source_volumes) + cmp = np.asarray(compressed_volumes) + if src.ndim == 3: + src = src[np.newaxis, ...] + if cmp.ndim == 3: + cmp = cmp[np.newaxis, ...] + + same_dice = float("nan") + if same_load_reconstructions is not None: + rec = np.asarray(same_load_reconstructions) + if rec.ndim == 3: + rec = rec[np.newaxis, ...] + dices: list[float] = [] + for s_v, r_v in zip(src, rec): + s_b = s_v >= 1 + r_b = r_v >= 1 + denom = int(s_b.sum()) + int(r_b.sum()) + dices.append(1.0 if denom == 0 else 2.0 * float(np.count_nonzero(s_b & r_b)) / float(denom)) + same_dice = float(np.mean(dices)) if dices else float("nan") + + def _corr2d(a: np.ndarray, b: np.ndarray) -> float: + af = a.ravel().astype(np.float64) + bf = b.ravel().astype(np.float64) + sa = float(np.std(af)) + sb = float(np.std(bf)) + if sa < 1e-12 or sb < 1e-12: + return 0.0 + return float(np.corrcoef(af, bf)[0, 1]) + + n = min(src.shape[0], cmp.shape[0]) + paired_corrs: list[float] = [] + unpaired_corrs: list[float] = [] + for i in range(n): + src_xy = (src[i] >= 1).mean(axis=0) + cmp_xy = (cmp[i] >= 1).mean(axis=0) + paired_corrs.append(_corr2d(src_xy, cmp_xy)) + if n >= 2: + other_xy = (src[(i + 1) % n] >= 1).mean(axis=0) + unpaired_corrs.append(_corr2d(other_xy, cmp_xy)) + + paired_mean = float(np.mean(paired_corrs)) if paired_corrs else float("nan") + unpaired_mean = float(np.mean(unpaired_corrs)) if unpaired_corrs else float("nan") + lift = ( + paired_mean - unpaired_mean + if (np.isfinite(paired_mean) and np.isfinite(unpaired_mean)) + else float("nan") + ) + return { + "same_load_solid_dice": same_dice, + "paired_xy_column_corr": paired_mean, + "unpaired_xy_column_corr": unpaired_mean, + "xy_correlation_lift": lift, + } + + +def _compute_tile_starts(full_len: int, patch_size: int, overlap: float) -> list[int]: + if full_len < patch_size: + raise ValueError(f"Target dimension {full_len} must be >= patch_size {patch_size}") + if full_len == patch_size: + return [0] + max_stride = max(1, int(round(patch_size * (1.0 - overlap)))) + span = full_len - patch_size + num_intervals = max(1, math.ceil(span / float(max_stride))) + raw_starts = [int(round(i * span / float(num_intervals))) for i in range(num_intervals + 1)] + starts: list[int] = [] + for s in raw_starts: + if not starts or s > starts[-1]: + starts.append(s) + if starts[-1] != span: + starts.append(span) + return starts + + +def _build_3d_tile_weight( + patch_size: int, + z0: int, + y0: int, + x0: int, + full_shape: tuple[int, int, int], + device: torch.device, +) -> torch.Tensor: + """Smooth 3D raised-cosine window with flat outer-border support for MultiDiffusion.""" + idx = np.arange(patch_size, dtype=np.float32) + hann = 0.5 * (1.0 - np.cos(2.0 * np.pi * (idx + 0.5) / float(patch_size))) + hann = np.clip(hann, 1e-3, 1.0) + half = patch_size // 2 + + def _axis_w(start: int, total: int) -> np.ndarray: + if total == patch_size: + return np.ones(patch_size, dtype=np.float32) + w = hann.copy() + if start == 0: + w[:half] = 1.0 + if start + patch_size == total: + w[half:] = 1.0 + return w + + wz = _axis_w(z0, full_shape[0]) + wy = _axis_w(y0, full_shape[1]) + wx = _axis_w(x0, full_shape[2]) + w3d = wz[:, None, None] * wy[None, :, None] * wx[None, None, :] + return torch.from_numpy(w3d).to(device=device, dtype=torch.float32).unsqueeze(0).unsqueeze(0) + + +@torch.inference_mode() +def generate_tiled_3d( + model: ContinuousLoadConditionedUNet, + scheduler: object, + target_shape: tuple[int, int, int], + patch_size: int, + load_value: float, + guidance_scale: float = 1.5, + overlap: float = 0.5, + batch_size: int = 1, + tile_batch_size: int = 4, + material_id: int = 0, + voxel_size_um: float = 1.0, + use_amp: bool = False, + amp_dtype: torch.dtype = torch.float32, + generator: torch.Generator | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Seamless 3D MultiDiffusion synthesizing `target_shape = (D, H, W)` >= `patch_size^3`.""" + if not 0.1 <= overlap <= 0.85: + raise ValueError(f"overlap must be in [0.1, 0.85], got {overlap}") + d_full, h_full, w_full = (int(s) for s in target_shape) + device = next(model.parameters()).device + data_ch = model.data_channels + + z_starts = _compute_tile_starts(d_full, patch_size, overlap) + y_starts = _compute_tile_starts(h_full, patch_size, overlap) + x_starts = _compute_tile_starts(w_full, patch_size, overlap) + coords = [(z0, y0, x0) for z0 in z_starts for y0 in y_starts for x0 in x_starts] + + tile_weights = [ + _build_3d_tile_weight(patch_size, z0, y0, x0, (d_full, h_full, w_full), device) + for (z0, y0, x0) in coords + ] + weight_norm = torch.zeros((1, 1, d_full, h_full, w_full), device=device, dtype=torch.float32) + for (z0, y0, x0), tw in zip(coords, tile_weights): + weight_norm[:, :, z0 : z0 + patch_size, y0 : y0 + patch_size, x0 : x0 + patch_size] += tw + weight_norm = weight_norm.clamp(min=1e-6) + + image = torch.randn( + (batch_size, data_ch, d_full, h_full, w_full), + generator=generator, + device=device, + dtype=torch.float32, + ) + + for timestep in scheduler.timesteps: # type: ignore[attr-defined] + t_int = int(timestep) + pred_accum = torch.zeros_like(image, dtype=torch.float32) + + # Evaluate overlapping 3D windows in memory-bounded mini-batches + flat_tiles: list[tuple[int, int, int, int, torch.Tensor, torch.Tensor]] = [] + for b_idx in range(batch_size): + for (z0, y0, x0), tw in zip(coords, tile_weights): + crop = image[ + b_idx : b_idx + 1, + :, + z0 : z0 + patch_size, + y0 : y0 + patch_size, + x0 : x0 + patch_size, + ] + flat_tiles.append((b_idx, z0, y0, x0, crop, tw)) + + for chunk_start in range(0, len(flat_tiles), max(1, tile_batch_size)): + sub = flat_tiles[chunk_start : chunk_start + max(1, tile_batch_size)] + sub_b = len(sub) + x_sub = torch.cat([item[4] for item in sub], dim=0) + t_sub = torch.full((sub_b,), t_int, device=device, dtype=torch.long) + l_sub = torch.full((sub_b,), float(load_value), device=device, dtype=torch.float32) + m_sub = ( + torch.full((sub_b,), int(material_id), device=device, dtype=torch.long) + if model.is_multimodal + else None + ) + v_sub = ( + torch.full((sub_b,), float(voxel_size_um), device=device, dtype=torch.float32) + if model.is_multimodal + else None + ) + + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred_sub = model.forward_cfg( + x_sub, + t_sub, + l_sub, + guidance_scale=guidance_scale, + material_ids=m_sub, + voxel_sizes=v_sub, + ).float() + + for item_i, (b_idx, z0, y0, x0, _, tw) in enumerate(sub): + pred_accum[ + b_idx : b_idx + 1, + :, + z0 : z0 + patch_size, + y0 : y0 + patch_size, + x0 : x0 + patch_size, + ] += pred_sub[item_i : item_i + 1] * tw + + blended_pred = pred_accum / weight_norm + image = scheduler.step(blended_pred, t_int, image, generator=generator)[0] # type: ignore[attr-defined] + + raw_np = image.float().cpu().numpy() + phases = decode_channels_to_phases(raw_np, enforce_solid_support=True, threshold=0.0) + probs = np.clip((raw_np[:, 0] + 1.0) * 0.5, 0.0, 1.0).astype(np.float32) + return phases, probs diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0729bbc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools>=64"] +build-backend = "setuptools.build_meta" + +[project] +name = "porousdiff" +version = "0.4.0" +description = "Conditional 3D Diffusion Surrogate Modelling for Compressed Porous Battery Electrodes" +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.10" +dependencies = [ + "numpy>=1.24", + "scipy>=1.10", + "scikit-image>=0.21", + "tifffile>=2023.7", + "matplotlib>=3.7", + "torch>=2.1", + "monai>=1.3", + "einops>=0.7", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4", + "ruff>=0.4", +] +transport = [ + "porespy>=2.3", +] + +[tool.setuptools.packages.find] +include = ["porousdiff*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-q" + diff --git a/requirements-isambard.txt b/requirements-isambard.txt index bded01b..08e46e5 100644 --- a/requirements-isambard.txt +++ b/requirements-isambard.txt @@ -1,46 +1,17 @@ -contourpy==1.3.3 -cuda-bindings==12.9.7 -cuda-pathfinder==1.6.0 -cuda-toolkit==12.8.1 -cycler==0.12.1 -filelock==3.32.3 -fonttools==4.66.1 -fsspec==2026.7.0 -ImageIO==2.38.0 -Jinja2==3.1.6 -kiwisolver==1.5.1 -lazy-loader==0.6 -MarkupSafe==3.0.3 -matplotlib==3.11.2 -mpmath==1.3.0 -networkx==3.6.1 -numpy==2.4.6 -nvidia-cublas-cu12==12.8.4.1 -nvidia-cuda-cupti-cu12==12.8.90 -nvidia-cuda-nvrtc-cu12==12.8.93 -nvidia-cuda-runtime-cu12==12.8.90 -nvidia-cudnn-cu12==9.19.0.56 -nvidia-cufft-cu12==11.3.3.83 -nvidia-cufile-cu12==1.13.1.3 -nvidia-curand-cu12==10.3.9.90 -nvidia-cusolver-cu12==11.7.3.90 -nvidia-cusparse-cu12==12.5.8.93 -nvidia-cusparselt-cu12==0.7.1 -nvidia-nccl-cu12==2.28.9 -nvidia-nvjitlink-cu12==12.8.93 -nvidia-nvshmem-cu12==3.4.5 -nvidia-nvtx-cu12==12.8.90 -packaging==26.3 -pillow==12.3.0 -pyparsing==3.3.3 -python-dateutil==2.9.0.post0 -scikit-image==0.26.0 -scipy==1.17.1 -six==1.17.0 -sympy==1.14.0 -tifffile==2026.3.3 -torch==2.11.0+cu128 -torchvision==0.26.0+cu128 -tqdm==4.70.1 -triton==3.6.0 -typing_extensions==4.16.0 +# Isambard-AI (NVIDIA GH200 Grace-Hopper, ARM64 aarch64 + CUDA 12.8) +# Install PyTorch for ARM64 CUDA 12.8 first if creating a fresh venv: +# pip install torch torchvision --index-url https://download.pytorch.org/whl/cu128 +# pip install -r requirements-isambard.txt +# pip install -e . + +monai>=1.4.0 +einops>=0.7.0 +numpy>=2.0.0 +scipy>=1.13.0 +scikit-image>=0.24.0 +tifffile>=2024.5.0 +matplotlib>=3.9.0 +imageio>=2.34.0 +pillow>=10.3.0 +tqdm>=4.66.0 +pytest>=8.0.0 diff --git a/scripts/evaluation/evaluate_beamtime_multiplier.py b/scripts/evaluation/evaluate_beamtime_multiplier.py new file mode 100755 index 0000000..7e76250 --- /dev/null +++ b/scripts/evaluation/evaluate_beamtime_multiplier.py @@ -0,0 +1,643 @@ +"""Head-to-head benchmark: Real Held-Out XCT vs. Numerical Z-Compression Baselines vs. PorousDiff. + +Evaluates the core scientific claims of the Beamtime Multiplier paper: + 1. Non-affine compression fidelity: Compares PorousDiff against standard literature heuristics + (`affine_z_compress_to_target` and `pillar_pore_deletion_compress` computed from each + material's uncompressed/lowest-load XCT scan) on held-out real synchrotron loads. + 2. Multi-phase coated-electrode electrochemistry: Evaluates 3-phase EASA (`ssa_pore_coating`), + triple-phase boundary density (`tpb_density`), wired coating fraction, chord anisotropy, + two-point correlation `S_2(r)` RMSE, percolation, and geodesic pore tortuosity. + 3. Synchrotron Beamtime Multiplier: Compares single-material sparse models, Few-Shot + Virgin-transferred models (`2` or `3` coated loads), and Full Oracle models side by side. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +import torch + +from porousdiff.baselines import ( + affine_z_compress_to_target, + compute_mirror_tiled_fraction, + pillar_pore_deletion_compress, +) +from porousdiff.data import ( + default_project_root, + load_split_patches, + load_tall_or_split_patches, + resolve_material_id, + resolve_patch_split_dir, + resolve_scan_load_value, + resolve_use_stress_kpa_from_checkpoints, +) +from porousdiff.losses import compute_coating_retention_diagnostics +from porousdiff.metrics import ( + directional_two_point_correlation, + evaluate_multiphase_physics, + export_openimpala_tiff, + summarize_metric_records, +) +from porousdiff.models import ( + ContinuousLoadConditionedUNet, + EMAModel, + build_noise_scheduler, + infer_use_ordinal_material_channel, +) +from porousdiff.preprocessing import infer_material_name, parse_load_value +from porousdiff.transport_and_xrd import ( + correlate_xrd_with_microstructure, + generate_openimpala_job_suite, + parse_openimpala_results, +) +from porousdiff.virtual_compression import ( + check_load_extrapolation, + compute_virtual_compression_fidelity, + virtual_compress_volumes, +) + + +def _load_generated_volumes(samples_dir: Path, state_name: str) -> np.ndarray: + state_dir = samples_dir / state_name + stacked = state_dir / "generated_samples.npy" + if stacked.is_file(): + return np.load(stacked) + files = sorted(state_dir.glob("sample_*_binary.npy")) + if not files: + raise FileNotFoundError(f"No generated samples found in {state_dir}") + return np.stack([np.load(p) for p in files], axis=0) + + +def _mean_s2_curves(volumes: np.ndarray, max_lag: int) -> dict[str, np.ndarray]: + per_vol = [directional_two_point_correlation(v >= 1, max_lag=max_lag) for v in volumes] + return { + "lags": per_vol[0]["lags"], + "s2_z": np.mean([c["s2_z"] for c in per_vol], axis=0), + "s2_xy": np.mean([0.5 * (c["s2_x"] + c["s2_y"]) for c in per_vol], axis=0), + } + + +def _s2_rmse(a: dict[str, np.ndarray], b: dict[str, np.ndarray]) -> float: + dz = float(np.sqrt(np.mean((a["s2_z"] - b["s2_z"]) ** 2))) + dxy = float(np.sqrt(np.mean((a["s2_xy"] - b["s2_xy"]) ** 2))) + return 0.5 * (dz + dxy) + + +def _parse_experiments(entries: list[str]) -> dict[str, str]: + """Parse `label=experiment_name` or bare `experiment_name`.""" + out: dict[str, str] = {} + for item in entries: + if "=" in item: + lbl, exp = item.split("=", 1) + out[lbl.strip()] = exp.strip() + else: + out[item.strip()] = item.strip() + return out + + +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Benchmark PorousDiff vs. Affine & Pillar Z-Compression Baselines on Real Held-Out XCT" + ) + parser.add_argument( + "--experiments", + nargs="+", + required=True, + help="Experiment(s) to evaluate, e.g. porousdiff=beamtime_fewshot oracle=beamtime_oracle", + ) + parser.add_argument("--data-root", type=Path, default=proj_root / "data" / "carbon_virgin") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--out-dir", type=Path, default=None) + parser.add_argument("--manifest-json", type=Path, default=None) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--states", nargs="*", default=None, help="Fallback state=load list if no --manifest-json") + parser.add_argument("--heldout-states", nargs="*", default=None) + parser.add_argument("--include-repeats", action="store_true") + parser.add_argument("--beamline", default=None, help="Optional beamline filter (e.g. DLS_K11_DIAD)") + parser.add_argument( + "--use-stress-kpa", + action="store_true", + help="Fallback load unit when no checkpoint exists under --output-root (otherwise read from checkpoint config)", + ) + parser.add_argument("--max-real-samples", type=int, default=20) + parser.add_argument("--max-openimpala-per-method", type=int, default=5) + parser.add_argument("--voxel-size-um", type=float, default=1.0) + parser.add_argument("--export-real-openimpala", action="store_true") + parser.add_argument( + "--generate-openimpala-suite", + action="store_true", + help="Generate OpenImpala INI configs and run_openimpala_batch.sh across Real, Baselines & PorousDiff", + ) + parser.add_argument( + "--openimpala-template-ini", + type=Path, + default=None, + help="Optional user OpenImpala INI template with {tiff_path}, {phase}, {transport_phase}, {output_prefix}", + ) + parser.add_argument( + "--xrd-csv", + type=Path, + default=None, + help="Optional DLS K11 DIAD co-registered XRD CSV to correlate against 3D microstructural observables", + ) + parser.add_argument( + "--include-virtual-compression", + action="store_true", + help="Virtually compress real lowest-load XCT patches via deterministic DDIM ODE inversion + target-stress denoising", + ) + parser.add_argument("--checkpoint", type=Path, default=None, help="Optional checkpoint override for --include-virtual-compression") + parser.add_argument("--virtual-steps", type=int, default=40) + parser.add_argument("--invert-strength", type=float, default=0.60) + parser.add_argument("--guidance-scale", type=float, default=1.5) + parser.add_argument("--allow-cpu", action="store_true") + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + exp_map = _parse_experiments(args.experiments) + first_exp = next(iter(exp_map.values())) + out_dir = ( + args.out_dir + if args.out_dir is not None + else (args.output_root / "results" / first_exp / "beamtime_benchmark") + ) + out_dir.mkdir(parents=True, exist_ok=True) + heldout_set = set(args.heldout_states or []) + + use_kpa, _ = resolve_use_stress_kpa_from_checkpoints( + output_root=args.output_root, + experiments=list(exp_map.values()), + explicit_checkpoint=args.checkpoint, + default=args.use_stress_kpa, + ) + unit_label = "kPa" if use_kpa else "mV/V" + + # Build state specification list grouped by (material, beamline) + state_specs: list[dict[str, object]] = [] + if args.manifest_json is not None: + manifest = json.loads(args.manifest_json.read_text()) + for sc in manifest.get("scans", []): + if not args.include_repeats and bool(sc.get("is_repeat_or_secondary", False)): + continue + sc_beam = str(sc.get("beamline", "primary")) + if args.beamline and sc_beam != args.beamline: + continue + lv_resolved = resolve_scan_load_value(sc, use_kpa) + state_specs.append( + { + "state": str(sc["state"]), + "material": str(sc.get("material", "virgin_carbon")), + "beamline": sc_beam, + "load_value": lv_resolved, + "voxel_size_um": float(sc.get("effective_voxel_size_um", args.voxel_size_um)), + "num_phases": int(sc.get("num_phases", 2)), + "patch_dir_name": sc.get("patch_dir_name"), + } + ) + elif args.states: + for item in args.states: + if "=" in item: + st, lv_s = item.split("=", 1) + st, lv = st.strip(), float(lv_s.strip()) + else: + st = item.strip() + lv = parse_load_value(st) + state_specs.append( + { + "state": st, + "material": infer_material_name(st), + "beamline": args.beamline or "primary", + "load_value": lv, + "voxel_size_um": args.voxel_size_um, + "num_phases": 3 if infer_material_name(st) != "virgin_carbon" else 2, + "patch_dir_name": None, + } + ) + else: + raise ValueError("Provide either --manifest-json or --states") + + state_specs.sort( + key=lambda s: (str(s["material"]), str(s["beamline"]), float(s["load_value"])) # type: ignore[arg-type] + ) + + # Identify lowest-load reference real validation patches per (material, beamline) + rng = np.random.default_rng(args.seed) + expected_shape = (args.patch_size, args.patch_size, args.patch_size) + real_cache: dict[str, np.ndarray] = {} + tall_ref_cache: dict[str, np.ndarray] = {} + lowest_state_by_group: dict[tuple[str, str], str] = {} + lowest_load_by_group: dict[tuple[str, str], float] = {} + + for spec in state_specs: + st = str(spec["state"]) + mat = str(spec["material"]) + beam = str(spec["beamline"]) + lv = float(spec["load_value"]) # type: ignore[arg-type] + pdir_hint = str(spec["patch_dir_name"]) if spec.get("patch_dir_name") else None + split_dir = resolve_patch_split_dir( + args.data_root, st, args.patch_size, args.version, "val", patch_dir_name=pdir_hint + ) + vols = load_split_patches(split_dir, expected_shape=expected_shape) + if vols.shape[0] > args.max_real_samples: + idx = np.sort(rng.choice(vols.shape[0], size=args.max_real_samples, replace=False)) + vols = vols[idx] + real_cache[st] = vols + tall_vols = load_tall_or_split_patches(split_dir, vols) + if tall_vols.shape[0] > args.max_real_samples: + tall_vols = tall_vols[: args.max_real_samples] + tall_ref_cache[st] = tall_vols + if (mat, beam) not in lowest_state_by_group: + lowest_state_by_group[(mat, beam)] = st + lowest_load_by_group[(mat, beam)] = lv + + # Optional: Virtual compression of real lowest-load scans via DDIM ODE inversion + virtual_cache_by_state: dict[str, np.ndarray] = {} + vc_same_load_by_group: dict[tuple[str, str], np.ndarray] = {} + vc_extrap_by_group: dict[tuple[str, str], dict[str, object]] = {} + if args.include_virtual_compression: + if torch.cuda.is_available(): + vc_device = torch.device("cuda") + elif args.allow_cpu: + vc_device = torch.device("cpu") + else: + raise RuntimeError("CUDA GPU required for --include-virtual-compression (or pass --allow-cpu)") + + ckpt_dir = args.output_root / "checkpoints" / first_exp + ckpt_path = ( + args.checkpoint + if args.checkpoint is not None + else ( + ckpt_dir / "best_val.pth" + if (ckpt_dir / "best_val.pth").is_file() + else ckpt_dir / "latest.pth" + ) + ) + if not ckpt_path.is_file(): + raise FileNotFoundError(f"Checkpoint not found for virtual compression: {ckpt_path}") + + saved_ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False) + vc_cfg = saved_ckpt["config"] + vc_num_materials = int(vc_cfg.get("num_materials", 1)) + vc_model = ContinuousLoadConditionedUNet( + channels=vc_cfg["channels"], + attention_levels=vc_cfg["attention_levels"], + num_res_blocks=int(vc_cfg["num_res_blocks"]), + norm_num_groups=int(vc_cfg["norm_groups"]), + load_min=float(vc_cfg["load_min"]), + load_max=float(vc_cfg["load_max"]), + use_flash_attention=(vc_device.type == "cuda"), + conditioning_mode=str(vc_cfg.get("conditioning_mode", "multiscale_fourier")), + num_phases=int(vc_cfg.get("num_phases", 2)), + num_materials=vc_num_materials, + ref_voxel_size_um=float(vc_cfg.get("ref_voxel_size_um", 1.0)), + use_ordinal_material_channel=infer_use_ordinal_material_channel(saved_ckpt), + ).to(vc_device) + if "ema" in saved_ckpt: + vc_ema = EMAModel(vc_model) + vc_ema.load_state_dict(saved_ckpt["ema"]) + vc_active = vc_ema.shadow_model.to(vc_device).eval() + else: + vc_model.load_state_dict(saved_ckpt["model"]) + vc_active = vc_model.eval() + + vc_sched = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=int(vc_cfg["timesteps"]), + schedule=str(vc_cfg["schedule"]), + prediction_type=str(vc_cfg["prediction_type"]), + ) + vc_use_amp = vc_device.type == "cuda" + vc_amp_dtype = ( + torch.bfloat16 + if (vc_device.type == "cuda" and torch.cuda.is_bf16_supported()) + else torch.float16 + ) + + specs_by_group: dict[tuple[str, str], list[dict[str, object]]] = {} + for sp in state_specs: + specs_by_group.setdefault((str(sp["material"]), str(sp["beamline"])), []).append(sp) + + for (mat, beam), grp_specs in specs_by_group.items(): + ref_st = lowest_state_by_group[(mat, beam)] + ref_lv = lowest_load_by_group[(mat, beam)] + ref_vols = real_cache[ref_st] + target_lvs = list( + dict.fromkeys([ref_lv, *[float(sp["load_value"]) for sp in grp_specs]]) # type: ignore[arg-type] + ) + vc_extrap_by_group[(mat, beam)] = check_load_extrapolation( + target_lvs, float(vc_cfg["load_min"]), float(vc_cfg["load_max"]) + ) + mat_id = resolve_material_id(mat) if vc_num_materials > 1 else 0 + vox_um = float(grp_specs[0]["voxel_size_um"]) # type: ignore[arg-type] + vc_map = virtual_compress_volumes( + model=vc_active, + alphas_cumprod=vc_sched.alphas_cumprod, + source_volumes=ref_vols, + source_load=ref_lv, + target_loads=target_lvs, + prediction_type=str(vc_cfg["prediction_type"]), + num_steps=args.virtual_steps, + invert_strength=args.invert_strength, + guidance_scale=args.guidance_scale, + material_id=mat_id, + voxel_size_um=vox_um, + batch_size=min(4, int(ref_vols.shape[0])), + use_amp=vc_use_amp, + amp_dtype=vc_amp_dtype, + ) + vc_same_load_by_group[(mat, beam)] = vc_map[ref_lv] + for sp in grp_specs: + st_k = str(sp["state"]) + lv_k = float(sp["load_value"]) # type: ignore[arg-type] + virtual_cache_by_state[st_k] = vc_map[lv_k] + + max_lag = min(32, args.patch_size // 2) + report: dict[str, object] = {} + csv_rows: list[dict[str, object]] = [] + openimpala_entries: list[dict[str, object]] = [] + want_openimpala = args.export_real_openimpala or args.generate_openimpala_suite + + key_metrics = [ + "solid_fraction", + "coating_fraction", + "ssa_pore_coating", + "tpb_density", + "wired_coating_fraction", + "coating_pinch_fraction", + "pore_anisotropy_xy_over_z", + "geodesic_tortuosity_z", + ] + + for spec in state_specs: + st = str(spec["state"]) + mat = str(spec["material"]) + beam = str(spec["beamline"]) + lv = float(spec["load_value"]) # type: ignore[arg-type] + vox = float(spec["voxel_size_um"]) # type: ignore[arg-type] + n_ph = int(spec["num_phases"]) # type: ignore[arg-type] + is_heldout = st in heldout_set + + real_vols = real_cache[st] + if want_openimpala: + for r_i, rv in enumerate(real_vols[: args.max_openimpala_per_method], start=1): + tif_p = out_dir / "openimpala_real" / st / f"real_val_{r_i:03d}.tif" + export_openimpala_tiff(rv, tif_p, preserve_multiphase=(n_ph == 3)) + openimpala_entries.append( + { + "method": "real", + "state": st, + "material": mat, + "beamline": beam, + "load_value": lv, + "tiff_path": tif_p, + "num_phases": n_ph, + } + ) + + real_recs = [evaluate_multiphase_physics(v, voxel_size=vox) for v in real_vols] + real_sum = summarize_metric_records(real_recs) + real_s2 = _mean_s2_curves(real_vols, max_lag=max_lag) + target_sf = float(real_sum["solid_fraction"]["mean"]) + + # Construct literature Z-compression baselines from tall clean uncompressed real sub-volumes + ref_st = lowest_state_by_group[(mat, beam)] + base_src_vols = tall_ref_cache[ref_st] + phi_0_src = float((base_src_vols >= 1).mean()) + mirror_frac = compute_mirror_tiled_fraction( + source_depth=int(base_src_vols.shape[1]), + output_depth=args.patch_size, + initial_solid_fraction=phi_0_src, + target_solid_fraction=max(target_sf, 1e-3), + ) + affine_vols = np.stack( + [ + affine_z_compress_to_target( + rv, target_solid_fraction=max(target_sf, 1e-3), output_depth=args.patch_size + ) + for rv in base_src_vols + ], + axis=0, + ) + pillar_vols = np.stack( + [ + pillar_pore_deletion_compress( + rv, + target_solid_fraction=max(target_sf, 1e-3), + output_depth=args.patch_size, + rng=np.random.default_rng(args.seed + i), + ) + for i, rv in enumerate(base_src_vols) + ], + axis=0, + ) + + methods_vols: dict[str, np.ndarray] = { + "affine_z_squash": affine_vols, + "pillar_pore_deletion": pillar_vols, + } + if st in virtual_cache_by_state: + methods_vols["virtual_compression"] = virtual_cache_by_state[st] + for lbl, exp_name in exp_map.items(): + samples_dir = args.output_root / "results" / exp_name / "samples" + if (samples_dir / st).is_dir(): + methods_vols[lbl] = _load_generated_volumes(samples_dir, st) + + if want_openimpala: + for m_name, m_vols in methods_vols.items(): + for b_i, mv_vol in enumerate(m_vols[: args.max_openimpala_per_method], start=1): + tif_p = out_dir / "openimpala_methods" / m_name / st / f"{m_name}_{b_i:03d}.tif" + export_openimpala_tiff(mv_vol, tif_p, preserve_multiphase=(n_ph == 3)) + openimpala_entries.append( + { + "method": m_name, + "state": st, + "material": mat, + "beamline": beam, + "load_value": lv, + "tiff_path": tif_p, + "num_phases": n_ph, + } + ) + + methods_summary: dict[str, object] = {} + for m_name, m_vols in methods_vols.items(): + m_recs = [evaluate_multiphase_physics(v, voxel_size=vox) for v in m_vols] + m_sum = summarize_metric_records(m_recs) + m_s2 = _mean_s2_curves(m_vols, max_lag=max_lag) + s2_err = _s2_rmse(real_s2, m_s2) + + methods_summary[m_name] = { + "num_volumes": int(m_vols.shape[0]), + "s2_rmse": s2_err, + "metrics": m_sum, + } + + row: dict[str, object] = { + "state": st, + "material": mat, + "beamline": beam, + "load_value": lv, + "load_unit": unit_label, + "is_heldout": is_heldout, + "method": m_name, + "s2_rmse": round(s2_err, 6), + "baseline_mirror_tiled_fraction": ( + round(mirror_frac, 6) + if m_name in ("affine_z_squash", "pillar_pore_deletion") + else 0.0 + ), + } + for km in key_metrics: + r_mu = real_sum[km]["mean"] + m_mu = m_sum[km]["mean"] + row[f"real_{km}"] = round(r_mu, 6) if np.isfinite(r_mu) else float("nan") + row[f"pred_{km}"] = round(m_mu, 6) if np.isfinite(m_mu) else float("nan") + row[f"abs_err_{km}"] = ( + round(abs(m_mu - r_mu), 6) + if (np.isfinite(r_mu) and np.isfinite(m_mu)) + else float("nan") + ) + csv_rows.append(row) + + st_entry: dict[str, object] = { + "material": mat, + "beamline": beam, + "load_value": lv, + "load_unit": unit_label, + "voxel_size_um": vox, + "is_heldout": is_heldout, + "reference_uncompressed_state": ref_st, + "baseline_source_depth": int(base_src_vols.shape[1]), + "baseline_mirror_tiled_fraction": round(mirror_frac, 6), + "baseline_oracle_porosity_note": ( + "Affine and pillar baselines receive the held-out target solid_fraction by construction; " + "evaluate morphological fidelity via S2(r), chord anisotropy, EASA/TPB, and tortuosity." + ), + "real": {"num_volumes": int(real_vols.shape[0]), "metrics": real_sum}, + "methods": methods_summary, + } + if st in virtual_cache_by_state: + vc_diag = dict( + compute_virtual_compression_fidelity( + source_volumes=real_cache[ref_st], + compressed_volumes=virtual_cache_by_state[st], + same_load_reconstructions=vc_same_load_by_group.get((mat, beam)), + ) + ) + vc_diag.update(vc_extrap_by_group.get((mat, beam), {})) + st_entry["virtual_compression_diagnostics"] = vc_diag + report[st] = st_entry + print( + f"[{st} | {mat} ({beam}) | heldout={is_heldout}] real φ_s={target_sf:.4f} | " + + " | ".join( + f"{m}: s2_rmse={methods_summary[m]['s2_rmse']:.4f}" # type: ignore[index] + for m in methods_summary + ), + flush=True, + ) + + retention_diag = compute_coating_retention_diagnostics(report) + with (out_dir / "coating_retention_diagnostics.json").open("w") as f: + json.dump(retention_diag, f, indent=2) + + with (out_dir / "beamtime_multiplier_report.json").open("w") as f: + json.dump(report, f, indent=2) + + if csv_rows: + with (out_dir / "baseline_vs_porousdiff_table.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(csv_rows[0].keys())) + writer.writeheader() + writer.writerows(csv_rows) + + if want_openimpala and openimpala_entries: + suite_info = generate_openimpala_job_suite( + tiff_entries=openimpala_entries, + suite_dir=out_dir / "openimpala_suite", + template_ini=args.openimpala_template_ini, + include_hetero_solid=True, + ) + print( + f"Generated OpenImpala batch suite ({suite_info['num_jobs']} jobs) -> {suite_info['script_path']}", + flush=True, + ) + if (out_dir / "openimpala_suite" / "outputs").is_dir(): + parse_openimpala_results( + suite_dir=out_dir / "openimpala_suite", + out_csv=out_dir / "openimpala_transport_parity.csv", + ) + + if args.xrd_csv is not None: + xrd_summary = correlate_xrd_with_microstructure( + xrd_csv_path=args.xrd_csv, + benchmark_report=report, + out_dir=out_dir / "diad_xrd_analysis", + ) + print( + f"Saved DIAD XRD correlation ({xrd_summary['num_matched_states']} states) -> {out_dir / 'diad_xrd_analysis'}", + flush=True, + ) + + # Plot Real XCT vs Baselines vs PorousDiff across loads per material + materials = sorted({str(s["material"]) for s in state_specs}) + fig, axes = plt.subplots(len(materials), 3, figsize=(14, 4.0 * len(materials)), squeeze=False) + plot_Specs = [ + ("pore_anisotropy_xy_over_z", "Pore Chord Anisotropy (XY / Z)"), + ("geodesic_tortuosity_z", "Through-Plane Geodesic Tortuosity τ_Z"), + ("ssa_pore_coating", "Active Coating EASA (1/µm)"), + ] + + for r_idx, mat in enumerate(materials): + mat_states = [str(s["state"]) for s in state_specs if str(s["material"]) == mat] + loads = [float(report[st]["load_value"]) for st in mat_states] # type: ignore[index] + + for c_idx, (m_key, m_label) in enumerate(plot_Specs): + ax = axes[r_idx, c_idx] + real_y = [report[st]["real"]["metrics"][m_key]["mean"] for st in mat_states] # type: ignore[index] + ax.plot(loads, real_y, "-o", color="black", lw=2.0, label="Real XCT (Val)") + + all_methods = list(report[mat_states[0]]["methods"].keys()) # type: ignore[index] + palette = ["tab:gray", "tab:orange", "tab:blue", "tab:green", "tab:red"] + for m_i, m_name in enumerate(all_methods): + my = [ + report[st]["methods"][m_name]["metrics"][m_key]["mean"] # type: ignore[index] + for st in mat_states + if m_name in report[st]["methods"] # type: ignore[index] + ] + mx = [ + float(report[st]["load_value"]) # type: ignore[index] + for st in mat_states + if m_name in report[st]["methods"] # type: ignore[index] + ] + style = "--s" if m_name in ("affine_z_squash", "pillar_pore_deletion") else "-^" + ax.plot(mx, my, style, color=palette[m_i % len(palette)], lw=1.5, label=m_name) + + ax.set_title(f"{mat} — {m_label}", fontsize=10) + ax.set_xlabel(f"Compressive Load / Stress ({unit_label})") + ax.grid(True, alpha=0.3) + if r_idx == 0 and c_idx == 0: + ax.legend(fontsize=8) + + fig.suptitle("Real Held-Out Synchrotron XCT vs. Numerical Z-Compression vs. PorousDiff", fontsize=13) + fig.tight_layout() + fig.savefig(out_dir / "beamtime_multiplier_comparison.png", dpi=240, bbox_inches="tight") + fig.savefig(out_dir / "beamtime_multiplier_comparison.pdf", bbox_inches="tight") + plt.close(fig) + print(f"Saved Beamtime Multiplier benchmark -> {out_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluation/evaluate_carbon_physics.py b/scripts/evaluation/evaluate_carbon_physics.py new file mode 100644 index 0000000..b7e5671 --- /dev/null +++ b/scripts/evaluation/evaluate_carbon_physics.py @@ -0,0 +1,253 @@ +"""Quantitative 3D physical validation comparing real XCT patches vs generated volumes. + +Evaluates both training loads and held-out interpolated loads across: + - Solid fraction / porosity vs mechanical load + - Specific surface area (total and directional) + - Pore & fiber chord-length distributions and XY/Z compression anisotropy + - Directional two-point correlation functions S_2(r_z), S_2(r_y), S_2(r_x) + - 3D solid network connectivity, pore percolation, and geodesic tortuosity +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.data import default_project_root, load_split_patches +from porousdiff.metrics import ( + directional_two_point_correlation, + evaluate_volume_physics, + export_openimpala_tiff, + summarize_metric_records, +) +from porousdiff.preprocessing import parse_load_value + + +def _load_generated_state_volumes(samples_dir: Path, state_name: str) -> np.ndarray: + state_dir = samples_dir / state_name + stacked = state_dir / "generated_samples.npy" + if stacked.is_file(): + return np.load(stacked) + files = sorted(state_dir.glob("*_binary.npy")) + if not files: + raise FileNotFoundError(f"No generated samples found in {state_dir}") + return np.stack([np.load(p) for p in files], axis=0) + + +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Compare 3D physical & transport metrics of real vs generated microstructures" + ) + parser.add_argument("--experiment", default="carbon_conditional_96_v2") + parser.add_argument("--data-root", type=Path, default=proj_root / "data" / "carbon_virgin") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--real-split", default="val", choices=["val", "train"]) + parser.add_argument("--max-real-samples", type=int, default=30) + parser.add_argument("--voxel-size-um", type=float, default=1.0) + parser.add_argument( + "--export-real-openimpala", + action="store_true", + help="Export real validation subvolumes as OpenImpala uint8 multi-page TIFF stacks", + ) + parser.add_argument( + "--num-openimpala", + type=int, + default=10, + help="Max real validation subvolumes per state to export when --export-real-openimpala is set", + ) + parser.add_argument( + "--states", + nargs="+", + required=True, + help="States to compare (e.g. Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033)", + ) + parser.add_argument( + "--heldout-states", + nargs="*", + default=[], + help="Optional state names that were held out from training (highlighted on plots)", + ) + args = parser.parse_args() + + states: dict[str, float] = {} + for item in args.states: + if "=" in item: + k, v = item.split("=", 1) + states[k.strip()] = float(v.strip()) + else: + states[item.strip()] = parse_load_value(item.strip()) + + heldout = set(args.heldout_states) + exp_dir = args.output_root / "results" / args.experiment + samples_dir = exp_dir / "samples" + eval_dir = exp_dir / "physics_validation" + eval_dir.mkdir(parents=True, exist_ok=True) + + rng = np.random.default_rng(42) + comparison_rows: list[dict[str, object]] = [] + full_report: dict[str, object] = {} + s2_store: dict[str, dict[str, np.ndarray]] = {} + + for state_name, load_val in sorted(states.items(), key=lambda kv: kv[1]): + real_split_dir = ( + args.data_root + / f"{state_name}_binary_patches{args.patch_size}_{args.version}" + / args.real_split + ) + real_vols = load_split_patches(real_split_dir) + if real_vols.shape[0] > args.max_real_samples: + idx = rng.choice(real_vols.shape[0], size=args.max_real_samples, replace=False) + real_vols = real_vols[np.sort(idx)] + + if args.export_real_openimpala: + real_oi_dir = eval_dir / "openimpala_real" / state_name + for r_idx, r_vol in enumerate(real_vols[: max(1, args.num_openimpala)], start=1): + export_openimpala_tiff(r_vol, real_oi_dir / f"real_val_{r_idx:03d}.tif") + + gen_vols = _load_generated_state_volumes(samples_dir, state_name) + + real_records = [evaluate_volume_physics(v, voxel_size=args.voxel_size_um) for v in real_vols] + gen_records = [evaluate_volume_physics(v, voxel_size=args.voxel_size_um) for v in gen_vols] + + real_sum = summarize_metric_records(real_records) + gen_sum = summarize_metric_records(gen_records) + + # Average directional S_2(r) curves + real_s2_list = [directional_two_point_correlation(v) for v in real_vols[:12]] + gen_s2_list = [directional_two_point_correlation(v) for v in gen_vols[:12]] + lags = real_s2_list[0]["lags"] + s2_entry = { + "lags": lags, + "real_s2_z": np.mean([c["s2_z"] for c in real_s2_list], axis=0), + "gen_s2_z": np.mean([c["s2_z"] for c in gen_s2_list], axis=0), + "real_s2_xy": 0.5 * ( + np.mean([c["s2_x"] for c in real_s2_list], axis=0) + + np.mean([c["s2_y"] for c in real_s2_list], axis=0) + ), + "gen_s2_xy": 0.5 * ( + np.mean([c["s2_x"] for c in gen_s2_list], axis=0) + + np.mean([c["s2_y"] for c in gen_s2_list], axis=0) + ), + } + s2_store[state_name] = s2_entry + + s2_rmse_z = float(np.sqrt(np.mean((s2_entry["real_s2_z"] - s2_entry["gen_s2_z"]) ** 2))) + s2_rmse_xy = float(np.sqrt(np.mean((s2_entry["real_s2_xy"] - s2_entry["gen_s2_xy"]) ** 2))) + + full_report[state_name] = { + "load_mv_per_v": load_val, + "is_heldout_interpolation": state_name in heldout, + "num_real": int(real_vols.shape[0]), + "num_generated": int(gen_vols.shape[0]), + "s2_rmse_z": s2_rmse_z, + "s2_rmse_xy": s2_rmse_xy, + "real": real_sum, + "generated": gen_sum, + } + + row: dict[str, object] = { + "state": state_name, + "load_mv_per_v": load_val, + "regime": "heldout_interp" if state_name in heldout else "train_load", + "real_solid_mean": round(real_sum["solid_fraction"]["mean"], 5), + "real_solid_std": round(real_sum["solid_fraction"]["std"], 5), + "gen_solid_mean": round(gen_sum["solid_fraction"]["mean"], 5), + "gen_solid_std": round(gen_sum["solid_fraction"]["std"], 5), + "real_ssa_mean": round(real_sum["ssa_total"]["mean"], 5), + "gen_ssa_mean": round(gen_sum["ssa_total"]["mean"], 5), + "real_pore_anisotropy": round(real_sum["pore_anisotropy_xy_over_z"]["mean"], 4), + "gen_pore_anisotropy": round(gen_sum["pore_anisotropy_xy_over_z"]["mean"], 4), + "real_tortuosity_z": round(real_sum["geodesic_tortuosity_z"]["mean"], 4), + "gen_tortuosity_z": round(gen_sum["geodesic_tortuosity_z"]["mean"], 4), + "real_solid_conn": round(real_sum["solid_connected_fraction"]["mean"], 4), + "gen_solid_conn": round(gen_sum["solid_connected_fraction"]["mean"], 4), + "s2_rmse_z": round(s2_rmse_z, 6), + "s2_rmse_xy": round(s2_rmse_xy, 6), + } + comparison_rows.append(row) + print(json.dumps(row), flush=True) + + with (eval_dir / "physics_validation_summary.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(comparison_rows[0].keys())) + writer.writeheader() + writer.writerows(comparison_rows) + + with (eval_dir / "physics_validation_report.json").open("w") as f: + json.dump(full_report, f, indent=2) + + # Plot 1: Multi-metric compression response curves (Real vs Generated) + loads_x = [float(r["load_mv_per_v"]) for r in comparison_rows] + fig, axes = plt.subplots(2, 2, figsize=(12, 9)) + + panels = [ + (axes[0, 0], "solid_fraction", "Solid Volume Fraction φ_s"), + (axes[0, 1], "ssa_total", f"Specific Surface Area (1/{args.voxel_size_um:g} µm)"), + (axes[1, 0], "pore_anisotropy_xy_over_z", "Pore Chord Anisotropy (XY / Z)"), + (axes[1, 1], "geodesic_tortuosity_z", "Through-Plane Geodesic Tortuosity τ_z"), + ] + ordered_names = [str(r["state"]) for r in comparison_rows] + for ax, metric_key, ylabel in panels: + r_m = [full_report[s]["real"][metric_key]["mean"] for s in ordered_names] # type: ignore[index] + r_s = [full_report[s]["real"][metric_key]["std"] for s in ordered_names] # type: ignore[index] + g_m = [full_report[s]["generated"][metric_key]["mean"] for s in ordered_names] # type: ignore[index] + g_s = [full_report[s]["generated"][metric_key]["std"] for s in ordered_names] # type: ignore[index] + + ax.errorbar(loads_x, r_m, yerr=r_s, fmt="-o", capsize=4, lw=1.8, label="Real XCT") + ax.errorbar(loads_x, g_m, yerr=g_s, fmt="--s", capsize=4, lw=1.8, label="PorousDiff Generated") + for x_val, s_name in zip(loads_x, ordered_names): + if s_name in heldout: + ax.axvline(x_val, color="tab:red", ls=":", alpha=0.5) + ax.set_xlabel("Load-Cell Compression Reading (mV/V)") + ax.set_ylabel(ylabel) + ax.grid(True, alpha=0.3) + ax.legend() + + fig.suptitle("PorousDiff Physics Validation Across Mechanical Compression", fontsize=13) + fig.tight_layout() + fig.savefig(eval_dir / "physics_compression_curves.png", dpi=240) + fig.savefig(eval_dir / "physics_compression_curves.pdf") + plt.close(fig) + + # Plot 2: Directional S_2(r) comparison across states + n_st = len(ordered_names) + fig, axes = plt.subplots(1, n_st, figsize=(4.5 * n_st, 4.2), sharey=False) + if n_st == 1: + axes = [axes] + for ax, s_name in zip(axes, ordered_names): + ent = s2_store[s_name] + r = ent["lags"] * args.voxel_size_um + ax.plot(r, ent["real_s2_xy"], "-", color="tab:blue", lw=1.8, label="Real S₂(XY)") + ax.plot(r, ent["gen_s2_xy"], "--", color="tab:blue", lw=1.8, label="Gen S₂(XY)") + ax.plot(r, ent["real_s2_z"], "-", color="tab:orange", lw=1.8, label="Real S₂(Z)") + ax.plot(r, ent["gen_s2_z"], "--", color="tab:orange", lw=1.8, label="Gen S₂(Z)") + tag = " [Held-Out]" if s_name in heldout else "" + ax.set_title(f"{s_name} ({states[s_name]:g} mV/V){tag}") + ax.set_xlabel(f"Lag r ({args.voxel_size_um:g} µm)" if args.voxel_size_um != 1.0 else "Lag r (voxels)") + ax.set_ylabel("Two-Point Probability S₂(r)") + ax.grid(True, alpha=0.3) + ax.legend(fontsize=8) + + fig.tight_layout() + fig.savefig(eval_dir / "s2_correlation_comparison.png", dpi=240) + fig.savefig(eval_dir / "s2_correlation_comparison.pdf") + plt.close(fig) + print(f"Saved validation reports & figures -> {eval_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluation/optimize_clamping_pressure.py b/scripts/evaluation/optimize_clamping_pressure.py new file mode 100644 index 0000000..d8f6039 --- /dev/null +++ b/scripts/evaluation/optimize_clamping_pressure.py @@ -0,0 +1,692 @@ +"""Find optimal electrode clamping pressure `sigma*` coupling PorousDiff & OpenImpala transport. + +Computes the continuous reaction-transport Figure of Merit `FoM(sigma)` and bootstrap 95% +confidence interval on the optimal cell clamping stress `sigma*` across electrode chemistries +(`virgin_carbon`, `tio2_anode`, `cuhcf_cathode`), comparing: + - Real synchrotron XCT ground truth (`real`), + - PorousDiff generative predictions (`porousdiff`, plus optional dense continuous stress sweep), + - Standard literature numerical compression heuristics (`affine_z_squash`, `pillar_pore_deletion`). + +Automatically ingests OpenImpala continuum solver outputs (`openimpala_transport_parity.csv`) +when present, and seamlessly falls back to exact 3D geodesic pore/solid tortuosity and +Carman-Kozeny permeability surrogates prior to running OpenImpala. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import torch + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.baselines import affine_z_compress_to_target, pillar_pore_deletion_compress +from porousdiff.data import ( + decode_channels_to_phases, + default_project_root, + load_split_patches, + load_tall_or_split_patches, + resolve_material_id, + resolve_patch_split_dir, + resolve_scan_load_value, + resolve_use_stress_kpa_from_checkpoints, +) +from porousdiff.metrics import evaluate_multiphase_physics +from porousdiff.models import ( + ContinuousLoadConditionedUNet, + EMAModel, + build_noise_scheduler, + infer_use_ordinal_material_channel, +) +from porousdiff.optimization import ( + compute_transport_and_fom_record, + find_optimal_clamping_pressure, + load_openimpala_lookup, +) +from porousdiff.preprocessing import infer_material_name, parse_load_value + + +def _parse_experiments(entries: list[str]) -> dict[str, str]: + out: dict[str, str] = {} + for item in entries: + if "=" in item: + lbl, exp = item.split("=", 1) + out[lbl.strip()] = exp.strip() + else: + out[item.strip()] = item.strip() + return out + + +def _load_generated_volumes(samples_dir: Path, state_name: str) -> np.ndarray: + state_dir = samples_dir / state_name + stacked = state_dir / "generated_samples.npy" + if stacked.is_file(): + return np.load(stacked) + files = sorted(state_dir.glob("sample_*_binary.npy")) + if not files: + raise FileNotFoundError(f"No generated samples found in {state_dir}") + return np.stack([np.load(p) for p in files], axis=0) + + +@torch.inference_mode() +def _sample_dense_sweep_for_material( + active_model: ContinuousLoadConditionedUNet, + scheduler: object, + dense_loads: np.ndarray, + num_samples_per_load: int, + patch_size: int, + guidance_scale: float, + material_id: int, + voxel_size_um: float, + seed: int, + use_amp: bool, + amp_dtype: torch.dtype, +) -> dict[float, list[dict[str, float]]]: + """Generate a continuous stress sweep with shared latent seeds to resolve `sigma*` finely.""" + device = next(active_model.parameters()).device + data_ch = active_model.data_channels + gen = torch.Generator(device=device).manual_seed(seed) + shared_noise = torch.randn( + (num_samples_per_load, data_ch, patch_size, patch_size, patch_size), + generator=gen, + device=device, + ) + records_by_load: dict[float, list[dict[str, float]]] = {} + for lv in dense_loads: + lv_f = float(lv) + img = shared_noise.clone() + loads_t = torch.full((num_samples_per_load,), lv_f, device=device, dtype=torch.float32) + mats_t = ( + torch.full((num_samples_per_load,), int(material_id), device=device, dtype=torch.long) + if active_model.is_multimodal + else None + ) + voxs_t = ( + torch.full( + (num_samples_per_load,), float(voxel_size_um), device=device, dtype=torch.float32 + ) + if active_model.is_multimodal + else None + ) + step_gen = torch.Generator(device=device).manual_seed(seed + 101) + for timestep in scheduler.timesteps: # type: ignore[attr-defined] + t_int = int(timestep) + t_batch = torch.full((num_samples_per_load,), t_int, device=device, dtype=torch.long) + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = active_model.forward_cfg( + img, + t_batch, + loads_t, + guidance_scale=guidance_scale, + material_ids=mats_t, + voxel_sizes=voxs_t, + ) + img = scheduler.step(pred.float(), t_int, img, generator=step_gen)[0] # type: ignore[attr-defined] + + vols = decode_channels_to_phases(img.float().cpu().numpy(), enforce_solid_support=True) + recs = [ + compute_transport_and_fom_record( + evaluate_multiphase_physics(v, voxel_size=voxel_size_um), + openimpala_metrics=None, + voxel_size_um=voxel_size_um, + ) + for v in vols + ] + records_by_load[lv_f] = recs + return records_by_load + + +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Discover optimal electrode clamping pressure sigma* across Real XCT, OpenImpala, and PorousDiff" + ) + parser.add_argument( + "--experiments", + nargs="+", + required=True, + help="Experiment(s), e.g. porousdiff=beamtime_multiplier_96", + ) + parser.add_argument("--data-root", type=Path, default=None, help="Optional real XCT patch root for ground-truth & baselines") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--out-dir", type=Path, default=None) + parser.add_argument("--manifest-json", type=Path, default=None) + parser.add_argument("--states", nargs="*", default=None) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--voxel-size-um", type=float, default=1.0) + parser.add_argument("--include-repeats", action="store_true") + parser.add_argument("--beamline", default=None) + parser.add_argument( + "--use-stress-kpa", + action="store_true", + help="Fallback load unit when no checkpoint exists under --output-root (otherwise read from checkpoint config)", + ) + parser.add_argument("--max-real-samples", type=int, default=20) + parser.add_argument( + "--openimpala-csv", + type=Path, + default=None, + help="Optional openimpala_transport_parity.csv from parse_openimpala_suite.py", + ) + parser.add_argument( + "--weights", + default="0.35,0.35,0.30", + help="Comma-separated weights 'ionic,electronic,kinetic' for harmonic cell conductance FoM", + ) + parser.add_argument("--dense-sweep-points", type=int, default=0, help=">0 runs continuous intermediate PorousDiff stress sweep") + parser.add_argument("--dense-samples-per-load", type=int, default=4) + parser.add_argument( + "--uncompressed-thickness-cm", + type=float, + default=0.30, + help="Uncompressed felt thickness L_0 in cm (default 0.30 cm = 3.0 mm) for 1D Newman ASR", + ) + parser.add_argument("--checkpoint", type=Path, default=None) + parser.add_argument("--steps", type=int, default=50) + parser.add_argument("--guidance-scale", type=float, default=1.5) + parser.add_argument("--bootstrap", type=int, default=200) + parser.add_argument("--allow-cpu", action="store_true") + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + w_parts = [float(x.strip()) for x in args.weights.split(",") if x.strip()] + if len(w_parts) != 3: + raise ValueError("--weights must have 3 comma-separated floats: ionic,electronic,kinetic") + weights = (w_parts[0], w_parts[1], w_parts[2]) + + exp_map = _parse_experiments(args.experiments) + first_exp = next(iter(exp_map.values())) + out_dir = ( + args.out_dir + if args.out_dir is not None + else (args.output_root / "results" / first_exp / "clamping_optimization") + ) + out_dir.mkdir(parents=True, exist_ok=True) + + use_kpa, _ = resolve_use_stress_kpa_from_checkpoints( + output_root=args.output_root, + experiments=list(exp_map.values()), + explicit_checkpoint=args.checkpoint, + default=args.use_stress_kpa, + ) + + auto_oi_csv = ( + args.openimpala_csv + if args.openimpala_csv is not None + else (args.output_root / "results" / first_exp / "beamtime_benchmark" / "openimpala_transport_parity.csv") + ) + oi_lookup = load_openimpala_lookup(auto_oi_csv) if auto_oi_csv.is_file() else {} + + # Build state specs from manifest, --states, or sampled directories + state_specs: list[dict[str, object]] = [] + if args.manifest_json is not None: + manifest = json.loads(args.manifest_json.read_text()) + for sc in manifest.get("scans", []): + if not args.include_repeats and bool(sc.get("is_repeat_or_secondary", False)): + continue + sc_beam = str(sc.get("beamline", "primary")) + if args.beamline and sc_beam != args.beamline: + continue + lv_resolved = resolve_scan_load_value(sc, use_kpa) + raw_sz = sc.get("strain_z") + state_specs.append( + { + "state": str(sc["state"]), + "material": str(sc.get("material", "virgin_carbon")), + "beamline": sc_beam, + "load_value": lv_resolved, + "strain_z": float(raw_sz) if raw_sz is not None else None, + "voxel_size_um": float(sc.get("effective_voxel_size_um", args.voxel_size_um)), + "patch_dir_name": sc.get("patch_dir_name"), + } + ) + elif args.states: + for item in args.states: + if "=" in item: + st, lv_s = item.split("=", 1) + st, lv = st.strip(), float(lv_s.strip()) + else: + st = item.strip() + lv = parse_load_value(st) + state_specs.append( + { + "state": st, + "material": infer_material_name(st), + "beamline": args.beamline or "primary", + "load_value": lv, + "strain_z": None, + "voxel_size_um": args.voxel_size_um, + "patch_dir_name": None, + } + ) + else: + # Discover from first experiment's per_sample_physics.csv + sample_csv = args.output_root / "results" / first_exp / "samples" / "per_sample_physics.csv" + if not sample_csv.is_file(): + raise ValueError("Provide --manifest-json, --states, or run sample_carbon_conditional_96.py first") + seen_st: set[str] = set() + with sample_csv.open("r", newline="") as f: + for row in csv.DictReader(f): + st = str(row["state"]) + if st in seen_st: + continue + seen_st.add(st) + state_specs.append( + { + "state": st, + "material": str(row.get("material", infer_material_name(st))), + "beamline": "primary", + "load_value": float(row["load_value"]), + "strain_z": None, + "voxel_size_um": float(row.get("voxel_size_um", args.voxel_size_um)), + "patch_dir_name": None, + } + ) + + state_specs.sort(key=lambda s: (str(s["material"]), float(s["load_value"]))) # type: ignore[arg-type] + + # Collect per-(material, method, load) transport records + rng = np.random.default_rng(args.seed) + expected_shape = (args.patch_size, args.patch_size, args.patch_size) + records_tree: dict[str, dict[str, dict[float, list[dict[str, float]]]]] = {} + + have_real_data = args.data_root is not None and args.data_root.is_dir() + real_cache: dict[str, np.ndarray] = {} + tall_real_cache: dict[str, np.ndarray] = {} + lowest_state_by_mat: dict[str, str] = {} + + if have_real_data: + for sp in state_specs: + st = str(sp["state"]) + mat = str(sp["material"]) + pdir_hint = str(sp["patch_dir_name"]) if sp.get("patch_dir_name") else None + try: + split_dir = resolve_patch_split_dir( + args.data_root, st, args.patch_size, args.version, "val", patch_dir_name=pdir_hint # type: ignore[arg-type] + ) + vols = load_split_patches(split_dir, expected_shape=expected_shape) + except (FileNotFoundError, RuntimeError): + continue + if vols.shape[0] > args.max_real_samples: + idx = np.sort(rng.choice(vols.shape[0], size=args.max_real_samples, replace=False)) + vols = vols[idx] + real_cache[st] = vols + tall_vols = load_tall_or_split_patches(split_dir, vols) + if tall_vols.shape[0] > args.max_real_samples: + tall_vols = tall_vols[: args.max_real_samples] + tall_real_cache[st] = tall_vols + if mat not in lowest_state_by_mat: + lowest_state_by_mat[mat] = st + + # Verify per-(material, method) OpenImpala coverage so curves never mix partial OpenImpala + surrogate + oi_method_complete: dict[tuple[str, str], bool] = {} + if oi_lookup: + mat_to_real_states: dict[str, list[str]] = {} + mat_to_exp_states: dict[tuple[str, str], list[str]] = {} + for sp in state_specs: + st = str(sp["state"]) + mat = str(sp["material"]) + if st in real_cache: + mat_to_real_states.setdefault(mat, []).append(st) + for lbl, exp_name in exp_map.items(): + if (args.output_root / "results" / exp_name / "samples" / st).is_dir(): + mat_to_exp_states.setdefault((mat, lbl), []).append(st) + + def _check_oi_complete(mat_k: str, meth_k: str, states_list: list[str]) -> None: + if not states_list: + return + cov = sum(1 for st_k in states_list if (meth_k, st_k) in oi_lookup) + if cov == len(states_list): + oi_method_complete[(mat_k, meth_k)] = True + else: + oi_method_complete[(mat_k, meth_k)] = False + if cov > 0: + print( + f"Warning: OpenImpala results cover only {cov}/{len(states_list)} states for " + f"({mat_k}, {meth_k}); falling back to uniform 26-connected geodesic estimator " + "across all loads to avoid mixing two transport estimators.", + flush=True, + ) + + for mat_k, r_sts in mat_to_real_states.items(): + for meth_k in ("real", "affine_z_squash", "pillar_pore_deletion"): + _check_oi_complete(mat_k, meth_k, r_sts) + for (mat_k, lbl_k), g_sts in mat_to_exp_states.items(): + _check_oi_complete(mat_k, lbl_k, g_sts) + + for sp in state_specs: + st = str(sp["state"]) + mat = str(sp["material"]) + lv = float(sp["load_value"]) # type: ignore[arg-type] + vox = float(sp["voxel_size_um"]) # type: ignore[arg-type] + sz_opt = float(sp["strain_z"]) if sp.get("strain_z") is not None else None # type: ignore[arg-type] + mat_bucket = records_tree.setdefault(mat, {}) + + if st in real_cache: + r_vols = real_cache[st] + r_oi = oi_lookup.get(("real", st)) if oi_method_complete.get((mat, "real"), False) else None + r_recs = [ + compute_transport_and_fom_record( + evaluate_multiphase_physics(v, voxel_size=vox), + openimpala_metrics=r_oi, + voxel_size_um=vox, + strain_z=sz_opt, + ) + for v in r_vols + ] + mat_bucket.setdefault("real", {}).setdefault(lv, []).extend(r_recs) + + target_sf = float(np.mean([r["solid_fraction"] for r in r_recs])) + ref_vols = tall_real_cache[lowest_state_by_mat[mat]] + aff_vols = [ + affine_z_compress_to_target(rv, target_solid_fraction=max(target_sf, 1e-3), output_depth=args.patch_size) + for rv in ref_vols + ] + pil_vols = [ + pillar_pore_deletion_compress( + rv, + target_solid_fraction=max(target_sf, 1e-3), + output_depth=args.patch_size, + rng=np.random.default_rng(args.seed + i), + ) + for i, rv in enumerate(ref_vols) + ] + aff_oi = ( + oi_lookup.get(("affine_z_squash", st)) + if oi_method_complete.get((mat, "affine_z_squash"), False) + else None + ) + pil_oi = ( + oi_lookup.get(("pillar_pore_deletion", st)) + if oi_method_complete.get((mat, "pillar_pore_deletion"), False) + else None + ) + mat_bucket.setdefault("affine_z_squash", {}).setdefault(lv, []).extend( + [ + compute_transport_and_fom_record( + evaluate_multiphase_physics(v, voxel_size=vox), + openimpala_metrics=aff_oi, + voxel_size_um=vox, + strain_z=sz_opt, + ) + for v in aff_vols + ] + ) + mat_bucket.setdefault("pillar_pore_deletion", {}).setdefault(lv, []).extend( + [ + compute_transport_and_fom_record( + evaluate_multiphase_physics(v, voxel_size=vox), + openimpala_metrics=pil_oi, + voxel_size_um=vox, + strain_z=sz_opt, + ) + for v in pil_vols + ] + ) + + for lbl, exp_name in exp_map.items(): + samples_dir = args.output_root / "results" / exp_name / "samples" + if (samples_dir / st).is_dir(): + g_vols = _load_generated_volumes(samples_dir, st) + g_oi = oi_lookup.get((lbl, st)) if oi_method_complete.get((mat, lbl), False) else None + g_recs = [ + compute_transport_and_fom_record( + evaluate_multiphase_physics(v, voxel_size=vox), + openimpala_metrics=g_oi, + voxel_size_um=vox, + strain_z=sz_opt, + ) + for v in g_vols + ] + mat_bucket.setdefault(lbl, {}).setdefault(lv, []).extend(g_recs) + + # Optional continuous dense PorousDiff sweep for high-resolution sigma* discovery + if args.dense_sweep_points >= 3: + device = ( + torch.device("cuda") + if torch.cuda.is_available() + else (torch.device("cpu") if args.allow_cpu else None) + ) + if device is None: + raise RuntimeError("CUDA required for --dense-sweep-points (or pass --allow-cpu)") + ckpt_dir = args.output_root / "checkpoints" / first_exp + ckpt_path = ( + args.checkpoint + if args.checkpoint is not None + else ( + ckpt_dir / "best_val.pth" + if (ckpt_dir / "best_val.pth").is_file() + else ckpt_dir / "latest.pth" + ) + ) + saved = torch.load(ckpt_path, map_location="cpu", weights_only=False) + cfg = saved["config"] + num_materials = int(cfg.get("num_materials", 1)) + model = ContinuousLoadConditionedUNet( + channels=cfg["channels"], + attention_levels=cfg["attention_levels"], + num_res_blocks=int(cfg["num_res_blocks"]), + norm_num_groups=int(cfg["norm_groups"]), + load_min=float(cfg["load_min"]), + load_max=float(cfg["load_max"]), + use_flash_attention=(device.type == "cuda"), + conditioning_mode=str(cfg.get("conditioning_mode", "multiscale_fourier")), + num_phases=int(cfg.get("num_phases", 2)), + num_materials=num_materials, + ref_voxel_size_um=float(cfg.get("ref_voxel_size_um", 1.0)), + use_ordinal_material_channel=infer_use_ordinal_material_channel(saved), + ).to(device) + if "ema" in saved: + ema = EMAModel(model) + ema.load_state_dict(saved["ema"]) + active_model = ema.shadow_model.to(device).eval() + else: + model.load_state_dict(saved["model"]) + active_model = model.eval() + sched = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=int(cfg["timesteps"]), + schedule=str(cfg["schedule"]), + prediction_type=str(cfg["prediction_type"]), + ) + sched.set_timesteps(args.steps, device=device) + use_amp = device.type == "cuda" + amp_dtype = ( + torch.bfloat16 + if (device.type == "cuda" and torch.cuda.is_bf16_supported()) + else torch.float16 + ) + + for mat, mat_bucket in records_tree.items(): + mat_loads = sorted({float(sp["load_value"]) for sp in state_specs if str(sp["material"]) == mat}) # type: ignore[arg-type] + if len(mat_loads) < 2: + continue + dense_grid = np.linspace(mat_loads[0], mat_loads[-1], args.dense_sweep_points) + vox_um = float( + next(sp["voxel_size_um"] for sp in state_specs if str(sp["material"]) == mat) # type: ignore[arg-type] + ) + mat_id = resolve_material_id(mat) if num_materials > 1 else 0 + mat_bucket["porousdiff_dense"] = _sample_dense_sweep_for_material( + active_model=active_model, + scheduler=sched, + dense_loads=dense_grid, + num_samples_per_load=args.dense_samples_per_load, + patch_size=int(cfg["patch_size"]), + guidance_scale=args.guidance_scale, + material_id=mat_id, + voxel_size_um=vox_um, + seed=args.seed, + use_amp=use_amp, + amp_dtype=amp_dtype, + ) + + summary_out: dict[str, dict[str, object]] = {} + csv_rows: list[dict[str, object]] = [] + + for mat, methods_map in sorted(records_tree.items()): + if not methods_map: + continue + mat_summary: dict[str, object] = {} + ref_scales: tuple[float, float, float] | None = None + if "real" in methods_map: + real_opt = find_optimal_clamping_pressure( + methods_map["real"], + weights=weights, + n_bootstrap=args.bootstrap, + seed=args.seed, + uncompressed_thickness_cm=args.uncompressed_thickness_cm, + ) + ns = real_opt["norm_scales"] # type: ignore[index] + ref_scales = ( + float(ns["d_eff_pore_z"]), # type: ignore[index] + float(ns["sigma_eff_solid_z"]), # type: ignore[index] + float(ns["active_wired_interface"]), # type: ignore[index] + ) + mat_summary["real"] = real_opt + + real_sigma_star = ( + float(mat_summary["real"]["optimal_load"]) # type: ignore[index] + if "real" in mat_summary + else float("nan") + ) + real_sigma_star_asr = ( + float(mat_summary["real"]["newman_asr"]["optimal_load_asr"]) # type: ignore[index] + if "real" in mat_summary + else float("nan") + ) + + for m_name, load_map in methods_map.items(): + if m_name == "real": + opt_res = mat_summary["real"] # type: ignore[assignment] + else: + opt_res = find_optimal_clamping_pressure( + load_map, + weights=weights, + n_bootstrap=args.bootstrap, + seed=args.seed, + norm_scales=ref_scales, + uncompressed_thickness_cm=args.uncompressed_thickness_cm, + ) + if np.isfinite(real_sigma_star): + opt_res["abs_err_optimal_load_vs_real"] = round( + abs(float(opt_res["optimal_load"]) - real_sigma_star), 6 + ) + if np.isfinite(real_sigma_star_asr): + opt_res["abs_err_optimal_load_asr_vs_real"] = round( + abs(float(opt_res["newman_asr"]["optimal_load_asr"]) - real_sigma_star_asr), 6 # type: ignore[index] + ) + mat_summary[m_name] = opt_res + + for pt in opt_res["discrete_points"]: # type: ignore[index] + row: dict[str, object] = { + "material": mat, + "method": m_name, + "optimal_load_method": opt_res["optimal_load"], # type: ignore[index] + "optimal_load_asr_method": opt_res["newman_asr"]["optimal_load_asr"], # type: ignore[index] + } + row.update(pt) + csv_rows.append(row) + + asr_info = opt_res["newman_asr"] # type: ignore[index] + print( + f"[{mat} | {m_name}] sigma* (FoM) = {opt_res['optimal_load']} " # type: ignore[index] + f"(95% CI {opt_res['optimal_load_ci95']}) | " # type: ignore[index] + f"sigma* (Newman ASR) = {asr_info['optimal_load_asr']} " # type: ignore[index] + f"(min ASR = {asr_info['min_asr_ohm_cm2']} Ohm*cm^2)", # type: ignore[index] + flush=True, + ) + summary_out[mat] = mat_summary + + (out_dir / "optimal_clamping_summary.json").write_text(json.dumps(summary_out, indent=2)) + if csv_rows: + with (out_dir / "clamping_optimization_curves.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(csv_rows[0].keys())) + writer.writeheader() + writer.writerows(csv_rows) + + # Render 4-panel publication figure per material + materials = list(summary_out.keys()) + if materials: + fig, axes = plt.subplots( + len(materials), 4, figsize=(16.5, 4.0 * len(materials)), squeeze=False + ) + style_map = { + "real": ("black", "-o", 2.2), + "affine_z_squash": ("tab:gray", "--s", 1.4), + "pillar_pore_deletion": ("tab:orange", "--d", 1.4), + } + extra_colors = ["tab:blue", "tab:green", "tab:purple", "tab:red"] + + for r_idx, mat in enumerate(materials): + mat_res = summary_out[mat] + extra_i = 0 + for m_name, m_obj in mat_res.items(): + pts = m_obj["discrete_points"] # type: ignore[index] + curve = m_obj["continuous_curve"] # type: ignore[index] + xs = [float(p["load_value"]) for p in pts] + if m_name in style_map: + color, marker_style, lw = style_map[m_name] + else: + color = extra_colors[extra_i % len(extra_colors)] + marker_style, lw = "-^", 2.0 + extra_i += 1 + + axes[r_idx, 0].plot( + xs, [float(p["d_eff_pore_z"]) for p in pts], marker_style, color=color, lw=lw, label=m_name + ) + axes[r_idx, 1].plot( + xs, [float(p["sigma_eff_solid_z"]) for p in pts], marker_style, color=color, lw=lw, label=m_name + ) + axes[r_idx, 2].plot( + xs, [float(p["active_wired_interface"]) for p in pts], marker_style, color=color, lw=lw, label=m_name + ) + + gx = np.asarray(curve["grid_loads"], dtype=np.float64) + gfom = np.asarray(curve["fom_mean"], dtype=np.float64) + glo = np.asarray(curve["fom_ci95_low"], dtype=np.float64) + ghi = np.asarray(curve["fom_ci95_high"], dtype=np.float64) + ls = "--" if m_name in ("affine_z_squash", "pillar_pore_deletion") else "-" + axes[r_idx, 3].plot(gx, gfom, ls, color=color, lw=lw, label=f"{m_name} (σ*={m_obj['optimal_load']:g})") # type: ignore[index] + axes[r_idx, 3].fill_between(gx, glo, ghi, color=color, alpha=0.14) + axes[r_idx, 3].axvline(float(m_obj["optimal_load"]), color=color, ls=":", lw=1.2) # type: ignore[index] + + titles = [ + f"{mat} — Ionic D_eff,Z / D_0", + f"{mat} — Electronic σ_eff,Z / σ_0", + f"{mat} — Wired Active Interface (1/µm)", + f"{mat} — Cell Conductance FoM & Optimal σ*", + ] + unit_lbl = "kPa" if use_kpa else "mV/V" + for c_idx, ttl in enumerate(titles): + axes[r_idx, c_idx].set_title(ttl, fontsize=10) + axes[r_idx, c_idx].set_xlabel(f"Compressive Load / Stress ({unit_lbl})") + axes[r_idx, c_idx].grid(True, alpha=0.3) + axes[r_idx, 3].legend(fontsize=7.5, loc="best") + if r_idx == 0: + axes[r_idx, 0].legend(fontsize=7.5, loc="best") + + fig.suptitle( + "Reaction-Transport Trade-Off & Optimal Cell Clamping Pressure σ* (OpenImpala + PorousDiff)", + fontsize=13, + ) + fig.tight_layout() + fig.savefig(out_dir / "optimal_clamping_pressure.png", dpi=240, bbox_inches="tight") + fig.savefig(out_dir / "optimal_clamping_pressure.pdf", bbox_inches="tight") + plt.close(fig) + + print(f"Saved optimal clamping-pressure study -> {out_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluation/parse_openimpala_suite.py b/scripts/evaluation/parse_openimpala_suite.py new file mode 100755 index 0000000..9295af7 --- /dev/null +++ b/scripts/evaluation/parse_openimpala_suite.py @@ -0,0 +1,67 @@ +"""Parse completed OpenImpala outputs into transport parity CSVs and figures.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +import sys + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.transport_and_xrd import parse_openimpala_results + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Aggregate OpenImpala TortuosityFD outputs across Real, Baselines & PorousDiff" + ) + parser.add_argument("--suite-dir", type=Path, required=True, help="Path to openimpala_suite folder") + parser.add_argument("--out-csv", type=Path, default=None) + args = parser.parse_args() + + out_csv = args.out_csv or (args.suite_dir / "openimpala_transport_parity.csv") + rows = parse_openimpala_results(suite_dir=args.suite_dir, out_csv=out_csv) + if not rows: + print(f"No completed OpenImpala output files found in {args.suite_dir / 'outputs'}", flush=True) + return + + print(f"Parsed {len(rows)} OpenImpala results -> {out_csv}", flush=True) + + # Plot through-plane effective diffusivity D_eff_z (if present) grouped by transport_phase + phases = sorted({str(r["transport_phase"]) for r in rows}) + if any("d_eff_z" in r for r in rows): + fig, axes = plt.subplots(1, len(phases), figsize=(5.2 * len(phases), 4.2), squeeze=False) + for idx, ph in enumerate(phases): + ax = axes[0, idx] + sub = [r for r in rows if str(r["transport_phase"]) == ph and "d_eff_z" in r] + methods = sorted({str(r["method"]) for r in sub}) + for m in methods: + m_rows = [r for r in sub if str(r["method"]) == m] + loads = sorted({float(r["load_value"]) for r in m_rows}) # type: ignore[arg-type] + means = [ + float(np.mean([float(r["d_eff_z"]) for r in m_rows if float(r["load_value"]) == lv])) # type: ignore[arg-type] + for lv in loads + ] + style = "-o" if m == "real" else ("--s" if "squash" in m or "pillar" in m else "-^") + ax.plot(loads, means, style, lw=1.6, label=m) + ax.set_title(f"{ph} — D_eff,Z / D_0") + ax.set_xlabel("Load / Stress") + ax.set_ylabel("Effective transport factor") + ax.grid(True, alpha=0.3) + ax.legend(fontsize=8) + fig.tight_layout() + fig_path = out_csv.parent / "openimpala_transport_parity.png" + fig.savefig(fig_path, dpi=220, bbox_inches="tight") + plt.close(fig) + print(f"Saved OpenImpala parity figure -> {fig_path}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/evaluation/sample_carbon_conditional_96.py b/scripts/evaluation/sample_carbon_conditional_96.py index fdf8e6e..3e291d8 100644 --- a/scripts/evaluation/sample_carbon_conditional_96.py +++ b/scripts/evaluation/sample_carbon_conditional_96.py @@ -1,10 +1,20 @@ +"""Sample trained PorousDiff 3D models across single-material or multi-electrode 3-phase states. + +Supports: + - 2-phase (`{0, 1}`) and hierarchical 3-phase (`{0: pore, 1: carbon, 2: coating}`) decoding, + - Multi-electrode (`--manifest-json` or `--material`) + voxel-size (`--voxel-size-um`) conditioning, + - Unseen held-out load interpolation, fixed-noise compression sweeps, DDIM/DDPM, CFG, EMA, + and OpenImpala multi-page TIFF export. """ -Sample a trained conditional 3D DDPM at 8, 18 and 33 mV. -Save continuous and binary volumes, figures and sampling metrics. -""" +from __future__ import annotations + +import argparse import csv +import json +import sys import time +from matplotlib.colors import ListedColormap from pathlib import Path import matplotlib @@ -13,173 +23,516 @@ import numpy as np import torch -from generative.networks.nets import DiffusionModelUNet -from generative.networks.schedulers import DDPMScheduler - -ROOT = Path("/projects/u6uc/asif") -EXPERIMENT = "carbon_conditional_96_v1" -CHECKPOINT = ROOT / "checkpoints" / EXPERIMENT / "latest.pth" -OUTPUT = ROOT / "results" / EXPERIMENT / "samples" -FIGURES = ROOT / "paper" / "figures" - -STATES = { - "Op008mVV": 8.0, - "Op018mVV": 18.0, - "Op033mVV": 33.0, -} - - -def main(): - if not torch.cuda.is_available(): - raise RuntimeError("GPU required; use Slurm.") - - OUTPUT.mkdir(parents=True, exist_ok=True) - FIGURES.mkdir(parents=True, exist_ok=True) - - device = torch.device("cuda") - torch.manual_seed(42) - torch.cuda.manual_seed_all(42) - - model = DiffusionModelUNet( - spatial_dims=3, - in_channels=1, - out_channels=1, - num_channels=(32, 64, 128), - attention_levels=(False, False, True), - num_res_blocks=1, - with_conditioning=True, - cross_attention_dim=32, - ).to(device) - - condition_encoder = torch.nn.Sequential( - torch.nn.Linear(1, 32), - torch.nn.SiLU(), - torch.nn.Linear(32, 32), - ).to(device) - - checkpoint = torch.load( - CHECKPOINT, map_location="cpu", weights_only=False +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.data import ( + decode_channels_to_phases, + default_project_root, + resolve_material_id, + resolve_scan_load_value, +) +from porousdiff.metrics import ( + evaluate_multiphase_physics, + evaluate_volume_physics, + export_openimpala_tiff, +) +from porousdiff.models import ( + ContinuousLoadConditionedUNet, + EMAModel, + build_noise_scheduler, + infer_use_ordinal_material_channel, +) +from porousdiff.preprocessing import infer_material_name, parse_load_value +from porousdiff.virtual_compression import generate_tiled_3d + +PHASE_CMAP = ListedColormap(["#0b0e14", "#38bdf8", "#facc15"]) + + +@torch.inference_mode() +def generate_batch( + model: ContinuousLoadConditionedUNet, + scheduler: object, + initial_noise: torch.Tensor, + load_value: float, + guidance_scale: float, + use_amp: bool, + amp_dtype: torch.dtype, + material_id: int = 0, + voxel_size_um: float = 1.0, + generator: torch.Generator | None = None, +) -> tuple[np.ndarray, np.ndarray]: + """Run reverse diffusion from `initial_noise` `[B, C, D, H, W]`.""" + device = initial_noise.device + bsz = initial_noise.shape[0] + image = initial_noise.clone() + loads = torch.full((bsz,), float(load_value), device=device, dtype=torch.float32) + mats = ( + torch.full((bsz,), int(material_id), device=device, dtype=torch.long) + if model.is_multimodal + else None ) - model.load_state_dict(checkpoint["model"]) - condition_encoder.load_state_dict( - checkpoint["condition_encoder"] + voxs = ( + torch.full((bsz,), float(voxel_size_um), device=device, dtype=torch.float32) + if model.is_multimodal + else None ) - model.eval() - condition_encoder.eval() + for timestep in scheduler.timesteps: # type: ignore[attr-defined] + t_int = int(timestep) + t_batch = torch.full((bsz,), t_int, device=device, dtype=torch.long) + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = model.forward_cfg( + image, + t_batch, + loads, + guidance_scale=guidance_scale, + material_ids=mats, + voxel_sizes=voxs, + ) + image = scheduler.step(pred.float(), t_int, image, generator=generator)[0] # type: ignore[attr-defined] + + raw_np = image.float().cpu().numpy() + phases = decode_channels_to_phases(raw_np, enforce_solid_support=True, threshold=0.0) + probs = np.clip((raw_np[:, 0] + 1.0) * 0.5, 0.0, 1.0).astype(np.float32) + return phases, probs + + +def save_orthoview_figure( + volume: np.ndarray, + title: str, + png_path: Path, + pdf_path: Path | None = None, + num_phases: int = 2, +) -> None: + mid_z, mid_y, mid_x = (s // 2 for s in volume.shape) + views = [ + (volume[mid_z, :, :], "XY (In-Plane Sheet)"), + (volume[:, mid_y, :], "XZ (Through-Plane Compression)"), + (volume[:, :, mid_x], "YZ (Through-Plane Compression)"), + ] + fig, axes = plt.subplots(1, 3, figsize=(12, 4)) + cmap = PHASE_CMAP if num_phases == 3 else "gray" + vmax = 2 if num_phases == 3 else 1 + for ax, (view, vtitle) in zip(axes, views): + ax.imshow(view, cmap=cmap, vmin=0, vmax=vmax, interpolation="nearest") + ax.set_title(vtitle) + ax.axis("off") + sf = float((volume >= 1).mean()) + extra = f", coat = {float((volume == 2).mean()):.3f}" if num_phases == 3 else "" + fig.suptitle(f"{title} (solid = {sf:.3f}{extra})") + fig.tight_layout() + fig.savefig(png_path, dpi=240, bbox_inches="tight") + if pdf_path is not None: + fig.savefig(pdf_path, bbox_inches="tight") + plt.close(fig) + + +def save_sweep_panel( + sweep_volumes: list[tuple[float, np.ndarray]], + png_path: Path, + pdf_path: Path, + num_phases: int = 2, + unit_label: str = "mV/V", +) -> None: + n_cols = len(sweep_volumes) + fig, axes = plt.subplots(2, n_cols, figsize=(3.2 * n_cols, 6.2)) + if n_cols == 1: + axes = np.expand_dims(axes, axis=1) + + cmap = PHASE_CMAP if num_phases == 3 else "gray" + vmax = 2 if num_phases == 3 else 1 + for col, (load_val, vol) in enumerate(sweep_volumes): + mz, my, _ = (s // 2 for s in vol.shape) + axes[0, col].imshow(vol[mz], cmap=cmap, vmin=0, vmax=vmax, interpolation="nearest") + sf = float((vol >= 1).mean()) + axes[0, col].set_title(f"{load_val:g} {unit_label}\nφ_s = {sf:.3f} (XY)") + axes[0, col].axis("off") + + axes[1, col].imshow(vol[:, my, :], cmap=cmap, vmin=0, vmax=vmax, interpolation="nearest") + axes[1, col].set_title("XZ (Compression ↓)") + axes[1, col].axis("off") + + fig.suptitle("Virtual Compression Sweep (Fixed Initial Gaussian Noise Seed)", fontsize=13) + fig.tight_layout() + fig.savefig(png_path, dpi=260, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + plt.close(fig) + + +def _build_eval_specs( + args: argparse.Namespace, + cfg: dict[str, object], +) -> list[dict[str, object]]: + ref_vox = float(cfg.get("ref_voxel_size_um", 1.0)) # type: ignore[arg-type] + use_kpa = bool(cfg.get("use_stress_kpa", False)) + default_mat = args.material + + if args.manifest_json is not None: + manifest = json.loads(args.manifest_json.read_text()) + filter_set = set(args.eval_states) if args.eval_states else None + specs: list[dict[str, object]] = [] + for scan in manifest.get("scans", []): + st = str(scan["state"]) + if filter_set is not None and st not in filter_set: + continue + if args.beamline and str(scan.get("beamline", "primary")) != args.beamline: + continue + if ( + filter_set is None + and not args.include_repeats + and bool(scan.get("is_repeat_or_secondary", False)) + ): + continue + lv = resolve_scan_load_value(scan, use_kpa) + specs.append( + { + "state": st, + "load_value": lv, + "material": str(scan.get("material", "virgin_carbon")), + "voxel_size_um": float(scan.get("effective_voxel_size_um", ref_vox)), + } + ) + return specs + + if args.eval_states: + specs = [] + for entry in args.eval_states: + if "=" in entry: + k, v = entry.split("=", 1) + st = k.strip() + lv = float(v.strip()) + else: + st = entry.strip() + lv = parse_load_value(st) + mat = infer_material_name(st) if default_mat == "auto" else default_mat + specs.append( + { + "state": st, + "load_value": lv, + "material": mat, + "voxel_size_um": args.voxel_size_um if args.voxel_size_um > 0 else ref_vox, + } + ) + return specs + + saved_specs = cfg.get("scan_specs") + if isinstance(saved_specs, list) and saved_specs: + return [dict(s) for s in saved_specs] # type: ignore[arg-type] + + states_map = cfg["states"] # type: ignore[index] + return [ + { + "state": str(k), + "load_value": float(v), + "material": infer_material_name(str(k)) if default_mat == "auto" else default_mat, + "voxel_size_um": args.voxel_size_um if args.voxel_size_um > 0 else ref_vox, + } + for k, v in states_map.items() # type: ignore[union-attr] + ] - print("Checkpoint:", CHECKPOINT, flush=True) - print("Completed epochs:", checkpoint["epoch"] + 1, flush=True) - print("GPU:", torch.cuda.get_device_name(0), flush=True) - scheduler = DDPMScheduler( - num_train_timesteps=1000, - schedule="scaled_linear_beta", +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Sample load-conditioned 3D microstructures across discrete and interpolated loads" ) - - # Identical initial noise for every condition. - initial_noise = torch.randn( - (1, 1, 96, 96, 96), device=device + parser.add_argument("--experiment", default="carbon_conditional_96_v2") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--checkpoint", type=Path, default=None) + parser.add_argument("--manifest-json", type=Path, default=None) + parser.add_argument("--no-ema", action="store_true") + parser.add_argument("--sampler", choices=["ddim", "ddpm"], default="ddim") + parser.add_argument("--steps", type=int, default=100) + parser.add_argument("--guidance-scale", type=float, default=1.5) + parser.add_argument("--num-samples", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--eval-states", nargs="+", default=None) + parser.add_argument("--include-repeats", action="store_true") + parser.add_argument("--beamline", default=None, help="Optional beamline filter (e.g. DLS_K11_DIAD)") + parser.add_argument( + "--material", + default="auto", + choices=["auto", "virgin_carbon", "tio2_anode", "cuhcf_cathode"], + ) + parser.add_argument("--voxel-size-um", type=float, default=0.0, help="0 = use checkpoint/manifest voxel size") + parser.add_argument( + "--sweep-loads", + default="0.008,0.015,0.023,0.033,0.043,0.053", + help="Comma-separated load/stress values for a fixed-noise sweep (empty to skip)", + ) + parser.add_argument("--export-openimpala", action="store_true") + parser.add_argument( + "--tiled-shape", + default="", + help="Optional seamless >patch_size^3 3D MultiDiffusion shape 'D,H,W' (e.g. '96,96,192' or '160,160,160') for OpenImpala REV convergence", + ) + parser.add_argument("--tiled-overlap", type=float, default=0.5) + parser.add_argument("--tile-batch-size", type=int, default=4) + parser.add_argument("--allow-cpu", action="store_true") + parser.add_argument("--seed", type=int, default=42) + args = parser.parse_args() + + if torch.cuda.is_available(): + device = torch.device("cuda") + elif args.allow_cpu: + device = torch.device("cpu") + else: + raise RuntimeError("CUDA GPU required (pass --allow-cpu for local CPU testing).") + + use_amp = device.type == "cuda" + amp_dtype = ( + torch.bfloat16 + if (device.type == "cuda" and torch.cuda.is_bf16_supported()) + else torch.float16 ) - results = [] - - for name, mv in STATES.items(): - torch.cuda.reset_peak_memory_stats() - torch.cuda.synchronize() - started = time.perf_counter() + ckpt_dir = args.output_root / "checkpoints" / args.experiment + if args.checkpoint is not None: + ckpt_path = args.checkpoint + elif (ckpt_dir / "best_val.pth").is_file(): + ckpt_path = ckpt_dir / "best_val.pth" + else: + ckpt_path = ckpt_dir / "latest.pth" + + if not ckpt_path.is_file(): + raise FileNotFoundError(f"Checkpoint not found: {ckpt_path}") + + saved = torch.load(ckpt_path, map_location="cpu", weights_only=False) + cfg = saved["config"] + patch_size = int(cfg["patch_size"]) + num_phases = int(cfg.get("num_phases", 2)) + num_materials = int(cfg.get("num_materials", 1)) + ref_vox = float(cfg.get("ref_voxel_size_um", 1.0)) + unit_label = "kPa" if bool(cfg.get("use_stress_kpa", False)) else "mV/V" + + tiled_shape: tuple[int, int, int] | None = None + if args.tiled_shape.strip(): + parts = [int(p.strip()) for p in args.tiled_shape.split(",") if p.strip()] + if len(parts) != 3 or any(p < patch_size for p in parts): + raise ValueError( + f"--tiled-shape must be 'D,H,W' with each dimension >= patch_size ({patch_size})" + ) + tiled_shape = (parts[0], parts[1], parts[2]) + + model = ContinuousLoadConditionedUNet( + channels=cfg["channels"], + attention_levels=cfg["attention_levels"], + num_res_blocks=int(cfg["num_res_blocks"]), + norm_num_groups=int(cfg["norm_groups"]), + load_min=float(cfg["load_min"]), + load_max=float(cfg["load_max"]), + use_flash_attention=(device.type == "cuda"), + conditioning_mode=str(cfg.get("conditioning_mode", "multiscale_fourier")), + num_phases=num_phases, + num_materials=num_materials, + ref_voxel_size_um=ref_vox, + use_ordinal_material_channel=infer_use_ordinal_material_channel(saved), + ).to(device) - condition = torch.tensor( - [[(mv - 20.5) / 12.5]], - device=device, - dtype=torch.float32, - ) + if not args.no_ema and "ema" in saved: + ema = EMAModel(model) + ema.load_state_dict(saved["ema"]) + active_model = ema.shadow_model.to(device).eval() + weight_source = "EMA" + else: + model.load_state_dict(saved["model"]) + active_model = model.eval() + weight_source = "online" + + scheduler = build_noise_scheduler( + sampler=args.sampler, + num_train_timesteps=int(cfg["timesteps"]), + schedule=str(cfg["schedule"]), + prediction_type=str(cfg["prediction_type"]), + ) + scheduler.set_timesteps(args.steps, device=device) - with torch.inference_mode(): - context = condition_encoder(condition).unsqueeze(1) - image = initial_noise.clone() + print( + f"Loaded {weight_source} weights from {ckpt_path} (epoch {int(saved['epoch']) + 1}) | " + f"phases={num_phases} | materials={num_materials} | sampler={args.sampler} ({args.steps} steps)" + + (f" | tiled={tiled_shape}" if tiled_shape else ""), + flush=True, + ) - for timestep in scheduler.timesteps: - t = torch.as_tensor( - [int(timestep)], device=device - ).long() + eval_specs = _build_eval_specs(args, cfg) + out_dir = args.output_root / "results" / args.experiment / "samples" + figures_dir = args.output_root / "results" / args.experiment / "figures" + out_dir.mkdir(parents=True, exist_ok=True) + figures_dir.mkdir(parents=True, exist_ok=True) - predicted_noise = model( - image, t, context=context - ) + per_sample_records: list[dict[str, object]] = [] + summary_records: list[dict[str, object]] = [] + data_ch = active_model.data_channels - image = scheduler.step( - predicted_noise, int(timestep), image - )[0] + for spec in eval_specs: + state_name = str(spec["state"]) + load_val = float(spec["load_value"]) # type: ignore[arg-type] + mat_name = str(spec.get("material", "virgin_carbon")) + mat_id = resolve_material_id(mat_name) if num_materials > 1 else 0 + vox_um = float(spec.get("voxel_size_um", ref_vox)) # type: ignore[arg-type] - torch.cuda.synchronize() - seconds = time.perf_counter() - started + state_dir = out_dir / state_name + state_dir.mkdir(parents=True, exist_ok=True) - continuous = image[0, 0].float().cpu().numpy() - normalized = np.clip( - (continuous + 1.0) / 2.0, 0, 1 - ).astype(np.float32) - binary = (normalized >= 0.5).astype(np.uint8) + gen = torch.Generator(device=device).manual_seed(args.seed) + collected_vols: list[np.ndarray] = [] + started = time.perf_counter() - np.save(OUTPUT / f"{name}_generated_binary.npy", binary) - np.save( - OUTPUT / f"{name}_generated_continuous.npy", - normalized, + remaining = args.num_samples + sample_idx = 0 + while remaining > 0: + cur_b = 1 if tiled_shape is not None else min(args.batch_size, remaining) + if tiled_shape is not None: + vols, probs = generate_tiled_3d( + model=active_model, + scheduler=scheduler, + target_shape=tiled_shape, + patch_size=patch_size, + load_value=load_val, + guidance_scale=args.guidance_scale, + overlap=args.tiled_overlap, + batch_size=cur_b, + tile_batch_size=args.tile_batch_size, + material_id=mat_id, + voxel_size_um=vox_um, + use_amp=use_amp, + amp_dtype=amp_dtype, + generator=gen, + ) + else: + noise = torch.randn( + (cur_b, data_ch, patch_size, patch_size, patch_size), + generator=gen, + device=device, + ) + vols, probs = generate_batch( + model=active_model, + scheduler=scheduler, + initial_noise=noise, + load_value=load_val, + guidance_scale=args.guidance_scale, + use_amp=use_amp, + amp_dtype=amp_dtype, + material_id=mat_id, + voxel_size_um=vox_um, + generator=gen, + ) + for b_vol, p_vol in zip(vols, probs): + sample_idx += 1 + collected_vols.append(b_vol) + np.save(state_dir / f"sample_{sample_idx:03d}_binary.npy", b_vol) + np.save(state_dir / f"sample_{sample_idx:03d}_prob.npy", p_vol) + + if args.export_openimpala: + export_openimpala_tiff( + b_vol, + state_dir / "openimpala" / f"sample_{sample_idx:03d}.tif", + preserve_multiphase=(num_phases == 3), + ) + + phys = ( + evaluate_multiphase_physics(b_vol, voxel_size=vox_um) + if num_phases == 3 + else evaluate_volume_physics(b_vol, voxel_size=vox_um) + ) + rec: dict[str, object] = { + "state": state_name, + "material": mat_name, + "load_value": load_val, + "voxel_size_um": vox_um, + "sample_id": sample_idx, + } + rec.update( + {k: round(float(v), 6) if np.isfinite(v) else float("nan") for k, v in phys.items()} + ) + per_sample_records.append(rec) + + if sample_idx == 1: + save_orthoview_figure( + b_vol, + title=f"{state_name} ({mat_name}, {load_val:g} {unit_label})", + png_path=figures_dir / f"figure_generated_{state_name}.png", + pdf_path=figures_dir / f"figure_generated_{state_name}.pdf", + num_phases=num_phases, + ) + remaining -= cur_b + + elapsed = time.perf_counter() - started + stacked_state = np.stack(collected_vols, axis=0) + np.save(state_dir / "generated_samples.npy", stacked_state) + + sfs = [float((v >= 1).mean()) for v in collected_vols] + cfs = [float((v == 2).mean()) for v in collected_vols] + summary_row = { + "state": state_name, + "material": mat_name, + "load_value": load_val, + "num_samples": args.num_samples, + "solid_fraction_mean": round(float(np.mean(sfs)), 6), + "solid_fraction_std": round(float(np.std(sfs)), 6), + "coating_fraction_mean": round(float(np.mean(cfs)), 6), + "sampling_seconds": round(elapsed, 2), + } + summary_records.append(summary_row) + print(json.dumps(summary_row), flush=True) + + if per_sample_records: + with (out_dir / "per_sample_physics.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(per_sample_records[0].keys())) + writer.writeheader() + writer.writerows(per_sample_records) + + if summary_records: + with (out_dir / "sampling_metrics.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(summary_records[0].keys())) + writer.writeheader() + writer.writerows(summary_records) + + if args.sweep_loads.strip(): + sweep_vals = [float(x.strip()) for x in args.sweep_loads.split(",") if x.strip()] + sweep_mat_name = "virgin_carbon" if args.material == "auto" else args.material + sweep_mat_id = resolve_material_id(sweep_mat_name) if num_materials > 1 else 0 + sweep_vox = args.voxel_size_um if args.voxel_size_um > 0 else ref_vox + + sweep_gen = torch.Generator(device=device).manual_seed(args.seed + 777) + shared_noise = torch.randn( + (1, data_ch, patch_size, patch_size, patch_size), + generator=sweep_gen, + device=device, ) - - fig, axes = plt.subplots(1, 3, figsize=(12, 4)) - views = [ - (binary[48], "XY"), - (binary[:, 48, :], "XZ"), - (binary[:, :, 48], "YZ"), - ] - - for ax, (view, title) in zip(axes, views): - ax.imshow( - view, cmap="gray", vmin=0, vmax=1, - interpolation="nearest" + sweep_dir = out_dir / "continuous_sweep" + sweep_dir.mkdir(parents=True, exist_ok=True) + + sweep_pairs: list[tuple[float, np.ndarray]] = [] + for mv in sweep_vals: + step_gen = torch.Generator(device=device).manual_seed(args.seed + 888) + b_vol, _ = generate_batch( + model=active_model, + scheduler=scheduler, + initial_noise=shared_noise, + load_value=mv, + guidance_scale=args.guidance_scale, + use_amp=use_amp, + amp_dtype=amp_dtype, + material_id=sweep_mat_id, + voxel_size_um=sweep_vox, + generator=step_gen, ) - ax.set_title(title) - ax.axis("off") - - fig.suptitle(f"Conditional DDPM — {mv:g} mV") - fig.tight_layout() - fig.savefig( - FIGURES / f"figure_generated_{name}.png", - dpi=300, - bbox_inches="tight", + vol = b_vol[0] + tag = f"{mv:.4f}".replace(".", "p") + np.save(sweep_dir / f"sweep_{tag}mVV.npy", vol) + sweep_pairs.append((mv, vol)) + + save_sweep_panel( + sweep_pairs, + png_path=figures_dir / "compression_sweep_orthoviews.png", + pdf_path=figures_dir / "compression_sweep_orthoviews.pdf", + num_phases=num_phases, + unit_label=unit_label, ) - fig.savefig( - FIGURES / f"figure_generated_{name}.pdf", - bbox_inches="tight", - ) - plt.close(fig) - - record = { - "state": name, - "load_cell_mv": mv, - "solid_fraction": float(binary.mean()), - "sampling_seconds": round(seconds, 2), - "peak_allocated_gib": round( - torch.cuda.max_memory_allocated() / 1024**3, 3 - ), - "peak_reserved_gib": round( - torch.cuda.max_memory_reserved() / 1024**3, 3 - ), - } - results.append(record) - print(record, flush=True) - - with (OUTPUT / "sampling_metrics.csv").open( - "w", newline="" - ) as f: - writer = csv.DictWriter( - f, fieldnames=results[0].keys() - ) - writer.writeheader() - writer.writerows(results) - - print("Conditional sampling completed.", flush=True) + print(f"Saved continuous compression sweep ({len(sweep_pairs)} loads) -> {figures_dir}", flush=True) if __name__ == "__main__": diff --git a/scripts/evaluation/virtual_compress_scan.py b/scripts/evaluation/virtual_compress_scan.py new file mode 100644 index 0000000..e95f08d --- /dev/null +++ b/scripts/evaluation/virtual_compress_scan.py @@ -0,0 +1,307 @@ +"""Virtual in-situ compression of a real uncompressed (0 kPa / benchtop XCT) 3D scan. + +Deterministically inverts an actual segmented 3D electrode volume (`{0, 1}` or `{0, 1, 2}`) +at `--source-load` along the conditional DDIM probability-flow ODE and denoises it back to +`t = 0` at any list of `--target-loads`, producing: + - Virtually compressed 3D `.npy` volumes and optional OpenImpala `.tif` stacks, + - `virtual_compression_metrics.csv` quantifying porosity, EASA, TPB, pinch-points, and `tau_Z`, + - Publication side-by-side comparison figure `virtual_compression_panel.{png,pdf}`. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from matplotlib.colors import ListedColormap +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import tifffile +import torch + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.data import default_project_root, resolve_material_id +from porousdiff.metrics import ( + evaluate_multiphase_physics, + evaluate_volume_physics, + export_openimpala_tiff, +) +from porousdiff.models import ( + ContinuousLoadConditionedUNet, + EMAModel, + build_noise_scheduler, + infer_use_ordinal_material_channel, +) +from porousdiff.virtual_compression import ( + check_load_extrapolation, + compute_virtual_compression_fidelity, + virtual_compress_volumes, +) + +PHASE_CMAP = ListedColormap(["#0b0e14", "#38bdf8", "#facc15"]) + + +def _load_input_volume(path: Path) -> np.ndarray: + ext = path.suffix.lower() + if ext == ".npy": + arr = np.load(path) + elif ext in (".tif", ".tiff"): + arr = tifffile.imread(path) + else: + raise ValueError(f"Unsupported input volume format {path}; expected .npy or .tif/.tiff") + if arr.ndim == 4: + arr = arr[0] + if arr.ndim != 3: + raise ValueError(f"Expected 3D volume (Z, Y, X), got shape {arr.shape}") + return arr.astype(np.uint8) + + +def _save_virtual_panel( + source_vol: np.ndarray, + source_load: float, + compressed_by_load: dict[float, np.ndarray], + png_path: Path, + pdf_path: Path, + num_phases: int, + unit_label: str, +) -> None: + items = [(source_load, source_vol, "Real Input")] + [ + (lv, vol, "Virtual PorousDiff") for lv, vol in sorted(compressed_by_load.items()) + ] + n_cols = len(items) + fig, axes = plt.subplots(2, n_cols, figsize=(3.3 * n_cols, 6.2), squeeze=False) + cmap = PHASE_CMAP if num_phases == 3 else "gray" + vmax = 2 if num_phases == 3 else 1 + + for col, (lv, vol, tag) in enumerate(items): + mz, my, _ = (s // 2 for s in vol.shape) + sf = float((vol >= 1).mean()) + axes[0, col].imshow(vol[mz], cmap=cmap, vmin=0, vmax=vmax, interpolation="nearest") + axes[0, col].set_title(f"{tag}\n{lv:g} {unit_label} (φ_s={sf:.3f})", fontsize=10) + axes[0, col].axis("off") + + axes[1, col].imshow(vol[:, my, :], cmap=cmap, vmin=0, vmax=vmax, interpolation="nearest") + axes[1, col].set_title("XZ (Compression ↓)", fontsize=9) + axes[1, col].axis("off") + + fig.suptitle("Conditional DDIM Microstructure Transfer Across Compressive Loads", fontsize=12) + fig.tight_layout() + fig.savefig(png_path, dpi=240, bbox_inches="tight") + fig.savefig(pdf_path, bbox_inches="tight") + plt.close(fig) + + +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Transfer a real low-load 3D XCT scan across target compressive loads via DDIM ODE inversion" + ) + parser.add_argument("--input", type=Path, required=True, help="Input 3D segmented .npy or .tif scan") + parser.add_argument("--experiment", default="beamtime_multiplier_96") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--checkpoint", type=Path, default=None) + parser.add_argument("--out-dir", type=Path, default=None) + parser.add_argument("--source-load", type=float, required=True, help="Physical load/stress of the input scan") + parser.add_argument( + "--target-loads", + required=True, + help="Comma-separated target loads/stresses (e.g. '50,150,300,500' or '0.018,0.033,0.053')", + ) + parser.add_argument( + "--material", + default="virgin_carbon", + choices=["virgin_carbon", "tio2_anode", "cuhcf_cathode"], + ) + parser.add_argument("--voxel-size-um", type=float, default=0.0) + parser.add_argument("--steps", type=int, default=50) + parser.add_argument("--invert-strength", type=float, default=0.60) + parser.add_argument("--guidance-scale", type=float, default=1.5) + parser.add_argument("--export-openimpala", action="store_true") + parser.add_argument("--allow-cpu", action="store_true") + args = parser.parse_args() + + if torch.cuda.is_available(): + device = torch.device("cuda") + elif args.allow_cpu: + device = torch.device("cpu") + else: + raise RuntimeError("CUDA GPU required (pass --allow-cpu for local CPU testing).") + + ckpt_dir = args.output_root / "checkpoints" / args.experiment + ckpt_path = ( + args.checkpoint + if args.checkpoint is not None + else ( + ckpt_dir / "best_val.pth" + if (ckpt_dir / "best_val.pth").is_file() + else ckpt_dir / "latest.pth" + ) + ) + if not ckpt_path.is_file(): + raise FileNotFoundError(f"Checkpoint not found: {ckpt_path}") + + saved = torch.load(ckpt_path, map_location="cpu", weights_only=False) + cfg = saved["config"] + num_phases = int(cfg.get("num_phases", 2)) + num_materials = int(cfg.get("num_materials", 1)) + ref_vox = float(cfg.get("ref_voxel_size_um", 1.0)) + vox_um = args.voxel_size_um if args.voxel_size_um > 0 else ref_vox + unit_label = "kPa" if bool(cfg.get("use_stress_kpa", False)) else "mV/V" + + model = ContinuousLoadConditionedUNet( + channels=cfg["channels"], + attention_levels=cfg["attention_levels"], + num_res_blocks=int(cfg["num_res_blocks"]), + norm_num_groups=int(cfg["norm_groups"]), + load_min=float(cfg["load_min"]), + load_max=float(cfg["load_max"]), + use_flash_attention=(device.type == "cuda"), + conditioning_mode=str(cfg.get("conditioning_mode", "multiscale_fourier")), + num_phases=num_phases, + num_materials=num_materials, + ref_voxel_size_um=ref_vox, + use_ordinal_material_channel=infer_use_ordinal_material_channel(saved), + ).to(device) + if "ema" in saved: + ema = EMAModel(model) + ema.load_state_dict(saved["ema"]) + active_model = ema.shadow_model.to(device).eval() + else: + model.load_state_dict(saved["model"]) + active_model = model.eval() + + scheduler = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=int(cfg["timesteps"]), + schedule=str(cfg["schedule"]), + prediction_type=str(cfg["prediction_type"]), + ) + + source_vol = _load_input_volume(args.input) + target_loads = [float(x.strip()) for x in args.target_loads.split(",") if x.strip()] + extrap = check_load_extrapolation( + [float(args.source_load), *target_loads], + float(cfg["load_min"]), + float(cfg["load_max"]), + ) + if bool(extrap["is_extrapolated"]): + print( + f"Warning: Requested loads {extrap['extrapolated_loads']} extrapolate outside trained range " + f"[{extrap['load_min']}, {extrap['load_max']}] ({unit_label}).", + flush=True, + ) + + mat_id = resolve_material_id(args.material) if num_materials > 1 else 0 + use_amp = device.type == "cuda" + amp_dtype = ( + torch.bfloat16 + if (device.type == "cuda" and torch.cuda.is_bf16_supported()) + else torch.float16 + ) + + all_eval_loads = list(dict.fromkeys([float(args.source_load), *target_loads])) + all_compressed = virtual_compress_volumes( + model=active_model, + alphas_cumprod=scheduler.alphas_cumprod, + source_volumes=source_vol, + source_load=args.source_load, + target_loads=all_eval_loads, + prediction_type=str(cfg["prediction_type"]), + num_steps=args.steps, + invert_strength=args.invert_strength, + guidance_scale=args.guidance_scale, + material_id=mat_id, + voxel_size_um=vox_um, + batch_size=1, + use_amp=use_amp, + amp_dtype=amp_dtype, + ) + same_load_vol = all_compressed[float(args.source_load)] + compressed = {float(tl): all_compressed[float(tl)] for tl in target_loads} + + out_dir = ( + args.out_dir + if args.out_dir is not None + else (args.output_root / "results" / args.experiment / "virtual_compression") + ) + out_dir.mkdir(parents=True, exist_ok=True) + + rows: list[dict[str, object]] = [] + fidelity_by_load: dict[str, dict[str, float]] = {} + eval_fn = evaluate_multiphase_physics if num_phases == 3 else evaluate_volume_physics + src_phys = eval_fn(source_vol, voxel_size=vox_um) + src_fid = compute_virtual_compression_fidelity( + source_vol, same_load_vol, same_load_reconstructions=same_load_vol + ) + src_row: dict[str, object] = { + "kind": "real_source", + "load_value": args.source_load, + "load_unit": unit_label, + "material": args.material, + "same_load_solid_dice": round(float(src_fid["same_load_solid_dice"]), 6), + "paired_xy_column_corr": 1.0, + } + src_row.update({k: round(float(v), 6) if np.isfinite(v) else float("nan") for k, v in src_phys.items()}) + rows.append(src_row) + + for lv, vol in sorted(compressed.items()): + tag = f"{lv:.4f}".replace(".", "p") + np.save(out_dir / f"virtual_{args.material}_{tag}.npy", vol) + if args.export_openimpala: + export_openimpala_tiff( + vol, + out_dir / "openimpala" / f"virtual_{args.material}_{tag}.tif", + preserve_multiphase=(num_phases == 3), + ) + phys = eval_fn(vol, voxel_size=vox_um) + fid = compute_virtual_compression_fidelity( + source_vol, vol, same_load_reconstructions=same_load_vol + ) + fidelity_by_load[f"{lv:g}"] = fid + row: dict[str, object] = { + "kind": "virtual_compressed", + "load_value": lv, + "load_unit": unit_label, + "material": args.material, + "same_load_solid_dice": round(float(fid["same_load_solid_dice"]), 6), + "paired_xy_column_corr": round(float(fid["paired_xy_column_corr"]), 6), + } + row.update({k: round(float(v), 6) if np.isfinite(v) else float("nan") for k, v in phys.items()}) + rows.append(row) + + with (out_dir / "virtual_compression_metrics.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + summary_payload = { + "load_unit": unit_label, + "invert_strength": float(args.invert_strength), + "extrapolation_check": extrap, + "same_load_solid_dice": round(float(src_fid["same_load_solid_dice"]), 6), + "fidelity_by_load": fidelity_by_load, + "records": rows, + } + (out_dir / "virtual_compression_summary.json").write_text(json.dumps(summary_payload, indent=2)) + _save_virtual_panel( + source_vol=source_vol, + source_load=args.source_load, + compressed_by_load=compressed, + png_path=out_dir / "virtual_compression_panel.png", + pdf_path=out_dir / "virtual_compression_panel.pdf", + num_phases=num_phases, + unit_label=unit_label, + ) + print(f"Saved virtual compression series ({len(compressed)} target loads) -> {out_dir}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/extract_carbon_3d_patches.py b/scripts/preprocessing/extract_carbon_3d_patches.py index 7ce97c1..54d677f 100644 --- a/scripts/preprocessing/extract_carbon_3d_patches.py +++ b/scripts/preprocessing/extract_carbon_3d_patches.py @@ -1,304 +1,294 @@ +"""Extract stratified 3D binary XCT patches inside the circular ROI across clean Z intervals. + +Fixes Z-depth starvation, exterior-air corner contamination, and adjacent train/val Z leakage, +and writes stacked `patches.npy` (`uint8`) for fast single-read RAM loading on HPC. +""" + +from __future__ import annotations import argparse import csv import json +import re +import sys from pathlib import Path import numpy as np import tifffile - -def main(): - parser = argparse.ArgumentParser( - description="Extract reusable 3D binary XCT patches" - ) - parser.add_argument("--dataset", required=True) - parser.add_argument("--version", default="v2") - parser.add_argument("--expected-slices", type=int, default=800) - parser.add_argument("--patch-size", type=int, default=64) - parser.add_argument("--stride-xy", type=int, default=128) - parser.add_argument("--stride-z", type=int, default=32) - parser.add_argument("--max-patches", type=int, default=1200) - parser.add_argument("--min-white", type=float, default=0.01) - parser.add_argument("--max-white", type=float, default=0.99) - parser.add_argument("--val-fraction", type=float, default=0.2) - parser.add_argument("--seed", type=int, default=42) - args = parser.parse_args() - - # -------------------------------------------------- - # 1. Validate arguments - # -------------------------------------------------- - if not args.dataset.isalnum(): - raise ValueError("Dataset must contain only letters and numbers") - - if not args.version.replace("_", "").isalnum(): - raise ValueError("Version must contain only letters and numbers") - - if not 0 < args.val_fraction < 1: - raise ValueError("val-fraction must be between 0 and 1") - - if not 0 <= args.min_white < args.max_white <= 1: - raise ValueError("Invalid white-fraction thresholds") - - if min( - args.expected_slices, - args.patch_size, - args.stride_xy, - args.stride_z, - args.max_patches, - ) <= 0: - raise ValueError("Sizes, strides and patch count must be positive") - - # -------------------------------------------------- - # 2. Input and output paths - # -------------------------------------------------- - root = Path("/projects/u6uc/asif/data/carbon_virgin") - - input_dir = root / ( - f"{args.dataset}_binary_corrected_{args.version}" - ) - - output_dir = root / ( - f"{args.dataset}_binary_patches" - f"{args.patch_size}_{args.version}" - ) +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.data import default_project_root +from porousdiff.preprocessing import ( + build_stratified_patch_plan, + enumerate_cylinder_xy_positions, + split_z_intervals_with_buffer, +) + + +def _validate_token(value: str, label: str) -> None: + if not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + raise ValueError(f"Invalid {label}: {value!r}") + + +def extract_dataset_patches( + dataset_name: str, + data_root: Path, + output_root: Path, + version: str, + expected_slices: int, + patch_size: int, + stride_xy: int, + stride_z: int, + max_patches: int, + min_white: float, + max_white: float, + val_fraction: float, + roi_radius_fraction: float, + ignore_valid_slices: bool, + save_individual_npy: bool, + seed: int, + overwrite: bool, +) -> dict[str, object]: + input_dir = data_root / f"{dataset_name}_binary_corrected_{version}" + output_dir = output_root / f"{dataset_name}_binary_patches{patch_size}_{version}" files = sorted(input_dir.glob("binary_*.tif")) + if not files: + raise FileNotFoundError(f"No binary_*.tif files found in {input_dir}") - if len(files) != args.expected_slices: - raise RuntimeError( - f"Expected {args.expected_slices} slices, " - f"found {len(files)}" - ) - - expected_names = [ - f"binary_{i:04d}.tif" - for i in range(1, args.expected_slices + 1) - ] + if expected_slices > 0 and len(files) != expected_slices: + raise RuntimeError(f"Expected {expected_slices} slices in {input_dir}, found {len(files)}") + expected_names = [f"binary_{i:04d}.tif" for i in range(1, len(files) + 1)] if [f.name for f in files] != expected_names: - raise RuntimeError("Binary TIFF sequence is incomplete") + raise RuntimeError(f"Binary TIFF sequence in {input_dir} is incomplete or non-contiguous") - # Never overwrite an existing experiment. - if output_dir.exists(): + if output_dir.exists() and not overwrite: raise FileExistsError( - f"Output already exists: {output_dir}. " - "Use a new version for a separate experiment." + f"Output already exists: {output_dir}. Use a new --version or --overwrite." ) - # -------------------------------------------------- - # 3. Spatial train/validation split - # -------------------------------------------------- - size = args.patch_size - total = len(files) - - boundary = int(total * (1 - args.val_fraction)) - - regions = { - "train": (0, boundary), - "val": (boundary, total), - } - - budgets = { - "train": int( - args.max_patches * (1 - args.val_fraction) - ) - } - - budgets["val"] = ( - args.max_patches - budgets["train"] + first_slice = tifffile.imread(files[0]) + shape_hw = (int(first_slice.shape[0]), int(first_slice.shape[1])) + + valid_slices_file = input_dir / "valid_slices.json" + if valid_slices_file.is_file() and not ignore_valid_slices: + payload = json.loads(valid_slices_file.read_text()) + clean_intervals = [ + (int(s), int(e)) for s, e in payload["valid_z_intervals_0based_half_open"] + ] + roi_radius_fraction = float(payload.get("roi_radius_fraction", roi_radius_fraction)) + else: + clean_intervals = [(0, len(files))] + + split_intervals = split_z_intervals_with_buffer( + intervals=clean_intervals, + patch_size=patch_size, + val_fraction=val_fraction, + buffer_slices=patch_size, ) - for split, (start, end) in regions.items(): - if end - start < size: - raise RuntimeError( - f"{split} region has fewer than " - f"{size} slices" - ) + xy_positions = enumerate_cylinder_xy_positions( + shape_hw=shape_hw, + patch_size=patch_size, + stride_xy=stride_xy, + radius_fraction=roi_radius_fraction, + ) + if not xy_positions: + raise RuntimeError( + f"No {patch_size}x{patch_size} XY patches fit inside circular ROI " + f"(slice shape={shape_hw}, radius_fraction={roi_radius_fraction})" + ) - # -------------------------------------------------- - # 4. Initialize reproducible sampling - # -------------------------------------------------- - rng = np.random.default_rng(args.seed) + budgets = {"train": int(round(max_patches * (1.0 - val_fraction)))} + budgets["val"] = max(1, max_patches - budgets["train"]) - counts = { - "train": 0, - "val": 0, - } + rng = np.random.default_rng(seed) + output_dir.mkdir(parents=True, exist_ok=True) + metadata_path = output_dir / "patch_metadata.csv" - output_dir.mkdir(parents=True) + counts = {"train": 0, "val": 0} + solid_stats: dict[str, list[float]] = {"train": [], "val": []} + sampled_z_windows: dict[str, set[int]] = {"train": set(), "val": set()} - metadata_path = output_dir / "patch_metadata.csv" + with metadata_path.open("w", newline="") as mf: + writer = csv.writer(mf) + writer.writerow( + ["filename", "split", "z_start", "z_end", "y_start", "x_start", "white_fraction"] + ) - # -------------------------------------------------- - # 5. Extract patches - # -------------------------------------------------- - with metadata_path.open("w", newline="") as metadata_file: - writer = csv.writer(metadata_file) - - writer.writerow([ - "filename", - "split", - "z_start", - "z_end", - "y_start", - "x_start", - "white_fraction", - ]) - - for split, (start, end) in regions.items(): + for split in ("train", "val"): split_dir = output_dir / split - split_dir.mkdir() - - # Shuffle depth windows. - z_positions = list( - range( - start, - end - size + 1, - args.stride_z, - ) + split_dir.mkdir(parents=True, exist_ok=True) + + plan = build_stratified_patch_plan( + z_intervals=split_intervals[split], + xy_positions=xy_positions, + patch_size=patch_size, + stride_z=stride_z, + max_patches=budgets[split], + rng=rng, ) - rng.shuffle(z_positions) + collected: list[np.ndarray] = [] + # Because `plan` is sorted by ascending `z`, maintain a sliding slab cache + cached_z: int | None = None + cached_slab: np.ndarray | None = None - print( - f"Starting {split}: " - f"{len(z_positions)} depth windows", - flush=True, - ) + for z, xy_list in plan: + if counts[split] >= budgets[split]: + break - for z in z_positions: - # Read only 64 consecutive TIFF slices. - volume = np.stack([ - tifffile.imread(f) - for f in files[z:z + size] - ]) - - if volume.dtype != np.uint8: - raise ValueError( - f"Unexpected data type at z={z}" - ) - - if volume.ndim != 3: - raise ValueError( - f"Expected 3D volume at z={z}" - ) - - if volume.shape[0] != size: - raise ValueError( - f"Unexpected depth at z={z}" - ) - - if not np.all( - (volume == 0) | (volume == 1) - ): - raise ValueError( - f"Non-binary values at z={z}" - ) - - # Build XY sampling positions. - xy_positions = [ - (y, x) - for y in range( - 0, - volume.shape[1] - size + 1, - args.stride_xy, - ) - for x in range( - 0, - volume.shape[2] - size + 1, - args.stride_xy, - ) - ] - - # Shuffle XY positions as well. - rng.shuffle(xy_positions) - - for y, x in xy_positions: - patch = volume[ - :, - y:y + size, - x:x + size, + if cached_slab is not None and cached_z is not None and z > cached_z and (z - cached_z) < patch_size: + overlap = patch_size - (z - cached_z) + new_slices = [ + tifffile.imread(f) + for f in files[cached_z + patch_size : z + patch_size] ] - - white_fraction = float(patch.mean()) - - # Reject nearly empty or nearly full patches. - if not ( - args.min_white - < white_fraction - < args.max_white - ): + slab = np.concatenate([cached_slab[-overlap:], np.stack(new_slices, axis=0)], axis=0) + else: + slab = np.stack([tifffile.imread(f) for f in files[z : z + patch_size]], axis=0) + + cached_z = z + cached_slab = slab + + if slab.dtype != np.uint8 or slab.shape != (patch_size, shape_hw[0], shape_hw[1]): + raise ValueError(f"Unexpected slab shape/dtype at z={z}: {slab.shape} {slab.dtype}") + if not np.all((slab == 0) | (slab == 1)): + raise ValueError(f"Non-binary values encountered at z={z}") + + for y, x in xy_list: + patch = slab[:, y : y + patch_size, x : x + patch_size] + wf = float(patch.mean()) + if not (min_white <= wf <= max_white): continue - filename = ( - f"{args.dataset}_binary_{split}_" - f"{counts[split]:05d}.npy" - ) - - np.save( - split_dir / filename, - patch, - ) - - writer.writerow([ - filename, - split, - z + 1, - z + size, - y, - x, - round(white_fraction, 6), - ]) + idx = counts[split] + filename = f"{dataset_name}_binary_{split}_{idx:05d}.npy" + if save_individual_npy: + np.save(split_dir / filename, patch) + collected.append(patch.copy()) + solid_stats[split].append(wf) + sampled_z_windows[split].add(z) + writer.writerow([filename, split, z + 1, z + patch_size, y, x, round(wf, 6)]) counts[split] += 1 if counts[split] >= budgets[split]: break - # Stop once this split's budget is reached. - if counts[split] >= budgets[split]: - break + if not collected: + raise RuntimeError(f"[{dataset_name}] Extracted 0 patches for split={split!r}") + stacked = np.stack(collected, axis=0).astype(np.uint8) + np.save(split_dir / "patches.npy", stacked) print( - f"{split}: " - f"{counts[split]}/{budgets[split]} patches", + f"[{dataset_name}] {split}: {counts[split]}/{budgets[split]} patches across " + f"{len(sampled_z_windows[split])} Z-windows (mean solid={np.mean(solid_stats[split]):.4f})", flush=True, ) - # -------------------------------------------------- - # 6. Save experiment report - # -------------------------------------------------- - report = { - "dataset": args.dataset, - "version": args.version, + report: dict[str, object] = { + "dataset": dataset_name, + "version": version, "input_directory": str(input_dir), "output_directory": str(output_dir), - "expected_slices": args.expected_slices, - "patch_size": size, - "stride_xy": args.stride_xy, - "stride_z": args.stride_z, - "max_patches": args.max_patches, - "min_white": args.min_white, - "max_white": args.max_white, - "val_fraction": args.val_fraction, - "seed": args.seed, - "train_z_range": [1, boundary], - "val_z_range": [boundary + 1, total], + "patch_size": patch_size, + "stride_xy": stride_xy, + "stride_z": stride_z, + "roi_radius_fraction": roi_radius_fraction, + "xy_candidates_per_z": len(xy_positions), + "clean_z_intervals": [[s, e] for s, e in clean_intervals], + "split_z_intervals_0based_half_open": { + k: [[s, e] for s, e in v] for k, v in split_intervals.items() + }, + "z_windows_sampled": {k: sorted(list(v)) for k, v in sampled_z_windows.items()}, + "min_white": min_white, + "max_white": max_white, + "val_fraction": val_fraction, + "seed": seed, "counts": counts, "total_patches": sum(counts.values()), + "mean_solid_fraction": { + k: float(np.mean(v)) if v else 0.0 for k, v in solid_stats.items() + }, + "std_solid_fraction": { + k: float(np.std(v)) if v else 0.0 for k, v in solid_stats.items() + }, } + with (output_dir / "patch_extraction_report.json").open("w") as rf: + json.dump(report, rf, indent=2) + + return report + + +def main() -> None: + default_root = default_project_root() / "data" / "carbon_virgin" + parser = argparse.ArgumentParser( + description="Extract Z-stratified 3D binary XCT patches inside circular sample ROI" + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--dataset", help="Single dataset name (e.g. Op008mVV)") + group.add_argument("--datasets", nargs="+", help="Multiple dataset names to extract") + + parser.add_argument("--data-root", type=Path, default=default_root) + parser.add_argument("--output-root", type=Path, default=None) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument("--expected-slices", type=int, default=0) + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--stride-xy", type=int, default=96) + parser.add_argument("--stride-z", type=int, default=32) + parser.add_argument("--max-patches", type=int, default=1200) + parser.add_argument("--min-white", type=float, default=0.002) + parser.add_argument("--max-white", type=float, default=0.95) + parser.add_argument("--val-fraction", type=float, default=0.2) + parser.add_argument("--roi-radius-fraction", type=float, default=0.70) + parser.add_argument("--ignore-valid-slices", action="store_true") + parser.add_argument( + "--no-individual-npy", + action="store_true", + help="Write only stacked patches.npy (recommended for large HPC runs)", + ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() - with ( - output_dir / "patch_extraction_report.json" - ).open("w") as report_file: - json.dump(report, report_file, indent=2) + datasets = [args.dataset] if args.dataset else list(args.datasets) + for ds in datasets: + _validate_token(ds, "dataset") + _validate_token(args.version, "version") - print("\nPatch extraction completed.") - print("Training patches:", counts["train"]) - print("Validation patches:", counts["val"]) - print("Total patches:", sum(counts.values())) - print("Output:", output_dir) + if not 0.0 < args.val_fraction < 1.0: + raise ValueError("--val-fraction must be between 0 and 1") + if not 0.0 <= args.min_white < args.max_white <= 1.0: + raise ValueError("Invalid white-fraction thresholds") + if min(args.patch_size, args.stride_xy, args.stride_z, args.max_patches) <= 0: + raise ValueError("Patch size, strides, and max-patches must be positive") + + out_root = args.output_root if args.output_root is not None else args.data_root + for ds in datasets: + extract_dataset_patches( + dataset_name=ds, + data_root=args.data_root, + output_root=out_root, + version=args.version, + expected_slices=args.expected_slices, + patch_size=args.patch_size, + stride_xy=args.stride_xy, + stride_z=args.stride_z, + max_patches=args.max_patches, + min_white=args.min_white, + max_white=args.max_white, + val_fraction=args.val_fraction, + roi_radius_fraction=args.roi_radius_fraction, + ignore_valid_slices=args.ignore_valid_slices, + save_individual_npy=not args.no_individual_npy, + seed=args.seed, + overwrite=args.overwrite, + ) if __name__ == "__main__": diff --git a/scripts/preprocessing/prepare_local_drive_bundle.py b/scripts/preprocessing/prepare_local_drive_bundle.py new file mode 100644 index 0000000..a9e8850 --- /dev/null +++ b/scripts/preprocessing/prepare_local_drive_bundle.py @@ -0,0 +1,1036 @@ +"""Stream-segment raw XCT scans (Virgin, TiO2 Anode, CuHCF Cathode, DIAD) into 3D patch bundles. + +Supports: + 1. 10-second `--preview-only` mode generating `qa/multiphase_overlay_diagnostic.png` (`pore=black`, + `carbon=cyan`, `active_coating=gold`) so 3-phase thresholds can be visually verified in Spyder. + 2. Automatic material inference (`virgin_carbon`, `tio2_anode`, `cuhcf_cathode`), 2-phase or + 3-phase pore-referenced segmentation, and bullseye/plunger Z-interval gating. + 3. Topology-preserving `--spatial-bin` (`1x` native or `2x` REV-scale block binning) + physical + calibration metadata (`voxel_size_um`, `stress_kpa`, `strain_z`, `beamline`). + 4. Single `.zip` archive output (`--zip`) with `primary_states.json` + `bundle_manifest.json`. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import zipfile +from matplotlib.colors import ListedColormap +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import tifffile + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.preprocessing import ( + CalibrationMetadata, + build_stratified_patch_plan, + calibrate_delta_threshold, + calibrate_multiphase_thresholds, + compute_pore_referenced_delta, + compute_radial_ring_score, + detect_valid_z_intervals, + downsample_volume_rev, + enumerate_cylinder_xy_positions, + infer_material_name, + load_overrides_table, + make_circular_mask, + make_collision_safe_state_name, + parse_load_with_unit, + resolve_tiff_directory, + segment_slice_multiphase, + split_xy_positions_with_buffer, + split_z_intervals_with_buffer, +) + +SPYDER_DRIVE_ROOT = Path("D:/dry_samples") +SPYDER_OUTPUT_DIR = Path("D:/porousdiff_upload_bundle") +PHASE_CMAP = ListedColormap(["#0b0e14", "#38bdf8", "#facc15"]) + + +def _is_repeat_or_secondary(folder_or_path: str) -> bool: + low = folder_or_path.lower() + return ("rep" in low) or ("virgin_2" in low) or ("sample2" in low) or ("diad" in low and "rep" in low) + + +def _save_multiphase_preview( + out_path: Path, + state_name: str, + material: str, + raw_slice: np.ndarray, + seg_slice: np.ndarray, + delta_slice: np.ndarray, + roi_mask: np.ndarray, + delta_carbon: float, + delta_coating: float | None, +) -> None: + h, w = raw_slice.shape + ch = min(h, max(96, h // 4)) + cw = min(w, max(96, w // 4)) + y0, x0 = (h - ch) // 2, (w - cw) // 2 + + raw_crop = raw_slice[y0 : y0 + ch, x0 : x0 + cw].astype(np.float32) + seg_crop = seg_slice[y0 : y0 + ch, x0 : x0 + cw] + vmin, vmax = np.percentile(raw_crop, [1.0, 99.5]) + + fig, axes = plt.subplots(1, 3, figsize=(14, 4.4)) + axes[0].imshow(raw_crop, cmap="gray", vmin=vmin, vmax=max(vmax, vmin + 1.0)) + axes[0].set_title(f"{state_name} ({material})\nRaw Center Crop ({ch}x{cw})") + axes[0].axis("off") + + axes[1].imshow(seg_crop, cmap=PHASE_CMAP, vmin=0, vmax=2, interpolation="nearest") + c_frac = float((seg_slice[roi_mask] == 1).mean()) + a_frac = float((seg_slice[roi_mask] == 2).mean()) + axes[1].set_title( + f"Segmented (Black=Pore, Cyan=Carbon {c_frac:.3f}, Gold=Coating {a_frac:.3f})" + ) + axes[1].axis("off") + + roi_delta = delta_slice[roi_mask] + lo, hi = np.percentile(roi_delta, [0.5, 99.9]) + clipped = roi_delta[(roi_delta >= lo) & (roi_delta <= hi)] + axes[2].hist(clipped, bins=120, color="slategray", log=True, alpha=0.85) + axes[2].axvline(delta_carbon, color="#0284c7", lw=1.8, label=f"delta_carbon={delta_carbon:.0f}") + if delta_coating is not None: + axes[2].axvline( + delta_coating, color="#eab308", lw=1.8, label=f"delta_coating={delta_coating:.0f}" + ) + axes[2].set_xlabel("Pore-referenced intensity delta") + axes[2].set_ylabel("Voxel count (log)") + axes[2].set_title("Phase Threshold Calibration") + axes[2].legend(loc="upper right", fontsize=8) + axes[2].grid(True, alpha=0.25) + + fig.tight_layout() + fig.savefig(out_path, dpi=180, bbox_inches="tight") + plt.close(fig) + + +def _save_z_plot( + out_path: Path, + state_name: str, + solid_fractions: list[float], + coating_fractions: list[float], + ring_scores: list[float], + slice_jumps: list[float], + valid_mask: np.ndarray, + intervals: list[tuple[int, int]], +) -> None: + z = np.arange(len(solid_fractions)) + fig, axes = plt.subplots(2, 1, figsize=(11, 6), sharex=True) + axes[0].plot(z, solid_fractions, color="tab:blue", lw=1.3, label="Total solid fraction") + if any(c > 0.0 for c in coating_fractions): + axes[0].plot(z, coating_fractions, color="goldenrod", lw=1.2, label="Active coating fraction") + for idx, (s, e) in enumerate(intervals): + axes[0].axvspan( + s, e - 1, color="tab:green", alpha=0.15, label="Accepted clean Z" if idx == 0 else None + ) + axes[0].set_ylabel("Volume fraction") + axes[0].set_title(f"{state_name} — Segmented Z Profile & Accepted Clean Intervals") + axes[0].legend(loc="upper right") + axes[0].grid(True, alpha=0.3) + + ax2 = axes[1] + ax2.plot(z, ring_scores, color="tab:red", lw=1.2, label="Radial ring score") + ax2_r = ax2.twinx() + ax2_r.plot(z, slice_jumps, color="tab:orange", lw=1.0, alpha=0.75, label="|dI/dz| slice jump") + bad_idx = np.where(~valid_mask)[0] + if bad_idx.size: + ax2.scatter( + bad_idx, + np.asarray(ring_scores)[bad_idx], + color="black", + s=6, + zorder=4, + label="Excluded artifact/plunger slices", + ) + ax2.set_xlabel("Slice index Z") + ax2.set_ylabel("Radial ring score") + ax2_r.set_ylabel("Mean |dI/dz|") + ax2.legend(loc="upper left") + ax2.grid(True, alpha=0.3) + fig.tight_layout() + fig.savefig(out_path, dpi=160) + plt.close(fig) + + +def _clean_ref_indices(num_files: int) -> list[int]: + """Select clean upper-middle Z slice indices (`20%`, `32%`, `45%`) avoiding top edge & bottom plunger.""" + if num_files <= 3: + return list(range(num_files)) + raw = [int(0.20 * num_files), int(0.32 * num_files), int(0.45 * num_files)] + return sorted({min(max(0, i), num_files - 1) for i in raw}) + + +def discover_scan_folders( + drive_root: Path, + max_depth: int = 3, + overrides: dict[str, tuple[float, str]] | None = None, +) -> list[Path]: + """Recursively discover XCT scan folders up to `max_depth` levels under `drive_root`.""" + candidates: list[Path] = [] + + def _walk(curr: Path, depth: int) -> None: + if depth > max_depth or not curr.is_dir(): + return + for entry in sorted(curr.iterdir()): + if not entry.is_dir() or entry.name.startswith(".") or entry.name in ("qa", "rec_16bit_Paganin"): + continue + rel_key = str(entry.relative_to(drive_root)).replace("\\", "/") + has_tiffs = False + try: + resolve_tiff_directory(entry) + has_tiffs = True + except FileNotFoundError: + has_tiffs = False + + if has_tiffs: + try: + parse_load_with_unit(rel_key, overrides=overrides) + candidates.append(entry) + continue + except ValueError: + pass + _walk(entry, depth + 1) + + _walk(drive_root, 1) + return candidates + + +def stream_bundle_single_scan( + scan_dir: Path, + output_dir: Path, + state_name: str | None = None, + version: str = "carbon_calibrated_v1", + material: str = "auto", + num_phases: int = 2, + auto_multiphase: bool = False, + auto_calibrate_carbon: bool = False, + delta_threshold: float = 5500.0, + delta_coating_threshold: float = 0.0, + precalibrated_thresholds: tuple[float, float | None] | None = None, + load_overrides: dict[str, tuple[float, str]] | None = None, + sigma: float = 32.0, + roi_radius_fraction: float = 0.70, + patch_size: int = 96, + spatial_bin: int = 1, + stride_xy: int = 96, + stride_z: int = 32, + max_patches: int = 1200, + min_white: float = 0.005, + max_white: float = 0.90, + val_fraction: float = 0.2, + min_interval_length: int = 96, + max_ring_score: float = 0.15, + seam_margin: int = 12, + calibration: CalibrationMetadata | None = None, + strip_halo_px: int = 1, + split_mode: str = "auto", + preview_only: bool = False, + resume: bool = False, + seed: int = 42, +) -> dict[str, object]: + if split_mode not in ("auto", "z", "xy"): + raise ValueError(f"split_mode must be 'auto', 'z', or 'xy', got {split_mode!r}") + calib = calibration or CalibrationMetadata(spatial_bin=spatial_bin) + resolved_material = infer_material_name(scan_dir) if material == "auto" else material + resolved_state = state_name or make_collision_safe_state_name( + scan_dir, material=resolved_material, beamline=calib.beamline + ) + load_val, load_unit = parse_load_with_unit(str(scan_dir), overrides=load_overrides) + stress_kpa = calib.resolve_stress_kpa(load_val, unit=load_unit) + load_mv = load_val if load_unit == "mv_per_v" else ( + (stress_kpa / calib.kpa_per_mvv) + if (stress_kpa is not None and calib.kpa_per_mvv) + else load_val + ) + + use_three_phase = (num_phases == 3) or (auto_multiphase and resolved_material != "virgin_carbon") + eff_num_phases = 3 if (use_three_phase and resolved_material != "virgin_carbon") else 2 + + patch_dir = output_dir / f"{resolved_state}_binary_patches{patch_size}_{version}" + qa_dir = patch_dir / "qa" + report_path = qa_dir / "segmentation_report.json" + + if ( + resume + and not preview_only + and (patch_dir / "train" / "patches.npy").is_file() + and (patch_dir / "val" / "patches.npy").is_file() + and report_path.is_file() + ): + cached = json.loads(report_path.read_text()) + print(f"[{resolved_state}] Resuming: reusing existing patches in {patch_dir.name}", flush=True) + return cached + + tiff_dir = resolve_tiff_directory(scan_dir) + files = sorted(list(tiff_dir.glob("*.tif")) + list(tiff_dir.glob("*.tiff"))) + raw_window = patch_size * max(1, spatial_bin) + min_required_slices = raw_window if split_mode in ("auto", "xy") else 2 * raw_window + if not preview_only and len(files) < min_required_slices: + raise RuntimeError( + f"[{resolved_state}] Need >= {min_required_slices} slices in {tiff_dir}, found {len(files)}" + ) + + with tifffile.TiffFile(files[0]) as first_tif: + shape_hw = tuple(first_tif.pages[0].shape) + dtype = first_tif.pages[0].dtype + + roi_mask = make_circular_mask(shape_hw, radius_fraction=roi_radius_fraction) + + if precalibrated_thresholds is not None: + eff_delta_carbon, delta_coating = precalibrated_thresholds + else: + ref_slices = [tifffile.imread(files[i]) for i in _clean_ref_indices(len(files))] + if eff_num_phases == 3: + eff_delta_carbon, auto_coat = calibrate_multiphase_thresholds( + ref_slices, + roi_mask=roi_mask, + sigma=sigma, + fixed_delta_carbon=None if auto_calibrate_carbon else delta_threshold, + ) + delta_coating = ( + float(delta_coating_threshold) + if delta_coating_threshold > eff_delta_carbon + else auto_coat + ) + else: + eff_delta_carbon = ( + calibrate_delta_threshold(ref_slices, roi_mask=roi_mask, sigma=sigma) + if auto_calibrate_carbon + else float(delta_threshold) + ) + delta_coating = None + + qa_dir.mkdir(parents=True, exist_ok=True) + + mid_idx = int(0.35 * len(files)) if len(files) >= 16 else (len(files) // 2) + mid_raw = tifffile.imread(files[mid_idx]) + mid_delta, _, _ = compute_pore_referenced_delta(mid_raw, roi_mask=roi_mask, sigma=sigma) + mid_seg, mid_stats = segment_slice_multiphase( + mid_raw, + delta_carbon=eff_delta_carbon, + delta_coating=delta_coating, + roi_mask=roi_mask, + sigma=sigma, + strip_halo_px=strip_halo_px, + ) + _save_multiphase_preview( + out_path=qa_dir / "multiphase_overlay_diagnostic.png", + state_name=resolved_state, + material=resolved_material, + raw_slice=mid_raw, + seg_slice=mid_seg, + delta_slice=mid_delta, + roi_mask=roi_mask, + delta_carbon=eff_delta_carbon, + delta_coating=delta_coating, + ) + + if preview_only: + preview_info: dict[str, object] = { + "state": resolved_state, + "material": resolved_material, + "beamline": calib.beamline, + "load_mv_per_v": load_mv, + "load_raw_value": load_val, + "load_unit": load_unit, + "stress_kpa": stress_kpa, + "effective_voxel_size_um": calib.effective_voxel_size_um, + "num_phases": eff_num_phases, + "delta_carbon": eff_delta_carbon, + "delta_coating": delta_coating, + "strip_halo_px": int(strip_halo_px), + "mid_slice_stats": mid_stats, + "preview_figure": str(qa_dir / "multiphase_overlay_diagnostic.png"), + } + with (qa_dir / "preview_summary.json").open("w") as f: + json.dump(preview_info, f, indent=2) + print(f"[{resolved_state}] Saved preview -> {qa_dir / 'multiphase_overlay_diagnostic.png'}", flush=True) + return preview_info + + raw_stride_xy = stride_xy * max(1, spatial_bin) + raw_stride_z = stride_z * max(1, spatial_bin) + xy_positions = enumerate_cylinder_xy_positions( + shape_hw=shape_hw, + patch_size=raw_window, + stride_xy=raw_stride_xy, + radius_fraction=roi_radius_fraction, + ) + if not xy_positions: + raise RuntimeError(f"[{resolved_state}] No valid {raw_window}^2 XY positions inside ROI") + + print( + f"[{resolved_state}] Pass 1/2: scanning {len(files)} slices " + f"(material={resolved_material}, phases={eff_num_phases}, load={load_val:.4f} {load_unit})...", + flush=True, + ) + solid_fractions: list[float] = [] + coating_fractions: list[float] = [] + pore_modes: list[float] = [] + ring_scores: list[float] = [] + slice_jumps: list[float] = [] + prev_sub: np.ndarray | None = None + + for idx, path in enumerate(files, start=1): + img = tifffile.imread(path) + _, stats = segment_slice_multiphase( + img, + delta_carbon=eff_delta_carbon, + delta_coating=delta_coating, + roi_mask=roi_mask, + sigma=sigma, + strip_halo_px=strip_halo_px, + ) + solid_fractions.append(stats["solid_fraction"]) + coating_fractions.append(stats["coating_fraction"]) + pore_modes.append(stats["pore_mode"]) + ring_scores.append(compute_radial_ring_score(img)) + + curr_sub = img[::4, ::4].astype(np.float32) + slice_jumps.append(0.0 if prev_sub is None else float(np.abs(curr_sub - prev_sub).mean())) + prev_sub = curr_sub + + if idx % 250 == 0 or idx == len(files): + print(f" [{resolved_state}] Scanned {idx}/{len(files)} slices", flush=True) + + if len(slice_jumps) > 1: + slice_jumps[0] = slice_jumps[1] + + eff_min_interval = max(min_interval_length, raw_window) + valid_mask, valid_intervals = detect_valid_z_intervals( + ring_scores=ring_scores, + pore_modes=pore_modes, + slice_jumps=slice_jumps, + min_interval_length=eff_min_interval, + max_ring_score=max_ring_score, + seam_margin=seam_margin, + ) + usable_z = [(s, e) for s, e in valid_intervals if (e - s) >= raw_window] + if not usable_z: + raise RuntimeError( + f"[{resolved_state}] No clean Z intervals of length >= {raw_window} available" + ) + max_single_z_span = max(e - s for s, e in usable_z) + + if split_mode == "xy": + split_xy = split_xy_positions_with_buffer( + xy_positions=xy_positions, + patch_size=raw_window, + val_fraction=val_fraction, + ) + split_intervals = {"train": list(usable_z), "val": list(usable_z)} + effective_split_mode = "xy" + elif split_mode == "z": + split_intervals = split_z_intervals_with_buffer( + intervals=usable_z, + patch_size=raw_window, + val_fraction=val_fraction, + buffer_slices=raw_window, + ) + split_xy = {"train": list(xy_positions), "val": list(xy_positions)} + effective_split_mode = "z" + elif max_single_z_span < 3 * raw_window and len(usable_z) < 2: + try: + split_xy = split_xy_positions_with_buffer( + xy_positions=xy_positions, + patch_size=raw_window, + val_fraction=val_fraction, + ) + split_intervals = {"train": list(usable_z), "val": list(usable_z)} + effective_split_mode = "xy" + print( + f" [{resolved_state}] Clean Z span ({max_single_z_span} slices < {3 * raw_window}) is thin; " + "switching to buffered XY-sector train/val split across full Z thickness.", + flush=True, + ) + except RuntimeError: + split_intervals = split_z_intervals_with_buffer( + intervals=usable_z, + patch_size=raw_window, + val_fraction=val_fraction, + buffer_slices=raw_window, + ) + split_xy = {"train": list(xy_positions), "val": list(xy_positions)} + effective_split_mode = "z" + else: + split_intervals = split_z_intervals_with_buffer( + intervals=usable_z, + patch_size=raw_window, + val_fraction=val_fraction, + buffer_slices=raw_window, + ) + split_xy = {"train": list(xy_positions), "val": list(xy_positions)} + effective_split_mode = "z" + + _save_z_plot( + out_path=qa_dir / "z_quality_diagnostic.png", + state_name=resolved_state, + solid_fractions=solid_fractions, + coating_fractions=coating_fractions, + ring_scores=ring_scores, + slice_jumps=slice_jumps, + valid_mask=valid_mask, + intervals=valid_intervals, + ) + + rng = np.random.default_rng(seed) + val_quota = max(1, int(round(max_patches * val_fraction))) + quotas = {"train": max_patches - val_quota, "val": val_quota} + split_counts: dict[str, int] = {} + split_sf_means: dict[str, float] = {} + split_coat_means: dict[str, float] = {} + + print(f"[{resolved_state}] Pass 2/2: streaming clean 3D patches -> {patch_dir.name}", flush=True) + for split_name in ("train", "val"): + plan = build_stratified_patch_plan( + z_intervals=split_intervals[split_name], + xy_positions=split_xy[split_name], + patch_size=raw_window, + stride_z=raw_stride_z, + max_patches=quotas[split_name], + rng=rng, + ) + + slice_cache: dict[int, np.ndarray] = {} + accepted: list[np.ndarray] = [] + + for z_start, xy_list in plan: + if len(accepted) >= quotas[split_name]: + break + for k in [kz for kz in slice_cache if kz < z_start]: + del slice_cache[k] + for z_idx in range(z_start, z_start + raw_window): + if z_idx not in slice_cache: + raw_slc = tifffile.imread(files[z_idx]) + seg_slc, _ = segment_slice_multiphase( + raw_slc, + delta_carbon=eff_delta_carbon, + delta_coating=delta_coating, + roi_mask=roi_mask, + sigma=sigma, + strip_halo_px=strip_halo_px, + ) + slice_cache[z_idx] = seg_slc + + slab = np.stack([slice_cache[z_idx] for z_idx in range(z_start, z_start + raw_window)], axis=0) + for y, x in xy_list: + if len(accepted) >= quotas[split_name]: + break + raw_patch = np.ascontiguousarray( + slab[:, y : y + raw_window, x : x + raw_window], dtype=np.uint8 + ) + patch = downsample_volume_rev(raw_patch, factor=spatial_bin) + wf = float((patch >= 1).mean()) + if min_white <= wf <= max_white: + accepted.append(patch) + + if not accepted: + raise RuntimeError(f"[{resolved_state}] 0 {split_name} patches passed filter [{min_white}, {max_white}]") + + stacked = np.stack(accepted, axis=0) + split_out = patch_dir / split_name + split_out.mkdir(parents=True, exist_ok=True) + np.save(split_out / "patches.npy", stacked) + split_counts[split_name] = int(stacked.shape[0]) + split_sf_means[split_name] = float((stacked >= 1).mean()) + split_coat_means[split_name] = float((stacked == 2).mean()) + print( + f" [{resolved_state}] {split_name}: {stacked.shape[0]} patches " + f"(solid={split_sf_means[split_name]:.4f}, coating={split_coat_means[split_name]:.4f})", + flush=True, + ) + + val_tall_depth = patch_size + if split_intervals["val"] and split_xy["val"]: + longest_val_iv = max(split_intervals["val"], key=lambda iv: iv[1] - iv[0]) + val_span_raw = int(longest_val_iv[1] - longest_val_iv[0]) + cand_tall_d = min(val_span_raw // spatial_bin, 3 * patch_size) + if cand_tall_d > patch_size: + tall_raw_d = cand_tall_d * spatial_bin + z0_tall = int(longest_val_iv[0]) + (val_span_raw - tall_raw_d) // 2 + val_xy_pool = list(split_xy["val"]) + rng.shuffle(val_xy_pool) + cand_xy = val_xy_pool[: min(len(val_xy_pool), max(val_quota, 4), 12)] + tall_raw_cols = np.empty( + (len(cand_xy), tall_raw_d, raw_window, raw_window), dtype=np.uint8 + ) + for z_rel, z_idx in enumerate(range(z0_tall, z0_tall + tall_raw_d)): + raw_slc = tifffile.imread(files[z_idx]) + seg_slc, _ = segment_slice_multiphase( + raw_slc, + delta_carbon=eff_delta_carbon, + delta_coating=delta_coating, + roi_mask=roi_mask, + sigma=sigma, + strip_halo_px=strip_halo_px, + ) + for c_idx, (y, x) in enumerate(cand_xy): + tall_raw_cols[c_idx, z_rel] = seg_slc[y : y + raw_window, x : x + raw_window] + + tall_accepted: list[np.ndarray] = [] + for c_idx in range(len(cand_xy)): + if len(tall_accepted) >= min(val_quota, 8): + break + tall_col = downsample_volume_rev(tall_raw_cols[c_idx], factor=spatial_bin) + wf_t = float((tall_col >= 1).mean()) + if min_white <= wf_t <= max_white: + tall_accepted.append(tall_col) + + if tall_accepted: + tall_stacked = np.stack(tall_accepted, axis=0) + np.save(patch_dir / "val" / "tall_patches.npy", tall_stacked) + val_tall_depth = int(tall_stacked.shape[1]) + print( + f" [{resolved_state}] val tall_patches: {tall_stacked.shape[0]} columns " + f"of shape {tuple(tall_stacked.shape[1:])} (mirror-free baseline source)", + flush=True, + ) + + sf_arr = np.asarray(solid_fractions, dtype=np.float64) + cf_arr = np.asarray(coating_fractions, dtype=np.float64) + clean_sf = sf_arr[valid_mask] if valid_mask.any() else sf_arr + clean_cf = cf_arr[valid_mask] if valid_mask.any() else cf_arr + clean_slices = int(valid_mask.sum()) + + summary: dict[str, object] = { + "state": resolved_state, + "material": resolved_material, + "beamline": calib.beamline, + "source_folder": str(scan_dir), + "patch_dir_name": patch_dir.name, + "load_mv_per_v": load_mv, + "load_raw_value": load_val, + "load_unit": load_unit, + "stress_kpa": stress_kpa, + "strain_z": calib.estimate_strain_z(clean_slices), + "voxel_size_um": calib.voxel_size_um, + "spatial_bin": spatial_bin, + "effective_voxel_size_um": calib.effective_voxel_size_um, + "num_phases": eff_num_phases, + "delta_carbon": eff_delta_carbon, + "delta_coating": delta_coating, + "strip_halo_px": int(strip_halo_px), + "split_mode_used": effective_split_mode, + "val_tall_depth": int(val_tall_depth), + "is_repeat_or_secondary": _is_repeat_or_secondary(resolved_state), + "version": version, + "patch_size": patch_size, + "input_dtype": str(dtype), + "slice_shape_hw": list(shape_hw), + "slices_total": len(files), + "slices_valid_clean": clean_slices, + "valid_z_intervals": [[int(s), int(e)] for s, e in valid_intervals], + "split_z_intervals": { + k: [[int(s), int(e)] for s, e in v] for k, v in split_intervals.items() + }, + "clean_solid_fraction_mean": float(clean_sf.mean()), + "clean_coating_fraction_mean": float(clean_cf.mean()), + "train_patches": split_counts["train"], + "val_patches": split_counts["val"], + "train_solid_fraction_mean": split_sf_means["train"], + "val_solid_fraction_mean": split_sf_means["val"], + "train_coating_fraction_mean": split_coat_means["train"], + "val_coating_fraction_mean": split_coat_means["val"], + } + with (qa_dir / "segmentation_report.json").open("w") as f: + json.dump(summary, f, indent=2) + with (qa_dir / "valid_slices.json").open("w") as f: + json.dump( + { + "dataset": resolved_state, + "material": resolved_material, + "load_mv_per_v": load_mv, + "stress_kpa": stress_kpa, + "total_slices": len(files), + "valid_slice_count": clean_slices, + "valid_z_intervals": [[int(s), int(e)] for s, e in valid_intervals], + }, + f, + indent=2, + ) + return summary + + +def _calibrate_group_thresholds( + ref_scan_dir: Path, + eff_num_phases: int, + auto_calibrate_carbon: bool, + delta_threshold: float, + delta_coating_threshold: float, + roi_radius_fraction: float, + sigma: float, +) -> tuple[float, float | None]: + """Calibrate one shared `(delta_carbon, delta_coating)` pair on a group's lowest-load scan.""" + tiff_dir = resolve_tiff_directory(ref_scan_dir) + files = sorted(list(tiff_dir.glob("*.tif")) + list(tiff_dir.glob("*.tiff"))) + with tifffile.TiffFile(files[0]) as first_tif: + shape_hw = tuple(first_tif.pages[0].shape) + roi_mask = make_circular_mask(shape_hw, radius_fraction=roi_radius_fraction) + ref_slices = [tifffile.imread(files[i]) for i in _clean_ref_indices(len(files))] + + if eff_num_phases == 3: + d_carb, auto_coat = calibrate_multiphase_thresholds( + ref_slices, + roi_mask=roi_mask, + sigma=sigma, + fixed_delta_carbon=None if auto_calibrate_carbon else delta_threshold, + ) + d_coat = ( + float(delta_coating_threshold) + if delta_coating_threshold > d_carb + else auto_coat + ) + return float(d_carb), float(d_coat) + + d_carb = ( + calibrate_delta_threshold(ref_slices, roi_mask=roi_mask, sigma=sigma) + if auto_calibrate_carbon + else float(delta_threshold) + ) + return float(d_carb), None + + +def build_bundle( + drive_root: Path, + output_dir: Path, + datasets: list[str] | None = None, + version: str = "carbon_calibrated_v1", + material: str = "auto", + num_phases: int = 2, + auto_multiphase: bool = False, + auto_calibrate_carbon: bool = False, + delta_threshold: float = 5500.0, + delta_coating_threshold: float = 0.0, + loads_csv: Path | None = None, + sigma: float = 32.0, + roi_radius_fraction: float = 0.70, + patch_size: int = 96, + spatial_bin: int = 1, + stride_xy: int = 96, + stride_z: int = 32, + max_patches: int = 1200, + min_interval_length: int = 96, + seam_margin: int = 12, + voxel_size_um: float = 1.0, + kpa_per_mvv: float | None = None, + punch_diameter_mm: float | None = None, + beamline: str = "primary", + strip_halo_px: int = 1, + split_mode: str = "auto", + preview_only: bool = False, + append: bool = False, + resume: bool = False, + make_zip: bool = True, +) -> Path: + overrides = load_overrides_table(loads_csv) if loads_csv is not None else None + scan_dirs = ( + [drive_root / d for d in datasets] + if datasets + else discover_scan_folders(drive_root, overrides=overrides) + ) + if not scan_dirs: + raise FileNotFoundError(f"No XCT scan folders discovered under {drive_root}") + + output_dir.mkdir(parents=True, exist_ok=True) + existing_scans_by_state: dict[str, dict[str, object]] = {} + manifest_path = output_dir / "bundle_manifest.json" + if append and not preview_only and manifest_path.is_file(): + prev_manifest = json.loads(manifest_path.read_text()) + for prev_sc in prev_manifest.get("scans", []): + existing_scans_by_state[str(prev_sc["state"])] = dict(prev_sc) + + # Plan canonical state names and group by (material, beamline) sorted by ascending load + seen_names: set[str] = set() + planned: list[dict[str, object]] = [] + for scan_dir in scan_dirs: + try: + rel_key = str(scan_dir.relative_to(drive_root)).replace("\\", "/") + except ValueError: + rel_key = str(scan_dir) + res_mat = infer_material_name(rel_key) if material == "auto" else material + st_name = make_collision_safe_state_name( + scan_dir, material=res_mat, beamline=beamline, existing_names=seen_names + ) + seen_names.add(st_name) + lv, lu = parse_load_with_unit(rel_key, overrides=overrides) + planned.append( + { + "scan_dir": scan_dir, + "state_name": st_name, + "material": res_mat, + "load_val": lv, + "load_unit": lu, + } + ) + + planned.sort(key=lambda item: (str(item["material"]), float(item["load_val"]))) # type: ignore[arg-type] + + # Calibrate ONE shared (delta_carbon, delta_coating) pair per (material, beamline) from lowest-load scan + group_thresholds: dict[tuple[str, str], tuple[float, float | None]] = {} + for item in planned: + res_mat = str(item["material"]) + grp_key = (res_mat, beamline) + if grp_key in group_thresholds: + continue + use_3p = (num_phases == 3) or (auto_multiphase and res_mat != "virgin_carbon") + eff_nph = 3 if (use_3p and res_mat != "virgin_carbon") else 2 + group_thresholds[grp_key] = _calibrate_group_thresholds( + ref_scan_dir=item["scan_dir"], # type: ignore[arg-type] + eff_num_phases=eff_nph, + auto_calibrate_carbon=auto_calibrate_carbon, + delta_threshold=delta_threshold, + delta_coating_threshold=delta_coating_threshold, + roi_radius_fraction=roi_radius_fraction, + sigma=sigma, + ) + + ref_thickness_by_group: dict[tuple[str, str], int] = {} + new_summaries: list[dict[str, object]] = [] + + for item in planned: + res_mat = str(item["material"]) + grp_key = (res_mat, beamline) + calib = CalibrationMetadata( + voxel_size_um=voxel_size_um, + spatial_bin=spatial_bin, + kpa_per_mvv=kpa_per_mvv, + punch_diameter_mm=punch_diameter_mm, + reference_thickness_slices=ref_thickness_by_group.get(grp_key), + beamline=beamline, + ) + summary = stream_bundle_single_scan( + scan_dir=item["scan_dir"], # type: ignore[arg-type] + output_dir=output_dir, + state_name=str(item["state_name"]), + version=version, + material=res_mat, + num_phases=num_phases, + auto_multiphase=auto_multiphase, + auto_calibrate_carbon=auto_calibrate_carbon, + delta_threshold=delta_threshold, + delta_coating_threshold=delta_coating_threshold, + precalibrated_thresholds=group_thresholds[grp_key], + load_overrides=overrides, + sigma=sigma, + roi_radius_fraction=roi_radius_fraction, + patch_size=patch_size, + spatial_bin=spatial_bin, + stride_xy=stride_xy, + stride_z=stride_z, + max_patches=max_patches, + min_interval_length=min_interval_length, + seam_margin=seam_margin, + calibration=calib, + strip_halo_px=strip_halo_px, + split_mode=split_mode, + preview_only=preview_only, + resume=resume, + ) + if not preview_only and grp_key not in ref_thickness_by_group: + ref_slices = int(summary["slices_valid_clean"]) # type: ignore[arg-type] + ref_thickness_by_group[grp_key] = ref_slices + summary["strain_z"] = 0.0 + qa_rep = output_dir / str(summary["patch_dir_name"]) / "qa" / "segmentation_report.json" + if qa_rep.is_file(): + with qa_rep.open("w") as f: + json.dump(summary, f, indent=2) + + new_summaries.append(summary) + + if preview_only: + preview_manifest = output_dir / "preview_manifest.json" + with preview_manifest.open("w") as f: + json.dump({"previews": new_summaries}, f, indent=2) + return preview_manifest + + for summary in new_summaries: + existing_scans_by_state[str(summary["state"])] = summary + scan_summaries = list(existing_scans_by_state.values()) + + primary_states: dict[str, float] = {} + repeat_states: dict[str, float] = {} + by_material: dict[str, dict[str, float]] = {} + for summary in scan_summaries: + st_name = str(summary["state"]) + mat_name = str(summary["material"]) + mv = float(summary["load_mv_per_v"]) # type: ignore[arg-type] + if summary["is_repeat_or_secondary"]: + repeat_states[st_name] = mv + else: + primary_states[st_name] = mv + by_material.setdefault(mat_name, {})[st_name] = mv + + with (output_dir / "primary_states.json").open("w") as f: + json.dump(dict(sorted(primary_states.items(), key=lambda kv: kv[1])), f, indent=2) + + manifest = { + "version": version, + "patch_size": patch_size, + "spatial_bin": spatial_bin, + "voxel_size_um": voxel_size_um, + "effective_voxel_size_um": float(voxel_size_um * max(1, spatial_bin)), + "kpa_per_mvv": kpa_per_mvv, + "punch_diameter_mm": punch_diameter_mm, + "beamline": beamline, + "delta_threshold": delta_threshold, + "strip_halo_px": int(strip_halo_px), + "split_mode": split_mode, + "primary_states": dict(sorted(primary_states.items(), key=lambda kv: kv[1])), + "states_by_material": { + m: dict(sorted(d.items(), key=lambda kv: kv[1])) for m, d in sorted(by_material.items()) + }, + "repeat_and_secondary_states": dict(sorted(repeat_states.items(), key=lambda kv: kv[1])), + "scans": scan_summaries, + } + with manifest_path.open("w") as f: + json.dump(manifest, f, indent=2) + + if make_zip: + zip_path = output_dir.parent / f"{output_dir.name}.zip" + print(f"Compressing bundle -> {zip_path} ...", flush=True) + with zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zf: + for file_path in sorted(output_dir.rglob("*")): + if file_path.is_file(): + zf.write(file_path, arcname=file_path.relative_to(output_dir)) + size_mb = zip_path.stat().st_size / (1024**2) + print(f"Bundle ready: {zip_path} ({size_mb:.1f} MiB)", flush=True) + return zip_path + + return manifest_path + + +def main() -> None: + if len(sys.argv) == 1 and SPYDER_DRIVE_ROOT.exists(): + build_bundle( + drive_root=SPYDER_DRIVE_ROOT, + output_dir=SPYDER_OUTPUT_DIR, + auto_multiphase=True, + spatial_bin=2, + resume=True, + make_zip=True, + ) + return + + parser = argparse.ArgumentParser( + description="Stream-segment raw multi-electrode XCT scans directly into uploadable 3D patch bundles" + ) + parser.add_argument("--drive-root", type=Path, required=True, help="Root folder (e.g. D:/dry_samples)") + parser.add_argument("--output-dir", type=Path, required=True, help="Output bundle directory") + parser.add_argument("--datasets", nargs="*", default=None, help="Optional subset of scan folders") + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument( + "--material", + default="auto", + choices=["auto", "virgin_carbon", "tio2_anode", "cuhcf_cathode"], + ) + parser.add_argument("--num-phases", type=int, default=2, choices=[2, 3]) + parser.add_argument( + "--auto-multiphase", + action="store_true", + help="Automatically segment TiO2 and CuHCF scans into 3 phases {pore, carbon, coating} and Virgin into 2 phases", + ) + parser.add_argument( + "--auto-calibrate-carbon", + action="store_true", + help="Auto-calibrate delta_carbon per (material, beamline) from lowest-load scan (recommended for K11 DIAD)", + ) + parser.add_argument("--delta-threshold", type=float, default=5500.0) + parser.add_argument( + "--delta-coating-threshold", + type=float, + default=0.0, + help="Explicit coating delta threshold (> delta-threshold); 0 auto-calibrates via 2-stage solid Otsu", + ) + parser.add_argument( + "--loads-csv", + type=Path, + default=None, + help="Optional CSV/JSON lookup table mapping folder names to load values/units", + ) + parser.add_argument("--sigma", type=float, default=32.0) + parser.add_argument("--roi-radius-fraction", type=float, default=0.70) + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--spatial-bin", type=int, default=1, help="Spatial block-binning factor (e.g. 2 for REV scale)") + parser.add_argument("--stride-xy", type=int, default=96) + parser.add_argument("--stride-z", type=int, default=32) + parser.add_argument("--max-patches", type=int, default=1200) + parser.add_argument("--min-interval-length", type=int, default=96) + parser.add_argument("--seam-margin", type=int, default=12) + parser.add_argument("--voxel-size-um", type=float, default=1.0) + parser.add_argument("--kpa-per-mvv", type=float, default=None) + parser.add_argument("--punch-diameter-mm", type=float, default=None) + parser.add_argument("--beamline", default="primary") + parser.add_argument( + "--strip-halo-px", + type=int, + default=1, + help="Width (px) of partial-volume carbon halo stripped around pore-coating interfaces (0 disables)", + ) + parser.add_argument( + "--split-mode", + default="auto", + choices=["auto", "z", "xy"], + help="Train/val spatial split strategy: 'auto' uses Z-slabs and falls back to buffered XY sectors on thin compressed scans", + ) + parser.add_argument( + "--preview-only", + action="store_true", + help="Fast 10s check: save multiphase_overlay_diagnostic.png without extracting 3D patches", + ) + parser.add_argument( + "--append", + action="store_true", + help="Merge newly bundled scans into an existing bundle_manifest.json (for multi-drive / K11 DIAD workflows)", + ) + parser.add_argument( + "--resume", + action="store_true", + help="Skip scans whose train/val patches.npy and segmentation_report.json already exist", + ) + parser.add_argument("--zip", action="store_true", help="Compress output directory into a single .zip archive") + args = parser.parse_args() + + build_bundle( + drive_root=args.drive_root, + output_dir=args.output_dir, + datasets=args.datasets, + version=args.version, + material=args.material, + num_phases=args.num_phases, + auto_multiphase=args.auto_multiphase, + auto_calibrate_carbon=args.auto_calibrate_carbon, + delta_threshold=args.delta_threshold, + delta_coating_threshold=args.delta_coating_threshold, + loads_csv=args.loads_csv, + sigma=args.sigma, + roi_radius_fraction=args.roi_radius_fraction, + patch_size=args.patch_size, + spatial_bin=args.spatial_bin, + stride_xy=args.stride_xy, + stride_z=args.stride_z, + max_patches=args.max_patches, + min_interval_length=args.min_interval_length, + seam_margin=args.seam_margin, + voxel_size_um=args.voxel_size_um, + kpa_per_mvv=args.kpa_per_mvv, + punch_diameter_mm=args.punch_diameter_mm, + beamline=args.beamline, + strip_halo_px=args.strip_halo_px, + split_mode=args.split_mode, + preview_only=args.preview_only, + append=args.append, + resume=args.resume, + make_zip=args.zip, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/preprocessing/segment_carbon_xct.py b/scripts/preprocessing/segment_carbon_xct.py index c122251..95e02da 100644 --- a/scripts/preprocessing/segment_carbon_xct.py +++ b/scripts/preprocessing/segment_carbon_xct.py @@ -1,111 +1,315 @@ +"""Calibrated binary XCT segmentation with circular ROI masking and Z-artifact gating. + +Preserves physical solid-fraction scaling across compression states by subtracting +each slice's in-cylinder pore background mode and applying a single shared threshold +across all loads, while flagging bullseye ring seams and rig plungers in `valid_slices.json`. +""" + +from __future__ import annotations + import argparse import json import re +import sys from pathlib import Path +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt import numpy as np import tifffile -from scipy.ndimage import gaussian_filter -from skimage.filters import threshold_otsu -DATA_ROOT = Path("/projects/u6uc/asif/data/carbon_virgin") +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) +from porousdiff.data import default_project_root +from porousdiff.preprocessing import ( + calibrate_delta_threshold, + compute_radial_ring_score, + detect_valid_z_intervals, + make_circular_mask, + resolve_tiff_directory, + segment_slice_calibrated, +) -def segment_binary(image, sigma=30): - image = image.astype(np.float32) - background = gaussian_filter(image, sigma=sigma) - corrected = image - background +def _validate_token(value: str, label: str) -> None: + if not re.fullmatch(r"[A-Za-z0-9_.-]+", value): + raise ValueError(f"Invalid {label}: {value!r}") - lo, hi = np.percentile(corrected, [1, 99]) - - norm = np.clip( - (corrected - lo) / (hi - lo + 1e-8), - 0, 1 - ) - threshold = threshold_otsu(norm) - return (norm > threshold).astype(np.uint8) +def _save_z_diagnostic_plot( + out_path: Path, + dataset_name: str, + solid_fractions: list[float], + ring_scores: list[float], + slice_jumps: list[float], + valid_mask: np.ndarray, + intervals: list[tuple[int, int]], +) -> None: + z = np.arange(len(solid_fractions)) + fig, axes = plt.subplots(2, 1, figsize=(11, 6), sharex=True) + axes[0].plot(z, solid_fractions, color="tab:blue", lw=1.3, label="In-cylinder solid fraction") + for idx, (s, e) in enumerate(intervals): + axes[0].axvspan( + s, e - 1, color="tab:green", alpha=0.15, label="Accepted clean Z" if idx == 0 else None + ) + axes[0].set_ylabel("Solid fraction") + axes[0].set_title(f"{dataset_name} — Segmented Z Profile & Accepted Clean Intervals") + axes[0].legend(loc="upper right") + axes[0].grid(True, alpha=0.3) -def main(): - parser = argparse.ArgumentParser( - description="Reusable binary XCT segmentation" - ) - parser.add_argument("--dataset", required=True) - parser.add_argument("--expected-slices", type=int, required=True) - parser.add_argument("--sigma", type=float, default=50) - parser.add_argument("--version", default="v2") - args = parser.parse_args() - - if not re.fullmatch(r"[A-Za-z0-9_-]+", args.dataset): - parser.error("Invalid dataset name") + ax2 = axes[1] + ax2.plot(z, ring_scores, color="tab:red", lw=1.2, label="Radial ring score") + ax2_r = ax2.twinx() + ax2_r.plot(z, slice_jumps, color="tab:orange", lw=1.0, alpha=0.75, label="|dI/dz| slice jump") + bad_idx = np.where(~valid_mask)[0] + if bad_idx.size: + ax2.scatter( + bad_idx, + np.asarray(ring_scores)[bad_idx], + color="black", + s=6, + zorder=4, + label="Excluded artifact/plunger slices", + ) + ax2.set_xlabel("Slice index Z") + ax2.set_ylabel("Radial ring score") + ax2_r.set_ylabel("Mean |dI/dz|") + ax2.legend(loc="upper left") + ax2.grid(True, alpha=0.3) - if args.expected_slices <= 0 or args.sigma <= 0: - parser.error("Expected slices and sigma must be positive") + fig.tight_layout() + fig.savefig(out_path, dpi=160) + plt.close(fig) - input_dir = DATA_ROOT / args.dataset - output_dir = DATA_ROOT / ( - f"{args.dataset}_binary_corrected_{args.version}" - ) - files = sorted(input_dir.glob("*.tif")) +def process_single_dataset( + dataset_name: str, + data_root: Path, + output_root: Path, + version: str, + delta_threshold: float, + sigma: float, + roi_radius_fraction: float, + expected_slices: int, + min_interval_length: int, + max_ring_score: float, + seam_margin: int, + overwrite: bool, +) -> dict[str, object]: + input_base = data_root / dataset_name + tiff_dir = resolve_tiff_directory(input_base) + files = sorted(list(tiff_dir.glob("*.tif")) + list(tiff_dir.glob("*.tiff"))) - if len(files) != args.expected_slices: + if expected_slices > 0 and len(files) != expected_slices: raise RuntimeError( - f"Expected {args.expected_slices} TIFFs, " - f"found {len(files)}" + f"[{dataset_name}] Expected {expected_slices} TIFF slices in {tiff_dir}, found {len(files)}" ) + if not files: + raise RuntimeError(f"[{dataset_name}] No TIFF slices found in {tiff_dir}") - if output_dir.exists(): + output_dir = output_root / f"{dataset_name}_binary_corrected_{version}" + if output_dir.exists() and not overwrite: raise FileExistsError( - f"Output already exists: {output_dir}" + f"Output already exists: {output_dir}. Pass --overwrite or a new --version." ) - # Validate input metadata before creating output. + with tifffile.TiffFile(files[0]) as first_tif: + shape_hw = tuple(first_tif.pages[0].shape) + dtype = first_tif.pages[0].dtype + if len(shape_hw) != 2: + raise ValueError(f"[{dataset_name}] Expected 2D TIFF slices, got shape {shape_hw}") + for path in files: with tifffile.TiffFile(path) as tif: page = tif.pages[0] - if page.shape != (2560, 2560) or page.dtype != np.uint16: - raise ValueError(f"Unexpected TIFF format: {path.name}") + if tuple(page.shape) != shape_hw or page.dtype != dtype: + raise ValueError(f"[{dataset_name}] Inconsistent TIFF format at {path.name}") - output_dir.mkdir(parents=True) + output_dir.mkdir(parents=True, exist_ok=True) + roi_mask = make_circular_mask(shape_hw, radius_fraction=roi_radius_fraction) - fractions = [] + solid_fractions: list[float] = [] + pore_modes: list[float] = [] + ring_scores: list[float] = [] + slice_jumps: list[float] = [] + prev_sub: np.ndarray | None = None - for i, path in enumerate(files, start=1): + for idx, path in enumerate(files, start=1): image = tifffile.imread(path) - binary = segment_binary(image, args.sigma) + binary, stats = segment_slice_calibrated( + image, + delta_threshold=delta_threshold, + roi_mask=roi_mask, + sigma=sigma, + ) + tifffile.imwrite(output_dir / f"binary_{idx:04d}.tif", binary) + + solid_fractions.append(stats["solid_fraction"]) + pore_modes.append(stats["pore_mode"]) + ring_scores.append(compute_radial_ring_score(image)) + + curr_sub = image[::4, ::4].astype(np.float32) + if prev_sub is None: + slice_jumps.append(0.0) + else: + slice_jumps.append(float(np.abs(curr_sub - prev_sub).mean())) + prev_sub = curr_sub + + if idx % 50 == 0 or idx == len(files): + print(f"[{dataset_name}] Processed {idx}/{len(files)} slices", flush=True) + + if len(slice_jumps) > 1: + slice_jumps[0] = slice_jumps[1] - destination = output_dir / f"binary_{i:04d}.tif" - tifffile.imwrite(destination, binary) + valid_mask, valid_intervals = detect_valid_z_intervals( + ring_scores=ring_scores, + pore_modes=pore_modes, + slice_jumps=slice_jumps, + min_interval_length=min_interval_length, + max_ring_score=max_ring_score, + seam_margin=seam_margin, + ) - fractions.append(float(binary.mean())) + sf_arr = np.asarray(solid_fractions, dtype=np.float64) + clean_sf = sf_arr[valid_mask] if valid_mask.any() else sf_arr - if i % 25 == 0: - print( - f"Processed {i}/{len(files)} slices", - flush=True - ) + valid_slices_payload = { + "dataset": dataset_name, + "version": version, + "total_slices": len(files), + "slice_shape_hw": list(shape_hw), + "roi_radius_fraction": roi_radius_fraction, + "min_interval_length": min_interval_length, + "seam_margin": seam_margin, + "max_ring_score": max_ring_score, + "valid_slice_count": int(valid_mask.sum()), + "valid_z_intervals_0based_half_open": [[int(s), int(e)] for s, e in valid_intervals], + } + with (output_dir / "valid_slices.json").open("w") as vf: + json.dump(valid_slices_payload, vf, indent=2) - report = { - "dataset": args.dataset, - "input_directory": str(input_dir), + _save_z_diagnostic_plot( + out_path=output_dir / "z_quality_diagnostic.png", + dataset_name=dataset_name, + solid_fractions=solid_fractions, + ring_scores=ring_scores, + slice_jumps=slice_jumps, + valid_mask=valid_mask, + intervals=valid_intervals, + ) + + report: dict[str, object] = { + "dataset": dataset_name, + "version": version, + "input_directory": str(tiff_dir), "output_directory": str(output_dir), - "slices": len(files), - "sigma": args.sigma, - "normalization_percentiles": [1, 99], - "threshold": "Otsu per slice", - "white_fraction_mean": float(np.mean(fractions)), - "white_fraction_min": float(np.min(fractions)), - "white_fraction_max": float(np.max(fractions)), - "white_fraction_std": float(np.std(fractions)), + "slices_total": len(files), + "slices_valid_clean": int(valid_mask.sum()), + "valid_z_intervals": [[int(s), int(e)] for s, e in valid_intervals], + "slice_shape_hw": list(shape_hw), + "input_dtype": str(dtype), + "sigma": sigma, + "delta_threshold": delta_threshold, + "roi_radius_fraction": roi_radius_fraction, + "threshold_method": "Pore-mode-referenced shared physical delta threshold", + "clean_solid_fraction_mean": float(clean_sf.mean()), + "clean_solid_fraction_std": float(clean_sf.std()), + "clean_solid_fraction_min": float(clean_sf.min()), + "clean_solid_fraction_max": float(clean_sf.max()), + "all_slices_solid_fraction_mean": float(sf_arr.mean()), } + with (output_dir / "segmentation_report.json").open("w") as rf: + json.dump(report, rf, indent=2) + + print(json.dumps(report, indent=2), flush=True) + return report + + +def main() -> None: + default_root = default_project_root() / "data" / "carbon_virgin" + parser = argparse.ArgumentParser( + description="Physics-consistent XCT segmentation with circular ROI and Z-artifact gating" + ) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--dataset", help="Single dataset folder name (e.g. Op008mVV)") + group.add_argument("--datasets", nargs="+", help="Multiple dataset folders to segment consistently") - with open(output_dir / "segmentation_report.json", "w") as f: - json.dump(report, f, indent=2) + parser.add_argument("--data-root", type=Path, default=default_root) + parser.add_argument("--output-root", type=Path, default=None) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument( + "--delta-threshold", + type=float, + default=5500.0, + help="Shared intensity threshold above local pore mode (used across all loads)", + ) + parser.add_argument( + "--auto-calibrate-from", + default=None, + help="Optional dataset name from which to auto-calibrate delta_threshold before segmenting", + ) + parser.add_argument("--sigma", type=float, default=32.0) + parser.add_argument("--roi-radius-fraction", type=float, default=0.70) + parser.add_argument( + "--expected-slices", + type=int, + default=0, + help="Expected slice count (0 = accept any complete sequence found)", + ) + parser.add_argument("--min-interval-length", type=int, default=96) + parser.add_argument("--max-ring-score", type=float, default=0.15) + parser.add_argument("--seam-margin", type=int, default=12) + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + datasets = [args.dataset] if args.dataset else list(args.datasets) + for ds in datasets: + _validate_token(ds, "dataset") + _validate_token(args.version, "version") - print(json.dumps(report, indent=2)) + if args.sigma < 0 or args.expected_slices < 0 or args.min_interval_length < 1: + parser.error("Invalid non-positive numeric parameter") + + out_root = args.output_root if args.output_root is not None else args.data_root + threshold = args.delta_threshold + + if args.auto_calibrate_from: + _validate_token(args.auto_calibrate_from, "auto-calibrate-from") + ref_dir = resolve_tiff_directory(args.data_root / args.auto_calibrate_from) + ref_files = sorted(list(ref_dir.glob("*.tif")) + list(ref_dir.glob("*.tiff"))) + if not ref_files: + raise RuntimeError(f"No reference TIFFs found in {ref_dir}") + step = max(1, len(ref_files) // 16) + sample_imgs = [ + tifffile.imread(p) + for p in ref_files[::step] + if compute_radial_ring_score(tifffile.imread(p)) <= args.max_ring_score + ] + if not sample_imgs: + sample_imgs = [tifffile.imread(ref_files[len(ref_files) // 4])] + roi = make_circular_mask(sample_imgs[0].shape, radius_fraction=args.roi_radius_fraction) + threshold = calibrate_delta_threshold(sample_imgs, roi_mask=roi, sigma=args.sigma) + print(f"Auto-calibrated shared delta_threshold = {threshold:.2f}", flush=True) + + for ds in datasets: + process_single_dataset( + dataset_name=ds, + data_root=args.data_root, + output_root=out_root, + version=args.version, + delta_threshold=threshold, + sigma=args.sigma, + roi_radius_fraction=args.roi_radius_fraction, + expected_slices=args.expected_slices, + min_interval_length=args.min_interval_length, + max_ring_score=args.max_ring_score, + seam_margin=args.seam_margin, + overwrite=args.overwrite, + ) if __name__ == "__main__": diff --git a/scripts/training/train_carbon_conditional_96.py b/scripts/training/train_carbon_conditional_96.py index ac7d08c..4c07746 100644 --- a/scripts/training/train_carbon_conditional_96.py +++ b/scripts/training/train_carbon_conditional_96.py @@ -1,261 +1,786 @@ - +"""Train PorousDiff continuous load/stress-conditioned 3D diffusion models (single or multi-electrode). + +Supports: + - Single-material binary training (`--states ...`) for backward-compatible Virgin Felt runs, + - Unified multi-electrode 3-phase training (`--manifest-json bundle_manifest.json`) across + `virgin_carbon`, `tio2_anode`, and `cuhcf_cathode` with balanced material/state sampling, + - Few-Shot "Beamtime Multiplier" filtering (`--include-states` / `--exclude-states`) holding + out intermediate coated-electrode loads while leveraging dense Virgin Felt scaffold physics, + - Zero-Terminal-SNR `v_prediction`, EMA, GH200 `bf16` AMP, and full RNG `--resume`. """ -PorousDiff — Carbon-felt conditional 3D DDPM (96³). -One shared model conditioned on load-cell readings: 8, 18, 33 mV. -Records losses, epoch timings, GPU memory, and resumable checkpoints. -""" +from __future__ import annotations + import argparse import csv import json import random +import re +import signal +import sys import time from pathlib import Path import numpy as np import torch import torch.nn.functional as F -from torch.utils.data import DataLoader, Dataset -from generative.networks.nets import DiffusionModelUNet -from generative.networks.schedulers import DDPMScheduler - -ROOT = Path("/projects/u6uc/asif") -DATA_ROOT = ROOT / "data/carbon_virgin" -EXPERIMENT = "carbon_conditional_96_v1" -RESULT_DIR = ROOT / "results" / EXPERIMENT -CHECKPOINT_DIR = ROOT / "checkpoints" / EXPERIMENT - -STATES = {"Op008mVV": 8.0, "Op018mVV": 18.0, "Op033mVV": 33.0} -VERSION = "carbon_sigma50_v1" - - -class CarbonDataset(Dataset): - def __init__(self, split): - self.samples = [] - for state, mv in STATES.items(): - folder = DATA_ROOT / ( - f"{state}_binary_patches96_{VERSION}" - ) / split - files = sorted(folder.glob("*.npy")) - if not files: - raise RuntimeError(f"No patches in {folder}") - self.samples.extend((p, mv) for p in files) - - def __len__(self): - return len(self.samples) - - def __getitem__(self, index): - path, mv = self.samples[index] - patch = np.load(path) - if patch.shape != (96, 96, 96) or patch.dtype != np.uint8: - raise ValueError(f"Invalid patch: {path}") - image = torch.from_numpy( - patch.astype(np.float32) - ).unsqueeze(0) * 2 - 1 - - # Normalize the sensor reading to approximately [-1, 1]. - condition = torch.tensor( - [(mv - 20.5) / 12.5], dtype=torch.float32 +from torch.utils.data import DataLoader, WeightedRandomSampler + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from porousdiff.data import ( + MATERIAL_TO_ID, + LoadConditionedPatchDataset, + MultiElectrodePatchDataset, + capture_rng_states, + default_project_root, + resolve_scan_load_value, + restore_rng_states, +) +from porousdiff.losses import PhysicsInformedDiffusionLoss, reconstruct_x0 +from porousdiff.models import ContinuousLoadConditionedUNet, EMAModel, build_noise_scheduler +from porousdiff.preprocessing import parse_load_value + +DEFAULT_CARBON_STATES = { + "Op008mVV": 0.008, + "Op018mVV": 0.018, + "Op033mVV": 0.033, +} + + +def parse_states_arg( + entries: list[str] | None, states_json: Path | None +) -> dict[str, float]: + if states_json is not None: + with states_json.open("r") as f: + raw = json.load(f) + return {str(k): float(v) for k, v in raw.items()} + if not entries: + return dict(DEFAULT_CARBON_STATES) + + mapping: dict[str, float] = {} + for item in entries: + if "=" in item: + name, val_s = item.split("=", 1) + mapping[name.strip()] = float(val_s.strip()) + else: + name = item.strip() + mapping[name] = parse_load_value(name) + return mapping + + +def load_manifest_specs( + manifest_path: Path, + data_root: Path, + include_states: set[str] | None = None, + exclude_states: set[str] | None = None, + include_repeats: bool = False, + use_stress_kpa: bool = False, +) -> tuple[list[dict[str, object]], int, float]: + with manifest_path.open("r") as f: + manifest = json.load(f) + + specs: list[dict[str, object]] = [] + max_phases = 2 + vox_list: list[float] = [] + + filtered_scans: list[dict[str, object]] = [] + for scan in manifest.get("scans", []): + st = str(scan["state"]) + if not include_repeats and bool(scan.get("is_repeat_or_secondary", False)): + if include_states is None or st not in include_states: + continue + if include_states is not None and st not in include_states: + continue + if exclude_states is not None and st in exclude_states: + continue + filtered_scans.append(scan) + + if len(filtered_scans) < 2: + raise ValueError(f"Manifest filtering yielded < 2 training states ({len(filtered_scans)} found)") + + eff_use_kpa = bool(use_stress_kpa) + units_seen = {str(sc.get("load_unit", "mv_per_v")).lower() for sc in filtered_scans} + beams_seen = {str(sc.get("beamline", "primary")) for sc in filtered_scans} + all_have_kpa = all(sc.get("stress_kpa") is not None for sc in filtered_scans) + + if eff_use_kpa and not all_have_kpa: + missing = [str(sc["state"]) for sc in filtered_scans if sc.get("stress_kpa") is None] + raise ValueError( + f"--use-stress-kpa requested, but manifest scans lack calibrated stress_kpa: {missing}. " + "Supply --kpa-per-mvv / --punch-diameter-mm or stress_kpa in --loads-csv." + ) + + if not eff_use_kpa and (len(units_seen) > 1 or len(beams_seen) > 1): + if all_have_kpa: + print( + f"Notice: Multi-unit ({sorted(units_seen)}) or multi-beamline ({sorted(beams_seen)}) " + "manifest detected; automatically conditioning on calibrated stress_kpa.", + flush=True, + ) + eff_use_kpa = True + else: + missing = [str(sc["state"]) for sc in filtered_scans if sc.get("stress_kpa") is None] + raise ValueError( + f"Cannot mix uncalibrated load units {sorted(units_seen)} or beamlines {sorted(beams_seen)} " + f"without stress_kpa (missing on {missing}). Calibrate stress_kpa via --kpa-per-mvv, " + "--punch-diameter-mm, or --loads-csv before multi-rig training." + ) + + for scan in filtered_scans: + st = str(scan["state"]) + load_val = resolve_scan_load_value(scan, eff_use_kpa) + + eff_vox = float(scan.get("effective_voxel_size_um", manifest.get("effective_voxel_size_um", 1.0))) # type: ignore[arg-type] + n_ph = int(scan.get("num_phases", 2)) # type: ignore[arg-type] + max_phases = max(max_phases, n_ph) + vox_list.append(eff_vox) + + patch_dir_name = scan.get("patch_dir_name") + patch_dir = (data_root / str(patch_dir_name)) if patch_dir_name else None + + specs.append( + { + "state": st, + "material": str(scan.get("material", "virgin_carbon")), + "load_value": load_val, + "used_stress_kpa": eff_use_kpa, + "voxel_size_um": eff_vox, + "num_phases": n_ph, + "patch_dir": str(patch_dir) if (patch_dir and patch_dir.is_dir()) else None, + } + ) + + ref_vox = float(np.median(vox_list)) if vox_list else 1.0 + return specs, max_phases, ref_vox + + +def validate_resume_config(saved_cfg: dict[str, object], current_cfg: dict[str, object]) -> None: + """Prevent held-out state leakage or load-unit drift when auto-resuming a checkpoint.""" + mismatches: list[str] = [] + for key in ("use_stress_kpa", "num_phases", "num_materials"): + if saved_cfg.get(key) != current_cfg.get(key): + mismatches.append(f"{key}: saved={saved_cfg.get(key)!r} vs current={current_cfg.get(key)!r}") + + for list_key in ("heldout_states", "include_states"): + if list_key in saved_cfg and saved_cfg.get(list_key) != current_cfg.get(list_key): + mismatches.append( + f"{list_key}: saved={saved_cfg.get(list_key)!r} vs current={current_cfg.get(list_key)!r}" + ) + + def _norm_states(d: object) -> dict[str, float]: + if not isinstance(d, dict): + return {} + return {str(k): round(float(v), 6) for k, v in sorted(d.items())} + + if _norm_states(saved_cfg.get("states")) != _norm_states(current_cfg.get("states")): + mismatches.append( + f"states: saved={_norm_states(saved_cfg.get('states'))} vs current={_norm_states(current_cfg.get('states'))}" ) - return image, condition + def _norm_specs(items: object) -> list[tuple[str, float, str]]: + if not isinstance(items, list): + return [] + return sorted( + (str(s["state"]), round(float(s["load_value"]), 6), str(s.get("material", "virgin_carbon"))) + for s in items + if isinstance(s, dict) + ) -def checkpoint(path, model, optimizer, epoch, history): - torch.save({ - "epoch": epoch, - "model": model.state_dict(), - "optimizer": optimizer.state_dict(), - "history": history, - "rng_torch": torch.get_rng_state(), - "rng_numpy": np.random.get_state(), - "rng_python": random.getstate(), - }, path) + if _norm_specs(saved_cfg.get("scan_specs")) != _norm_specs(current_cfg.get("scan_specs")): + mismatches.append("scan_specs (training states, loads, or materials) differ from checkpoint") + if mismatches: + raise ValueError( + "Refusing --resume because checkpoint configuration does not match current invocation " + "(risk of held-out data leakage or unit drift): " + "; ".join(mismatches) + + ". Use a new --experiment name or pass --allow-config-mismatch to override." + ) + + +def atomic_torch_save(payload: dict[str, object], target: Path) -> None: + tmp = target.with_suffix(target.suffix + ".tmp") + torch.save(payload, tmp) + tmp.replace(target) + + +def main() -> None: + proj_root = default_project_root() + parser = argparse.ArgumentParser( + description="Train continuous load/stress-conditioned 3D diffusion model (PorousDiff)" + ) + parser.add_argument("--experiment", default="carbon_conditional_96_v2") + parser.add_argument("--data-root", type=Path, default=proj_root / "data" / "carbon_virgin") + parser.add_argument("--output-root", type=Path, default=proj_root) + parser.add_argument("--version", default="carbon_calibrated_v1") + parser.add_argument( + "--states", + nargs="+", + default=None, + help="Dataset=load pairs (e.g. Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033) or folder names", + ) + parser.add_argument("--states-json", type=Path, default=None) + parser.add_argument( + "--manifest-json", + type=Path, + default=None, + help="Multi-electrode bundle_manifest.json activating material + 3-phase + voxel-size conditioning", + ) + parser.add_argument("--include-states", nargs="*", default=None, help="Explicit subset of manifest states to train on") + parser.add_argument("--exclude-states", nargs="*", default=None, help="Held-out manifest states to exclude from training") + parser.add_argument("--include-repeats", action="store_true", help="Include replicate/secondary scans from manifest") + parser.add_argument("--use-stress-kpa", action="store_true", help="Condition on calibrated stress_kpa from manifest") + parser.add_argument("--num-phases", type=int, default=0, choices=[0, 2, 3], help="0=infer from manifest (or 2 for legacy --states)") + parser.add_argument("--balanced-sampling", action="store_true", help="Equalize gradient weight across materials and states") + + parser.add_argument("--patch-size", type=int, default=96) + parser.add_argument("--channels", default="32,64,128") + parser.add_argument("--attention-levels", default="0,0,1") + parser.add_argument("--num-res-blocks", type=int, default=2) + parser.add_argument("--norm-groups", type=int, default=16) + + parser.add_argument("--schedule", default="zero_snr_scaled_linear") + parser.add_argument("--prediction-type", default="v_prediction", choices=["v_prediction", "epsilon"]) + parser.add_argument( + "--conditioning-mode", + default="multiscale_fourier", + choices=["multiscale_fourier", "legacy_crossattn"], + ) + parser.add_argument("--timesteps", type=int, default=1000) + parser.add_argument("--cfg-drop-prob", type=float, default=0.10) + parser.add_argument("--ema-decay", type=float, default=0.9995) + parser.add_argument( + "--physics-loss", + action="store_true", + help="Enable physics-informed x0_hat training loss (Min-SNR-gamma + 3D interfacial gradient + solid support + catalyst loading conservation)", + ) + parser.add_argument("--snr-gamma", type=float, default=5.0) + parser.add_argument("--interface-weight", type=float, default=0.05) + parser.add_argument("--support-weight", type=float, default=0.05) + parser.add_argument("--loading-weight", type=float, default=0.05) -def main(): - parser = argparse.ArgumentParser() parser.add_argument("--epochs", type=int, default=100) - parser.add_argument("--batch-size", type=int, default=1) + parser.add_argument("--batch-size", type=int, default=4) + parser.add_argument("--grad-accum", type=int, default=2) parser.add_argument("--lr", type=float, default=1e-4) + parser.add_argument("--weight-decay", type=float, default=1e-4) + parser.add_argument("--grad-clip", type=float, default=1.0) + parser.add_argument("--num-workers", type=int, default=2) + parser.add_argument("--no-augment", action="store_true") + parser.add_argument( + "--no-z-flip", + action="store_true", + help="Disable Z-axis reflection augmentation when plunger-vs-base compaction is asymmetric", + ) + parser.add_argument( + "--ordinal-material-channel", + action="store_true", + help="Legacy flag to concatenate an ordinal chemistry scalar spatial channel alongside learned embedding", + ) + parser.add_argument("--amp", choices=["auto", "bf16", "fp16", "off"], default="auto") + parser.add_argument("--allow-cpu", action="store_true") + + parser.add_argument("--save-every", type=int, default=10) parser.add_argument("--resume", action="store_true") - parser.add_argument("--seed", type=int, default=42) - parser.add_argument("--max-steps", type=int, default=0, - help="Limit training batches per epoch for smoke tests") - parser.add_argument("--max-val-steps", type=int, default=0) parser.add_argument( - "--experiment", - default="carbon_conditional_96_v1", - help="Separate experiment name for checkpoints and results", + "--allow-config-mismatch", + action="store_true", + help="Override --resume safety check if training states/units intentionally changed", ) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--max-steps", type=int, default=0, help="Cap train steps/epoch for smoke tests") + parser.add_argument("--max-val-steps", type=int, default=0) args = parser.parse_args() - if not args.experiment.replace("_", "").isalnum(): - parser.error("Invalid experiment name") - - global RESULT_DIR, CHECKPOINT_DIR - RESULT_DIR = ROOT / "results" / args.experiment - CHECKPOINT_DIR = ROOT / "checkpoints" / args.experiment - - if not torch.cuda.is_available(): - raise RuntimeError("GPU required; submit through Slurm") + if not re.fullmatch(r"[A-Za-z0-9_.-]+", args.experiment): + parser.error("Invalid --experiment name") + if args.epochs < 1 or args.batch_size < 1 or args.grad_accum < 1: + parser.error("--epochs, --batch-size, and --grad-accum must be >= 1") + if not 0.0 <= args.cfg_drop_prob < 1.0: + parser.error("--cfg-drop-prob must be in [0, 1)") + + if torch.cuda.is_available(): + device = torch.device("cuda") + torch.backends.cudnn.benchmark = True + elif args.allow_cpu: + device = torch.device("cpu") + else: + raise RuntimeError("CUDA GPU required (pass --allow-cpu only for local smoke tests).") + + use_amp = False + amp_dtype = torch.float32 + if device.type == "cuda" and args.amp != "off": + if args.amp == "bf16" or (args.amp == "auto" and torch.cuda.is_bf16_supported()): + use_amp = True + amp_dtype = torch.bfloat16 + else: + use_amp = True + amp_dtype = torch.float16 + scaler = torch.amp.GradScaler("cuda", enabled=(use_amp and amp_dtype == torch.float16)) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) - torch.cuda.manual_seed_all(args.seed) - - RESULT_DIR.mkdir(parents=True, exist_ok=True) - CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True) - - device = torch.device("cuda") - print("GPU:", torch.cuda.get_device_name(0), flush=True) - - train_data = CarbonDataset("train") - val_data = CarbonDataset("val") - print("Training patches:", len(train_data), flush=True) - print("Validation patches:", len(val_data), flush=True) - - train_loader = DataLoader( - train_data, batch_size=args.batch_size, - shuffle=True, num_workers=2, pin_memory=True - ) - val_loader = DataLoader( - val_data, batch_size=args.batch_size, - shuffle=False, num_workers=2, pin_memory=True + if device.type == "cuda": + torch.cuda.manual_seed_all(args.seed) + + channels = tuple(int(c.strip()) for c in args.channels.split(",")) + attention_levels = tuple(bool(int(a.strip())) for a in args.attention_levels.split(",")) + if len(channels) != len(attention_levels): + raise ValueError("--channels and --attention-levels must have matching length") + + result_dir = args.output_root / "results" / args.experiment + checkpoint_dir = args.output_root / "checkpoints" / args.experiment + result_dir.mkdir(parents=True, exist_ok=True) + checkpoint_dir.mkdir(parents=True, exist_ok=True) + + print( + f"Device: {torch.cuda.get_device_name(0) if device.type == 'cuda' else 'CPU'} | " + f"AMP: {amp_dtype if use_amp else 'fp32'}", + flush=True, ) - model = DiffusionModelUNet( - spatial_dims=3, - in_channels=1, - out_channels=1, - num_channels=(32, 64, 128), - attention_levels=(False, False, True), - num_res_blocks=1, - with_conditioning=True, - cross_attention_dim=32, - ).to(device) + pin = device.type == "cuda" + workers = args.num_workers if device.type == "cuda" else 0 + is_multielectrode = args.manifest_json is not None + + if is_multielectrode: + inc_set = set(args.include_states) if args.include_states else None + exc_set = set(args.exclude_states) if args.exclude_states else None + scan_specs, inferred_phases, ref_vox = load_manifest_specs( + manifest_path=args.manifest_json, + data_root=args.data_root, + include_states=inc_set, + exclude_states=exc_set, + include_repeats=args.include_repeats, + use_stress_kpa=args.use_stress_kpa, + ) + if scan_specs and bool(scan_specs[0].get("used_stress_kpa", False)): + args.use_stress_kpa = True + num_phases = args.num_phases if args.num_phases in (2, 3) else inferred_phases + num_materials = len(MATERIAL_TO_ID) + states = {str(s["state"]): float(s["load_value"]) for s in scan_specs} # type: ignore[arg-type] + load_min = min(states.values()) + load_max = max(states.values()) + if load_max <= load_min: + load_max = load_min + 1.0 + + train_data_mm = MultiElectrodePatchDataset( + data_root=args.data_root, + scan_specs=scan_specs, + split="train", + patch_size=args.patch_size, + version=args.version, + num_phases=num_phases, + augment=not args.no_augment, + allow_z_flip=not args.no_z_flip, + ) + val_data_mm = MultiElectrodePatchDataset( + data_root=args.data_root, + scan_specs=scan_specs, + split="val", + patch_size=args.patch_size, + version=args.version, + num_phases=num_phases, + augment=False, + ) + print( + f"Multi-electrode train: {len(train_data_mm)} patches | materials={train_data_mm.material_counts}", + flush=True, + ) + print(f"Multi-electrode val: {len(val_data_mm)} patches", flush=True) + + if args.balanced_sampling: + weights = train_data_mm.build_balanced_weights() + sampler = WeightedRandomSampler(weights, num_samples=len(train_data_mm), replacement=True) + train_loader: DataLoader = DataLoader( + train_data_mm, + batch_size=args.batch_size, + sampler=sampler, + num_workers=workers, + pin_memory=pin, + ) + else: + train_loader = DataLoader( + train_data_mm, + batch_size=args.batch_size, + shuffle=True, + num_workers=workers, + pin_memory=pin, + ) + val_loader: DataLoader = DataLoader( + val_data_mm, + batch_size=args.batch_size, + shuffle=False, + num_workers=workers, + pin_memory=pin, + ) + else: + states = parse_states_arg(args.states, args.states_json) + if len(states) < 2: + raise ValueError("Conditional training requires at least 2 distinct load states") + load_min = min(states.values()) + load_max = max(states.values()) + num_phases = args.num_phases if args.num_phases in (2, 3) else 2 + num_materials = 1 + ref_vox = 1.0 + scan_specs = [] + + train_data = LoadConditionedPatchDataset( + data_root=args.data_root, + states=states, + split="train", + patch_size=args.patch_size, + version=args.version, + augment=not args.no_augment, + num_phases=num_phases, + allow_z_flip=not args.no_z_flip, + ) + val_data = LoadConditionedPatchDataset( + data_root=args.data_root, + states=states, + split="val", + patch_size=args.patch_size, + version=args.version, + augment=False, + num_phases=num_phases, + ) + print(f"Train patches: {len(train_data)} ({train_data.state_counts})", flush=True) + print(f"Val patches: {len(val_data)} ({val_data.state_counts})", flush=True) + + train_loader = DataLoader( + train_data, + batch_size=args.batch_size, + shuffle=True, + num_workers=workers, + pin_memory=pin, + drop_last=False, + ) + val_loader = DataLoader( + val_data, + batch_size=args.batch_size, + shuffle=False, + num_workers=workers, + pin_memory=pin, + ) - # Embed the scalar load-cell reading into cross-attention context. - condition_encoder = torch.nn.Sequential( - torch.nn.Linear(1, 32), - torch.nn.SiLU(), - torch.nn.Linear(32, 32), + model = ContinuousLoadConditionedUNet( + channels=channels, + attention_levels=attention_levels, + num_res_blocks=args.num_res_blocks, + norm_num_groups=args.norm_groups, + load_min=load_min, + load_max=load_max, + use_flash_attention=(device.type == "cuda" and use_amp), + conditioning_mode=args.conditioning_mode, + num_phases=num_phases, + num_materials=num_materials, + ref_voxel_size_um=ref_vox, + use_ordinal_material_channel=args.ordinal_material_channel, ).to(device) - scheduler = DDPMScheduler( - num_train_timesteps=1000, - schedule="scaled_linear_beta", + ema = EMAModel(model, decay=args.ema_decay) + scheduler = build_noise_scheduler( + sampler="ddpm", + num_train_timesteps=args.timesteps, + schedule=args.schedule, + prediction_type=args.prediction_type, ) - optimizer = torch.optim.AdamW( - list(model.parameters()) + - list(condition_encoder.parameters()), + model.parameters(), lr=args.lr, + weight_decay=args.weight_decay, ) - def loss_fn(batch): - images, conditions = batch - images = images.to(device, non_blocking=True) - conditions = conditions.to(device, non_blocking=True) - context = condition_encoder(conditions).unsqueeze(1) + physics_criterion: PhysicsInformedDiffusionLoss | None = None + if args.physics_loss: + physics_criterion = PhysicsInformedDiffusionLoss( + alphas_cumprod=scheduler.alphas_cumprod, + prediction_type=args.prediction_type, + snr_gamma=args.snr_gamma, + interface_weight=args.interface_weight, + support_weight=args.support_weight, + loading_weight=args.loading_weight, + ).to(device) + print( + f"Physics-informed loss enabled: snr_gamma={args.snr_gamma}, " + f"iface={args.interface_weight}, support={args.support_weight}, loading={args.loading_weight}", + flush=True, + ) - noise = torch.randn_like(images) - t = torch.randint( - 0, 1000, (images.shape[0],), device=device - ).long() - noisy = scheduler.add_noise(images, noise, t) - predicted = model(noisy, t, context=context) - return F.mse_loss(predicted.float(), noise.float()) - - latest = CHECKPOINT_DIR / "latest.pth" - history = [] + model_config = { + "patch_size": args.patch_size, + "channels": list(channels), + "attention_levels": list(attention_levels), + "num_res_blocks": args.num_res_blocks, + "norm_groups": args.norm_groups, + "load_min": load_min, + "load_max": load_max, + "schedule": args.schedule, + "prediction_type": args.prediction_type, + "conditioning_mode": args.conditioning_mode, + "num_phases": num_phases, + "num_materials": num_materials, + "ref_voxel_size_um": ref_vox, + "use_stress_kpa": args.use_stress_kpa, + "allow_z_flip": not args.no_z_flip, + "use_ordinal_material_channel": bool(args.ordinal_material_channel), + "heldout_states": sorted(args.exclude_states) if args.exclude_states else [], + "include_states": sorted(args.include_states) if args.include_states else None, + "physics_loss": args.physics_loss, + "snr_gamma": args.snr_gamma if args.physics_loss else 0.0, + "timesteps": args.timesteps, + "states": states, + "scan_specs": scan_specs, + "version": args.version, + } + + latest_path = checkpoint_dir / "latest.pth" + best_path = checkpoint_dir / "best_val.pth" + history: list[dict[str, object]] = [] start_epoch = 0 + best_val_loss = float("inf") if args.resume: - saved = torch.load(latest, map_location=device, - weights_only=False) + if not latest_path.is_file(): + raise FileNotFoundError(f"Cannot --resume; missing {latest_path}") + saved = torch.load(latest_path, map_location=device, weights_only=False) + if not args.allow_config_mismatch: + validate_resume_config(saved.get("config", {}), model_config) model.load_state_dict(saved["model"]) - condition_encoder.load_state_dict(saved["condition_encoder"]) + ema.load_state_dict(saved["ema"]) optimizer.load_state_dict(saved["optimizer"]) - history = saved["history"] - start_epoch = saved["epoch"] + 1 - torch.set_rng_state(saved["rng_torch"].cpu()) - np.random.set_state(saved["rng_numpy"]) - random.setstate(saved["rng_python"]) - elif latest.exists(): + if "scaler" in saved and scaler.is_enabled(): + scaler.load_state_dict(saved["scaler"]) + history = list(saved["history"]) + start_epoch = int(saved["epoch"]) + 1 + best_val_loss = float(saved.get("best_val_loss", float("inf"))) + restore_rng_states(saved["rng_states"]) + print(f"Resumed from epoch {start_epoch} (best_val={best_val_loss:.6f})", flush=True) + elif latest_path.exists(): raise FileExistsError( - f"{latest} exists; use --resume" + f"{latest_path} already exists; pass --resume or a new --experiment name." ) + stop_requested = False + + def _handle_preempt(signum: int, _frame: object) -> None: + nonlocal stop_requested + print(f"Received signal {signum}; will checkpoint cleanly at end of epoch.", flush=True) + stop_requested = True + + signal.signal(signal.SIGTERM, _handle_preempt) + if hasattr(signal, "SIGUSR1"): + signal.signal(signal.SIGUSR1, _handle_preempt) + + def _unpack_batch( + batch: tuple[torch.Tensor, ...], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor | None, torch.Tensor | None]: + if len(batch) == 4: + imgs, lds, mats, voxs = batch + return ( + imgs.to(device, non_blocking=True), + lds.to(device, non_blocking=True), + mats.to(device, non_blocking=True), + voxs.to(device, non_blocking=True), + ) + imgs, lds = batch + return imgs.to(device, non_blocking=True), lds.to(device, non_blocking=True), None, None + + def compute_batch_loss( + batch: tuple[torch.Tensor, ...], + is_train: bool, + ) -> torch.Tensor: + images, loads, mats, voxs = _unpack_batch(batch) + bsz = images.shape[0] + noise = torch.randn_like(images) + timesteps = torch.randint(0, args.timesteps, (bsz,), device=device).long() + if is_train and args.cfg_drop_prob > 0.0: + keep = torch.rand(bsz, device=device) >= args.cfg_drop_prob + cond_mask = keep.to(torch.float32) + else: + cond_mask = torch.ones(bsz, device=device, dtype=torch.float32) + + noisy = scheduler.add_noise(original_samples=images, noise=noise, timesteps=timesteps) + target = ( + scheduler.get_velocity(images, noise, timesteps) + if args.prediction_type == "v_prediction" + else noise + ) + + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = model( + noisy, + timesteps, + loads, + cond_mask=cond_mask, + material_ids=mats, + voxel_sizes=voxs, + ) + if physics_criterion is not None: + loss, _ = physics_criterion( + model_pred=pred, + target=target, + x_t=noisy, + x_0=images, + timesteps=timesteps, + ) + else: + loss = F.mse_loss(pred.float(), target.float()) + return loss + for epoch in range(start_epoch, args.epochs): started = time.perf_counter() - torch.cuda.reset_peak_memory_stats() + if device.type == "cuda": + torch.cuda.reset_peak_memory_stats() + model.train() - condition_encoder.train() + optimizer.zero_grad(set_to_none=True) train_total = 0.0 train_steps = 0 - for images, conditions in train_loader: - optimizer.zero_grad(set_to_none=True) - loss = loss_fn((images, conditions)) - loss.backward() - optimizer.step() - train_total += loss.item() + for step_idx, batch in enumerate(train_loader, start=1): + raw_loss = compute_batch_loss(batch, is_train=True) + scaled_loss = raw_loss / args.grad_accum + + if scaler.is_enabled(): + scaler.scale(scaled_loss).backward() + else: + scaled_loss.backward() + + train_total += float(raw_loss.item()) train_steps += 1 - if train_steps % 25 == 0: - print(f"Epoch {epoch+1} step {train_steps}: " - f"loss={loss.item():.6f}", flush=True) + + is_last_batch = (step_idx == len(train_loader)) or ( + bool(args.max_steps) and train_steps >= args.max_steps + ) + if (train_steps % args.grad_accum == 0) or is_last_batch: + if scaler.is_enabled(): + scaler.unscale_(optimizer) + if args.grad_clip > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + scaler.step(optimizer) + scaler.update() + else: + if args.grad_clip > 0: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.grad_clip) + optimizer.step() + optimizer.zero_grad(set_to_none=True) + ema.update(model) + + if train_steps % 50 == 0: + print( + f"Epoch {epoch + 1}/{args.epochs} step {train_steps}: loss={raw_loss.item():.6f}", + flush=True, + ) if args.max_steps and train_steps >= args.max_steps: break - model.eval() - condition_encoder.eval() + ema_model = ema.shadow_model + ema_model.eval() + val_gen = torch.Generator(device=device).manual_seed(args.seed + 10_007) val_total = 0.0 + val_sf_mae_total = 0.0 val_steps = 0 + with torch.no_grad(): for batch in val_loader: - val_total += loss_fn(batch).item() + images, loads, mats, voxs = _unpack_batch(batch) + bsz = images.shape[0] + noise = torch.randn(images.shape, generator=val_gen, device=device, dtype=images.dtype) + timesteps = torch.randint( + 0, args.timesteps, (bsz,), generator=val_gen, device=device + ).long() + noisy = scheduler.add_noise(original_samples=images, noise=noise, timesteps=timesteps) + target = ( + scheduler.get_velocity(images, noise, timesteps) + if args.prediction_type == "v_prediction" + else noise + ) + ones = torch.ones(bsz, device=device, dtype=torch.float32) + with torch.amp.autocast(device_type=device.type, dtype=amp_dtype, enabled=use_amp): + pred = ema_model( + noisy, + timesteps, + loads, + cond_mask=ones, + material_ids=mats, + voxel_sizes=voxs, + ) + v_loss = F.mse_loss(pred.float(), target.float()) + x0_hat = reconstruct_x0( + x_t=noisy, + model_pred=pred, + timesteps=timesteps, + alphas_cumprod=scheduler.alphas_cumprod, + prediction_type=args.prediction_type, + ) + sf_hat = (x0_hat[:, 0] > 0.0).float().mean(dim=(1, 2, 3)) + sf_true = (images[:, 0] > 0.0).float().mean(dim=(1, 2, 3)) + val_sf_mae_total += float((sf_hat - sf_true).abs().mean().item()) + val_total += float(v_loss.item()) val_steps += 1 if args.max_val_steps and val_steps >= args.max_val_steps: break - torch.cuda.synchronize() - record = { + if device.type == "cuda": + torch.cuda.synchronize() + + val_loss = val_total / max(1, val_steps) + val_x0_sf_mae = val_sf_mae_total / max(1, val_steps) + is_best = val_loss < best_val_loss + if is_best: + best_val_loss = val_loss + + record: dict[str, object] = { "epoch": epoch + 1, - "train_loss": train_total / train_steps, - "val_loss": val_total / val_steps, - "epoch_seconds": round( - time.perf_counter() - started, 2 - ), - "peak_allocated_gib": round( - torch.cuda.max_memory_allocated() / 1024**3, 3 + "train_loss": round(train_total / max(1, train_steps), 6), + "val_loss_ema": round(val_loss, 6), + "val_x0_sf_mae": round(val_x0_sf_mae, 6), + "best_val_loss": round(best_val_loss, 6), + "epoch_seconds": round(time.perf_counter() - started, 2), + "peak_allocated_gib": ( + round(torch.cuda.max_memory_allocated() / 1024**3, 3) + if device.type == "cuda" + else 0.0 ), - "peak_reserved_gib": round( - torch.cuda.max_memory_reserved() / 1024**3, 3 + "peak_reserved_gib": ( + round(torch.cuda.max_memory_reserved() / 1024**3, 3) + if device.type == "cuda" + else 0.0 ), } history.append(record) print(json.dumps(record), flush=True) - with (RESULT_DIR / "losses.csv").open("w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=record.keys()) + with (result_dir / "losses.csv").open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(record.keys())) writer.writeheader() writer.writerows(history) - state = { + ckpt_payload: dict[str, object] = { "epoch": epoch, + "best_val_loss": best_val_loss, + "config": model_config, "model": model.state_dict(), - "condition_encoder": condition_encoder.state_dict(), + "ema": ema.state_dict(), "optimizer": optimizer.state_dict(), + "scaler": scaler.state_dict(), "history": history, - "rng_torch": torch.get_rng_state(), - "rng_numpy": np.random.get_state(), - "rng_python": random.getstate(), + "rng_states": capture_rng_states(), } - temporary = latest.with_suffix(".pth.tmp") - torch.save(state, temporary) - temporary.replace(latest) - print("Checkpoint:", latest, flush=True) + + atomic_torch_save(ckpt_payload, latest_path) + if is_best: + atomic_torch_save(ckpt_payload, best_path) + if args.save_every > 0 and (epoch + 1) % args.save_every == 0: + atomic_torch_save(ckpt_payload, checkpoint_dir / f"epoch_{epoch + 1:03d}.pth") + + if stop_requested: + print(f"Clean exit after saving epoch {epoch + 1} checkpoint.", flush=True) + return print("Training completed.", flush=True) diff --git a/slurm/benchmark_carbon_conditional_96.slurm b/slurm/benchmark_carbon_conditional_96.slurm index 05f281b..1d5e253 100644 --- a/slurm/benchmark_carbon_conditional_96.slurm +++ b/slurm/benchmark_carbon_conditional_96.slurm @@ -6,24 +6,38 @@ #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G #SBATCH --time=00:45:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-bench-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-bench-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" + +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" + +STATES_STR="${STATES:-Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033}" +read -r -a STATES_ARR <<< "$STATES_STR" +VERSION="${VERSION:-carbon_calibrated_v1}" echo "Started: $(date -u)" nvidia-smi python -u scripts/training/train_carbon_conditional_96.py \ --experiment carbon_conditional_96_benchmark \ + --version "$VERSION" \ + --states "${STATES_ARR[@]}" \ --epochs 1 \ - --batch-size 1 \ + --batch-size 4 \ + --grad-accum 2 \ + --amp bf16 \ + --num-workers 4 \ --max-steps 100 \ --max-val-steps 20 diff --git a/slurm/carbon_cond_to_100.slurm b/slurm/carbon_cond_to_100.slurm deleted file mode 100644 index aef6e2d..0000000 --- a/slurm/carbon_cond_to_100.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 100 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/carbon_cond_to_28.slurm b/slurm/carbon_cond_to_28.slurm deleted file mode 100644 index a657277..0000000 --- a/slurm/carbon_cond_to_28.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 28 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/carbon_cond_to_46.slurm b/slurm/carbon_cond_to_46.slurm deleted file mode 100644 index 33dffe1..0000000 --- a/slurm/carbon_cond_to_46.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 46 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/carbon_cond_to_64.slurm b/slurm/carbon_cond_to_64.slurm deleted file mode 100644 index 78e9e4e..0000000 --- a/slurm/carbon_cond_to_64.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 64 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/carbon_cond_to_82.slurm b/slurm/carbon_cond_to_82.slurm deleted file mode 100644 index bb47bf6..0000000 --- a/slurm/carbon_cond_to_82.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 82 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/extract_carbon_96.slurm b/slurm/extract_carbon_96.slurm index 88c7d11..216006c 100644 --- a/slurm/extract_carbon_96.slurm +++ b/slurm/extract_carbon_96.slurm @@ -5,33 +5,50 @@ #SBATCH --qos=normal #SBATCH --nodes=1 #SBATCH --ntasks=1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=16G +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G #SBATCH --time=02:00:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-patches-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-patches-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" -DATASET="${1:?Provide dataset name}" +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" -echo "Dataset: $DATASET" -echo "Started: $(date)" +# Accept positional dataset(s) or DATASETS env var +if [ "$#" -gt 0 ]; then + DATASET_ARR=("$@") +else + DATASETS_STR="${DATASETS:-Op008mVV Op018mVV Op033mVV}" + read -r -a DATASET_ARR <<< "$DATASETS_STR" +fi + +VERSION="${VERSION:-carbon_calibrated_v1}" + +echo "Job ID: ${SLURM_JOB_ID:-local}" +echo "Datasets: ${DATASET_ARR[*]}" +echo "Started: $(date -u)" python -u scripts/preprocessing/extract_carbon_3d_patches.py \ - --dataset "$DATASET" \ - --version carbon_sigma50_v1 \ - --expected-slices 800 \ + --datasets "${DATASET_ARR[@]}" \ + --version "$VERSION" \ + --expected-slices "${EXPECTED_SLICES:-0}" \ --patch-size 96 \ - --stride-xy 128 \ + --stride-xy 96 \ --stride-z 32 \ - --max-patches 1200 \ - --min-white 0.0 \ - --max-white 1.0 \ + --max-patches "${MAX_PATCHES:-1200}" \ + --min-white 0.005 \ + --max-white 0.90 \ --val-fraction 0.2 \ - --seed 42 + --roi-radius-fraction 0.70 \ + --no-individual-npy \ + --seed 42 \ + --overwrite -echo "Finished: $(date)" +echo "Finished: $(date -u)" diff --git a/slurm/resume_carbon_conditional_100.slurm b/slurm/resume_carbon_conditional_100.slurm deleted file mode 100644 index aef6e2d..0000000 --- a/slurm/resume_carbon_conditional_100.slurm +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=carbon-cond-100 -#SBATCH --account=brics.u6uc -#SBATCH --partition=workq -#SBATCH --qos=normal -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=06:00:00 -#SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-100-%j.out -#SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-100-%j.err - -set -euo pipefail - -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate - -echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" - -python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 100 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 \ - --resume - -echo "Finished: $(date -u)" diff --git a/slurm/run_beamtime_multiplier.slurm b/slurm/run_beamtime_multiplier.slurm new file mode 100755 index 0000000..5520685 --- /dev/null +++ b/slurm/run_beamtime_multiplier.slurm @@ -0,0 +1,126 @@ +#!/bin/bash +#SBATCH --job-name=beamtime-mult +#SBATCH --account=brics.u6uc +#SBATCH --partition=workq +#SBATCH --qos=normal +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --gres=gpu:1 +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G +#SBATCH --time=06:00:00 +#SBATCH --output=/projects/u6uc/asif/logs/beamtime-mult-%j.out +#SBATCH --error=/projects/u6uc/asif/logs/beamtime-mult-%j.err + +set -euo pipefail + +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" +DATA_ROOT="${DATA_ROOT:-${POROUSDIFF_ROOT}/data/carbon_virgin}" +MANIFEST="${MANIFEST:-${DATA_ROOT}/bundle_manifest.json}" + +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" + +EXPERIMENT="${EXPERIMENT:-beamtime_multiplier_96}" +VERSION="${VERSION:-carbon_calibrated_v1}" +EPOCHS="${EPOCHS:-100}" +PATCH_SIZE="${PATCH_SIZE:-96}" + +INCLUDE_ARGS=() +if [ -n "${INCLUDE_STATES:-}" ]; then + read -r -a INC_ARR <<< "$INCLUDE_STATES" + INCLUDE_ARGS+=(--include-states "${INC_ARR[@]}") +fi + +EXCLUDE_ARGS=() +HELDOUT_ARGS=() +if [ -n "${HELDOUT_STATES:-}" ]; then + read -r -a EXC_ARR <<< "$HELDOUT_STATES" + EXCLUDE_ARGS+=(--exclude-states "${EXC_ARR[@]}") + HELDOUT_ARGS+=(--heldout-states "${EXC_ARR[@]}") +fi + +RESUME_ARGS=() +if [ -f "${POROUSDIFF_ROOT}/checkpoints/${EXPERIMENT}/latest.pth" ]; then + echo "Existing checkpoint detected; resuming automatically." + RESUME_ARGS+=(--resume) +fi + +PHYSICS_ARGS=() +if [ "${PHYSICS_LOSS:-1}" = "1" ]; then + PHYSICS_ARGS+=(--physics-loss) +fi + +STRESS_ARGS=() +if [ "${USE_STRESS_KPA:-auto}" = "1" ]; then + STRESS_ARGS+=(--use-stress-kpa) +elif [ "${USE_STRESS_KPA:-auto}" = "auto" ]; then + if [ -f "${POROUSDIFF_ROOT}/checkpoints/${EXPERIMENT}/latest.pth" ]; then + if python -c "import sys, torch; c = torch.load(sys.argv[1], map_location='cpu', weights_only=False).get('config', {}); sys.exit(0 if c.get('use_stress_kpa', False) else 1)" "${POROUSDIFF_ROOT}/checkpoints/${EXPERIMENT}/latest.pth"; then + STRESS_ARGS+=(--use-stress-kpa) + fi + elif [ -f "$MANIFEST" ]; then + if python -c "import json, sys; s = json.load(open(sys.argv[1])).get('scans', []); sys.exit(0 if (s and all(x.get('stress_kpa') is not None for x in s)) else 1)" "$MANIFEST"; then + echo "All manifest scans carry calibrated stress_kpa; enabling --use-stress-kpa." + STRESS_ARGS+=(--use-stress-kpa) + fi + fi +fi + +echo "Started Beamtime Multiplier pipeline: $(date -u)" +echo "Manifest: $MANIFEST | Experiment: $EXPERIMENT" +nvidia-smi + +python -u scripts/training/train_carbon_conditional_96.py \ + --experiment "$EXPERIMENT" \ + --data-root "$DATA_ROOT" \ + --manifest-json "$MANIFEST" \ + --version "$VERSION" \ + --patch-size "$PATCH_SIZE" \ + --balanced-sampling \ + --epochs "$EPOCHS" \ + --batch-size "${BATCH_SIZE:-4}" \ + --grad-accum "${GRAD_ACCUM:-2}" \ + --lr "${LR:-2e-4}" \ + --amp bf16 \ + --num-workers 4 \ + "${STRESS_ARGS[@]}" \ + "${PHYSICS_ARGS[@]}" \ + "${INCLUDE_ARGS[@]}" \ + "${EXCLUDE_ARGS[@]}" \ + "${RESUME_ARGS[@]}" + +python -u scripts/evaluation/sample_carbon_conditional_96.py \ + --experiment "$EXPERIMENT" \ + --manifest-json "$MANIFEST" \ + --sampler "${SAMPLER:-ddim}" \ + --steps "${STEPS:-100}" \ + --guidance-scale "${GUIDANCE_SCALE:-1.5}" \ + --num-samples "${NUM_SAMPLES:-20}" \ + --batch-size "${BATCH_SIZE:-4}" \ + --sweep-loads "" \ + --export-openimpala + +python -u scripts/evaluation/evaluate_beamtime_multiplier.py \ + --experiments "porousdiff=${EXPERIMENT}" \ + --data-root "$DATA_ROOT" \ + --manifest-json "$MANIFEST" \ + --version "$VERSION" \ + --patch-size "$PATCH_SIZE" \ + --include-virtual-compression \ + --generate-openimpala-suite \ + --export-real-openimpala \ + "${HELDOUT_ARGS[@]}" + +python -u scripts/evaluation/optimize_clamping_pressure.py \ + --experiments "porousdiff=${EXPERIMENT}" \ + --data-root "$DATA_ROOT" \ + --manifest-json "$MANIFEST" \ + --version "$VERSION" \ + --patch-size "$PATCH_SIZE" \ + --dense-sweep-points "${DENSE_SWEEP_POINTS:-15}" + +echo "Finished: $(date -u)" diff --git a/slurm/sample_carbon_conditional_96.slurm b/slurm/sample_carbon_conditional_96.slurm index 7f31b6c..09001ac 100644 --- a/slurm/sample_carbon_conditional_96.slurm +++ b/slurm/sample_carbon_conditional_96.slurm @@ -6,21 +6,56 @@ #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G -#SBATCH --time=04:00:00 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=48G +#SBATCH --time=02:00:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-sample-96-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-sample-96-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" + +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" + +EXPERIMENT="${EXPERIMENT:-carbon_conditional_96_v2}" +VERSION="${VERSION:-carbon_calibrated_v1}" +EVAL_STATES_STR="${EVAL_STATES:-Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033}" +read -r -a EVAL_STATES_ARR <<< "$EVAL_STATES_STR" +HELDOUT_STR="${HELDOUT_STATES:-}" +read -r -a HELDOUT_ARR <<< "$HELDOUT_STR" +SWEEP_LOADS="${SWEEP_LOADS:-0.008,0.013,0.018,0.023,0.028,0.033,0.038,0.043,0.048,0.053}" echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" +echo "Job ID: ${SLURM_JOB_ID:-local}" nvidia-smi -python -u scripts/evaluation/sample_carbon_conditional_96.py +python -u scripts/evaluation/sample_carbon_conditional_96.py \ + --experiment "$EXPERIMENT" \ + --sampler "${SAMPLER:-ddim}" \ + --steps "${STEPS:-100}" \ + --guidance-scale "${GUIDANCE_SCALE:-1.5}" \ + --num-samples "${NUM_SAMPLES:-20}" \ + --batch-size "${BATCH_SIZE:-4}" \ + --eval-states "${EVAL_STATES_ARR[@]}" \ + --sweep-loads "$SWEEP_LOADS" \ + --export-openimpala + +HELDOUT_ARGS=() +if [ "${#HELDOUT_ARR[@]}" -gt 0 ]; then + HELDOUT_ARGS+=(--heldout-states "${HELDOUT_ARR[@]}") +fi + +python -u scripts/evaluation/evaluate_carbon_physics.py \ + --experiment "$EXPERIMENT" \ + --version "$VERSION" \ + --patch-size "${PATCH_SIZE:-96}" \ + --export-real-openimpala \ + --states "${EVAL_STATES_ARR[@]}" \ + "${HELDOUT_ARGS[@]}" echo "Finished: $(date -u)" diff --git a/slurm/segment_carbon_xct.slurm b/slurm/segment_carbon_xct.slurm index e4ce76d..95a4aaf 100644 --- a/slurm/segment_carbon_xct.slurm +++ b/slurm/segment_carbon_xct.slurm @@ -5,34 +5,41 @@ #SBATCH --qos=normal #SBATCH --nodes=1 #SBATCH --ntasks=1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=16G +#SBATCH --cpus-per-task=8 +#SBATCH --mem=32G #SBATCH --time=04:00:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-segment-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-segment-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" -echo "Job ID: $SLURM_JOB_ID" -echo "Started: $(date)" -echo "Segmentation: Gaussian sigma=50, Otsu threshold" +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" -for dataset in Op008mVV Op018mVV Op033mVV; do - echo "==================================" - echo "Processing: $dataset" - echo "==================================" +# Override via `sbatch --export=ALL,DATASETS="0p008mVV 0p013mVV ..."` when all loads are staged +DATASETS_STR="${DATASETS:-Op008mVV Op018mVV Op033mVV}" +read -r -a DATASET_ARR <<< "$DATASETS_STR" +VERSION="${VERSION:-carbon_calibrated_v1}" +DELTA_THRESHOLD="${DELTA_THRESHOLD:-5500}" - python -u scripts/preprocessing/segment_carbon_xct.py \ - --dataset "$dataset" \ - --expected-slices 800 \ - --sigma 50 \ - --version carbon_sigma50_v1 +echo "Job ID: ${SLURM_JOB_ID:-local}" +echo "Started: $(date -u)" +echo "Datasets: ${DATASET_ARR[*]}" +echo "Version: ${VERSION} (delta_threshold=${DELTA_THRESHOLD}, circular ROI + Z-artifact gating)" - echo "Completed: $dataset" -done +python -u scripts/preprocessing/segment_carbon_xct.py \ + --datasets "${DATASET_ARR[@]}" \ + --version "$VERSION" \ + --delta-threshold "$DELTA_THRESHOLD" \ + --sigma 32 \ + --roi-radius-fraction 0.70 \ + --expected-slices "${EXPECTED_SLICES:-0}" \ + --overwrite -echo "All carbon datasets processed." -echo "Finished: $(date)" +echo "All carbon datasets segmented successfully." +echo "Finished: $(date -u)" diff --git a/slurm/test_carbon_conditional_96.slurm b/slurm/test_carbon_conditional_96.slurm index 9d3116d..607e3aa 100644 --- a/slurm/test_carbon_conditional_96.slurm +++ b/slurm/test_carbon_conditional_96.slurm @@ -6,25 +6,38 @@ #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G +#SBATCH --cpus-per-task=8 +#SBATCH --mem=48G #SBATCH --time=00:30:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-test-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-test-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" + +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" + +STATES_STR="${STATES:-Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033}" +read -r -a STATES_ARR <<< "$STATES_STR" +VERSION="${VERSION:-carbon_calibrated_v1}" echo "Started: $(date -u)" nvidia-smi python -u scripts/training/train_carbon_conditional_96.py \ --experiment carbon_conditional_96_smoke \ + --version "$VERSION" \ + --states "${STATES_ARR[@]}" \ --epochs 1 \ - --batch-size 1 \ - --max-steps 3 \ + --batch-size 2 \ + --grad-accum 1 \ + --amp bf16 \ + --max-steps 4 \ --max-val-steps 2 echo "Finished: $(date -u)" diff --git a/slurm/train_carbon_conditional_96.slurm b/slurm/train_carbon_conditional_96.slurm index 0226e5d..1d26485 100644 --- a/slurm/train_carbon_conditional_96.slurm +++ b/slurm/train_carbon_conditional_96.slurm @@ -6,27 +6,58 @@ #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --gres=gpu:1 -#SBATCH --cpus-per-task=4 -#SBATCH --mem=32G +#SBATCH --cpus-per-task=16 +#SBATCH --mem=64G #SBATCH --time=06:00:00 #SBATCH --output=/projects/u6uc/asif/logs/carbon-cond-96-%j.out #SBATCH --error=/projects/u6uc/asif/logs/carbon-cond-96-%j.err set -euo pipefail -cd /projects/u6uc/asif/code/PorousDiff -source /projects/u6uc/asif/.venv/bin/activate +export POROUSDIFF_ROOT="${POROUSDIFF_ROOT:-/projects/u6uc/asif}" +CODE_DIR="${CODE_DIR:-${POROUSDIFF_ROOT}/code/PorousDiff}" +VENV_DIR="${VENV_DIR:-${POROUSDIFF_ROOT}/.venv}" + +cd "$CODE_DIR" +source "$VENV_DIR/bin/activate" +export PYTHONPATH="${CODE_DIR}:${PYTHONPATH:-}" + +EXPERIMENT="${EXPERIMENT:-carbon_conditional_96_v2}" +VERSION="${VERSION:-carbon_calibrated_v1}" +EPOCHS="${EPOCHS:-100}" +STATES_STR="${STATES:-Op008mVV=0.008 Op018mVV=0.018 Op033mVV=0.033}" +read -r -a STATES_ARR <<< "$STATES_STR" + +RESUME_ARGS=() +if [ -f "${POROUSDIFF_ROOT}/checkpoints/${EXPERIMENT}/latest.pth" ]; then + echo "Existing checkpoint detected; resuming automatically." + RESUME_ARGS+=(--resume) +fi echo "Started: $(date -u)" -echo "Job ID: ${SLURM_JOB_ID}" +echo "Job ID: ${SLURM_JOB_ID:-local}" echo "Node: $(hostname)" +echo "Experiment: ${EXPERIMENT} (target epochs=${EPOCHS})" +echo "Training states: ${STATES_ARR[*]}" nvidia-smi python -u scripts/training/train_carbon_conditional_96.py \ - --experiment carbon_conditional_96_v1 \ - --epochs 10 \ - --batch-size 1 \ - --lr 1e-4 \ - --seed 42 + --experiment "$EXPERIMENT" \ + --version "$VERSION" \ + --states "${STATES_ARR[@]}" \ + --patch-size "${PATCH_SIZE:-96}" \ + --channels "${CHANNELS:-32,64,128}" \ + --schedule "${SCHEDULE:-zero_snr_scaled_linear}" \ + --prediction-type "${PREDICTION_TYPE:-v_prediction}" \ + --conditioning-mode "${CONDITIONING_MODE:-multiscale_fourier}" \ + --epochs "$EPOCHS" \ + --batch-size "${BATCH_SIZE:-4}" \ + --grad-accum "${GRAD_ACCUM:-2}" \ + --lr "${LR:-2e-4}" \ + --amp bf16 \ + --num-workers 4 \ + --save-every 10 \ + --seed "${SEED:-42}" \ + "${RESUME_ARGS[@]}" echo "Finished: $(date -u)" diff --git a/tests/test_beamtime_multiplier.py b/tests/test_beamtime_multiplier.py new file mode 100644 index 0000000..57f2fa3 --- /dev/null +++ b/tests/test_beamtime_multiplier.py @@ -0,0 +1,685 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import tifffile +import torch + +from porousdiff.baselines import affine_z_compress_to_target, pillar_pore_deletion_compress +from porousdiff.data import decode_channels_to_phases, encode_phases_to_channels +from porousdiff.metrics import evaluate_multiphase_physics, export_openimpala_tiff +from porousdiff.models import ContinuousLoadConditionedUNet +from porousdiff.preprocessing import ( + CalibrationMetadata, + calibrate_multiphase_thresholds, + downsample_volume_rev, + infer_material_name, + make_circular_mask, + segment_slice_multiphase, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_multiphase_segmentation_and_rev_binning() -> None: + rng = np.random.default_rng(101) + h, w = 128, 128 + roi = make_circular_mask((h, w), radius_fraction=0.80) + + # Synthetic 3-phase TiO2-on-carbon slice: pore (~30,000), carbon (~38,500), TiO2 coating (~49,500) + raw = (30_000.0 + rng.normal(0.0, 300.0, size=(h, w))).astype(np.float32) + u = rng.random((h, w)) + carbon_gt = roi & (u >= 0.80) & (u < 0.93) + coat_gt = roi & (u >= 0.93) + raw[carbon_gt] += 8_500.0 + raw[coat_gt] += 19_500.0 + raw_u16 = raw.clip(0, 65535).astype(np.uint16) + + d_carb, d_coat = calibrate_multiphase_thresholds([raw_u16], roi_mask=roi, sigma=16.0) + assert 3_000.0 < d_carb < 12_000.0 + assert d_carb < d_coat < 18_500.0 + + seg3, stats3 = segment_slice_multiphase( + raw_u16, delta_carbon=d_carb, delta_coating=d_coat, roi_mask=roi, sigma=16.0 + ) + assert set(np.unique(seg3)) == {0, 1, 2} + assert stats3["carbon_fraction"] == pytest.approx(float(carbon_gt[roi].mean()), abs=0.02) + assert stats3["coating_fraction"] == pytest.approx(float(coat_gt[roi].mean()), abs=0.02) + + # Virgin 2-phase backward compatibility when delta_coating=None + seg2, stats2 = segment_slice_multiphase( + raw_u16, delta_carbon=d_carb, delta_coating=None, roi_mask=roi, sigma=16.0 + ) + assert set(np.unique(seg2)) == {0, 1} + assert stats2["coating_fraction"] == 0.0 + + # Topology- & volume-fraction-preserving 2x REV block binning on 3-phase 3D volume + vol3 = np.zeros((64, 64, 64), dtype=np.uint8) + vol3[4:60, 10::8, :] = 1 # Thin carbon fibers + vol3[4:60, 11::16, ::2] = 2 # Attached active coating deposits + binned = downsample_volume_rev(vol3, factor=2) + assert binned.shape == (32, 32, 32) + assert set(np.unique(binned)) == {0, 1, 2} + assert float((binned >= 1).mean()) == pytest.approx(float((vol3 >= 1).mean()), abs=0.01) + assert float((binned == 2).mean()) == pytest.approx(float((vol3 == 2).mean()), abs=0.01) + + calib = CalibrationMetadata(voxel_size_um=1.6, spatial_bin=2, kpa_per_mvv=8500.0) + assert calib.effective_voxel_size_um == pytest.approx(3.2) + assert calib.load_to_stress_kpa(0.020) == pytest.approx(170.0) + assert infer_material_name("TiO2_felt_0p018mVV") == "tio2_anode" + assert infer_material_name("CuHCF_cathode_0p033mVV") == "cuhcf_cathode" + assert infer_material_name("crbon_virgin_1_0p008mVV") == "virgin_carbon" + + +def test_numerical_compression_baselines() -> None: + rng = np.random.default_rng(7) + uncomp = np.zeros((32, 24, 24), dtype=np.uint8) + p = rng.random(uncomp.shape) + uncomp[(p >= 0.86) & (p < 0.96)] = 1 + uncomp[p >= 0.96] = 2 + + target_sf = 0.28 + aff = affine_z_compress_to_target(uncomp, target_solid_fraction=target_sf, output_depth=32) + pil = pillar_pore_deletion_compress( + uncomp, target_solid_fraction=target_sf, output_depth=32, rng=rng + ) + + assert aff.shape == (32, 24, 24) + assert pil.shape == (32, 24, 24) + assert set(np.unique(aff)).issubset({0, 1, 2}) + assert set(np.unique(pil)).issubset({0, 1, 2}) + assert float((aff >= 1).mean()) == pytest.approx(target_sf, abs=0.025) + assert float((pil >= 1).mean()) == pytest.approx(target_sf, abs=0.04) + assert np.any(aff == 2) and np.any(pil == 2) + + +def test_hierarchical_phase_encoding_and_multielectrode_unet() -> None: + # 1. Lossless 3-phase hierarchical roundtrip {0, 1, 2} <-> [-1, +1]^2 + vol = np.array([[[0, 1], [2, 0]], [[1, 2], [0, 1]]], dtype=np.uint8) + enc = encode_phases_to_channels(vol, num_phases=3) + assert enc.shape == (2, 2, 2, 2) + dec = decode_channels_to_phases(enc, enforce_solid_support=True) + assert np.array_equal(dec, vol) + + # Verify floating coating in empty pore (c_solid=-1, c_coat=+1) snaps cleanly to pore (0) + impossible = torch.tensor([[[[-1.0]]], [[[+1.0]]]]) # [2, 1, 1, 1] + assert int(decode_channels_to_phases(impossible, enforce_solid_support=True).item()) == 0 + + # 2. Gradient flow across ALL parameters in MultiModalConditionEmbedder + 3-phase UNet + torch.manual_seed(12) + model = ContinuousLoadConditionedUNet( + channels=(16, 32), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=8, + load_min=10.0, + load_max=400.0, + num_phases=3, + num_materials=3, + ref_voxel_size_um=3.0, + ) + opt = torch.optim.AdamW(model.parameters(), lr=1e-3) + for step in range(3): + opt.zero_grad() + x = torch.randn(4, 2, 12, 12, 12) + target = torch.randn_like(x) + t = torch.tensor([15, 220, 610, 990], dtype=torch.long) + loads = torch.tensor([10.0, 150.0, 320.0, 400.0], dtype=torch.float32) + mats = torch.tensor([0, 1, 2, 1], dtype=torch.long) + voxs = torch.tensor([1.6, 3.2, 3.2, 5.0], dtype=torch.float32) + mask = torch.tensor([1.0, 1.0, 1.0, 0.0], dtype=torch.float32) + + pred = model(x, t, loads, cond_mask=mask, material_ids=mats, voxel_sizes=voxs) + assert pred.shape == x.shape + loss = torch.nn.functional.mse_loss(pred, target) + loss.backward() + if step < 2: + opt.step() + + dead = [ + n + for n, p in model.named_parameters() + if p.requires_grad and (p.grad is None or float(p.grad.abs().max().item()) == 0.0) + ] + assert dead == [], f"Parameters received zero gradient: {dead}" + + # Verify distinct material predictions & CFG + model.eval() + with torch.no_grad(): + x1 = torch.randn(1, 2, 12, 12, 12) + t1 = torch.tensor([350], dtype=torch.long) + lv1 = torch.tensor([120.0], dtype=torch.float32) + vx1 = torch.tensor([3.2], dtype=torch.float32) + out_virgin = model.forward_cfg( + x1, t1, lv1, guidance_scale=1.5, material_ids=torch.tensor([0]), voxel_sizes=vx1 + ) + out_tio2 = model.forward_cfg( + x1, t1, lv1, guidance_scale=1.5, material_ids=torch.tensor([1]), voxel_sizes=vx1 + ) + assert not torch.allclose(out_virgin, out_tio2) + + +def test_multiphase_electrochemical_metrics(tmp_path: Path) -> None: + vol = np.zeros((16, 16, 16), dtype=np.uint8) + # Z-spanning carbon fiber backbone at y=6..9, x=6..9 + vol[:, 6:10, 6:10] = 1 + # Wired active coating slab attached on x=10 across z=4..11, y=6..9 (32 voxels) + vol[4:12, 6:10, 10] = 2 + # Electrically isolated coating island in pore space (8 voxels) + vol[7:9, 1:3, 1:3] = 2 + + m = evaluate_multiphase_physics(vol, voxel_size=2.0) + assert m["wired_coating_fraction"] == pytest.approx(32.0 / 40.0) + # Exact 8 x 4 = 32 carbon-coating contact faces normalized by (16^3 * 2.0) + assert m["ssa_carbon_coating"] == pytest.approx(32.0 / (4096.0 * 2.0)) + # Exact perimeter of 8 x 4 contact rectangle = 2*(8 + 4) = 24 TPB edges / (16^3 * 2.0^2) + assert m["tpb_density"] == pytest.approx(24.0 / (4096.0 * 4.0)) + assert m["ssa_pore_coating"] > 0.0 + + tif_path = export_openimpala_tiff(vol, tmp_path / "mp.tif", preserve_multiphase=True) + reloaded = tifffile.imread(tif_path) + assert set(np.unique(reloaded)) == {0, 1, 2} + assert np.array_equal(reloaded, vol) + + +def test_end_to_end_beamtime_multiplier_cli(tmp_path: Path) -> None: + drive_root = tmp_path / "synchrotron_drive" + bundle_dir = tmp_path / "multielectrode_bundle" + out_root = tmp_path / "runs" + + rng = np.random.default_rng(2027) + yy, xx = np.indices((64, 64)) + in_circle = ((yy - 32.0) ** 2 + (xx - 32.0) ** 2) <= (27.0**2) + + scans = [ + ("crbon_virgin_1_0p008mVV_", 0.07, 0.00), + ("crbon_virgin_1_0p053mVV_", 0.24, 0.00), + ("tio2_anode_0p010mVV_", 0.10, 0.04), + ("tio2_anode_0p028mVV_", 0.16, 0.06), # Held-out coated load! + ("tio2_anode_0p048mVV_", 0.24, 0.08), + ] + for folder, p_carb, p_coat in scans: + rec = drive_root / folder / "rec_16bit_Paganin" + rec.mkdir(parents=True, exist_ok=True) + vol = (30_000.0 + rng.normal(0.0, 300.0, size=(72, 64, 64))).astype(np.float32) + u = rng.random((72, 64, 64)) + vol[(u < p_carb)] += 8_500.0 + vol[(u >= p_carb) & (u < p_carb + p_coat)] += 19_500.0 + vol_u16 = vol.clip(0, 65535).astype(np.uint16) + for z in range(vol_u16.shape[0]): + slc = vol_u16[z].copy() + slc[~in_circle] = 30_100 + tifffile.imwrite(rec / f"slice_{z:04d}.tif", slc) + + # 1. Fast --preview-only check + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/prepare_local_drive_bundle.py"), + "--drive-root", + str(drive_root), + "--output-dir", + str(bundle_dir), + "--datasets", + "tio2_anode_0p028mVV_", + "--auto-multiphase", + "--sigma", + "10", + "--roi-radius-fraction", + "0.80", + "--preview-only", + ], + check=True, + ) + assert ( + bundle_dir + / "tio2_anode_0p028mVV_binary_patches96_carbon_calibrated_v1" + / "qa" + / "multiphase_overlay_diagnostic.png" + ).is_file() + + # 2. Full 3-phase multi-electrode bundle + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/prepare_local_drive_bundle.py"), + "--drive-root", + str(drive_root), + "--output-dir", + str(bundle_dir), + "--version", + "bm_v1", + "--auto-multiphase", + "--delta-threshold", + "5000", + "--delta-coating-threshold", + "13500", + "--sigma", + "10", + "--roi-radius-fraction", + "0.80", + "--patch-size", + "24", + "--stride-xy", + "12", + "--stride-z", + "8", + "--max-patches", + "8", + "--min-interval-length", + "24", + "--seam-margin", + "2", + "--voxel-size-um", + "1.6", + "--kpa-per-mvv", + "8000", + ], + check=True, + ) + manifest_path = bundle_dir / "bundle_manifest.json" + assert manifest_path.is_file() + + # 3. Train Few-Shot Beamtime Multiplier holding out intermediate `tio2_anode_0p028mVV` + exp = "bm_fewshot_24" + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/training/train_carbon_conditional_96.py"), + "--experiment", + exp, + "--data-root", + str(bundle_dir), + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--exclude-states", + "tio2_anode_0p028mVV", + "--use-stress-kpa", + "--balanced-sampling", + "--version", + "bm_v1", + "--patch-size", + "24", + "--channels", + "16,32", + "--attention-levels", + "0,1", + "--num-res-blocks", + "1", + "--norm-groups", + "8", + "--epochs", + "1", + "--batch-size", + "2", + "--grad-accum", + "1", + "--max-steps", + "3", + "--max-val-steps", + "1", + "--amp", + "off", + "--allow-cpu", + ], + check=True, + ) + + # 4. Sample all states in manifest (including unseen held-out `tio2_anode_0p028mVV`) + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/evaluation/sample_carbon_conditional_96.py"), + "--experiment", + exp, + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--sampler", + "ddim", + "--steps", + "4", + "--num-samples", + "2", + "--batch-size", + "2", + "--sweep-loads", + "", + "--export-openimpala", + "--allow-cpu", + ], + check=True, + ) + + # 5. Benchmark PorousDiff vs Affine Z-Squash & Pillar Pore-Deletion Baselines + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/evaluation/evaluate_beamtime_multiplier.py"), + "--experiments", + f"porousdiff_fewshot={exp}", + "--data-root", + str(bundle_dir), + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--version", + "bm_v1", + "--patch-size", + "24", + "--heldout-states", + "tio2_anode_0p028mVV", + "--export-real-openimpala", + ], + check=True, + ) + + bm_dir = out_root / "results" / exp / "beamtime_benchmark" + bm_report = json.loads((bm_dir / "beamtime_multiplier_report.json").read_text()) + assert bm_report["tio2_anode_0p028mVV"]["is_heldout"] is True + assert set(bm_report["tio2_anode_0p028mVV"]["methods"].keys()) == { + "affine_z_squash", + "pillar_pore_deletion", + "porousdiff_fewshot", + } + assert (bm_dir / "baseline_vs_porousdiff_table.csv").is_file() + assert (bm_dir / "beamtime_multiplier_comparison.png").is_file() + assert (bm_dir / "openimpala_suite" / "openimpala_jobs.json").is_file() + assert (bm_dir / "openimpala_suite" / "run_openimpala_batch.sh").is_file() + + +def _write_synthetic_multiphase_stack( + scan_dir: Path, + n_slices: int, + hw: int, + carbon_prob: float, + coating_prob: float, + rng: np.random.Generator, +) -> None: + import tifffile + + scan_dir.mkdir(parents=True, exist_ok=True) + yy, xx = np.indices((hw, hw)) + c = hw / 2.0 + r = 0.44 * hw + in_circle = ((yy - c) ** 2 + (xx - c) ** 2) <= (r**2) + + vol = (30_000.0 + rng.normal(0.0, 280.0, size=(n_slices, hw, hw))).astype(np.float32) + u = rng.random((n_slices, hw, hw)) + vol[u < carbon_prob] += 8_500.0 + vol[(u >= carbon_prob) & (u < carbon_prob + coating_prob)] += 20_500.0 + vol_u16 = vol.clip(0, 65535).astype(np.uint16) + for z_idx in range(n_slices): + slc = vol_u16[z_idx].copy() + slc[~in_circle] = 30_050 + tifffile.imwrite(scan_dir / f"slice_{z_idx:04d}.tif", slc) + + +def test_nested_discovery_group_calibration_append_and_resume(tmp_path: Path) -> None: + """Verify nested Op001mVV collision safety, shared lowest-load threshold calibration + (immune to high-load bottom plunger), --resume cache reuse, and --append K11 DIAD merge.""" + rng = np.random.default_rng(77) + i12_root = tmp_path / "i12_drive" + diad_root = tmp_path / "diad_drive" + bundle_dir = tmp_path / "unified_bundle" + + # Nested folders with identical leaf names (`Op001mVV`) under `TiO2/` and `CuHCF/`, + # plus high-load `TiO2/Op045mVV` containing a bright solid metal plunger in bottom 30% of Z! + _write_synthetic_multiphase_stack( + i12_root / "TiO2_Anode" / "Op001mVV", + n_slices=52, + hw=56, + carbon_prob=0.14, + coating_prob=0.06, + rng=rng, + ) + high_load_dir = i12_root / "TiO2_Anode" / "Op045mVV" + _write_synthetic_multiphase_stack( + high_load_dir, + n_slices=52, + hw=56, + carbon_prob=0.28, + coating_prob=0.11, + rng=rng, + ) + # Contaminate Z >= 36 (`3N/4 = 39`) of high-load scan with a saturated 62,000-count plunger + import tifffile + + for z_idx in range(36, 52): + p = high_load_dir / f"slice_{z_idx:04d}.tif" + tifffile.imwrite(p, np.full((56, 56), 62_000, dtype=np.uint16)) + + _write_synthetic_multiphase_stack( + i12_root / "CuHCF_Cathode" / "Op001mVV", + n_slices=52, + hw=56, + carbon_prob=0.15, + coating_prob=0.07, + rng=rng, + ) + + base_cmd = [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/prepare_local_drive_bundle.py"), + "--output-dir", + str(bundle_dir), + "--version", + "v03_test", + "--auto-multiphase", + "--delta-threshold", + "4500", + "--sigma", + "10", + "--roi-radius-fraction", + "0.80", + "--patch-size", + "16", + "--stride-xy", + "12", + "--stride-z", + "8", + "--max-patches", + "6", + "--min-interval-length", + "16", + "--seam-margin", + "2", + ] + + subprocess.run( + base_cmd + + [ + "--drive-root", + str(i12_root), + "--voxel-size-um", + "1.6", + "--kpa-per-mvv", + "8000", + "--beamline", + "DLS_I12", + ], + check=True, + ) + + m1 = json.loads((bundle_dir / "bundle_manifest.json").read_text()) + scans_by_state = {sc["state"]: sc for sc in m1["scans"]} + assert "DLS_I12__tio2_anode__Op001mVV" in scans_by_state + assert "DLS_I12__tio2_anode__Op045mVV" in scans_by_state + assert "DLS_I12__cuhcf_cathode__Op001mVV" in scans_by_state + + # Both TiO2 loads MUST share the exact same delta_coating calibrated on Op001mVV (not plunger!) + t_low = scans_by_state["DLS_I12__tio2_anode__Op001mVV"] + t_high = scans_by_state["DLS_I12__tio2_anode__Op045mVV"] + assert t_low["delta_coating"] == t_high["delta_coating"] + assert 9_000.0 < float(t_high["delta_coating"]) < 19_000.0 + assert float(t_low["strain_z"]) == 0.0 + assert float(t_high["strain_z"]) > 0.15 # Plunger slices gated out -> positive axial compression strain + + # Test --resume (should reuse existing patches instantaneously) + res_proc = subprocess.run( + base_cmd + + [ + "--drive-root", + str(i12_root), + "--voxel-size-um", + "1.6", + "--kpa-per-mvv", + "8000", + "--beamline", + "DLS_I12", + "--resume", + ], + check=True, + capture_output=True, + text=True, + ) + assert "Resuming: reusing existing patches" in res_proc.stdout + + # Test --append from a second beamline (K11 DIAD) with Newton load (`50N` on `10 mm` punch) + --auto-calibrate-carbon + _write_synthetic_multiphase_stack( + diad_root / "TiO2" / "scan_50N", + n_slices=52, + hw=56, + carbon_prob=0.20, + coating_prob=0.08, + rng=rng, + ) + subprocess.run( + base_cmd + + [ + "--drive-root", + str(diad_root), + "--voxel-size-um", + "0.54", + "--punch-diameter-mm", + "10.0", + "--beamline", + "DLS_K11_DIAD", + "--auto-calibrate-carbon", + "--append", + ], + check=True, + ) + + m2 = json.loads((bundle_dir / "bundle_manifest.json").read_text()) + merged_states = {sc["state"]: sc for sc in m2["scans"]} + assert len(merged_states) == 4 + diad_sc = merged_states["DLS_K11_DIAD__tio2_anode__scan_50N"] + # 50 N over pi * (5 mm)^2 = 636.62 kPa + assert np.isclose(float(diad_sc["stress_kpa"]), 50.0 / (np.pi * (0.005**2)) * 1e-3, rtol=1e-3) + assert np.isclose(float(diad_sc["effective_voxel_size_um"]), 0.54) + + +def test_openimpala_suite_and_diad_xrd_correlation(tmp_path: Path) -> None: + """Verify OpenImpala output ingestion CLI and K11 DIAD XRD correlation analysis.""" + from porousdiff.metrics import export_openimpala_tiff + from porousdiff.transport_and_xrd import ( + correlate_xrd_with_microstructure, + generate_openimpala_job_suite, + ) + + vol = np.zeros((16, 16, 16), dtype=np.uint8) + vol[:, 4:10, 4:10] = 1 + vol[:, 6:8, 6:8] = 2 + tif_path = tmp_path / "tiffs" / "sample_001.tif" + export_openimpala_tiff(vol, tif_path, preserve_multiphase=True) + + suite_dir = tmp_path / "openimpala_suite" + info = generate_openimpala_job_suite( + [ + { + "method": "real", + "state": "diad_s1", + "material": "tio2_anode", + "beamline": "DLS_K11_DIAD", + "load_value": 100.0, + "tiff_path": tif_path, + "num_phases": 3, + }, + { + "method": "porousdiff", + "state": "diad_s1", + "material": "tio2_anode", + "beamline": "DLS_K11_DIAD", + "load_value": 100.0, + "tiff_path": tif_path, + "num_phases": 3, + }, + ], + suite_dir=suite_dir, + ) + assert info["num_jobs"] == 6 # 2 methods * 3 phases (pore_ionic, carbon_electronic, coating_active) + + # Write mock OpenImpala solver outputs (both JSON and key=value log formats) + jobs = json.loads((suite_dir / "openimpala_jobs.json").read_text())["jobs"] + Path(jobs[0]["output_json"]).write_text( + json.dumps({"d_eff_z": 0.42, "tau_z": 1.65, "volume_fraction": 0.70}) + ) + (suite_dir / "outputs" / f"{jobs[3]['job_id']}.log").write_text( + "volume_fraction = 0.69\nd_eff_z = 0.41\ntortuosity_z = 1.68\n" + ) + + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/evaluation/parse_openimpala_suite.py"), + "--suite-dir", + str(suite_dir), + ], + check=True, + ) + assert (suite_dir / "openimpala_transport_parity.csv").is_file() + assert (suite_dir / "openimpala_transport_parity.png").is_file() + + # Test DIAD XRD correlation across 4 compression states + mock_report: dict[str, object] = {} + xrd_lines = ["state,lattice_strain_pct,fwhm_deg"] + for idx, lv in enumerate([50.0, 150.0, 300.0, 500.0]): + st = f"DLS_K11_DIAD__tio2_s{idx}" + easa_real = 0.30 - 0.04 * idx + easa_pred = 0.295 - 0.039 * idx + tpb_real = 0.12 - 0.015 * idx + tau_real = 1.15 + 0.12 * idx + metrics_real = { + "solid_fraction": {"mean": 0.10 + 0.05 * idx}, + "coating_fraction": {"mean": 0.04 + 0.01 * idx}, + "ssa_pore_coating": {"mean": easa_real}, + "tpb_density": {"mean": tpb_real}, + "wired_coating_fraction": {"mean": 0.85 + 0.03 * idx}, + "geodesic_tortuosity_z": {"mean": tau_real}, + } + metrics_pred = { + "solid_fraction": {"mean": 0.105 + 0.05 * idx}, + "coating_fraction": {"mean": 0.041 + 0.01 * idx}, + "ssa_pore_coating": {"mean": easa_pred}, + "tpb_density": {"mean": tpb_real - 0.002}, + "wired_coating_fraction": {"mean": 0.84 + 0.03 * idx}, + "geodesic_tortuosity_z": {"mean": tau_real + 0.01}, + } + mock_report[st] = { + "material": "tio2_anode", + "load_value": lv, + "real": {"metrics": metrics_real}, + "methods": {"porousdiff": {"metrics": metrics_pred}}, + } + xrd_lines.append(f"tio2_s{idx},{0.05 + 0.08 * idx:.4f},{0.22 + 0.03 * idx:.4f}") + + xrd_csv = tmp_path / "diad_xrd.csv" + xrd_csv.write_text("\n".join(xrd_lines) + "\n") + xrd_out = tmp_path / "xrd_out" + corr = correlate_xrd_with_microstructure(xrd_csv, mock_report, xrd_out) + assert corr["num_matched_states"] == 4 + r_easa = corr["correlations"]["lattice_strain_pct"]["pred_ssa_pore_coating"]["pearson_r"] + assert r_easa < -0.99 + assert (xrd_out / "xrd_microstructure_merged.csv").is_file() + assert (xrd_out / "diad_xrd_microstructure_correlation.png").is_file() + diff --git a/tests/test_metrics_and_e2e.py b/tests/test_metrics_and_e2e.py new file mode 100644 index 0000000..5dfdf71 --- /dev/null +++ b/tests/test_metrics_and_e2e.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest +import tifffile + +from porousdiff.metrics import ( + connectivity_and_percolation, + directional_two_point_correlation, + evaluate_volume_physics, + export_openimpala_tiff, + geodesic_pore_tortuosity, + solid_fraction, + specific_surface_area, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_physics_metrics_exact_identities(tmp_path: Path) -> None: + # 1. Straight open pore channel vs baffled tortuous pore channel vs blocked wall + open_vol = np.zeros((24, 16, 16), dtype=np.uint8) + assert geodesic_pore_tortuosity(open_vol, axis=0) == pytest.approx(1.0) + + baffled = np.zeros((21, 16, 16), dtype=np.uint8) + baffled[7, :12, :] = 1 + baffled[14, 4:, :] = 1 + tau_baffled = geodesic_pore_tortuosity(baffled, axis=0) + assert tau_baffled > 1.15 + + blocked = np.zeros((16, 16, 16), dtype=np.uint8) + blocked[8, :, :] = 1 + assert np.isnan(geodesic_pore_tortuosity(blocked, axis=0)) + conn_blocked = connectivity_and_percolation(blocked) + assert conn_blocked["pore_percolates_z"] == 0.0 + assert conn_blocked["solid_connected_fraction"] == pytest.approx(1.0) + + # 2. Two-point correlation S_2(0) == solid_fraction + rng = np.random.default_rng(99) + rand_vol = (rng.random((24, 24, 24)) < 0.27).astype(np.uint8) + sf = solid_fraction(rand_vol) + s2 = directional_two_point_correlation(rand_vol, max_lag=8) + assert s2["s2_z"][0] == pytest.approx(sf, abs=1e-6) + assert s2["s2_y"][0] == pytest.approx(sf, abs=1e-6) + assert s2["s2_x"][0] == pytest.approx(sf, abs=1e-6) + + ssa = specific_surface_area(rand_vol, voxel_size=2.0) + assert ssa["ssa_total"] > 0.0 + + full = evaluate_volume_physics(baffled, voxel_size=1.5) + assert full["porosity"] == pytest.approx(1.0 - full["solid_fraction"]) + + # 3. OpenImpala multi-page TIFF export + tiff_path = export_openimpala_tiff(baffled, tmp_path / "openimpala.tif") + reloaded = tifffile.imread(tiff_path) + assert reloaded.shape == baffled.shape + assert reloaded.dtype == np.uint8 + assert np.array_equal(reloaded, baffled) + + +def _stage_mini_xct_datasets(raw_root: Path) -> list[str]: + """Create 3 realistic 128-slice x 96 x 96 uint16 XCT stacks (using real XCT crops when present).""" + real_dir = Path("/tmp/compression_data") + real_map = { + "0p008mVV": real_dir / "sample_crbon_virgin_1_0p008mVV.npz", + "0p028mVV": real_dir / "sample_crbon_virgin_1_0p028mVV.npz", + "0p053mVV": real_dir / "sample_crbon_virgin_1_0p053mVV.npz", + } + have_real = all(p.is_file() for p in real_map.values()) + rng = np.random.default_rng(2026) + + yy, xx = np.indices((96, 96)) + in_circle = ((yy - 48.0) ** 2 + (xx - 48.0) ** 2) <= (42.0**2) + + for idx, (state, npz_path) in enumerate(real_map.items()): + ds_dir = raw_root / state / "rec_16bit_Paganin" + ds_dir.mkdir(parents=True, exist_ok=True) + if have_real: + with np.load(npz_path) as d: + # Downsample XY 192->96 to run fast on CPU while preserving real synchrotron intensities + vol = d["crop_3d"][:, ::2, ::2].copy() # (128, 96, 96) + if state == "0p028mVV": + # z >= 15 in 0p028 crop_3d hits the bullseye seam; bounce-tile clean z < 15 + clean = vol[:15] + pair = np.concatenate([clean[:-1], clean[:0:-1]], axis=0) + vol = np.tile(pair, (5, 1, 1))[:128] + else: + # Synthetic fallback for standalone CI + fiber_p = [0.06, 0.19, 0.31][idx] + vol = (30_000.0 + rng.normal(0.0, 400.0, size=(128, 96, 96))).astype(np.float32) + vol[rng.random((128, 96, 96)) < fiber_p] += 9500.0 + vol = vol.clip(0, 65535).astype(np.uint16) + + # Surround circular sample punch with gray exterior air (~30,000) like real 2560^2 XCT slices + for z in range(vol.shape[0]): + slc = vol[z].copy() + slc[~in_circle] = 30_100 + tifffile.imwrite(ds_dir / f"slice_{z:04d}.tif", slc) + + return list(real_map.keys()) + + +def test_end_to_end_cli_pipeline_with_heldout_load_interpolation(tmp_path: Path) -> None: + raw_root = tmp_path / "data" / "raw" + proc_root = tmp_path / "data" / "processed" + res_root = tmp_path / "results" + states = _stage_mini_xct_datasets(raw_root) + + # 1. Segment all 3 compression states consistently + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/segment_carbon_xct.py"), + "--datasets", + *states, + "--data-root", + str(raw_root), + "--output-root", + str(proc_root), + "--version", + "e2e_v1", + "--delta-threshold", + "5500", + "--sigma", + "16", + "--roi-radius-fraction", + "0.82", + "--min-interval-length", + "32", + "--seam-margin", + "2", + ], + check=True, + ) + + mean_sfs = [] + for st in states: + seg_dir = proc_root / f"{st}_binary_corrected_e2e_v1" + valid_meta = json.loads((seg_dir / "valid_slices.json").read_text()) + assert valid_meta["valid_slice_count"] >= 96 + report = json.loads((seg_dir / "segmentation_report.json").read_text()) + mean_sfs.append(report["clean_solid_fraction_mean"]) + assert mean_sfs[0] < mean_sfs[1] < mean_sfs[2] + + # 2. Extract 32^3 Z-stratified patches with buffered train/val split + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/extract_carbon_3d_patches.py"), + "--datasets", + *states, + "--data-root", + str(proc_root), + "--output-root", + str(proc_root), + "--version", + "e2e_v1", + "--patch-size", + "32", + "--stride-xy", + "12", + "--stride-z", + "8", + "--max-patches", + "20", + "--roi-radius-fraction", + "0.82", + "--no-individual-npy", + ], + check=True, + ) + + # 3. Train on outer loads ONLY (0.008 and 0.053), holding out intermediate 0.028 mV/V! + exp = "e2e_cond_32" + train_cmd_base = [ + sys.executable, + str(REPO_ROOT / "scripts/training/train_carbon_conditional_96.py"), + "--experiment", + exp, + "--data-root", + str(proc_root), + "--output-root", + str(res_root), + "--version", + "e2e_v1", + "--states", + "0p008mVV=0.008", + "0p053mVV=0.053", + "--patch-size", + "32", + "--channels", + "16,32,64", + "--num-res-blocks", + "1", + "--norm-groups", + "8", + "--batch-size", + "2", + "--grad-accum", + "1", + "--amp", + "off", + "--allow-cpu", + "--max-steps", + "3", + "--max-val-steps", + "1", + ] + subprocess.run([*train_cmd_base, "--epochs", "1"], check=True) + # Verify exact checkpoint resumption to epoch 2 + subprocess.run([*train_cmd_base, "--epochs", "2", "--resume"], check=True) + + losses_lines = (res_root / "results" / exp / "losses.csv").read_text().strip().splitlines() + assert len(losses_lines) == 3 # header + 2 epochs + assert (res_root / "checkpoints" / exp / "best_val.pth").is_file() + + # 4. Sample all 3 loads (including UNSEEN interpolated 0.028 mV/V) + continuous sweep + OpenImpala TIFF + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/evaluation/sample_carbon_conditional_96.py"), + "--experiment", + exp, + "--output-root", + str(res_root), + "--sampler", + "ddim", + "--steps", + "5", + "--guidance-scale", + "1.5", + "--num-samples", + "2", + "--batch-size", + "2", + "--eval-states", + "0p008mVV=0.008", + "0p028mVV=0.028", + "0p053mVV=0.053", + "--sweep-loads", + "0.008,0.028,0.053", + "--export-openimpala", + "--allow-cpu", + ], + check=True, + ) + + exp_res_dir = res_root / "results" / exp + assert (exp_res_dir / "samples" / "0p028mVV" / "openimpala" / "sample_001.tif").is_file() + assert (exp_res_dir / "samples" / "continuous_sweep" / "sweep_0p0280mVV.npy").is_file() + assert (exp_res_dir / "figures" / "compression_sweep_orthoviews.png").is_file() + + # 5. Run full physics evaluation comparing real vs generated across seen & held-out loads + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/evaluation/evaluate_carbon_physics.py"), + "--experiment", + exp, + "--data-root", + str(proc_root), + "--output-root", + str(res_root), + "--version", + "e2e_v1", + "--patch-size", + "32", + "--states", + "0p008mVV=0.008", + "0p028mVV=0.028", + "0p053mVV=0.053", + "--heldout-states", + "0p028mVV", + "--export-real-openimpala", + "--num-openimpala", + "2", + ], + check=True, + ) + + eval_dir = exp_res_dir / "physics_validation" + phys_report = json.loads((eval_dir / "physics_validation_report.json").read_text()) + assert set(phys_report.keys()) == {"0p008mVV", "0p028mVV", "0p053mVV"} + assert phys_report["0p028mVV"]["is_heldout_interpolation"] is True + assert phys_report["0p008mVV"]["is_heldout_interpolation"] is False + assert (eval_dir / "physics_compression_curves.png").is_file() + assert (eval_dir / "s2_correlation_comparison.png").is_file() + assert (eval_dir / "openimpala_real" / "0p028mVV" / "real_val_001.tif").is_file() + + +def test_prepare_local_drive_bundle_cli_and_zip(tmp_path: Path) -> None: + import zipfile + + drive_root = tmp_path / "dry_samples" + bundle_dir = tmp_path / "porousdiff_upload_bundle" + rng = np.random.default_rng(123) + yy, xx = np.indices((64, 64)) + in_circle = ((yy - 32.0) ** 2 + (xx - 32.0) ** 2) <= (27.0**2) + + folders = [ + ("crbon_virgin_1_0p008mVV_", 0.08), + ("crbon_virgin_1_0p015mVV_rep2_", 0.14), + ("crbon_virgin_1_0p053mVV_", 0.30), + ] + for folder_name, fiber_p in folders: + rec_dir = drive_root / folder_name / "rec_16bit_Paganin" + rec_dir.mkdir(parents=True, exist_ok=True) + vol = (30_000.0 + rng.normal(0.0, 350.0, size=(80, 64, 64))).astype(np.float32) + vol[rng.random((80, 64, 64)) < fiber_p] += 9500.0 + vol_u16 = vol.clip(0, 65535).astype(np.uint16) + for z in range(vol_u16.shape[0]): + slc = vol_u16[z].copy() + slc[~in_circle] = 30_100 + tifffile.imwrite(rec_dir / f"slice_{z:04d}.tif", slc) + + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/preprocessing/prepare_local_drive_bundle.py"), + "--drive-root", + str(drive_root), + "--output-dir", + str(bundle_dir), + "--version", + "bundle_v1", + "--sigma", + "12", + "--roi-radius-fraction", + "0.80", + "--patch-size", + "24", + "--stride-xy", + "12", + "--stride-z", + "8", + "--max-patches", + "10", + "--min-interval-length", + "24", + "--seam-margin", + "2", + "--zip", + ], + check=True, + ) + + primary = json.loads((bundle_dir / "primary_states.json").read_text()) + manifest = json.loads((bundle_dir / "bundle_manifest.json").read_text()) + assert primary == { + "crbon_virgin_1_0p008mVV": 0.008, + "crbon_virgin_1_0p053mVV": 0.053, + } + assert manifest["repeat_and_secondary_states"] == { + "crbon_virgin_1_0p015mVV_rep2": 0.015, + } + p_low = np.load(bundle_dir / "crbon_virgin_1_0p008mVV_binary_patches24_bundle_v1" / "train" / "patches.npy") + p_high = np.load(bundle_dir / "crbon_virgin_1_0p053mVV_binary_patches24_bundle_v1" / "train" / "patches.npy") + assert p_low.shape[1:] == (24, 24, 24) + assert float(p_low.mean()) < float(p_high.mean()) + + zip_path = tmp_path / "porousdiff_upload_bundle.zip" + assert zip_path.is_file() + with zipfile.ZipFile(zip_path, "r") as zf: + names = set(zf.namelist()) + assert "primary_states.json" in names + assert "bundle_manifest.json" in names + diff --git a/tests/test_models_and_training.py b/tests/test_models_and_training.py new file mode 100644 index 0000000..6559c35 --- /dev/null +++ b/tests/test_models_and_training.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import numpy as np +import pytest +import torch + +from porousdiff.data import apply_compression_safe_symmetry, capture_rng_states, restore_rng_states +from porousdiff.metrics import directional_chord_metrics +from porousdiff.models import ( + ContinuousLoadConditionedUNet, + EMAModel, + build_noise_scheduler, +) + + +def test_zero_snr_schedule_and_ddim_ddpm_stepping_finite() -> None: + ddpm = build_noise_scheduler( + sampler="ddpm", + num_train_timesteps=1000, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + assert float(ddpm.alphas_cumprod[-1].item()) == pytest.approx(0.0, abs=1e-10) + assert float(ddpm.alphas_cumprod[0].item()) > 0.99 + + ddim = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=1000, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + ddim.set_timesteps(20) + x = torch.randn(2, 1, 16, 16, 16) + for t in ddim.timesteps: + v_pred = torch.zeros_like(x) + x, x0 = ddim.step(model_output=v_pred, timestep=int(t), sample=x) + assert torch.isfinite(x).all() + assert torch.isfinite(x0).all() + + +def test_continuous_load_conditioned_unet_all_params_get_gradients_and_cfg_distinct() -> None: + torch.manual_seed(0) + model = ContinuousLoadConditionedUNet( + channels=(16, 32, 64), + attention_levels=(False, False, True), + num_res_blocks=1, + norm_num_groups=8, + load_min=0.008, + load_max=0.053, + ) + + # MONAI zero-initializes `out.2.conv` and every `resnet.conv2` (`zero_module`). + # After 2 optimizer steps, both zero-init gates open and every parameter in the + # entire UNet + ContinuousScalarLoadEmbedder + attention blocks must get non-zero gradients. + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3) + for step in range(3): + optimizer.zero_grad() + x = torch.randn(4, 1, 16, 16, 16) + target = torch.randn_like(x) + t = torch.tensor([10, 250, 600, 999], dtype=torch.long) + loads = torch.tensor([0.008, 0.023, 0.0305, 0.053], dtype=torch.float32) + cond_mask = torch.tensor([1.0, 1.0, 0.0, 1.0], dtype=torch.float32) + + out = model(x, t, loads, cond_mask=cond_mask) + assert out.shape == x.shape + loss = torch.nn.functional.mse_loss(out, target) + loss.backward() + if step < 2: + optimizer.step() + + zero_grad_params = [ + name + for name, p in model.named_parameters() + if p.requires_grad and (p.grad is None or float(p.grad.abs().max().item()) == 0.0) + ] + assert zero_grad_params == [], f"Dead parameters with zero gradient: {zero_grad_params}" + + # Verify midpoint load (normalized to 0.0, mask=1) is strictly distinct from unconditional CFG (mask=0) + model.eval() + with torch.no_grad(): + x1 = torch.randn(1, 1, 16, 16, 16) + t1 = torch.tensor([400], dtype=torch.long) + mid_load = torch.tensor([0.5 * (0.008 + 0.053)], dtype=torch.float32) + + out_mid = model(x1, t1, mid_load, cond_mask=torch.ones(1)) + out_uncond = model(x1, t1, mid_load, cond_mask=torch.zeros(1)) + assert not torch.allclose(out_mid, out_uncond) + + # Smooth interpolation check: tiny load perturbation produces tiny output delta + out_near = model(x1, t1, mid_load + 1e-4, cond_mask=torch.ones(1)) + out_far = model(x1, t1, torch.tensor([0.053]), cond_mask=torch.ones(1)) + diff_near = float((out_near - out_mid).abs().mean().item()) + diff_far = float((out_far - out_mid).abs().mean().item()) + assert diff_near < diff_far * 0.05 + + # CFG scale != 1 runs batched conditional + unconditional forward cleanly + out_cfg = model.forward_cfg(x1, t1, mid_load, guidance_scale=1.75) + assert out_cfg.shape == x1.shape + assert not torch.allclose(out_cfg, out_mid) + + +def test_legacy_crossattn_ablation_mode_runs_and_trains() -> None: + torch.manual_seed(3) + legacy = ContinuousLoadConditionedUNet( + channels=(16, 32), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=8, + load_min=0.008, + load_max=0.053, + conditioning_mode="legacy_crossattn", + ) + x = torch.randn(2, 1, 16, 16, 16) + t = torch.tensor([50, 500], dtype=torch.long) + loads = torch.tensor([0.008, 0.053], dtype=torch.float32) + cond_mask = torch.tensor([1.0, 0.0], dtype=torch.float32) + + out = legacy(x, t, loads, cond_mask=cond_mask) + assert out.shape == x.shape + loss = out.pow(2).mean() + loss.backward() + + with torch.no_grad(): + cfg_out = legacy.forward_cfg(x[:1], t[:1], loads[:1], guidance_scale=1.5) + assert cfg_out.shape == (1, 1, 16, 16, 16) + + +def test_ema_and_rng_roundtrip() -> None: + lin = torch.nn.Linear(8, 4) + ema = EMAModel(lin, decay=0.99) + with torch.no_grad(): + lin.weight.add_(5.0) + ema.update(lin) + state = ema.state_dict() + + ema2 = EMAModel(torch.nn.Linear(8, 4), decay=0.5) + ema2.load_state_dict(state) + assert ema2.num_updates == 1 + assert torch.allclose( + ema.shadow_model.weight, ema2.shadow_model.weight + ) + + saved_rng = capture_rng_states() + a1 = torch.randn(5) + n1 = np.random.rand(5) + restore_rng_states(saved_rng) + assert torch.allclose(a1, torch.randn(5)) + assert np.allclose(n1, np.random.rand(5)) + + +def test_compression_safe_symmetry_never_mixes_z_with_xy() -> None: + # Create an anisotropic volume compressed along Z (thin horizontal sheets every 4 Z slices, long in XY) + vol = np.zeros((32, 32, 32), dtype=np.uint8) + vol[2::4, 4:28, 4:28] = 1 + base_metrics = directional_chord_metrics(vol, phase=1) + assert base_metrics["solid_anisotropy_xy_over_z"] > 5.0 + + seen_variants = set() + for sym_id in range(16): + aug = apply_compression_safe_symmetry(vol, sym_id) + assert aug.shape == vol.shape + m = directional_chord_metrics(aug, phase=1) + assert m["solid_chord_z"] == pytest.approx(base_metrics["solid_chord_z"]) + assert m["solid_anisotropy_xy_over_z"] == pytest.approx( + base_metrics["solid_anisotropy_xy_over_z"] + ) + seen_variants.add(aug.tobytes()) + assert len(seen_variants) >= 2 + + +def test_conditional_model_learns_monotonic_load_interpolation() -> None: + """Verify end-to-end learning: training on sparse (0.008) & dense (0.053) loads produces + monotonic generated solid fractions across low, unseen interpolated midpoint, and high loads. + """ + torch.manual_seed(7) + model = ContinuousLoadConditionedUNet( + channels=(8, 16), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=4, + load_min=0.008, + load_max=0.053, + ) + train_sched = build_noise_scheduler( + sampler="ddpm", + num_train_timesteps=200, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + optimizer = torch.optim.AdamW(model.parameters(), lr=3e-3) + + gen = torch.Generator().manual_seed(11) + x_low = (torch.rand((4, 1, 8, 8, 8), generator=gen) < 0.06).float() * 2.0 - 1.0 + x_high = (torch.rand((4, 1, 8, 8, 8), generator=gen) < 0.62).float() * 2.0 - 1.0 + x_all = torch.cat([x_low, x_high], dim=0) + loads_all = torch.tensor([0.008] * 4 + [0.053] * 4, dtype=torch.float32) + + for _ in range(60): + optimizer.zero_grad() + noise = torch.randn_like(x_all) + t = torch.randint(0, 200, (8,)) + noisy = train_sched.add_noise(original_samples=x_all, noise=noise, timesteps=t) + target = train_sched.get_velocity(x_all, noise, t) + cond_mask = (torch.rand(8) > 0.15).float() + pred = model(noisy, t, loads_all, cond_mask=cond_mask) + loss = torch.nn.functional.mse_loss(pred, target) + loss.backward() + optimizer.step() + + model.eval() + ddim = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=200, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + ddim.set_timesteps(15) + init_noise = torch.randn((4, 1, 8, 8, 8), generator=torch.Generator().manual_seed(99)) + + generated_sf: dict[float, float] = {} + with torch.no_grad(): + for load_val in (0.008, 0.0305, 0.053): + cur = init_noise.clone() + lv_tensor = torch.full((4,), load_val, dtype=torch.float32) + for ts in ddim.timesteps: + t_batch = torch.full((4,), int(ts), dtype=torch.long) + v_pred = model.forward_cfg(cur, t_batch, lv_tensor, guidance_scale=1.5) + cur, _ = ddim.step(model_output=v_pred, timestep=int(ts), sample=cur) + generated_sf[load_val] = float((cur > 0.0).float().mean().item()) + + # Strictly monotonic across low -> unseen interpolated midpoint -> high compression load + assert generated_sf[0.008] < generated_sf[0.0305] < generated_sf[0.053] + assert generated_sf[0.053] - generated_sf[0.008] > 0.25 + diff --git a/tests/test_next_level_novelty.py b/tests/test_next_level_novelty.py new file mode 100644 index 0000000..2bfdd39 --- /dev/null +++ b/tests/test_next_level_novelty.py @@ -0,0 +1,1090 @@ +"""Unit and end-to-end CLI tests for the 5 v0.4 scientific upgrades: + 1. Deterministic DDIM ODE virtual compression of real uncompressed 3D scans, + 2. Seamless >patch_size^3 Hann-weighted 3D MultiDiffusion volume stitching for OpenImpala REV, + 3. Physics-informed x0_hat diffusion loss + coating delamination diagnostics, + 4. Optimal cell clamping-pressure (sigma*) discovery coupled with OpenImpala transport, + 5. 3D compressive pinch-point & coating coordination metrics wired into K11 DIAD XRD correlation. +""" + +from __future__ import annotations + +import csv +import json +import subprocess +import sys +from pathlib import Path + +import numpy as np +import torch + +from porousdiff.data import encode_phases_to_channels +from porousdiff.losses import ( + PhysicsInformedDiffusionLoss, + compute_coating_retention_diagnostics, + hierarchical_support_penalty, + reconstruct_x0, +) +from porousdiff.metrics import evaluate_multiphase_physics, summarize_metric_records +from porousdiff.models import ContinuousLoadConditionedUNet, build_noise_scheduler +from porousdiff.optimization import ( + compute_transport_and_fom_record, + find_optimal_clamping_pressure, + load_openimpala_lookup, +) +from porousdiff.transport_and_xrd import correlate_xrd_with_microstructure +from porousdiff.virtual_compression import generate_tiled_3d, virtual_compress_volumes + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + + + +def test_pinch_point_and_solid_tortuosity_metrics(tmp_path: Path) -> None: + vol = np.zeros((16, 16, 16), dtype=np.uint8) + # Through-plane percolating carbon column at (y=8, x=8) + vol[:, 8, 8] = 1 + # Sandwiched coating pinched between z=4 and z=6 carbon voxels + vol[5, 8, 8] = 2 + # Non-sandwiched side coating attached laterally to carbon at (z=10, y=8, x=9) + vol[10, 8, 9] = 2 + + m = evaluate_multiphase_physics(vol, voxel_size=1.5) + assert m["z_pinch_contact_density"] > 0.0 + assert abs(m["coating_pinch_fraction"] - 0.5) < 1e-6 + assert m["mean_coating_coordination"] >= 1.5 + assert np.isfinite(m["geodesic_solid_tortuosity_z"]) + assert abs(m["geodesic_solid_tortuosity_z"] - 1.0) < 1e-6 + + # Verify K11 DIAD XRD correlation automatically correlates pinch-point & solid tortuosity + report: dict[str, object] = {} + xrd_csv = tmp_path / "diad_pinch.csv" + with xrd_csv.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=["state", "fwhm_deg"]) + writer.writeheader() + for idx, pinch_extra in enumerate([0, 1, 2, 3]): + st = f"cuhcf_load_{idx}" + v_i = vol.copy() + for k in range(pinch_extra): + z_pos = 2 + 2 * k + v_i[z_pos - 1 : z_pos + 2, 4, 4] = 1 + v_i[z_pos, 4, 4] = 2 + rec = evaluate_multiphase_physics(v_i, voxel_size=1.5) + summ = summarize_metric_records([rec]) + report[st] = { + "material": "cuhcf_cathode", + "load_value": float(50 * (idx + 1)), + "real": {"metrics": summ}, + "methods": {"porousdiff": {"metrics": summ}}, + } + writer.writerow({"state": st, "fwhm_deg": round(0.12 + 0.03 * idx, 4)}) + + corr_out = correlate_xrd_with_microstructure( + xrd_csv_path=xrd_csv, + benchmark_report=report, + out_dir=tmp_path / "xrd_out", + ) + fwhm_corrs = corr_out["correlations"]["fwhm_deg"] + assert "real_coating_pinch_fraction" in fwhm_corrs + assert "real_z_pinch_contact_density" in fwhm_corrs + assert fwhm_corrs["real_coating_pinch_fraction"]["pearson_r"] > 0.90 + assert (tmp_path / "xrd_out" / "diad_xrd_microstructure_correlation.png").is_file() + + +def test_physics_informed_loss_and_delamination_diagnostics() -> None: + sched = build_noise_scheduler( + sampler="ddpm", + num_train_timesteps=100, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + # Verify exact x0 reconstruction even at Zero-Terminal-SNR endpoint t = T - 1 (alpha_bar == 0) + x0 = torch.randn(2, 2, 8, 8, 8).clamp(-1.0, 1.0) + noise = torch.randn_like(x0) + t_ext = torch.tensor([0, 99], dtype=torch.long) + x_t = sched.add_noise(original_samples=x0, noise=noise, timesteps=t_ext) + v_true = sched.get_velocity(x0, noise, t_ext) + x0_rec = reconstruct_x0( + x_t=x_t, + model_pred=v_true, + timesteps=t_ext, + alphas_cumprod=sched.alphas_cumprod, + prediction_type="v_prediction", + clamp_range=None, + ) + assert torch.allclose(x0_rec, x0, atol=1e-4) + + # Hierarchical support penalty detects floating coating (c_coating > c_solid) + w = torch.ones(2, dtype=torch.float32) + valid_hier = torch.stack([torch.ones(2, 4, 4, 4), -torch.ones(2, 4, 4, 4)], dim=1) + floating_hier = torch.stack([-torch.ones(2, 4, 4, 4), torch.ones(2, 4, 4, 4)], dim=1) + assert float(hierarchical_support_penalty(valid_hier, w).item()) == 0.0 + assert float(hierarchical_support_penalty(floating_hier, w).item()) > 1.5 + + # Full PhysicsInformedDiffusionLoss backward pass with 3-phase UNet + model = ContinuousLoadConditionedUNet( + channels=(8, 16), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=4, + load_min=10.0, + load_max=400.0, + num_phases=3, + num_materials=3, + ) + criterion = PhysicsInformedDiffusionLoss( + alphas_cumprod=sched.alphas_cumprod, + prediction_type="v_prediction", + ) + loads = torch.tensor([25.0, 250.0]) + mats = torch.tensor([1, 2]) + voxs = torch.tensor([1.6, 1.6]) + pred = model(x_t, t_ext, loads, material_ids=mats, voxel_sizes=voxs) + loss, stats = criterion(pred, v_true, x_t, x0, t_ext) + loss.backward() + assert np.isfinite(stats["loss_total"]) + assert all(p.grad is not None and torch.isfinite(p.grad).all() for p in model.parameters() if p.requires_grad) + + # Coating delamination diagnostics + mock_report = { + "tio2_low": { + "material": "tio2_anode", + "beamline": "DLS_I12", + "load_value": 20.0, + "real": {"metrics": {"coating_to_carbon_ratio": {"mean": 0.40}}}, + }, + "tio2_high": { + "material": "tio2_anode", + "beamline": "DLS_I12", + "load_value": 400.0, + "real": {"metrics": {"coating_to_carbon_ratio": {"mean": 0.28}}}, + }, + } + diag = compute_coating_retention_diagnostics(mock_report, spallation_drop_threshold=0.15) + grp = diag["tio2_anode__DLS_I12"] + assert grp["any_spallation_detected"] is True + assert grp["states"]["tio2_high"]["possible_spallation"] is True # type: ignore[index] + + +def test_ddim_virtual_compression_trained_fidelity_and_target_shift() -> None: + from porousdiff.virtual_compression import compute_virtual_compression_fidelity + + torch.manual_seed(17) + np.random.seed(17) + + # Construct two distinct specimens (A: top-left fibre columns; B: bottom-right fibre columns) + # at low load (50 kPa, ~11% solid) and compacted high load (300 kPa, ~36% solid) + vol_a_low = np.zeros((12, 12, 12), dtype=np.uint8) + vol_a_low[:, 1:5, 1:5] = 1 + vol_b_low = np.zeros((12, 12, 12), dtype=np.uint8) + vol_b_low[:, 7:11, 7:11] = 1 + + vol_a_high = vol_a_low.copy() + vol_a_high[2:10, 0:7, 0:7] = 1 + vol_b_high = vol_b_low.copy() + vol_b_high[2:10, 5:12, 5:12] = 1 + + train_vols = np.stack([vol_a_low, vol_b_low, vol_a_high, vol_b_high], axis=0) + x0_batch = torch.stack([encode_phases_to_channels(v, num_phases=2) for v in train_vols], dim=0) + loads_batch = torch.tensor([50.0, 50.0, 300.0, 300.0], dtype=torch.float32) + + model = ContinuousLoadConditionedUNet( + channels=(8, 16), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=4, + load_min=50.0, + load_max=300.0, + num_phases=2, + num_materials=1, + ) + sched = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=60, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + opt = torch.optim.AdamW(model.parameters(), lr=1.5e-3, weight_decay=1e-4) + model.train() + for step in range(240): + opt.zero_grad(set_to_none=True) + noise = torch.randn_like(x0_batch) + t = torch.randint(0, 60, (4,), dtype=torch.long) + mask = (torch.rand(4) >= 0.20).to(torch.float32) + noisy = sched.add_noise(original_samples=x0_batch, noise=noise, timesteps=t) + v_target = sched.get_velocity(x0_batch, noise, t) + pred = model(noisy, t, loads_batch, cond_mask=mask) + loss = torch.nn.functional.mse_loss(pred, v_target) + loss.backward() + opt.step() + + model.eval() + sources = np.stack([vol_a_low, vol_b_low], axis=0) + out_060 = virtual_compress_volumes( + model=model, + alphas_cumprod=sched.alphas_cumprod, + source_volumes=sources, + source_load=50.0, + target_loads=[50.0, 300.0], + prediction_type="v_prediction", + num_steps=20, + invert_strength=0.60, + guidance_scale=2.0, + interface_smooth_sigma=0.0, + ) + out_075 = virtual_compress_volumes( + model=model, + alphas_cumprod=sched.alphas_cumprod, + source_volumes=sources, + source_load=50.0, + target_loads=[50.0, 300.0], + prediction_type="v_prediction", + num_steps=20, + invert_strength=0.75, + guidance_scale=2.0, + interface_smooth_sigma=0.0, + ) + + fid_060 = compute_virtual_compression_fidelity( + source_volumes=sources, + compressed_volumes=out_060[300.0], + same_load_reconstructions=out_060[50.0], + ) + fid_075 = compute_virtual_compression_fidelity( + source_volumes=sources, + compressed_volumes=out_075[300.0], + same_load_reconstructions=out_075[50.0], + ) + sf_low = float((out_060[50.0] >= 1).mean()) + sf_high_060 = float((out_060[300.0] >= 1).mean()) + sf_high_075 = float((out_075[300.0] >= 1).mean()) + + # 1. Same-load roundtrip preserves solid fibres (measured via solid Dice, not sparse background accuracy) + assert fid_060["same_load_solid_dice"] > 0.85, f"Low roundtrip solid Dice: {fid_060}" + assert fid_075["same_load_solid_dice"] > 0.85, f"Low roundtrip solid Dice: {fid_075}" + # 2. Target-load 300 kPa genuinely increases solid fraction vs 50 kPa + assert sf_high_060 > sf_low + 0.015, f"Expected compaction at 0.60: {sf_low:.3f} -> {sf_high_060:.3f}" + assert sf_high_075 > sf_low + 0.030, f"Expected stronger compaction at 0.75: {sf_low:.3f} -> {sf_high_075:.3f}" + # 3. Compressed outputs retain specimen-specific XY structure (A vs B) rather than random resampling + assert fid_060["xy_correlation_lift"] > 0.50, f"Expected specimen-specific XY lift: {fid_060}" + assert fid_075["xy_correlation_lift"] > 0.50, f"Expected specimen-specific XY lift: {fid_075}" + + +def test_seamless_tiled_3d_multidiffusion() -> None: + torch.manual_seed(11) + model = ContinuousLoadConditionedUNet( + channels=(8, 16), + attention_levels=(False, True), + num_res_blocks=1, + norm_num_groups=4, + load_min=0.0, + load_max=500.0, + num_phases=3, + num_materials=3, + ).eval() + sched = build_noise_scheduler( + sampler="ddim", + num_train_timesteps=50, + schedule="zero_snr_scaled_linear", + prediction_type="v_prediction", + ) + sched.set_timesteps(4, device=torch.device("cpu")) + gen = torch.Generator(device="cpu").manual_seed(99) + + phases, probs = generate_tiled_3d( + model=model, + scheduler=sched, + target_shape=(16, 16, 32), + patch_size=16, + load_value=150.0, + guidance_scale=1.2, + overlap=0.5, + batch_size=1, + tile_batch_size=2, + material_id=2, + voxel_size_um=1.6, + generator=gen, + ) + assert phases.shape == (1, 16, 16, 32) + assert probs.shape == (1, 16, 16, 32) + assert np.isfinite(probs).all() + assert set(np.unique(phases)).issubset({0, 1, 2}) + # Verify encoded channels have valid shape too + enc = encode_phases_to_channels(phases[0], num_phases=3) + assert enc.shape == (2, 16, 16, 32) + + +def test_optimal_clamping_pressure_and_full_cli_pipeline(tmp_path: Path) -> None: + # 1. Unit test sweet-spot finder on interior-peaked reaction-transport curve + rng = np.random.default_rng(123) + synthetic_loads = [20.0, 100.0, 200.0, 350.0, 500.0] + recs_by_load: dict[float, list[dict[str, float]]] = {} + for lv in synthetic_loads: + # Ionic D_eff decreases with compression; Electronic sigma_eff rises steeply then saturates; + # Active wired interface peaks near ~200 kPa before over-compression occludes pores. + d_eff = max(0.02, 0.55 * np.exp(-lv / 320.0)) + sig_eff = 0.08 * (1.0 - np.exp(-lv / 95.0)) + act_if = 0.25 * (1.0 - np.exp(-lv / 80.0)) * np.exp(-max(0.0, lv - 180.0) / 450.0) + recs_by_load[lv] = [ + { + "solid_fraction": 0.10 + 0.0005 * lv, + "porosity": 0.90 - 0.0005 * lv, + "d_eff_pore_z": float(d_eff * (1.0 + rng.normal(0.0, 0.03))), + "sigma_eff_solid_z": float(sig_eff * (1.0 + rng.normal(0.0, 0.03))), + "permeability_ck_um2": float(10.0 * d_eff), + "active_wired_interface": float(act_if * (1.0 + rng.normal(0.0, 0.03))), + "wired_coating_fraction": 0.9, + "tpb_density": 0.05, + "coating_pinch_fraction": 0.02 * (lv / 100.0), + "used_openimpala": 1.0, + } + for _ in range(6) + ] + + opt = find_optimal_clamping_pressure(recs_by_load, n_bootstrap=100, seed=42) + assert 120.0 <= float(opt["optimal_load"]) <= 280.0 # type: ignore[arg-type] + ci_lo, ci_hi = opt["optimal_load_ci95"] # type: ignore[misc] + assert ci_lo <= float(opt["optimal_load"]) <= ci_hi # type: ignore[arg-type] + + # 2. End-to-end CLI smoke test across physics-loss training, tiled sampling, + # virtual compression, benchmark with virtual compression, and clamping optimization! + data_root = tmp_path / "data" + out_root = tmp_path / "out" + patch_size = 16 + version = "v04_smoke" + + scan_defs = [ + ("virgin_low", "virgin_carbon", 20.0, 2, 0.12), + ("virgin_mid", "virgin_carbon", 180.0, 2, 0.22), + ("virgin_high", "virgin_carbon", 400.0, 2, 0.34), + ("tio2_low", "tio2_anode", 25.0, 3, 0.15), + ("tio2_mid", "tio2_anode", 200.0, 3, 0.25), + ("tio2_high", "tio2_anode", 420.0, 3, 0.36), + ] + manifest_scans = [] + for st, mat, kpa, n_ph, sf in scan_defs: + pdir_name = f"patches_{st}_{patch_size}_{version}" + for split, n_p in (("train", 4), ("val", 2)): + sdir = data_root / pdir_name / split + sdir.mkdir(parents=True, exist_ok=True) + arr = np.zeros((n_p, patch_size, patch_size, patch_size), dtype=np.uint8) + for i in range(n_p): + v = (rng.random((patch_size, patch_size, patch_size)) < sf).astype(np.uint8) + # Add percolating Z pillar so tortuosities are well-defined + v[:, 4, 4] = 1 + if n_ph == 3: + coat_mask = (v == 1) & (rng.random(v.shape) < 0.30) + v[coat_mask] = 2 + arr[i] = v + np.save(sdir / "patches.npy", arr) + manifest_scans.append( + { + "state": st, + "material": mat, + "beamline": "DLS_I12", + "is_repeat_or_secondary": False, + "load_mv_per_v": round(kpa / 10000.0, 5), + "stress_kpa": kpa, + "effective_voxel_size_um": 3.2, + "num_phases": n_ph, + "patch_dir_name": pdir_name, + } + ) + + manifest_path = data_root / "bundle_manifest.json" + manifest_path.write_text( + json.dumps({"version": version, "patch_size": patch_size, "scans": manifest_scans}, indent=2) + ) + + exp = "novelty_e2e_16" + # Train with --physics-loss + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "training" / "train_carbon_conditional_96.py"), + "--experiment", + exp, + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--version", + version, + "--patch-size", + str(patch_size), + "--channels", + "8,16", + "--attention-levels", + "0,1", + "--num-res-blocks", + "1", + "--norm-groups", + "4", + "--timesteps", + "40", + "--epochs", + "1", + "--batch-size", + "2", + "--grad-accum", + "1", + "--use-stress-kpa", + "--balanced-sampling", + "--physics-loss", + "--allow-cpu", + ], + check=True, + ) + + # Sample standard 16^3 patches for all states + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "sample_carbon_conditional_96.py"), + "--experiment", + exp, + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--steps", + "4", + "--num-samples", + "2", + "--batch-size", + "2", + "--sweep-loads", + "", + "--allow-cpu", + ], + check=True, + ) + + # Sample seamless >16^3 tiled volume into a separate test check via virtual_compress_scan CLI too + input_scan_path = tmp_path / "lab_ct_0kpa.npy" + src_patch = np.load(data_root / f"patches_tio2_low_{patch_size}_{version}" / "val" / "patches.npy")[0] + np.save(input_scan_path, src_patch) + vc_out = tmp_path / "vc_cli_out" + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "virtual_compress_scan.py"), + "--input", + str(input_scan_path), + "--experiment", + exp, + "--output-root", + str(out_root), + "--out-dir", + str(vc_out), + "--source-load", + "25.0", + "--target-loads", + "200.0,420.0", + "--material", + "tio2_anode", + "--steps", + "6", + "--export-openimpala", + "--allow-cpu", + ], + check=True, + ) + assert (vc_out / "virtual_compression_panel.png").is_file() + assert (vc_out / "virtual_compression_metrics.csv").is_file() + + # Benchmark with --include-virtual-compression + bench_dir = out_root / "results" / exp / "beamtime_benchmark" + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "evaluate_beamtime_multiplier.py"), + "--experiments", + f"porousdiff={exp}", + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--version", + version, + "--patch-size", + str(patch_size), + "--heldout-states", + "tio2_mid", + "--include-virtual-compression", + "--virtual-steps", + "6", + "--allow-cpu", + ], + check=True, + ) + bench_report = json.loads((bench_dir / "beamtime_multiplier_report.json").read_text()) + assert "virtual_compression" in bench_report["tio2_mid"]["methods"] + assert (bench_dir / "coating_retention_diagnostics.json").is_file() + + # Mock OpenImpala transport CSV to test seamless OpenImpala ingestion in optimize_clamping_pressure.py + oi_csv = bench_dir / "openimpala_transport_parity.csv" + with oi_csv.open("w", newline="") as f: + writer = csv.DictWriter( + f, fieldnames=["method", "state", "material", "transport_phase", "d_eff_z"] + ) + writer.writeheader() + writer.writerow( + { + "method": "real", + "state": "tio2_mid", + "material": "tio2_anode", + "transport_phase": "pore_ionic", + "d_eff_z": "0.31", + } + ) + writer.writerow( + { + "method": "real", + "state": "tio2_mid", + "material": "tio2_anode", + "transport_phase": "carbon_electronic", + "d_eff_z": "0.09", + } + ) + oi_map = load_openimpala_lookup(oi_csv) + assert abs(oi_map[("real", "tio2_mid")]["pore_d_eff_z"] - 0.31) < 1e-6 + rec_with_oi = compute_transport_and_fom_record( + evaluate_multiphase_physics(src_patch, voxel_size=3.2), + openimpala_metrics=oi_map[("real", "tio2_mid")], + voxel_size_um=3.2, + ) + assert rec_with_oi["used_openimpala"] == 1.0 + + # Run Optimal Clamping Pressure CLI with OpenImpala CSV + dense continuous sweep + opt_dir = out_root / "results" / exp / "clamping_optimization" + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "optimize_clamping_pressure.py"), + "--experiments", + f"porousdiff={exp}", + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(manifest_path), + "--version", + version, + "--patch-size", + str(patch_size), + "--openimpala-csv", + str(oi_csv), + "--dense-sweep-points", + "4", + "--dense-samples-per-load", + "2", + "--steps", + "4", + "--bootstrap", + "30", + "--allow-cpu", + ], + check=True, + ) + opt_summary = json.loads((opt_dir / "optimal_clamping_summary.json").read_text()) + assert "tio2_anode" in opt_summary + assert "porousdiff" in opt_summary["tio2_anode"] + assert "porousdiff_dense" in opt_summary["tio2_anode"] + assert opt_summary["tio2_anode"]["real"]["estimator_mode"] == "geodesic_surrogate" + assert "weight_sensitivity" in opt_summary["tio2_anode"]["porousdiff"] + assert "newman_asr" in opt_summary["tio2_anode"]["porousdiff"] + assert "regime_sensitivity" in opt_summary["tio2_anode"]["porousdiff"]["newman_asr"] + assert (opt_dir / "clamping_optimization_curves.csv").is_file() + assert (opt_dir / "optimal_clamping_pressure.png").is_file() + + +def test_partial_volume_carbon_halo_stripping_preserves_real_tpb() -> None: + from scipy.ndimage import gaussian_filter + from porousdiff.preprocessing import segment_slice_multiphase + + # Synthesize a Paganin/PSF-blurred 3-phase slice with: + # (A) an isolated TiO2 particle in pore space [y=20:32, x=20:32] (no real carbon fibre), + # (B) a real 6-px-wide carbon fibre [y=65:105, x=60:66] with TiO2 coating on its right flank [y=75:95, x=66:73]. + base = np.full((128, 128), 10000.0, dtype=np.float32) + base[20:32, 20:32] = 28000.0 # Isolated bright coating particle in pore + base[65:105, 60:66] = 18000.0 # Real 6-px carbon fibre + base[75:95, 66:73] = 28000.0 # Attached coating deposit creating real 0-1-2 TPBs + blurred = gaussian_filter(base, sigma=1.1) + + seg_raw, _ = segment_slice_multiphase( + blurred, delta_carbon=4000.0, delta_coating=12000.0, sigma=0.0, strip_halo_px=0 + ) + seg_clean, _ = segment_slice_multiphase( + blurred, delta_carbon=4000.0, delta_coating=12000.0, sigma=0.0, strip_halo_px=1 + ) + + # Without cleanup, blur creates a fake phase-1 carbon rind around the isolated coating blob + isolated_raw = seg_raw[15:37, 15:37] + isolated_clean = seg_clean[15:37, 15:37] + assert int((isolated_raw == 1).sum()) > 10 + assert int((isolated_clean == 1).sum()) == 0 + assert int((isolated_clean == 2).sum()) > 50 + + # Real >= 3 px carbon fibre and its 0-1-2 junction stay intact + fibre_clean = seg_clean[65:105, 58:76] + assert int((fibre_clean == 1).sum()) > 180 + assert int((fibre_clean == 2).sum()) > 90 + # Verify genuine 3-phase TPB 2x2 plaquettes exist at the top/bottom coating-fibre-pore corners + found_tpb = False + for y in range(70, 100): + for x in range(58, 74): + patch2x2 = seg_clean[y : y + 2, x : x + 2] + if set(np.unique(patch2x2)) == {0, 1, 2}: + found_tpb = True + break + assert found_tpb + + +def test_thin_scan_auto_xy_sector_split_zero_overlap(tmp_path: Path) -> None: + import tifffile + from porousdiff.preprocessing import enumerate_cylinder_xy_positions, split_xy_positions_with_buffer + from scripts.preprocessing.prepare_local_drive_bundle import stream_bundle_single_scan + + # Verify strict pixel-level non-overlap in split_xy_positions_with_buffer + xy_pts = enumerate_cylinder_xy_positions((128, 128), patch_size=32, stride_xy=16, radius_fraction=0.95) + xy_split = split_xy_positions_with_buffer(xy_pts, patch_size=32, val_fraction=0.25) + assert len(xy_split["train"]) >= 1 and len(xy_split["val"]) >= 1 + for ty, tx in xy_split["train"]: + for vy, vx in xy_split["val"]: + overlap_y = (ty < vy + 32) and (vy < ty + 32) + overlap_x = (tx < vx + 32) and (vx < tx + 32) + assert not (overlap_y and overlap_x) + + # Synthesize a heavily compressed thin scan (only 40 slices < 3 * raw_window = 96) + scan_dir = tmp_path / "CuHCF_450kPa" + scan_dir.mkdir(parents=True) + rng = np.random.default_rng(7) + for z in range(40): + slc = np.full((128, 128), 10000, dtype=np.uint16) + fibres = rng.random((128, 128)) < 0.18 + coat = fibres & (rng.random((128, 128)) < 0.35) + slc[fibres] = 18000 + slc[coat] = 28000 + tifffile.imwrite(scan_dir / f"slice_{z:04d}.tif", slc) + + out_dir = tmp_path / "bundle_thin" + summary = stream_bundle_single_scan( + scan_dir=scan_dir, + output_dir=out_dir, + version="thin_v1", + auto_multiphase=True, + delta_threshold=4000.0, + delta_coating_threshold=12000.0, + sigma=0.0, + roi_radius_fraction=0.95, + patch_size=16, + spatial_bin=2, + stride_xy=8, + stride_z=4, + max_patches=20, + min_interval_length=16, + seam_margin=0, + split_mode="auto", + ) + assert summary["split_mode_used"] == "xy" + pdir = out_dir / str(summary["patch_dir_name"]) + assert (pdir / "train" / "patches.npy").is_file() + assert (pdir / "val" / "patches.npy").is_file() + + +def test_newman_asr_dimensional_sweet_spot_and_regime_sensitivity() -> None: + loads = [0.0, 80.0, 180.0, 320.0, 500.0] + recs_by_load: dict[float, list[dict[str, float]]] = {} + for i, lv in enumerate(loads): + strain_z = min(0.55, lv / 900.0) + d_eff = max(0.015, 0.58 * np.exp(-lv / 240.0)) + sig_eff = 0.01 + 0.14 * (1.0 - np.exp(-lv / 90.0)) + act_if = 0.04 + 0.22 * (1.0 - np.exp(-lv / 70.0)) * np.exp(-max(0.0, lv - 160.0) / 380.0) + recs_by_load[lv] = [ + { + "solid_fraction": 0.12 / (1.0 - strain_z), + "porosity": 1.0 - 0.12 / (1.0 - strain_z), + "strain_z": strain_z, + "d_eff_pore_z": d_eff, + "sigma_eff_solid_z": sig_eff, + "permeability_ck_um2": 18.0 * (d_eff**1.5), + "active_wired_interface": act_if, + "z_pinch_contact_density": 0.0005 * lv, + } + for _ in range(4) + ] + + res = find_optimal_clamping_pressure(recs_by_load, n_bootstrap=40, uncompressed_thickness_cm=0.30) + asr = res["newman_asr"] # type: ignore[index] + assert 40.0 <= float(asr["optimal_load_asr"]) <= 350.0 # type: ignore[index] + assert float(asr["min_asr_ohm_cm2"]) > 0.0 # type: ignore[index] + thick_grid = asr["thickness_cm_grid"] # type: ignore[index] + assert thick_grid[0] > thick_grid[-1] + regimes = asr["regime_sensitivity"] # type: ignore[index] + # Mass-transport-limited cells suffer from pore closure earlier than kinetics-limited cells + assert float(regimes["mass_transport_limited"]["optimal_load_asr"]) <= float( # type: ignore[index] + regimes["kinetics_limited"]["optimal_load_asr"] # type: ignore[index] + ) + + +def test_mixed_unit_manifest_guard_and_openimpala_template_and_tile_symmetry(tmp_path: Path) -> None: + import pytest + from porousdiff.transport_and_xrd import generate_openimpala_job_suite + from porousdiff.virtual_compression import _compute_tile_starts + from scripts.training.train_carbon_conditional_96 import load_manifest_specs + + # 1. Symmetric even MultiDiffusion tile spacing + assert _compute_tile_starts(160, 96, overlap=0.25) == [0, 64] + starts_200 = _compute_tile_starts(200, 96, overlap=0.5) + assert starts_200 == [0, 35, 69, 104] + + # 2. Mixed mechanical unit / multi-beamline guard in load_manifest_specs + bad_manifest = tmp_path / "mixed_uncalibrated.json" + bad_manifest.write_text( + json.dumps( + { + "scans": [ + {"state": "i12_a", "beamline": "DLS_I12", "load_unit": "mv_per_v", "load_mv_per_v": 0.01, "stress_kpa": None}, + {"state": "k11_b", "beamline": "DLS_K11_DIAD", "load_unit": "newton", "load_mv_per_v": 15.0, "stress_kpa": None}, + ] + } + ) + ) + with pytest.raises(ValueError, match="Cannot mix uncalibrated load units"): + load_manifest_specs(bad_manifest, tmp_path, use_stress_kpa=False) + + # Auto-promotion to stress_kpa (including 0.0 kPa!) when calibrated stress_kpa is present + good_manifest = tmp_path / "mixed_calibrated.json" + good_manifest.write_text( + json.dumps( + { + "scans": [ + {"state": "i12_0kpa", "beamline": "DLS_I12", "load_unit": "mv_per_v", "load_mv_per_v": 0.001, "stress_kpa": 0.0}, + {"state": "k11_250kpa", "beamline": "DLS_K11_DIAD", "load_unit": "newton", "load_mv_per_v": 12.0, "stress_kpa": 250.0}, + ] + } + ) + ) + specs, _, _ = load_manifest_specs(good_manifest, tmp_path, use_stress_kpa=False) + assert [s["load_value"] for s in specs] == [0.0, 250.0] + assert all(bool(s["used_stress_kpa"]) for s in specs) + + # 3. Custom OpenImpala user INI template + 3-phase solid_hetero_electronic job + tpl = tmp_path / "custom_tortuosity.ini" + tpl.write_text( + "FILE={tiff_path}\nPHASE={phase}\nLABEL={transport_phase}\nCOND={phase_conductivities}\nOUT={output_prefix}\n" + ) + fake_tif = tmp_path / "sample.tif" + fake_tif.write_bytes(b"II*\x00") + suite = generate_openimpala_job_suite( + tiff_entries=[ + { + "method": "porousdiff", + "state": "tio2_200kpa", + "material": "tio2_anode", + "beamline": "DLS_I12", + "load_value": 200.0, + "tiff_path": fake_tif, + "num_phases": 3, + } + ], + suite_dir=tmp_path / "oi_suite", + template_ini=tpl, + include_hetero_solid=True, + ) + assert suite["num_jobs"] == 4 + hetero_ini = (tmp_path / "oi_suite" / "configs" / "porousdiff__tio2_200kpa__sample__solid_hetero_electronic.ini").read_text() + assert "COND=1:1.0,2:0.01" in hetero_ini + + +def test_peer_review_bugfixes_units_resume_tortuosity_tall_baselines_and_stats(tmp_path: Path) -> None: + import pytest + import tifffile + from porousdiff.baselines import compute_mirror_tiled_fraction + from porousdiff.data import resolve_scan_load_value + from porousdiff.metrics import geodesic_pore_tortuosity + from porousdiff.preprocessing import audit_binning_topology_preservation + from scripts.preprocessing.prepare_local_drive_bundle import stream_bundle_single_scan + from scripts.training.train_carbon_conditional_96 import load_manifest_specs + + # 1. Bug #6: 26-connected Euclidean geodesic tortuosity recovers exact sqrt(2) on a 45-degree pore channel + diag_vol = np.ones((24, 26, 24), dtype=np.uint8) + for z in range(24): + diag_vol[z, z : z + 2, 10:12] = 0 + tau_26 = geodesic_pore_tortuosity(diag_vol, axis=0, connectivity=26) + tau_6 = geodesic_pore_tortuosity(diag_vol, axis=0, connectivity=6) + assert abs(tau_26 - float(np.sqrt(2.0))) < 0.015, f"Expected ~1.4142, got {tau_26}" + assert tau_6 > 1.90 + + # 2. Bug #1 & #2: Strict load-unit resolution (preserves 0.0 kPa; raises on missing stress_kpa; + # reads checkpoint use_stress_kpa=False even when manifest carries stress_kpa!) + assert resolve_scan_load_value({"state": "s0", "stress_kpa": 0.0, "load_mv_per_v": 0.008}, True) == 0.0 + missing_kpa_manifest = tmp_path / "missing_kpa.json" + missing_kpa_manifest.write_text( + json.dumps( + { + "scans": [ + {"state": "s1", "load_mv_per_v": 0.008, "stress_kpa": 68.0}, + {"state": "s2", "load_mv_per_v": 0.033, "stress_kpa": None}, + ] + } + ) + ) + with pytest.raises(ValueError, match="lack calibrated stress_kpa"): + load_manifest_specs(missing_kpa_manifest, tmp_path, use_stress_kpa=True) + + # Train a model WITHOUT --use-stress-kpa (conditioned on mV/V: 0.010, 0.020, 0.033) on a manifest + # that ALSO has stress_kpa (85.0, 170.0, 280.5), then verify evaluate & optimize use mV/V (not kPa!) + rng = np.random.default_rng(99) + data_root = tmp_path / "unit_data" + out_root = tmp_path / "unit_out" + p_sz = 16 + ver = "unit_v1" + mv_scans = [] + for st, mv, sf in [("v_low", 0.010, 0.12), ("v_mid", 0.020, 0.20), ("v_high", 0.033, 0.30)]: + pdir = f"patches_{st}_{p_sz}_{ver}" + for sp, n_p in (("train", 3), ("val", 2)): + sd = data_root / pdir / sp + sd.mkdir(parents=True, exist_ok=True) + arr = (rng.random((n_p, p_sz, p_sz, p_sz)) < sf).astype(np.uint8) + arr[:, :, 4, 4] = 1 + np.save(sd / "patches.npy", arr) + mv_scans.append( + { + "state": st, + "material": "virgin_carbon", + "beamline": "DLS_I12", + "is_repeat_or_secondary": False, + "load_unit": "mv_per_v", + "load_mv_per_v": mv, + "stress_kpa": round(mv * 8500.0, 2), + "effective_voxel_size_um": 1.6, + "num_phases": 2, + "patch_dir_name": pdir, + } + ) + mv_manifest = data_root / "bundle_manifest.json" + mv_manifest.write_text(json.dumps({"version": ver, "patch_size": p_sz, "scans": mv_scans}, indent=2)) + + exp_mv = "mv_unit_check" + base_train_cmd = [ + sys.executable, + str(REPO_ROOT / "scripts" / "training" / "train_carbon_conditional_96.py"), + "--experiment", + exp_mv, + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(mv_manifest), + "--version", + ver, + "--patch-size", + str(p_sz), + "--channels", + "8,16", + "--attention-levels", + "0,1", + "--num-res-blocks", + "1", + "--norm-groups", + "4", + "--timesteps", + "20", + "--epochs", + "1", + "--batch-size", + "2", + "--grad-accum", + "1", + "--exclude-states", + "v_mid", + "--allow-cpu", + ] + subprocess.run(base_train_cmd, check=True) + + # 3. Bug #3: --resume MUST refuse if heldout_states (--exclude-states) or --use-stress-kpa changes! + leaked_cmd = [arg for arg in base_train_cmd if arg not in ("--exclude-states", "v_mid")] + [ + "--epochs", + "2", + "--resume", + ] + res_leak = subprocess.run(leaked_cmd, capture_output=True, text=True) + assert res_leak.returncode != 0 + assert "Refusing --resume" in (res_leak.stderr + res_leak.stdout) + + # Sample & evaluate on manifest that contains BOTH stress_kpa and load_mv_per_v + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "sample_carbon_conditional_96.py"), + "--experiment", + exp_mv, + "--output-root", + str(out_root), + "--manifest-json", + str(mv_manifest), + "--steps", + "4", + "--num-samples", + "2", + "--batch-size", + "2", + "--sweep-loads", + "", + "--allow-cpu", + ], + check=True, + ) + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "evaluate_beamtime_multiplier.py"), + "--experiments", + f"porousdiff={exp_mv}", + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(mv_manifest), + "--version", + ver, + "--patch-size", + str(p_sz), + "--heldout-states", + "v_mid", + "--include-virtual-compression", + "--virtual-steps", + "4", + "--allow-cpu", + ], + check=True, + ) + mv_rep = json.loads( + (out_root / "results" / exp_mv / "beamtime_benchmark" / "beamtime_multiplier_report.json").read_text() + ) + assert mv_rep["v_mid"]["load_unit"] == "mV/V" + assert abs(float(mv_rep["v_mid"]["load_value"]) - 0.020) < 1e-6 + assert mv_rep["v_mid"]["virtual_compression_diagnostics"]["is_extrapolated"] is False + + subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts" / "evaluation" / "optimize_clamping_pressure.py"), + "--experiments", + f"porousdiff={exp_mv}", + "--data-root", + str(data_root), + "--output-root", + str(out_root), + "--manifest-json", + str(mv_manifest), + "--version", + ver, + "--patch-size", + str(p_sz), + "--bootstrap", + "15", + "--allow-cpu", + ], + check=True, + ) + mv_opt = json.loads( + (out_root / "results" / exp_mv / "clamping_optimization" / "optimal_clamping_summary.json").read_text() + ) + assert 0.009 <= float(mv_opt["virgin_carbon"]["porousdiff"]["optimal_load"]) <= 0.034 + + # 4. Bug #4: Tall clean validation columns eliminate baseline mirror tiling (mirror_frac == 0.0) + tall_scan = tmp_path / "Virgin_0p008mVV" + tall_scan.mkdir(parents=True) + for z in range(80): + slc = np.full((96, 96), 10000, dtype=np.uint16) + slc[rng.random((96, 96)) < 0.14] = 18000 + tifffile.imwrite(tall_scan / f"slice_{z:04d}.tif", slc) + tall_sum = stream_bundle_single_scan( + scan_dir=tall_scan, + output_dir=tmp_path / "tall_bundle", + version="tall_v1", + delta_threshold=4000.0, + sigma=0.0, + roi_radius_fraction=0.95, + patch_size=16, + spatial_bin=1, + stride_xy=16, + stride_z=8, + max_patches=12, + min_interval_length=16, + seam_margin=0, + val_fraction=0.45, + ) + assert int(tall_sum["val_tall_depth"]) >= 32 # type: ignore[arg-type] + tall_pdir = tmp_path / "tall_bundle" / str(tall_sum["patch_dir_name"]) / "val" / "tall_patches.npy" + assert tall_pdir.is_file() + assert ( + compute_mirror_tiled_fraction( + source_depth=int(tall_sum["val_tall_depth"]), # type: ignore[arg-type] + output_depth=16, + initial_solid_fraction=0.14, + target_solid_fraction=0.28, + ) + == 0.0 + ) + + # 5. Mixed-estimator guard in find_optimal_clamping_pressure + partial XRD correlation + topology audit + mixed_recs = { + 10.0: [{"d_eff_pore_z": 0.5, "sigma_eff_solid_z": 0.05, "active_wired_interface": 0.1, "used_openimpala": 1.0}], + 50.0: [{"d_eff_pore_z": 0.3, "sigma_eff_solid_z": 0.09, "active_wired_interface": 0.2, "used_openimpala": 0.0}], + } + with pytest.raises(ValueError, match="Refusing to mix OpenImpala"): + find_optimal_clamping_pressure(mixed_recs, n_bootstrap=10) + + # Partial correlation controlling for solid_fraction separates pure sf proxies from true pinch excess + xrd_csv2 = tmp_path / "xrd_partial.csv" + sfs = [0.10, 0.16, 0.22, 0.28, 0.34] + pinch_excess = [1.1, 1.8, 1.3, 2.4, 2.0] # Orthogonal-ish variation to sf + mock_rep2: dict[str, object] = {} + with xrd_csv2.open("w", newline="") as f: + wr = csv.DictWriter(f, fieldnames=["state", "pure_sf_signal", "true_pinch_signal"]) + wr.writeheader() + for i, (sf_i, pe_i) in enumerate(zip(sfs, pinch_excess)): + st_i = f"k11_{i}" + mock_rep2[st_i] = { + "material": "cuhcf_cathode", + "load_value": float(50 * i), + "real": { + "metrics": { + "solid_fraction": {"mean": sf_i}, + "z_pinch_contact_density": {"mean": sf_i * 2.0 + 0.001 * pe_i}, + "specific_z_pinch_excess": {"mean": pe_i}, + } + }, + "methods": {}, + } + wr.writerow( + { + "state": st_i, + "pure_sf_signal": round(10.0 * sf_i, 5), + "true_pinch_signal": round(2.0 * sf_i + 3.0 * pe_i, 5), + } + ) + p_corr = correlate_xrd_with_microstructure(xrd_csv2, mock_rep2, tmp_path / "xrd_partial_out") + assert abs(p_corr["correlations"]["pure_sf_signal"]["real_z_pinch_contact_density"]["partial_pearson_r_given_sf"]) < 0.05 + assert p_corr["correlations"]["true_pinch_signal"]["real_specific_z_pinch_excess"]["partial_pearson_r_given_sf"] > 0.95 + + from porousdiff.preprocessing import downsample_volume_rev + + raw_3d = np.zeros((16, 16, 16), dtype=np.uint8) + raw_3d[:, 6:10, 6:10] = 1 + binned_3d = downsample_volume_rev(raw_3d, factor=2) + topo_audit = audit_binning_topology_preservation(raw_3d, binned_3d) + assert topo_audit["pore_percolation_agreement"] == 1.0 + assert abs(topo_audit["solid_fraction_abs_err"]) < 1e-6 + diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py new file mode 100644 index 0000000..d8a57d9 --- /dev/null +++ b/tests/test_preprocessing.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pytest + +from porousdiff.preprocessing import ( + build_stratified_patch_plan, + calibrate_delta_threshold, + compute_radial_ring_score, + detect_valid_z_intervals, + enumerate_cylinder_xy_positions, + make_circular_mask, + parse_load_value, + segment_slice_calibrated, + split_z_intervals_with_buffer, +) + + +def test_parse_load_value_variants() -> None: + assert parse_load_value("crbon_virgin_1_0p048mVV_") == pytest.approx(0.048) + assert parse_load_value("sample_crbon_virgin_1_0p008mVV") == pytest.approx(0.008) + assert parse_load_value("Op008mVV") == pytest.approx(0.008) + assert parse_load_value("Op033mVV") == pytest.approx(0.033) + with pytest.raises(ValueError): + parse_load_value("unknown_folder_name") + + +def test_circular_roi_and_xy_positions_stay_strictly_inside() -> None: + shape = (256, 256) + r_frac = 0.70 + mask = make_circular_mask(shape, radius_fraction=r_frac) + assert mask[128, 128] + assert not mask[0, 0] + assert not mask[0, 255] + + patch_size = 32 + positions = enumerate_cylinder_xy_positions( + shape_hw=shape, + patch_size=patch_size, + stride_xy=16, + radius_fraction=r_frac, + ) + assert len(positions) > 10 + for y, x in positions: + assert mask[y : y + patch_size, x : x + patch_size].all() + + +def test_calibrated_segmentation_rejects_empty_pore_slice_and_preserves_monotonicity() -> None: + rng = np.random.default_rng(123) + shape = (192, 192) + roi = make_circular_mask(shape, radius_fraction=0.80) + + # Empty pore slice with realistic Paganin radial cupping + synchrotron beam offset + yy, xx = np.indices(shape) + cupping = 600.0 * (((yy - 96) / 96.0) ** 2 + ((xx - 96) / 96.0) ** 2) + empty_slice = (30_000.0 + cupping + rng.normal(0.0, 450.0, size=shape)).astype(np.uint16) + + bin_empty, stats_empty = segment_slice_calibrated( + empty_slice, delta_threshold=5500.0, roi_mask=roi, sigma=24.0 + ) + assert stats_empty["solid_fraction"] < 0.005 + assert bin_empty[~roi].sum() == 0 + + # Increasing fiber density across 3 synthetic compression states (with drifting beam baseline) + fracs = [] + for beam_offset, fiber_prob in [(0.0, 0.06), (1800.0, 0.18), (-1200.0, 0.32)]: + slc = (30_000.0 + beam_offset + cupping + rng.normal(0.0, 450.0, size=shape)).astype( + np.float32 + ) + fibers = rng.random(shape) < fiber_prob + slc[fibers] += 9500.0 + _, st = segment_slice_calibrated( + slc.clip(0, 65535).astype(np.uint16), + delta_threshold=5500.0, + roi_mask=roi, + sigma=24.0, + ) + fracs.append(st["solid_fraction"]) + + assert fracs[0] < fracs[1] < fracs[2] + + # Auto-calibration smoke check + auto_thr = calibrate_delta_threshold([slc.clip(0, 65535).astype(np.uint16)], roi_mask=roi, sigma=24.0) + assert 2000.0 < auto_thr < 8500.0 + + +def test_real_xct_npz_monotonicity_and_ring_rejection_if_present() -> None: + data_dir = Path("/tmp/compression_data") + files = [ + data_dir / "sample_crbon_virgin_1_0p008mVV.npz", + data_dir / "sample_crbon_virgin_1_0p028mVV.npz", + data_dir / "sample_crbon_virgin_1_0p053mVV.npz", + ] + if not all(f.is_file() for f in files): + pytest.skip("Real XCT sample .npz files not staged in /tmp/compression_data") + + crop_fracs = [] + mid_ring_scores = {} + for label, path in zip(["0p008", "0p028", "0p053"], files): + with np.load(path) as d: + crop = d["crop_3d"] # (128, 192, 192) uint16 inside sample core + mid_xy = d["xy_mid"] + mid_ring_scores[label] = compute_radial_ring_score(mid_xy) + # In 0p028, z >= 40 intersects the real mid-stack bullseye stitch band; z < 36 is clean felt + z_limit = 36 if label == "0p028" else crop.shape[0] + slice_sfs = [ + segment_slice_calibrated(crop[z], delta_threshold=5500.0, sigma=24.0)[1][ + "solid_fraction" + ] + for z in range(0, z_limit, 6) + ] + crop_fracs.append(float(np.mean(slice_sfs))) + + # Clean mid-XY slices (0p008, 0p053) pass (< 0.05), while 0p028's bullseye mid-XY slice spikes (> 0.20) + assert mid_ring_scores["0p008"] < 0.05 + assert mid_ring_scores["0p053"] < 0.05 + assert mid_ring_scores["0p028"] > 0.20 + + # Solid fraction must strictly increase with mechanical load: 0.008 < 0.028 < 0.053 mV/V + assert 0.02 < crop_fracs[0] < crop_fracs[1] < crop_fracs[2] < 0.55 + + +def test_ring_artifact_and_plunger_z_interval_gating() -> None: + rng = np.random.default_rng(7) + h, w = 256, 256 + yy, xx = np.indices((h, w)) + rr = np.sqrt((yy - 128.0) ** 2 + (xx - 128.0) ** 2) + + clean_slice = 30_000.0 + rng.normal(0.0, 900.0, size=(h, w)) + bullseye_slice = clean_slice + 2200.0 * np.sin(rr * 0.45) + + assert compute_radial_ring_score(clean_slice) < 0.06 + assert compute_radial_ring_score(bullseye_slice) > 0.25 + + # Simulate a 400-slice stack with a bullseye seam at Z=150..165 and plunger at Z=330..399 + n = 400 + ring_scores = np.full(n, 0.02, dtype=np.float32) + pore_modes = np.full(n, 30_200.0, dtype=np.float32) + jumps = np.full(n, 80.0, dtype=np.float32) + + ring_scores[150:165] = 0.42 + jumps[150:165] = 350.0 + pore_modes[330:] = 42_000.0 + ring_scores[330:] = 0.65 + + valid_mask, intervals = detect_valid_z_intervals( + ring_scores=ring_scores, + pore_modes=pore_modes, + slice_jumps=jumps, + min_interval_length=64, + seam_margin=8, + ) + assert not valid_mask[145:170].any() + assert not valid_mask[325:].any() + assert len(intervals) == 2 + assert intervals[0][1] <= 142 + assert intervals[1][0] >= 173 and intervals[1][1] <= 322 + + +def test_buffered_split_and_stratified_patch_plan_cover_all_z() -> None: + split = split_z_intervals_with_buffer( + intervals=[(0, 500)], + patch_size=64, + val_fraction=0.2, + buffer_slices=64, + ) + train_s, train_e = split["train"][0] + val_s, val_e = split["val"][0] + assert train_e - train_s >= 64 + assert val_e - val_s >= 64 + # Strictly >= patch_size buffer between train and val slabs + assert val_s - train_e >= 64 + + xy_pos = [(y, x) for y in range(0, 256, 64) for x in range(0, 256, 64)] + plan = build_stratified_patch_plan( + z_intervals=[(0, 320)], + xy_positions=xy_pos, + patch_size=64, + stride_z=32, + max_patches=27, + rng=np.random.default_rng(42), + ) + planned_z = [z for z, _ in plan] + # Every Z window from 0 to 256 (9 windows) must be represented, not just the first 2! + assert len(planned_z) == 9 + assert min(planned_z) == 0 + assert max(planned_z) == 256