diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..a0fecdc
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,90 @@
+name: Release
+
+on:
+ push:
+ tags:
+ - "v*.*.*"
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+jobs:
+ build-and-verify:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install build & verification dependencies
+ run: |
+ python -m pip install --upgrade pip build twine ruff pytest
+ python -m pip install -e ".[notebook]"
+
+ - name: Verify lint and unit test suite
+ run: |
+ ruff check braggtrack/ tests/ scripts/
+ pytest tests/ -q
+
+ - name: Build sdist and wheel
+ run: python -m build
+
+ - name: Validate distribution metadata with Twine
+ run: twine check dist/*
+
+ - name: Verify clean wheel installation in isolated venv
+ run: |
+ python -m venv /tmp/release-smoke-venv
+ /tmp/release-smoke-venv/bin/pip install dist/*.whl
+ /tmp/release-smoke-venv/bin/braggtrack-inspect --help
+ /tmp/release-smoke-venv/bin/braggtrack-watch --help
+ /tmp/release-smoke-venv/bin/braggtrack-run --help
+
+ - name: Upload distribution artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ github-release:
+ needs: build-and-verify
+ if: startsWith(github.ref, 'refs/tags/v')
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Create GitHub Release
+ uses: softprops/action-gh-release@v2
+ with:
+ files: dist/*
+ generate_release_notes: true
+
+ publish-pypi:
+ needs: build-and-verify
+ if: startsWith(github.ref, 'refs/tags/v') && vars.ENABLE_PYPI_PUBLISH == 'true'
+ runs-on: ubuntu-latest
+ environment:
+ name: pypi
+ url: https://pypi.org/p/braggtrack
+ permissions:
+ id-token: write
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Publish distribution to PyPI via Trusted Publisher
+ uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fffb35..2c8f50d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,29 @@ All notable changes to BraggTrack will be documented in this file.
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.4.0] - 2026-11-17
+
+### Added
+- **Analytical Poisson + quantization centroid/strain uncertainties & ancestor-referenced kinematics (`braggtrack.segmentation.features`, `braggtrack.tracking.kinematics`, `braggtrack.tracking.assignment`)**:
+ - Emits per-spot `centroid_mu_err`, `centroid_chi_err`, and `centroid_d_err` combining Poisson photon-counting variance and `1/12` voxel quantization variance.
+ - Propagates `strain_err` ($\sigma_\varepsilon$), ancestor-referenced `root_track_id`, `lineage_strain`, and `lineage_strain_err` across `1 -> 2+` fracture splits, plus causal velocities (`rate_mu`, `rate_chi`, `rate_d`, `strain_rate`) and Hungarian runner-up `assignment_margin`.
+- **Operando electrochemistry & external stimulus synchronization (`braggtrack.io.operando_log`, `braggtrack.viz.operando`)**:
+ - Parses `.csv`/`.tsv` and Biologic EC-Lab `.mpt` potentiostat logs (`load_operando_log`, `synchronize_stimulus_to_scans`), computing Savitzky–Golay regularized differential capacity $\mathrm{d}Q/\mathrm{d}V$ and per-grain electro-strain susceptibility `d_strain_d_v` ($\partial\varepsilon/\partial V$).
+ - Added 3-panel publication plot `plot_operando_strain_and_events` and wired `--operando-log` / `--operando-tz-offset` into `braggtrack-report`, `braggtrack-run`, and `report.html`.
+- **Batched `DINOv2` ViT inference & zero-config `darfix` / ESRF BLISS NeXus auto-calibration (`braggtrack.semantic.dino`, `braggtrack.io.nexus`)**:
+ - Added `embed_batch` and `extract_patch_features_batch` across `MultiviewEncoder` and `PatchFeatureEncoder` (with CPU L3-friendly micro-batching), accelerating `braggtrack-embed-dataset`, `compute_fracture_precursors`, and `segment_dino` while guaranteeing bit-identical cosine parity.
+ - Added `NXdata` `@default` / `@signal` / `@axes` (`darfix` processed HDF5) and ESRF BLISS `instrument/positioners` auto-calibration in `discover_primary_volume_path` and `extract_axis_calibration`.
+- **Online causal streaming, Gaussian Process trajectory forecasting & Bayesian "Scout-and-Zoom" DFXM engine (`braggtrack.autonomous`, `braggtrack-watch`)**:
+ - Added `forecast_1d_gp`, `forecast_grain_trajectory_gp`, `evaluate_acquisition`, and `score_candidates_gp` (`gp.py`), combining inverse-variance-weighted linear trend estimation with a heteroscedastic `Matérn-3/2` Gaussian Process (`alpha = sigma_pos**2`) for smooth `(mu, chi, d)` setpoint forecasting and `UCB` / `EI` / `max_variance` Bayesian target ranking.
+ - Added `Bluesky-Adaptive` style `tell()` / `ask()` / `tell_zoom_outcome()` / `report()` methods and $O(1)$ bounded sliding-window incremental tracking (`sliding_window_scans=24`) in `CausalStreamingTracker` (`streaming.py`), maintaining constant per-scan latency across `1,000+` scan operando runs while preserving global track IDs and root strain anchors.
+ - Added staged execution modes (`--mode {replay, shadow, watch, active}`), `--acquisition {ucb, ei, max_variance}`, diffractometer motor soft-limit clamping, electrochemical voltage bounds, per-lineage X-ray dose caps, synchrotron beam-dump suppression, and live HDF5 file-quiescence + `SWMR` polling in `braggtrack-watch`, emitting `bliss_dfxm_zoom.v1` commands (`zoom_triggers.jsonl`) and baseline replay metrics including `mean_setpoint_mae`.
+- **Crystallographic physics & Red-Team hardening (`segmentation`, `semantic`, `tracking`, `viz`, `cli`)**:
+ - Added outer-perimeter background pedestal subtraction (`background_level`, `net_intensity`), gain-aware Poisson counting variance (`detector_gain`), standardized Pearson rocking-curve excess kurtosis (`kurtosis_mu`, `kurtosis_chi`, `kurtosis_d`), dimensionless voxel eigenvalues (`vox_eig_1..3`), and boundary truncation flags (`touches_border`, `touches_rocking_border`) in `extract_instance_table`, plus `float32` classical segmentation memory reduction (`segment_classical`).
+ - Added step-normalized physical-unit invariance across `PositionShapeCost`, `resolve_track_gaps`, and `resolve_splits_and_merges` so calibrated `AxisCalibration` runs (`deg`, `Å`) produce identical trajectories to uncalibrated voxel space, plus `mechanical_split` vs. `phase_split` (`H2 -> H3`) split classification and wall-clock `strain_rate_per_s`.
+ - Added aspect-ratio-preserving square letterbox padding for global `DINOv2` MIP embeddings, strict `--config` key validation in `braggtrack-run`, and Content-Security-Policy (CSP) + HTML entity escaping against DOM XSS in `generate_html_report`.
+- **Ecosystem positioning, autonomous facility bibliography & release workflow (`docs/positioning.md`, `.github/workflows/release.yml`)**:
+ - Added `docs/positioning.md` positioning `BraggTrack` alongside `darfix`, `ImageD11`, `HEXRD`, `BraggNN`, `TrackMate 7`, `LapTrack`, `gpCAM`, `Bluesky-Adaptive`, `ANDiE`, and the Facility Autonomy Levels (FAL) taxonomy, updated `paper.md` and `paper.bib` with verified DOIs (`@GarrigaFerrer2023`, `@Isern2025`, `@Bernier2011`, `@Liu2022`, `@Fukai2023`, `@Ershov2022`, `@Drnec2025`, `@LeHoux2026`, `@Lu2025`, `@LeHouxFAL2026`, `@Szymanski2023`, `@McDannald2022`, `@Noack2021`, `@Maffettone2023`, `@Kalinin2021`), bumped metadata to `0.4.0`, and added tag-triggered sdist/wheel + Twine + GitHub Release / PyPI Trusted Publisher workflow.
+
## [0.3.0] - 2026-11-17
### Added
diff --git a/CITATION.cff b/CITATION.cff
index 8d1e56b..a2f1b07 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -2,21 +2,25 @@ cff-version: 1.2.0
message: "If you use this software, please cite it as below."
title: "BraggTrack"
type: software
-version: 0.3.0
+version: 0.4.0
date-released: "2026-05-16"
license: MIT
repository-code: "https://github.com/BASE-Laboratory/BraggTrack"
url: "https://github.com/BASE-Laboratory/BraggTrack"
abstract: >-
- Semantic 4D kinematics, multi-scan gap closing, and chemomechanical
- fracture tracking for operando X-ray diffraction. BraggTrack segments
- Bragg reflections in 3D reciprocal-space volumes, tracks trajectories
- and split/merge lineage using classical crystallography and
- vision-transformer embeddings, and exports ImageD11, NeXus HDF5, and
+ Semantic 4D kinematics, multi-scan gap closing, chemomechanical fracture
+ tracking, operando electrochemistry synchronization, and online autonomous
+ Dark-Field X-ray Microscopy (DFXM) target selection for operando X-ray
+ diffraction. BraggTrack segments Bragg reflections in 3D reciprocal-space
+ volumes, tracks trajectories and split/merge lineage with analytical
+ uncertainties using classical crystallography and batched vision-transformer
+ embeddings, and exports ImageD11, NeXus HDF5, BLISS zoom triggers, and
interactive HTML 4D lineage reports.
keywords:
- X-ray diffraction
+ - dark-field X-ray microscopy
- operando crystallography
+ - autonomous experiments
- 3D segmentation
- object tracking
- grain fracture
diff --git a/README.md b/README.md
index d61abd5..daf0df6 100644
--- a/README.md
+++ b/README.md
@@ -12,15 +12,16 @@ Semantic 4D kinematics, multi-scan beam-dropout gap closing, grain lineage (`1 -
## Key Capabilities
-- **Beamline-scale NeXus/HDF5 I/O, lazy ROI streaming & crystallographic mapping (`braggtrack.io`)** — Auto-discovers 3D detector cubes across standard NeXus paths, ESRF-ID03 LIMA files (`/entry_0000/ESRF-ID03/pco_nf/data`), and arbitrary HDF5 trees (`auto_discover=True`). Streams `z`-slabs via `LazyVolume` (`otsu_threshold_chunked`, `read_roi`), calibrates `(mu, chi, d)` via `AxisCalibration`, maps transmission geometry `(2theta, eta, d, |Q|)` via `DetectorGeometry`, and exports directly to **`ImageD11` (`.flt`)** and **NeXus `NXprocess` HDF5 (`.nxs`)**.
-- **Stabilised 3D Bragg reflection segmentation (`braggtrack.segmentation`)** — Streaming Otsu foreground masking with cross-scan rolling-median smoothing (`smooth_thresholds`), 3D LoG peak enhancement, robust `h`-maxima seeding, 3D watershed, nearby-basin merging, and physical or voxel-space covariance/eigenvalue extraction. Includes an experimental dense patch-feature + HDBSCAN slice-stitching segmenter (`--method dino`).
-- **Multi-view foundation vision descriptors (`braggtrack.semantic`)** — Orthogonal $(\mu, \chi, d)$ maximum-intensity projections (MIPs) encoded with ungated **`facebook/dinov2-small`** by default (zero HuggingFace login required), drop-in support for **DINOv3** (`facebook/dinov3-vitb16-pretrain-lvd1689m`), or a deterministic zero-GPU `mock` backend for CI.
-- **4D grain kinematics, multi-scan gap closing, split/merge lineage & precursors (`braggtrack.tracking`)** — Vectorised geometry + semantic Hungarian association, Jaqaman-style multi-scan gap closing (`--max-gap`) across transient beam dropouts, per-grain lattice strain ($\Delta d / d_0$) and angular misorientation ($\Delta\mu$, $\Delta\chi$), intensity-partitioned `1 -> 2+` split and `2+ -> 1` merge DAG edges, and persistent multi-scan fracture precursor ranking combining 3D shape anisotropy, covariance spread growth, and spot ROI sub-peak / DINO patch-token dispersion.
-- **Publication figures & offline interactive HTML 4D Explorer (`braggtrack.viz`, `braggtrack-report`)** — Generates Okabe–Ito styled lineage DAGs, strain/misorientation dashboards, precursor heatmaps, 3-view MIP cards, and a single-file interactive `report.html` with embedded base64 MIP thumbnails that opens offline on air-gapped beamline networks.
-- **One-command reproducible pipeline (`braggtrack-run`)** — Runs validation, segmentation, semantic embedding, tracking, kinematics, fracture event analysis, HTML reporting, and `ImageD11`/NeXus export end-to-end, emitting SHA-256 input checksums, stage timings, and `run_manifest.json`.
+- **Beamline-scale NeXus/HDF5 I/O, `darfix`/BLISS auto-calibration & operando sync (`braggtrack.io`)** — Auto-discovers 3D detector cubes across `darfix` `NXdata` (`@default` / `@signal` / `@axes`), ESRF-ID03 LIMA files (`/entry_0000/ESRF-ID03/pco_nf/data`), ESRF BLISS `instrument/positioners`, and arbitrary HDF5 trees (`auto_discover=True`). Streams `z`-slabs via `LazyVolume`, synchronizes external potentiostat / load-frame logs (`CSV`/`TSV` and Biologic EC-Lab `.mpt`, regularized $\mathrm{d}Q/\mathrm{d}V$, $\partial\varepsilon/\partial V$), maps transmission geometry `(2theta, eta, d, |Q|)` via `DetectorGeometry`, and exports directly to **`ImageD11` (`.flt`)** and **NeXus `NXprocess` HDF5 (`.nxs`)**.
+- **Stabilised 3D Bragg reflection segmentation (`braggtrack.segmentation`)** — Streaming Otsu foreground masking with cross-scan rolling-median smoothing (`smooth_thresholds`), 3D LoG peak enhancement, robust `h`-maxima seeding, 3D watershed, nearby-basin merging, analytical Poisson + quantization centroid uncertainties (`centroid_*_err`), and physical or voxel-space covariance/eigenvalue extraction. Includes an experimental batched dense patch-feature + HDBSCAN slice-stitching segmenter (`--method dino`).
+- **Batched multi-view foundation vision descriptors (`braggtrack.semantic`)** — Orthogonal $(\mu, \chi, d)$ maximum-intensity projections (MIPs) encoded in batched forward passes (`embed_batch`, `extract_patch_features_batch`) with ungated **`facebook/dinov2-small`** by default (zero HuggingFace login required), drop-in support for **DINOv3** (`facebook/dinov3-vitb16-pretrain-lvd1689m`), or a deterministic zero-GPU `mock` backend for CI.
+- **4D grain kinematics, uncertainty propagation, split/merge lineage & precursors (`braggtrack.tracking`)** — Vectorised geometry + semantic Hungarian association with runner-up `assignment_margin`, Jaqaman-style multi-scan gap closing (`--max-gap`), local strain ($\Delta d / d_0$), ancestor-referenced `lineage_strain` across splits, propagated $\sigma_\varepsilon$, causal angular/lattice velocities (`rate_mu`, `rate_chi`, `rate_d`, `strain_rate`), intensity-partitioned `1 -> 2+` split and `2+ -> 1` merge DAG edges, and persistent multi-scan fracture precursor ranking.
+- **Online causal streaming & autonomous DFXM "Scout-and-Zoom" (`braggtrack.autonomous`, `braggtrack-watch`)** — Strictly causal trailing-window Otsu streaming tracker (`CausalStreamingTracker`) with $O(1)$-volume-RAM cached ROI precursors, velocity-extrapolated diffractometer setpoint prediction during rapid phase transitions (e.g., Ni92 `H2 -> H3` collapse), `bliss_dfxm_zoom.v1` JSONL trigger emission (`zoom_triggers.jsonl`), and offline causal replay benchmarking against naive baselines.
+- **Publication figures, operando $\mathrm{d}Q/\mathrm{d}V$ dashboards & offline HTML 4D Explorer (`braggtrack.viz`, `braggtrack-report`)** — Generates Okabe–Ito styled lineage DAGs, strain/misorientation dashboards, 3-panel operando electrochemistry + fracture synchronization plots, precursor heatmaps, 3-view MIP cards, and a single-file interactive `report.html` that opens offline on air-gapped beamline networks.
+- **One-command reproducible pipeline (`braggtrack-run`)** — Runs validation, operando stimulus sync, segmentation, semantic embedding, tracking, kinematics, fracture event analysis, HTML reporting, and `ImageD11`/NeXus export end-to-end, emitting SHA-256 input checksums, stage timings, and `run_manifest.json`.
-
+
---
@@ -67,7 +68,7 @@ Produces:
- `artifacts/pipeline/results.nxs` & `artifacts/pipeline/peaks.flt`
- `artifacts/pipeline/run_manifest.json` (SHA-256 input checksums, git SHA, config, stage timings)
-### Stage-by-stage CLI commands (9 Entry Points)
+### Stage-by-stage CLI commands (10 Entry Points)
| CLI Entry Point | Module Equivalent | Purpose |
|---|---|---|
@@ -75,11 +76,12 @@ Produces:
| `braggtrack-validate` | `python -m braggtrack.cli.validate_dataset` | Validate sequence ordering, monotonic timestamps, and axis contracts |
| `braggtrack-segment-synthetic` | `python -m braggtrack.cli.segment_synthetic` | Run fast synthetic 3D segmentation smoke test |
| `braggtrack-segment-dataset` | `python -m braggtrack.cli.segment_dataset` | Segment 3D Bragg spots via chunked Otsu + smoothed thresholds & optional calibration |
-| `braggtrack-embed-dataset` | `python -m braggtrack.cli.embed_dataset` | Stream lazy 3D spot ROIs & compute orthogonal MIP embeddings (`auto`, `mock`, `torch`) |
+| `braggtrack-embed-dataset` | `python -m braggtrack.cli.embed_dataset` | Stream lazy 3D spot ROIs & compute batched MIP embeddings (`auto`, `mock`, `torch`) |
| `braggtrack-track-dataset` | `python -m braggtrack.cli.track_dataset` | Build trajectory DAG (`--max-gap`), export `tracks.csv`, `kinematics.csv`, `.flt`, `.nxs` |
-| `braggtrack-events` | `python -m braggtrack.cli.detect_events` | Resolve `1 -> 2+` splits, `2+ -> 1` merges, and lazy-ROI fracture precursors |
-| `braggtrack-report` | `python -m braggtrack.cli.report_dataset` | Render publication PNG/PDF figures and self-contained interactive `report.html` |
-| `braggtrack-run` | `python -m braggtrack.cli.run_pipeline` | Execute the full pipeline with optional `--config config.json`, `--report`, and exports |
+| `braggtrack-events` | `python -m braggtrack.cli.detect_events` | Resolve `1 -> 2+` splits, `2+ -> 1` merges, and batched lazy-ROI fracture precursors |
+| `braggtrack-report` | `python -m braggtrack.cli.report_dataset` | Render publication PNG/PDF figures (`--operando-log`) and self-contained `report.html` |
+| `braggtrack-watch` | `python -m braggtrack.cli.watch_autonomous` | Live causal streaming watcher (`--mode watch`) & autonomous DFXM replay (`--mode replay`) |
+| `braggtrack-run` | `python -m braggtrack.cli.run_pipeline` | Execute the full pipeline with optional `--config`, `--operando-log`, `--report`, and exports |
---
@@ -162,9 +164,9 @@ python scripts/pre_pr_check.py
## Repository Layout & Contributing
-- **`braggtrack/`** — Installable Python library (`io`, `segmentation`, `semantic`, `tracking`, `viz`, `cli`).
+- **`braggtrack/`** — Installable Python library (`io`, `segmentation`, `semantic`, `tracking`, `autonomous`, `viz`, `cli`).
- **`data/sample_operando/`** — Bundled ESRF-ID03 operando test scans (`scan0001`–`scan0003`).
-- **`docs/`** — [Architecture](docs/architecture.md), [Benchmarks](docs/benchmarks.md), [Coding Practices](docs/coding-practices.md), and publication figures (`docs/figures/`).
+- **`docs/`** — [Ecosystem Positioning & Autonomous DFXM Guide](docs/positioning.md), [Architecture](docs/architecture.md), [Benchmarks](docs/benchmarks.md), [Coding Practices](docs/coding-practices.md), and publication figures (`docs/figures/`).
- **`scripts/`** — 9 acceptance gates, 5-scenario benchmark suite (`benchmark_suite.py`), semantic α/β ablation (`ablation_semantic.py`), and CI runner (`ci_report.py`).
- **`artifacts/`** — Generated pipeline outputs (gitignored; never commit).
diff --git a/braggtrack/__init__.py b/braggtrack/__init__.py
index 14eeacb..6a570d3 100644
--- a/braggtrack/__init__.py
+++ b/braggtrack/__init__.py
@@ -5,6 +5,6 @@
try:
__version__ = version("braggtrack")
except PackageNotFoundError:
- __version__ = "0.3.0.dev0"
+ __version__ = "0.4.0.dev0"
-__all__ = ["io", "segmentation", "semantic", "tracking", "viz"]
+__all__ = ["autonomous", "io", "segmentation", "semantic", "tracking", "viz"]
diff --git a/braggtrack/autonomous/__init__.py b/braggtrack/autonomous/__init__.py
new file mode 100644
index 0000000..ec7d0f8
--- /dev/null
+++ b/braggtrack/autonomous/__init__.py
@@ -0,0 +1,40 @@
+"""Online causal streaming, GP forecasting, and autonomous DFXM "Scout-and-Zoom" engine."""
+
+from .gp import (
+ AcquisitionKind,
+ GPForecast,
+ evaluate_acquisition,
+ forecast_1d_gp,
+ forecast_grain_trajectory_gp,
+ score_candidates_gp,
+)
+from .replay import PolicyReplayMetrics, ReplayBenchmarkResult, evaluate_causal_replay
+from .streaming import CausalStreamingTracker, StreamingStepResult
+from .trigger import (
+ ExecutionMode,
+ ZoomTarget,
+ ZoomTriggerPolicy,
+ clamp_setpoint_bounds,
+ compute_target_geometry,
+ select_zoom_targets,
+)
+
+__all__ = [
+ "AcquisitionKind",
+ "CausalStreamingTracker",
+ "ExecutionMode",
+ "GPForecast",
+ "PolicyReplayMetrics",
+ "ReplayBenchmarkResult",
+ "StreamingStepResult",
+ "ZoomTarget",
+ "ZoomTriggerPolicy",
+ "clamp_setpoint_bounds",
+ "compute_target_geometry",
+ "evaluate_acquisition",
+ "evaluate_causal_replay",
+ "forecast_1d_gp",
+ "forecast_grain_trajectory_gp",
+ "score_candidates_gp",
+ "select_zoom_targets",
+]
diff --git a/braggtrack/autonomous/gp.py b/braggtrack/autonomous/gp.py
new file mode 100644
index 0000000..5bef07d
--- /dev/null
+++ b/braggtrack/autonomous/gp.py
@@ -0,0 +1,277 @@
+"""Gaussian Process trajectory forecasting and Bayesian acquisition for autonomous DFXM.
+
+Implements standard surrogate modelling and active-learning primitives inspired by
+synchrotron, neutron, and microscopy autonomous experimentation frameworks
+(``gpCAM`` — Noack et al., 2021; ``ANDiE`` — McDannald et al., 2022; Kalinin et al., 2021)
+using ``scikit-learn`` and ``scipy`` with zero extra dependencies:
+
+- **Universal Kriging trajectory forecaster** (:func:`forecast_grain_trajectory_gp`):
+ combines weighted linear trend estimation with a heteroscedastic ``Matérn(nu=1.5)``
+ Gaussian Process over residuals using analytical Poisson centroid variances
+ ``alpha = sigma_pos**2`` to forecast smooth ``(mu, chi, d)`` setpoints and posterior
+ ``1-sigma`` uncertainties across uniform or non-uniform timestamps.
+- **Bayesian acquisition functions** (:func:`evaluate_acquisition`, :func:`score_candidates_gp`):
+ ranks candidate grains via Upper Confidence Bound (``ucb``), Expected Improvement (``ei``),
+ or pure Kriging variance exploration (``max_variance``), optionally conditioned on
+ high-resolution DFXM zoom outcomes supplied through ``tell_zoom_outcome``.
+"""
+
+from __future__ import annotations
+
+import math
+import warnings
+from dataclasses import dataclass
+from typing import Any, Literal
+
+import numpy as np
+from scipy.stats import norm
+from sklearn.exceptions import ConvergenceWarning
+from sklearn.gaussian_process import GaussianProcessRegressor
+from sklearn.gaussian_process.kernels import ConstantKernel, Matern
+
+AcquisitionKind = Literal["ucb", "ei", "max_variance"]
+
+
+@dataclass(frozen=True)
+class GPForecast:
+ """Posterior predictive mean and 1-sigma uncertainty for ``(mu, chi, d)`` at ``target_time``."""
+
+ mean: tuple[float, float, float]
+ std: tuple[float, float, float]
+
+
+def forecast_1d_gp(
+ times: np.ndarray,
+ values: np.ndarray,
+ obs_std: np.ndarray,
+ target_time: float,
+ *,
+ min_std_floor: float = 1e-6,
+) -> tuple[float, float]:
+ """Forecast scalar trajectory ``y(target_time)`` via weighted linear trend + Matérn-3/2 GP.
+
+ Universal Kriging formulation (Rasmussen & Williams, Sec. 2.7):
+ detrends ``values`` with inverse-variance-weighted linear regression so steady
+ crystallographic drift (e.g. ``H2 -> H3`` contraction) never mean-reverts toward
+ zero during extrapolation, then fits a heteroscedastic ``Matérn(nu=1.5)`` GP with
+ inhomogeneous noise ``alpha = obs_std**2`` to capture smooth nonlinear curvature.
+ """
+ t_arr = np.asarray(times, dtype=np.float64).ravel()
+ y_arr = np.asarray(values, dtype=np.float64).ravel()
+ s_arr = np.maximum(np.asarray(obs_std, dtype=np.float64).ravel(), float(min_std_floor))
+ n = len(t_arr)
+
+ if n == 0:
+ return 0.0, float(min_std_floor)
+ if n == 1:
+ return float(y_arr[0]), float(max(s_arr[0], min_std_floor))
+
+ # Normalize time axis relative to latest observation using median step size
+ diffs = np.diff(t_arr)
+ pos_diffs = diffs[diffs > 1e-12]
+ dt_scale = float(np.median(pos_diffs)) if len(pos_diffs) > 0 else 1.0
+ t_last = float(t_arr[-1])
+ tau = (t_arr - t_last) / dt_scale
+ tau_star = (float(target_time) - t_last) / dt_scale
+
+ # Weighted linear basis H = [1, tau] with mild ridge on slope when n == 2
+ weights = 1.0 / np.maximum(s_arr * s_arr, 1e-18)
+ H = np.column_stack([np.ones_like(tau), tau])
+ HWH = (H.T * weights) @ H
+ ridge = np.diag([1e-12, 1e-9 if n >= 3 else 1e-6])
+ cov_beta = np.linalg.pinv(HWH + ridge)
+ beta = cov_beta @ ((H.T * weights) @ y_arr)
+
+ h_star = np.array([1.0, tau_star], dtype=np.float64)
+ trend_pred = float(h_star @ beta)
+ residuals = y_arr - (H @ beta)
+
+ # Parametric WLS trend variance; fit Matérn-3/2 residual GP only when residuals exceed noise
+ dof = max(1, n - 2)
+ chi2_red = float(np.sum(weights * residuals * residuals)) / dof if n >= 3 else 1.0
+ param_var = float(h_star @ cov_beta @ h_star)
+ res_scale = float(np.std(residuals))
+
+ if n < 3 or chi2_red <= 1.25 or res_scale <= 1e-12:
+ total_std = max(math.sqrt(max(param_var, 0.0)), float(min_std_floor))
+ return trend_pred, total_std
+
+ y_norm = residuals / res_scale
+ alpha_norm = np.maximum((s_arr / res_scale) ** 2, 1e-6)
+ kernel = ConstantKernel(1.0, (1e-2, 1e2)) * Matern(length_scale=2.0, length_scale_bounds=(0.5, 15.0), nu=1.5)
+ gpr = GaussianProcessRegressor(
+ kernel=kernel,
+ alpha=alpha_norm,
+ optimizer="fmin_l_bfgs_b" if n >= 4 else None,
+ n_restarts_optimizer=0,
+ normalize_y=False,
+ random_state=0,
+ )
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", category=ConvergenceWarning)
+ gpr.fit(tau.reshape(-1, 1), y_norm)
+ r_mean_norm, r_std_norm = gpr.predict(np.array([[tau_star]], dtype=np.float64), return_std=True)
+
+ res_pred = float(r_mean_norm[0]) * res_scale
+ res_var = (float(r_std_norm[0]) * res_scale) ** 2
+ total_std = max(math.sqrt(max(param_var + res_var, 0.0)), float(min_std_floor))
+ return trend_pred + res_pred, total_std
+
+
+def forecast_grain_trajectory_gp(
+ times: np.ndarray,
+ mu_vals: np.ndarray,
+ chi_vals: np.ndarray,
+ d_vals: np.ndarray,
+ mu_errs: np.ndarray,
+ chi_errs: np.ndarray,
+ d_errs: np.ndarray,
+ target_time: float,
+ *,
+ step_floors: tuple[float, float, float] = (1e-6, 1e-6, 1e-6),
+) -> GPForecast:
+ """Forecast 3D ``(mu, chi, d)`` grain setpoint and 1-sigma uncertainty via heteroscedastic GP."""
+ pm, sm = forecast_1d_gp(times, mu_vals, mu_errs, target_time, min_std_floor=0.35 * max(step_floors[0], 1e-12))
+ pc, sc = forecast_1d_gp(times, chi_vals, chi_errs, target_time, min_std_floor=0.35 * max(step_floors[1], 1e-12))
+ pd, sd = forecast_1d_gp(times, d_vals, d_errs, target_time, min_std_floor=0.35 * max(step_floors[2], 1e-12))
+ return GPForecast(mean=(pm, pc, pd), std=(sm, sc, sd))
+
+
+def evaluate_acquisition(
+ mean: np.ndarray,
+ std: np.ndarray,
+ *,
+ kind: AcquisitionKind = "ucb",
+ beta: float = 1.96,
+ best_observed: float = 0.0,
+ xi: float = 0.01,
+) -> np.ndarray:
+ """Evaluate standard Bayesian acquisition scores over posterior ``(mean, std)``."""
+ mu = np.asarray(mean, dtype=np.float64)
+ sig = np.maximum(np.asarray(std, dtype=np.float64), 0.0)
+
+ if kind == "ucb":
+ return mu + float(beta) * sig
+ if kind == "max_variance":
+ return sig
+ if kind == "ei":
+ imp = mu - float(best_observed) - float(xi)
+ safe_sig = np.where(sig > 1e-12, sig, 1.0)
+ z = np.where(sig > 1e-12, imp / safe_sig, 0.0)
+ ei = imp * norm.cdf(z) + safe_sig * norm.pdf(z)
+ return np.where(sig > 1e-12, np.maximum(ei, 0.0), np.maximum(imp, 0.0))
+ raise ValueError(f"Unsupported acquisition kind: {kind!r}. Expected 'ucb', 'ei', or 'max_variance'.")
+
+
+def _standardize_features(X: np.ndarray) -> np.ndarray:
+ if len(X) <= 1:
+ return np.zeros_like(X, dtype=np.float64)
+ med = np.median(X, axis=0)
+ mad = 1.4826 * np.median(np.abs(X - med), axis=0)
+ scale = np.where(mad > 1e-9, mad, np.maximum(np.std(X, axis=0), 1.0))
+ return (X - med) / scale
+
+
+def score_candidates_gp(
+ candidate_records: list[dict[str, Any]],
+ *,
+ acquisition: AcquisitionKind = "ucb",
+ beta: float = 1.96,
+ xi: float = 0.01,
+ lead_scans: float = 1.0,
+ zoom_outcomes: dict[int, tuple[np.ndarray, float]] | None = None,
+) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
+ """Compute GP posterior ``(mean, std, acquisition_score)`` across candidate grains.
+
+ Each entry in ``candidate_records`` provides:
+ - ``score_times`` / ``score_history``: causal precursor score time series for that grain.
+ - ``feature_vec``: ``[abs_lineage_strain, abs_strain_rate, log_anisotropy, subpeak_norm, prec_score]``.
+
+ When ``zoom_outcomes`` (from ``tell_zoom_outcome``) has ``>= 1`` measured high-resolution
+ DFXM observations, a ``Matérn(nu=2.5)`` GP in descriptor space blends real DFXM feedback
+ into the posterior mean and epistemic uncertainty.
+ """
+ n_cand = len(candidate_records)
+ if n_cand == 0:
+ empty = np.zeros(0, dtype=np.float64)
+ return empty, empty, empty
+
+ temp_mu = np.zeros(n_cand, dtype=np.float64)
+ temp_std = np.zeros(n_cand, dtype=np.float64)
+ feats = np.vstack([np.asarray(rec["feature_vec"], dtype=np.float64) for rec in candidate_records])
+
+ for i, rec in enumerate(candidate_records):
+ s_times = np.asarray(rec["score_times"], dtype=np.float64)
+ s_vals = np.asarray(rec["score_history"], dtype=np.float64)
+ if len(s_vals) == 0:
+ temp_mu[i] = float(rec.get("prec_score", 0.0))
+ temp_std[i] = 0.15
+ elif len(s_vals) == 1:
+ temp_mu[i] = float(s_vals[0])
+ temp_std[i] = 0.12
+ else:
+ diffs = np.diff(s_times)
+ dt = float(np.median(diffs[diffs > 0])) if np.any(diffs > 0) else 1.0
+ noise_est = np.full_like(s_vals, 0.05, dtype=np.float64)
+ pm, ps = forecast_1d_gp(
+ s_times,
+ s_vals,
+ noise_est,
+ float(s_times[-1]) + float(lead_scans) * dt,
+ min_std_floor=0.02,
+ )
+ temp_mu[i] = float(np.clip(pm, 0.0, 1.5))
+ temp_std[i] = float(np.clip(ps, 0.02, 0.50))
+
+ # Cross-grain epistemic uncertainty in standardized descriptor space
+ if zoom_outcomes and len(zoom_outcomes) >= 1:
+ obs_X = np.vstack([v[0] for v in zoom_outcomes.values()])
+ obs_y = np.asarray([v[1] for v in zoom_outcomes.values()], dtype=np.float64)
+ all_X = np.vstack([obs_X, feats])
+ all_X_std = _standardize_features(all_X)
+ obs_X_std = all_X_std[: len(obs_X)]
+ cand_X_std = all_X_std[len(obs_X) :]
+
+ kernel = ConstantKernel(1.0, (1e-2, 1e2)) * Matern(length_scale=1.5, length_scale_bounds=(0.2, 10.0), nu=2.5)
+ gpr = GaussianProcessRegressor(
+ kernel=kernel,
+ alpha=0.02,
+ optimizer="fmin_l_bfgs_b" if len(obs_X) >= 3 else None,
+ normalize_y=len(obs_X) >= 2,
+ random_state=0,
+ )
+ with warnings.catch_warnings():
+ warnings.simplefilter("ignore", category=ConvergenceWarning)
+ gpr.fit(obs_X_std, obs_y)
+ ens_mu, ens_std = gpr.predict(cand_X_std, return_std=True)
+
+ post_mu = 0.5 * temp_mu + 0.5 * np.asarray(ens_mu, dtype=np.float64)
+ post_std = np.sqrt(temp_std**2 + np.maximum(ens_std, 0.02) ** 2)
+ best_obs = float(np.max(obs_y))
+ else:
+ cand_X_std = _standardize_features(feats)
+ if n_cand >= 2:
+ # Matérn-like kernel similarity to ensemble peers: isolated outlier grains carry higher epistemic std
+ sq_dists = np.sum((cand_X_std[:, None, :] - cand_X_std[None, :, :]) ** 2, axis=-1)
+ dists = np.sqrt(np.maximum(sq_dists, 0.0))
+ k_mat = (1.0 + math.sqrt(3.0) * dists / 2.0) * np.exp(-math.sqrt(3.0) * dists / 2.0)
+ np.fill_diagonal(k_mat, 0.0)
+ mean_peer_sim = np.sum(k_mat, axis=1) / max(1, n_cand - 1)
+ epistemic_std = 0.12 * np.sqrt(np.clip(1.0 - mean_peer_sim, 0.0, 1.0))
+ else:
+ epistemic_std = np.full(n_cand, 0.05, dtype=np.float64)
+
+ post_mu = temp_mu
+ post_std = np.sqrt(temp_std**2 + epistemic_std**2)
+ # Baseline reference for EI when no zoom outcomes exist yet: median precursor level of ensemble
+ best_obs = float(np.median(post_mu)) if n_cand >= 2 else 0.25
+
+ acq_scores = evaluate_acquisition(
+ post_mu,
+ post_std,
+ kind=acquisition,
+ beta=beta,
+ best_observed=best_obs,
+ xi=xi,
+ )
+ return post_mu, post_std, acq_scores
diff --git a/braggtrack/autonomous/replay.py b/braggtrack/autonomous/replay.py
new file mode 100644
index 0000000..5f7e316
--- /dev/null
+++ b/braggtrack/autonomous/replay.py
@@ -0,0 +1,356 @@
+"""Closed-loop retrospective causal replay benchmark for autonomous DFXM zoom.
+
+Replays a recorded 4D operando diffraction dataset strictly scan-by-scan through
+:class:`CausalStreamingTracker` and evaluates whether GP-guided zoom triggers fire
+*before* actual ``1 -> 2+`` grain fracture events, whether GP-extrapolated motor
+setpoints accurately center the grain at ``t + 1`` (reporting physical setpoint MAE
+alongside hit rate), and how ``BraggTrack`` compares against naive beamline trigger
+baselines (``random``, ``brightest_intensity``, and ``strain_only``) under an
+identical zoom acquisition budget.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from braggtrack.tracking.kinematics import compute_grain_kinematics
+from braggtrack.tracking.precursors import PrecursorConfig
+
+from .streaming import CausalStreamingTracker, StreamingStepResult
+from .trigger import ZoomTarget, ZoomTriggerPolicy, clamp_setpoint_bounds, compute_target_geometry
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.models import AxisCalibration
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.semantic.dino import BackendName
+
+
+@dataclass(frozen=True)
+class PolicyReplayMetrics:
+ """Quantitative closed-loop zoom performance for one trigger policy."""
+
+ policy_name: str
+ n_zooms_fired: int
+ n_fracture_lineages_total: int
+ n_fracture_lineages_captured_pre_split: int
+ fracture_capture_recall: float
+ fracture_capture_precision: float
+ mean_lead_scans_before_split: float
+ mean_lead_voltage_mv: float | None
+ setpoint_hit_rate_next_scan: float
+ static_centroid_hit_rate_next_scan: float
+ mean_setpoint_mae: tuple[float, float, float] | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "policy_name": self.policy_name,
+ "n_zooms_fired": int(self.n_zooms_fired),
+ "n_fracture_lineages_total": int(self.n_fracture_lineages_total),
+ "n_fracture_lineages_captured_pre_split": int(self.n_fracture_lineages_captured_pre_split),
+ "fracture_capture_recall": round(float(self.fracture_capture_recall), 4),
+ "fracture_capture_precision": round(float(self.fracture_capture_precision), 4),
+ "mean_lead_scans_before_split": round(float(self.mean_lead_scans_before_split), 3),
+ "mean_lead_voltage_mv": (
+ round(float(self.mean_lead_voltage_mv), 2) if self.mean_lead_voltage_mv is not None else None
+ ),
+ "setpoint_hit_rate_next_scan": round(float(self.setpoint_hit_rate_next_scan), 4),
+ "static_centroid_hit_rate_next_scan": round(float(self.static_centroid_hit_rate_next_scan), 4),
+ "mean_setpoint_mae": (
+ [round(float(x), 6) for x in self.mean_setpoint_mae] if self.mean_setpoint_mae is not None else None
+ ),
+ }
+
+
+@dataclass(frozen=True)
+class ReplayBenchmarkResult:
+ """Complete causal replay trajectory and baseline comparison table."""
+
+ n_scans: int
+ mean_step_latency_sec: float
+ max_step_latency_sec: float
+ steps: list[StreamingStepResult]
+ zoom_targets: list[ZoomTarget]
+ policy_comparison: dict[str, PolicyReplayMetrics]
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "n_scans": int(self.n_scans),
+ "mean_step_latency_sec": round(float(self.mean_step_latency_sec), 4),
+ "max_step_latency_sec": round(float(self.max_step_latency_sec), 4),
+ "n_zoom_targets": len(self.zoom_targets),
+ "zoom_targets": [t.to_bliss_command() for t in self.zoom_targets],
+ "policy_comparison": {k: v.to_dict() for k, v in self.policy_comparison.items()},
+ "steps": [s.to_dict() for s in self.steps],
+ }
+
+
+def _score_trigger_events(
+ policy_name: str,
+ triggers: list[tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]],
+ *,
+ split_scan_by_root: dict[int, int],
+ obs_by_scan_and_root: dict[tuple[int, int], list[tuple[float, float, float]]],
+ voltage_by_scan: dict[int, float],
+) -> PolicyReplayMetrics:
+ """Evaluate a list of ``(scan_idx, root_tid, cur_c, pred_c, half_w)`` zoom actions."""
+ n_frac_total = len(split_scan_by_root)
+ captured_roots: dict[int, int] = {}
+ pre_split_hits = 0
+
+ next_scan_eval_count = 0
+ extrap_hits = 0
+ static_hits = 0
+ abs_errs: list[tuple[float, float, float]] = []
+
+ for trig_scan, root_tid, cur_c, pred_c, half_w in triggers:
+ sp_scan = split_scan_by_root.get(root_tid)
+ if sp_scan is not None and trig_scan < sp_scan:
+ pre_split_hits += 1
+ if root_tid not in captured_roots or trig_scan < captured_roots[root_tid]:
+ captured_roots[root_tid] = trig_scan
+
+ next_obs = obs_by_scan_and_root.get((trig_scan + 1, root_tid), [])
+ if next_obs:
+ next_scan_eval_count += 1
+ hm, hc, hd = half_w
+ pm, pc, pd = pred_c
+ cm, cc, cd = cur_c
+ if any(abs(m - pm) <= hm and abs(c - pc) <= hc and abs(d - pd) <= hd for m, c, d in next_obs):
+ extrap_hits += 1
+ if any(abs(m - cm) <= hm and abs(c - cc) <= hc and abs(d - cd) <= hd for m, c, d in next_obs):
+ static_hits += 1
+ closest = min(next_obs, key=lambda o: (o[0] - pm) ** 2 + (o[1] - pc) ** 2 + (o[2] - pd) ** 2)
+ abs_errs.append((abs(closest[0] - pm), abs(closest[1] - pc), abs(closest[2] - pd)))
+
+ lead_scans = [split_scan_by_root[r] - s for r, s in captured_roots.items()]
+ lead_mv: list[float] = []
+ for r, s in captured_roots.items():
+ sp = split_scan_by_root[r]
+ if s in voltage_by_scan and sp in voltage_by_scan:
+ lead_mv.append(abs(voltage_by_scan[sp] - voltage_by_scan[s]) * 1000.0)
+
+ n_fired = len(triggers)
+ recall = (len(captured_roots) / n_frac_total) if n_frac_total > 0 else 0.0
+ precision = (pre_split_hits / n_fired) if n_fired > 0 else 0.0
+ mean_lead = float(np.mean(lead_scans)) if lead_scans else 0.0
+ mean_mv = float(np.mean(lead_mv)) if lead_mv else None
+ extrap_rate = (extrap_hits / next_scan_eval_count) if next_scan_eval_count > 0 else 0.0
+ static_rate = (static_hits / next_scan_eval_count) if next_scan_eval_count > 0 else 0.0
+ mae = (
+ (
+ float(np.mean([e[0] for e in abs_errs])),
+ float(np.mean([e[1] for e in abs_errs])),
+ float(np.mean([e[2] for e in abs_errs])),
+ )
+ if abs_errs
+ else None
+ )
+
+ return PolicyReplayMetrics(
+ policy_name=policy_name,
+ n_zooms_fired=n_fired,
+ n_fracture_lineages_total=n_frac_total,
+ n_fracture_lineages_captured_pre_split=len(captured_roots),
+ fracture_capture_recall=recall,
+ fracture_capture_precision=precision,
+ mean_lead_scans_before_split=mean_lead,
+ mean_lead_voltage_mv=mean_mv,
+ setpoint_hit_rate_next_scan=extrap_rate,
+ static_centroid_hit_rate_next_scan=static_rate,
+ mean_setpoint_mae=mae,
+ )
+
+
+def evaluate_causal_replay(
+ scans: Sequence[np.ndarray | str | Path],
+ *,
+ scan_names: Sequence[str] | None = None,
+ stimuli: Sequence[ScanStimulusPoint | None] | None = None,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ backend: BackendName | None = "mock",
+ precursor_config: PrecursorConfig | None = None,
+ trigger_policy: ZoomTriggerPolicy | None = None,
+ zoom_budget: int | None = None,
+ seed: int = 0,
+) -> ReplayBenchmarkResult:
+ """Run strictly causal scan-by-scan replay and benchmark against naive trigger baselines."""
+ pol = trigger_policy or ZoomTriggerPolicy()
+ tracker = CausalStreamingTracker(
+ backend=backend,
+ calibration=calibration,
+ detector=detector,
+ precursor_config=precursor_config,
+ trigger_policy=pol,
+ )
+
+ steps: list[StreamingStepResult] = []
+ causal_pools: list[
+ list[
+ tuple[
+ int,
+ float,
+ float,
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ ]
+ ]
+ ] = []
+
+ for idx, item in enumerate(scans):
+ s_name = scan_names[idx] if scan_names is not None and idx < len(scan_names) else None
+ stim = stimuli[idx] if stimuli is not None and idx < len(stimuli) else None
+ res = tracker.step(item, scan_name=s_name, stimulus=stim)
+ steps.append(res)
+
+ kin_by_tid = {k.track_id: k for k in tracker.kinematics}
+ pool_t: list[
+ tuple[
+ int,
+ float,
+ float,
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ ]
+ ] = []
+ for obs in tracker._window_rows:
+ if int(obs["scan_idx"]) != idx:
+ continue
+ role = str(obs.get("lineage_role", ""))
+ if pol.exclude_post_split_daughters and role in ("split_child", "split_daughter"):
+ continue
+ tid = int(obs["track_id"])
+ kin = kin_by_tid.get(tid)
+ if kin is None or idx not in kin.scan_indices:
+ continue
+ pos = kin.scan_indices.index(idx)
+ if (pos + 1) < pol.min_track_age:
+ continue
+ root_tid = int(kin.root_track_id) if kin.root_track_id > 0 else tid
+ inten = float(obs.get("integrated_intensity", 0.0))
+ if inten < pol.min_intensity:
+ continue
+ l_strain = abs(float(kin.lineage_strain[pos])) if len(kin.lineage_strain) > pos else 0.0
+ cur_c, pred_c, half_w, _, _, _ = compute_target_geometry(
+ obs,
+ kin,
+ pos,
+ lead_scans=pol.lead_scans,
+ window_sigma=pol.window_sigma,
+ calibration=tracker.calibration,
+ detector=detector,
+ max_extrapolation_steps=pol.max_extrapolation_steps,
+ scan_elapsed_s=tracker._scan_elapsed_s if tracker._scan_elapsed_s else None,
+ use_gp=pol.use_gp_extrapolation,
+ )
+ pred_c, half_w, _ = clamp_setpoint_bounds(
+ pred_c,
+ half_w,
+ mu_bounds=pol.mu_bounds,
+ chi_bounds=pol.chi_bounds,
+ d_bounds=pol.d_bounds,
+ )
+ pool_t.append((root_tid, inten, l_strain, cur_c, pred_c, half_w))
+ causal_pools.append(pool_t)
+
+ full_table = tracker.track_table
+ final_kin = compute_grain_kinematics(full_table) if full_table else []
+ final_kin_by_tid = {k.track_id: k for k in final_kin}
+ split_scan_by_root: dict[int, int] = {}
+ for ev in tracker.graph.graph.get("lineage_events", []):
+ if ev.event_type == "split":
+ for ptid in ev.parent_track_ids:
+ pk = final_kin_by_tid.get(int(ptid))
+ rtid = int(pk.root_track_id) if pk is not None and pk.root_track_id > 0 else int(ptid)
+ if rtid not in split_scan_by_root or ev.scan_idx_to < split_scan_by_root[rtid]:
+ split_scan_by_root[rtid] = int(ev.scan_idx_to)
+
+ obs_by_scan_and_root: dict[tuple[int, int], list[tuple[float, float, float]]] = {}
+ for obs in full_table:
+ s_i = int(obs["scan_idx"])
+ tid = int(obs["track_id"])
+ pk = final_kin_by_tid.get(tid)
+ rtid = int(pk.root_track_id) if pk is not None and pk.root_track_id > 0 else tid
+ obs_by_scan_and_root.setdefault((s_i, rtid), []).append(
+ (float(obs["centroid_mu"]), float(obs["centroid_chi"]), float(obs["centroid_d"]))
+ )
+
+ voltage_by_scan: dict[int, float] = {}
+ if stimuli is not None:
+ for idx, stim in enumerate(stimuli):
+ if stim is not None and stim.voltage_v is not None:
+ voltage_by_scan[idx] = float(stim.voltage_v)
+
+ bt_targets = list(tracker.all_zoom_targets)
+ if zoom_budget is not None and zoom_budget >= 0:
+ bt_targets = bt_targets[:zoom_budget]
+
+ effective_budget = len(bt_targets) if zoom_budget is None else max(0, int(zoom_budget))
+
+ bt_tuples = [
+ (t.scan_idx, t.root_track_id, t.current_centroid, t.predicted_setpoint, t.scan_half_widths) for t in bt_targets
+ ]
+
+ rng = np.random.default_rng(seed)
+
+ def _run_baseline(
+ mode: str,
+ ) -> list[tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]]:
+ fired: list[
+ tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]
+ ] = []
+ cd: dict[int, int] = {}
+ for t_idx, pool in enumerate(causal_pools):
+ if len(fired) >= effective_budget or not pool:
+ continue
+ eligible = [p for p in pool if (p[0] not in cd or (t_idx - cd[p[0]]) >= pol.cooldown_scans)]
+ if not eligible:
+ continue
+ if mode == "random":
+ choice = eligible[int(rng.integers(0, len(eligible)))]
+ elif mode == "brightest_intensity":
+ choice = max(eligible, key=lambda p: p[1])
+ elif mode == "strain_only":
+ choice = max(eligible, key=lambda p: p[2])
+ else:
+ raise ValueError(f"Unknown baseline mode: {mode}")
+ rtid, _, _, cur_c, pred_c, half_w = choice
+ cd[rtid] = t_idx
+ fired.append((t_idx, rtid, cur_c, pred_c, half_w))
+ return fired
+
+ comparison: dict[str, PolicyReplayMetrics] = {
+ "braggtrack_autonomous": _score_trigger_events(
+ "braggtrack_autonomous",
+ bt_tuples,
+ split_scan_by_root=split_scan_by_root,
+ obs_by_scan_and_root=obs_by_scan_and_root,
+ voltage_by_scan=voltage_by_scan,
+ ),
+ }
+ for base_name in ("brightest_intensity", "strain_only", "random"):
+ comparison[base_name] = _score_trigger_events(
+ base_name,
+ _run_baseline(base_name),
+ split_scan_by_root=split_scan_by_root,
+ obs_by_scan_and_root=obs_by_scan_and_root,
+ voltage_by_scan=voltage_by_scan,
+ )
+
+ latencies = [s.step_latency_sec for s in steps] if steps else [0.0]
+ return ReplayBenchmarkResult(
+ n_scans=len(steps),
+ mean_step_latency_sec=float(np.mean(latencies)),
+ max_step_latency_sec=float(np.max(latencies)),
+ steps=steps,
+ zoom_targets=bt_targets,
+ policy_comparison=comparison,
+ )
diff --git a/braggtrack/autonomous/streaming.py b/braggtrack/autonomous/streaming.py
new file mode 100644
index 0000000..a4cf7b3
--- /dev/null
+++ b/braggtrack/autonomous/streaming.py
@@ -0,0 +1,628 @@
+"""Strictly causal online scan-by-scan tracker with ``tell / ask / report`` agent protocol.
+
+Processes incoming 3D rocking-curve scans ``t = 0, 1, 2, ...`` without future lookahead,
+adopting the ``tell()`` / ``ask()`` / ``report()`` agent grammar standardized in synchrotron
+autonomous frameworks (``Bluesky-Adaptive`` — Maffettone et al., 2023; ``gpCAM`` — Noack et al., 2021):
+
+- ``tell(scan, ...)``: ingests one new 3D scan with causal trailing-window Otsu thresholding,
+ ``float32`` LoG + watershed segmentation, outer-ring pedestal subtraction, batched ``DINOv2``
+ encoding, and bounded $O(1)$ sliding-window trajectory DAG updates.
+- ``ask(n, acquisition=...)``: queries the Gaussian Process trajectory forecaster and Bayesian
+ acquisition policy (``ucb``, ``ei``, ``max_variance``) for high-resolution DFXM zoom setpoints.
+- ``tell_zoom_outcome(track_id, outcome)``: optional multi-fidelity hook feeding back measured
+ DFXM zoom metrics (e.g. ``darfix`` mosaicity spread) into the ensemble GP.
+- ``step(scan, ...)``: single-call convenience wrapper executing ``tell()`` followed by ``ask()``.
+"""
+
+from __future__ import annotations
+
+import math
+import time
+from dataclasses import dataclass, replace
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import networkx as nx
+import numpy as np
+
+from braggtrack.io.lazy import open_lazy_volume, otsu_threshold_chunked
+from braggtrack.io.models import AxisCalibration
+from braggtrack.io.nexus import extract_axis_calibration, extract_scan_metadata
+from braggtrack.segmentation import (
+ extract_instance_table,
+ otsu_threshold,
+ relabel_sequential,
+ remove_small_objects,
+ segment_classical,
+)
+from braggtrack.semantic import DEFAULT_DINO_MODEL, make_multiview_encoder, make_patch_encoder
+from braggtrack.semantic.mips import crop_spot_cube, orthogonal_mips
+from braggtrack.tracking.cost import GeometrySemanticCost, PositionShapeCost
+from braggtrack.tracking.events import LineageEvent, SplitMergeConfig, events_to_table
+from braggtrack.tracking.gap_closing import GapClosingConfig
+from braggtrack.tracking.kinematics import GrainKinematics, compute_grain_kinematics
+from braggtrack.tracking.lifecycle import build_tracks, tracks_to_table
+from braggtrack.tracking.precursors import (
+ ObservationPrecursorScore,
+ PrecursorCandidate,
+ PrecursorConfig,
+ compute_fracture_precursors,
+)
+
+from .gp import AcquisitionKind
+from .trigger import ZoomTarget, ZoomTriggerPolicy, select_zoom_targets
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.semantic.dino import BackendName
+
+
+@dataclass(frozen=True)
+class StreamingStepResult:
+ """Snapshot of online causal tracking and autonomous DFXM zoom targets after scan ``scan_idx``."""
+
+ scan_idx: int
+ scan_name: str
+ threshold_raw: float
+ threshold_causal: float
+ n_spots: int
+ n_active_tracks: int
+ n_lineage_events_total: int
+ new_lineage_events: list[LineageEvent]
+ precursor_candidates: list[PrecursorCandidate]
+ zoom_targets: list[ZoomTarget]
+ step_latency_sec: float
+ stimulus: ScanStimulusPoint | None = None
+ beam_ok: bool = True
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "scan_idx": int(self.scan_idx),
+ "scan_name": self.scan_name,
+ "threshold_raw": round(float(self.threshold_raw), 5),
+ "threshold_causal": round(float(self.threshold_causal), 5),
+ "n_spots": int(self.n_spots),
+ "n_active_tracks": int(self.n_active_tracks),
+ "n_lineage_events_total": int(self.n_lineage_events_total),
+ "new_lineage_events": events_to_table(self.new_lineage_events),
+ "n_precursor_candidates": len(self.precursor_candidates),
+ "zoom_targets": [t.to_bliss_command() for t in self.zoom_targets],
+ "step_latency_sec": round(float(self.step_latency_sec), 4),
+ "beam_ok": bool(self.beam_ok),
+ "stimulus": self.stimulus.to_dict() if self.stimulus is not None else None,
+ }
+
+
+class CausalStreamingTracker:
+ """Stateful causal 4D reflection tracker and GP-driven ``tell / ask / report`` DFXM agent."""
+
+ def __init__(
+ self,
+ *,
+ backend: BackendName | None = "auto",
+ model_name: str = DEFAULT_DINO_MODEL,
+ smooth_window: int = 5,
+ seed_peak_fraction: float = 0.08,
+ seed_response_percentile: float = 0.0,
+ min_spot_voxels: int = 4,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ cost_alpha: float = 1.0,
+ cost_beta: float = 0.25,
+ max_cost: float = float("inf"),
+ max_gap: int = 0,
+ split_max_distance: float = 25.0,
+ precursor_config: PrecursorConfig | None = None,
+ trigger_policy: ZoomTriggerPolicy | None = None,
+ sliding_window_scans: int | None = 24,
+ swmr: bool = False,
+ batch_size: int = 32,
+ use_patch_encoder: bool = True,
+ ) -> None:
+ self.smooth_window = max(1, int(smooth_window))
+ self.seed_peak_fraction = float(seed_peak_fraction)
+ self.seed_response_percentile = float(seed_response_percentile)
+ self.min_spot_voxels = max(1, int(min_spot_voxels))
+ self.calibration = calibration
+ self.detector = detector
+ self.cost_alpha = float(cost_alpha)
+ self.cost_beta = float(cost_beta)
+ self.max_cost = float(max_cost)
+ self.batch_size = max(1, int(batch_size))
+ self.swmr = bool(swmr)
+ self.sliding_window_scans = max(6, int(sliding_window_scans)) if sliding_window_scans is not None else None
+
+ self.gap_config = GapClosingConfig(max_gap=max(0, int(max_gap))) if max_gap > 0 else None
+ self.split_merge_config = SplitMergeConfig(max_distance=float(split_max_distance))
+ self.precursor_config = precursor_config or PrecursorConfig()
+ self.trigger_policy = trigger_policy or ZoomTriggerPolicy()
+
+ self._multiview_encoder = (
+ make_multiview_encoder(backend, model_name=model_name) if self.cost_beta > 0.0 else None
+ )
+ self._patch_encoder = make_patch_encoder(backend, model_name=model_name) if use_patch_encoder else None
+ self._cost_fn = GeometrySemanticCost(
+ PositionShapeCost(),
+ cost_alpha=self.cost_alpha,
+ cost_beta=self.cost_beta,
+ )
+
+ # Strictly causal state accumulated across scans 0 .. t
+ self._raw_thresholds: list[float] = []
+ self._causal_thresholds: list[float] = []
+ self._nominal_flux_history: list[float] = []
+ self._scan_names: list[str] = []
+ self._scan_timestamps: dict[int, str | None] = {}
+ self._scan_elapsed_s: dict[int, float] = {}
+ self._scan_tables: list[list[dict[str, Any]]] = []
+ self._stimuli: dict[int, ScanStimulusPoint] = {}
+ self._roi_cache: dict[tuple[int, int], tuple[int, float]] = {}
+ self._cooldown_by_lineage: dict[int, int] = {}
+ self._zoom_counts_by_lineage: dict[int, int] = {}
+ self._zoom_outcomes: dict[int, tuple[np.ndarray, float]] = {}
+ self._step_latencies: list[float] = []
+ self._last_beam_ok: bool = True
+ self._last_active_cal: AxisCalibration = calibration or AxisCalibration()
+
+ # Sliding-window state preserving global track_ids and root strain anchors in O(1)/step
+ self._next_global_tid: int = 1
+ self._spot_to_global_tid: dict[tuple[int, int], int] = {}
+ self._birth_anchor_by_tid: dict[int, dict[str, Any]] = {}
+ self._committed_rows: list[dict[str, Any]] = []
+ self._committed_events: list[LineageEvent] = []
+ self._committed_up_to_scan: int = -1
+
+ # Latest DAG & derived tables
+ self._graph: nx.DiGraph = nx.DiGraph()
+ self._window_rows: list[dict[str, Any]] = []
+ self._kinematics: list[GrainKinematics] = []
+ self._obs_precursor_scores: list[ObservationPrecursorScore] = []
+ self._precursor_candidates: list[PrecursorCandidate] = []
+ self._all_zoom_targets: list[ZoomTarget] = []
+
+ @property
+ def n_scans_processed(self) -> int:
+ return len(self._scan_tables)
+
+ @property
+ def graph(self) -> nx.DiGraph:
+ return self._graph
+
+ @property
+ def track_table(self) -> list[dict[str, Any]]:
+ return [*self._committed_rows, *self._window_rows]
+
+ @property
+ def kinematics(self) -> list[GrainKinematics]:
+ return list(self._kinematics)
+
+ @property
+ def obs_precursor_scores(self) -> list[ObservationPrecursorScore]:
+ return list(self._obs_precursor_scores)
+
+ @property
+ def all_zoom_targets(self) -> list[ZoomTarget]:
+ return list(self._all_zoom_targets)
+
+ def _resolve_volume_and_meta(
+ self,
+ volume_or_path: np.ndarray | str | Path,
+ scan_name: str | None,
+ timestamp: str | None,
+ ) -> tuple[np.ndarray, float, str, str | None, AxisCalibration]:
+ if isinstance(volume_or_path, (str, Path)):
+ p = Path(volume_or_path)
+ if p.is_dir():
+ h5s = sorted([*p.glob("*.h5"), *p.glob("*.hdf5"), *p.glob("*.nxs")])
+ if not h5s:
+ raise FileNotFoundError(f"No HDF5/NeXus file found in scan directory {p}")
+ file_path = h5s[0]
+ default_name = p.name
+ else:
+ file_path = p
+ default_name = p.stem
+ with open_lazy_volume(file_path, swmr=self.swmr) as lazy_vol:
+ raw_thr = float(otsu_threshold_chunked(lazy_vol))
+ vol = np.asarray(lazy_vol._dataset[()], dtype=np.float32)
+ meta = extract_scan_metadata(file_path, swmr=self.swmr)
+ ts = timestamp if timestamp is not None else meta.get("start_time")
+ cal = (
+ self.calibration
+ if self.calibration is not None
+ else extract_axis_calibration(file_path, swmr=self.swmr)
+ )
+ if self.calibration is None and cal.calibrated:
+ self.calibration = cal
+ return vol, raw_thr, (scan_name or default_name), ts, cal
+
+ arr = np.asarray(volume_or_path)
+ vol = np.asarray(arr, dtype=np.float32 if arr.dtype.itemsize <= 4 else np.float64)
+ raw_thr = float(otsu_threshold(vol))
+ cal = self.calibration if self.calibration is not None else AxisCalibration()
+ name = scan_name or f"scan_{len(self._scan_tables) + 1:04d}"
+ return vol, raw_thr, name, timestamp, cal
+
+ def _check_beam_flux(self, vol: np.ndarray) -> bool:
+ """Return False during synchrotron beam dumps or closed-shutter scans."""
+ flux = max(0.0, float(np.sum(vol, dtype=np.float64)))
+ if not self._nominal_flux_history:
+ if flux > 0.0:
+ self._nominal_flux_history.append(flux)
+ return flux > 0.0
+ baseline = float(np.median(self._nominal_flux_history[-self.smooth_window :]))
+ if baseline <= 0.0:
+ return True
+ ok = (flux / baseline) >= float(self.trigger_policy.min_beam_fraction)
+ if ok and flux > 0.0:
+ self._nominal_flux_history.append(flux)
+ return ok
+
+ def _build_window_dag(self, scan_idx: int) -> tuple[nx.DiGraph, list[dict[str, Any]], list[LineageEvent]]:
+ """Build causal DAG over active window ``[win_start, scan_idx]`` with immutable global track IDs."""
+ n_total = scan_idx + 1
+ win_start = (
+ 0
+ if self.sliding_window_scans is None or n_total <= self.sliding_window_scans
+ else n_total - self.sliding_window_scans
+ )
+
+ if win_start > 0 and self._committed_up_to_scan < win_start - 1:
+ for row in self._window_rows:
+ s_i = int(row["scan_idx"])
+ if self._committed_up_to_scan < s_i < win_start:
+ self._committed_rows.append(row)
+ prev_events: list[LineageEvent] = list(self._graph.graph.get("window_lineage_events", []))
+ for ev in prev_events:
+ if self._committed_up_to_scan < ev.scan_idx_to < win_start:
+ self._committed_events.append(ev)
+ self._committed_up_to_scan = win_start - 1
+ stale_keys = [k for k in self._roi_cache if k[0] < win_start - 1]
+ for k in stale_keys:
+ del self._roi_cache[k]
+
+ if win_start == 0:
+ G = build_tracks(
+ self._scan_tables,
+ cost_fn=self._cost_fn,
+ max_cost=self.max_cost,
+ gap_closing_config=self.gap_config,
+ split_merge_config=self.split_merge_config,
+ scan_timestamps=self._scan_timestamps,
+ )
+ w_rows = tracks_to_table(G)
+ for row in w_rows:
+ s_i = int(row["scan_idx"])
+ sp_i = int(row["spot_idx"])
+ tid = int(row["track_id"])
+ self._spot_to_global_tid[(s_i, sp_i)] = tid
+ if tid >= self._next_global_tid:
+ self._next_global_tid = tid + 1
+ if tid not in self._birth_anchor_by_tid or s_i < int(self._birth_anchor_by_tid[tid]["scan_idx"]):
+ self._birth_anchor_by_tid[tid] = dict(row)
+ w_events = list(G.graph.get("lineage_events", []))
+ G.graph["window_lineage_events"] = w_events
+ return G, w_rows, w_events
+
+ sub_tables = self._scan_tables[win_start : scan_idx + 1]
+ sub_ts = {k - win_start: v for k, v in self._scan_timestamps.items() if k >= win_start}
+ G_local = build_tracks(
+ sub_tables,
+ cost_fn=self._cost_fn,
+ max_cost=self.max_cost,
+ gap_closing_config=self.gap_config,
+ split_merge_config=self.split_merge_config,
+ scan_timestamps=sub_ts,
+ )
+
+ nodes_by_local_tid: dict[int, list[tuple[int, int, str]]] = {}
+ for nid, data in G_local.nodes(data=True):
+ ltid = int(data["track_id"])
+ nodes_by_local_tid.setdefault(ltid, []).append((int(data["scan_idx"]), int(data["spot_idx"]), nid))
+
+ local_to_global: dict[int, int] = {}
+ used_globals: set[int] = set()
+ for ltid in sorted(nodes_by_local_tid):
+ obs_list = sorted(nodes_by_local_tid[ltid])
+ s0_loc, sp0, _ = obs_list[0]
+ s0_glob = s0_loc + win_start
+ cand = self._spot_to_global_tid.get((s0_glob, sp0))
+ if cand is not None and cand not in used_globals:
+ gtid = cand
+ else:
+ gtid = self._next_global_tid
+ self._next_global_tid += 1
+ used_globals.add(gtid)
+ local_to_global[ltid] = gtid
+
+ G_global = nx.DiGraph()
+ nid_map: dict[str, str] = {}
+ for nid, data in G_local.nodes(data=True):
+ d = dict(data)
+ s_glob = int(d["scan_idx"]) + win_start
+ sp_i = int(d["spot_idx"])
+ ltid = int(d["track_id"])
+ gtid = local_to_global[ltid]
+ gnid = f"s{s_glob}_i{sp_i}"
+ nid_map[nid] = gnid
+ d["scan_idx"] = s_glob
+ d["track_id"] = gtid
+ if "parent_track_ids" in d and isinstance(d["parent_track_ids"], list):
+ d["parent_track_ids"] = [local_to_global.get(int(p), int(p)) for p in d["parent_track_ids"]]
+ elif gtid in self._birth_anchor_by_tid:
+ anchor = self._birth_anchor_by_tid[gtid]
+ if anchor.get("parent_track_ids"):
+ raw_p = anchor["parent_track_ids"]
+ if isinstance(raw_p, str):
+ d["parent_track_ids"] = [int(x) for x in raw_p.split(";") if x.strip()]
+ elif isinstance(raw_p, list):
+ d["parent_track_ids"] = list(raw_p)
+ if anchor.get("lineage_role") and not d.get("lineage_role"):
+ d["lineage_role"] = anchor["lineage_role"]
+ G_global.add_node(gnid, **d)
+ self._spot_to_global_tid[(s_glob, sp_i)] = gtid
+
+ for u, v, edata in G_local.edges(data=True):
+ ed = dict(edata)
+ if "parent_track_id" in ed:
+ ed["parent_track_id"] = local_to_global.get(int(ed["parent_track_id"]), int(ed["parent_track_id"]))
+ if "child_track_id" in ed:
+ ed["child_track_id"] = local_to_global.get(int(ed["child_track_id"]), int(ed["child_track_id"]))
+ G_global.add_edge(nid_map[u], nid_map[v], **ed)
+
+ remapped_win_events: list[LineageEvent] = []
+ for ev in G_local.graph.get("lineage_events", []):
+ remapped_win_events.append(
+ LineageEvent(
+ event_type=ev.event_type,
+ scan_idx_from=ev.scan_idx_from + win_start,
+ scan_idx_to=ev.scan_idx_to + win_start,
+ parent_track_ids=[local_to_global.get(int(p), int(p)) for p in ev.parent_track_ids],
+ child_track_ids=[local_to_global.get(int(c), int(c)) for c in ev.child_track_ids],
+ intensity_ratio=ev.intensity_ratio,
+ mean_distance=ev.mean_distance,
+ confidence=ev.confidence,
+ timestamp_from=ev.timestamp_from,
+ timestamp_to=ev.timestamp_to,
+ split_subtype=ev.split_subtype,
+ delta_orientation_norm=ev.delta_orientation_norm,
+ delta_d_norm=ev.delta_d_norm,
+ )
+ )
+ all_events = [*self._committed_events, *remapped_win_events]
+ G_global.graph["window_lineage_events"] = remapped_win_events
+ G_global.graph["lineage_events"] = all_events
+ G_global.graph["gap_closed_count"] = G_local.graph.get("gap_closed_count", 0)
+
+ w_rows = tracks_to_table(G_global)
+ for row in w_rows:
+ gtid = int(row["track_id"])
+ s_i = int(row["scan_idx"])
+ if gtid not in self._birth_anchor_by_tid or s_i < int(self._birth_anchor_by_tid[gtid]["scan_idx"]):
+ self._birth_anchor_by_tid[gtid] = dict(row)
+ return G_global, w_rows, all_events
+
+ def tell(
+ self,
+ volume_or_path: np.ndarray | str | Path,
+ *,
+ scan_name: str | None = None,
+ timestamp: str | None = None,
+ stimulus: ScanStimulusPoint | None = None,
+ ) -> StreamingStepResult:
+ """Ingest one new 3D scan ``t`` and update causal tracking, kinematics, and precursors."""
+ t0 = time.perf_counter()
+ scan_idx = len(self._scan_tables)
+
+ vol, raw_thr, resolved_name, resolved_ts, active_cal = self._resolve_volume_and_meta(
+ volume_or_path, scan_name, timestamp
+ )
+ if stimulus is not None and resolved_ts is None and stimulus.timestamp is not None:
+ resolved_ts = stimulus.timestamp
+
+ beam_ok = self._check_beam_flux(vol)
+ self._last_beam_ok = beam_ok
+ self._last_active_cal = active_cal
+
+ if beam_ok or not self._raw_thresholds:
+ self._raw_thresholds.append(raw_thr)
+ win_lo = max(0, len(self._raw_thresholds) - self.smooth_window)
+ causal_thr = float(np.median(self._raw_thresholds[win_lo:]))
+ self._causal_thresholds.append(causal_thr)
+
+ seg = segment_classical(
+ vol,
+ threshold=causal_thr,
+ seed_peak_fraction=self.seed_peak_fraction,
+ seed_response_percentile=self.seed_response_percentile,
+ )
+ labels = relabel_sequential(remove_small_objects(seg.labeled_volume, min_size=self.min_spot_voxels))
+ spots = extract_instance_table(labels, vol, calibration=active_cal) if beam_ok else []
+
+ if self._multiview_encoder is not None and spots:
+ triplets: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = []
+ for row in spots:
+ lbl = int(row["label"])
+ masked_cube, _ = crop_spot_cube(
+ vol,
+ labels,
+ lbl,
+ row,
+ margin=self.precursor_config.roi_margin,
+ )
+ triplets.append(orthogonal_mips(masked_cube))
+ embs = self._multiview_encoder.embed_batch(triplets, batch_size=self.batch_size)
+ for row, vec in zip(spots, embs, strict=True):
+ row["embedding"] = vec
+
+ self._scan_names.append(resolved_name)
+ self._scan_timestamps[scan_idx] = resolved_ts
+ self._scan_tables.append(spots)
+ if stimulus is not None:
+ self._stimuli[scan_idx] = stimulus
+ if stimulus.elapsed_s is not None and math.isfinite(stimulus.elapsed_s):
+ self._scan_elapsed_s[scan_idx] = float(stimulus.elapsed_s)
+
+ G, w_rows, lineage_events = self._build_window_dag(scan_idx)
+ new_events = [ev for ev in lineage_events if ev.scan_idx_to == scan_idx]
+
+ win_min_scan = min((int(r["scan_idx"]) for r in w_rows), default=scan_idx)
+ anchor_rows: list[dict[str, Any]] = []
+ if win_min_scan > 0:
+ queue = list({int(r["track_id"]) for r in w_rows})
+ visited_anc: set[int] = set()
+ while queue:
+ cur_t = queue.pop()
+ if cur_t in visited_anc:
+ continue
+ visited_anc.add(cur_t)
+ anc = self._birth_anchor_by_tid.get(cur_t)
+ if anc is not None:
+ if int(anc["scan_idx"]) < win_min_scan:
+ anchor_rows.append(anc)
+ raw_p = anc.get("parent_track_ids", "")
+ ptids = (
+ [int(x) for x in raw_p.split(";") if x.strip()]
+ if isinstance(raw_p, str)
+ else [int(x) for x in raw_p]
+ if isinstance(raw_p, list)
+ else []
+ )
+ for pt in ptids:
+ if pt not in visited_anc:
+ queue.append(pt)
+
+ kin_list = compute_grain_kinematics(
+ [*anchor_rows, *w_rows],
+ scan_elapsed_s=self._scan_elapsed_s if self._scan_elapsed_s else None,
+ )
+
+ prec_candidates, obs_scores = compute_fracture_precursors(
+ w_rows,
+ self.precursor_config,
+ volumes={scan_idx: vol},
+ label_volumes={scan_idx: labels},
+ patch_encoder=self._patch_encoder,
+ lineage_events=lineage_events,
+ roi_cache=self._roi_cache,
+ batch_size=self.batch_size,
+ )
+
+ self._graph = G
+ self._window_rows = w_rows
+ self._kinematics = kin_list
+ self._obs_precursor_scores = obs_scores
+ self._precursor_candidates = prec_candidates
+
+ n_active = sum(1 for row in w_rows if int(row["scan_idx"]) == scan_idx)
+ dt = time.perf_counter() - t0
+ self._step_latencies.append(dt)
+
+ return StreamingStepResult(
+ scan_idx=scan_idx,
+ scan_name=resolved_name,
+ threshold_raw=raw_thr,
+ threshold_causal=causal_thr,
+ n_spots=len(spots),
+ n_active_tracks=n_active,
+ n_lineage_events_total=len(lineage_events),
+ new_lineage_events=new_events,
+ precursor_candidates=prec_candidates,
+ zoom_targets=[],
+ step_latency_sec=dt,
+ stimulus=stimulus,
+ beam_ok=beam_ok,
+ )
+
+ def ask(
+ self,
+ n: int | None = None,
+ *,
+ acquisition: AcquisitionKind | None = None,
+ commit: bool = True,
+ ) -> list[ZoomTarget]:
+ """Query the GP surrogate and Bayesian acquisition policy for top ``n`` DFXM zoom targets."""
+ if not self._scan_tables:
+ return []
+ scan_idx = len(self._scan_tables) - 1
+ targets = select_zoom_targets(
+ current_scan_idx=scan_idx,
+ scan_name=self._scan_names[scan_idx],
+ track_table=self._window_rows,
+ kinematics=self._kinematics,
+ obs_precursor_scores=self._obs_precursor_scores,
+ policy=self.trigger_policy,
+ acquisition_override=acquisition,
+ max_targets_override=n,
+ calibration=self._last_active_cal,
+ detector=self.detector,
+ stimulus=self._stimuli.get(scan_idx),
+ scan_elapsed_s=self._scan_elapsed_s if self._scan_elapsed_s else None,
+ last_triggered_scan_by_lineage=self._cooldown_by_lineage,
+ zoom_counts_by_lineage=self._zoom_counts_by_lineage,
+ zoom_outcomes=self._zoom_outcomes,
+ beam_ok=self._last_beam_ok,
+ commit=commit,
+ )
+ if commit and targets:
+ self._all_zoom_targets.extend(targets)
+ return targets
+
+ def tell_zoom_outcome(self, track_id: int, outcome: float) -> None:
+ """Record a measured high-resolution DFXM zoom scalar (e.g. ``darfix`` mosaicity) for GP conditioning."""
+ kin_by_tid = {k.track_id: k for k in self._kinematics}
+ kin = kin_by_tid.get(int(track_id))
+ if kin is None or not kin.scan_indices:
+ return
+ pos = len(kin.scan_indices) - 1
+ root_tid = int(kin.root_track_id) if kin.root_track_id > 0 else int(track_id)
+ l_strain = float(kin.lineage_strain[pos]) if len(kin.lineage_strain) > pos else float(kin.strain[pos])
+ s_rate = float(kin.strain_rate[pos]) if len(kin.strain_rate) > pos else 0.0
+ aniso = float(kin.anisotropy[pos]) if len(kin.anisotropy) > pos else 1.0
+
+ sc_matching = [s for s in self._obs_precursor_scores if s.track_id == int(track_id)]
+ prec_score = float(sc_matching[-1].precursor_score) if sc_matching else 0.0
+ subpeaks = int(sc_matching[-1].roi_subpeak_count) if sc_matching else 1
+
+ f_vec = np.array(
+ [
+ abs(l_strain),
+ abs(s_rate),
+ math.log1p(max(0.0, aniso - 1.0)),
+ float(max(0, subpeaks - 1)),
+ prec_score,
+ ],
+ dtype=np.float64,
+ )
+ self._zoom_outcomes[root_tid] = (f_vec, float(outcome))
+
+ def report(self) -> dict[str, Any]:
+ """Return a concise ``Bluesky-Adaptive`` style status summary of the streaming agent."""
+ scan_idx = len(self._scan_tables) - 1
+ n_active = sum(1 for row in self._window_rows if int(row["scan_idx"]) == scan_idx) if scan_idx >= 0 else 0
+ return {
+ "n_scans_processed": len(self._scan_tables),
+ "n_active_tracks": n_active,
+ "n_zoom_targets_emitted": len(self._all_zoom_targets),
+ "n_zoom_outcomes_recorded": len(self._zoom_outcomes),
+ "acquisition": self.trigger_policy.acquisition,
+ "execution_mode": self.trigger_policy.execution_mode,
+ "beam_ok": self._last_beam_ok,
+ "mean_step_latency_sec": round(float(np.mean(self._step_latencies)), 4) if self._step_latencies else 0.0,
+ }
+
+ def step(
+ self,
+ volume_or_path: np.ndarray | str | Path,
+ *,
+ scan_name: str | None = None,
+ timestamp: str | None = None,
+ stimulus: ScanStimulusPoint | None = None,
+ ) -> StreamingStepResult:
+ """Convenience wrapper executing ``tell(volume_or_path, ...)`` followed by ``ask()``."""
+ t0 = time.perf_counter()
+ tell_res = self.tell(volume_or_path, scan_name=scan_name, timestamp=timestamp, stimulus=stimulus)
+ targets = self.ask(commit=True)
+ total_dt = time.perf_counter() - t0
+ self._step_latencies[-1] = total_dt
+ return replace(tell_res, zoom_targets=targets, step_latency_sec=total_dt)
diff --git a/braggtrack/autonomous/trigger.py b/braggtrack/autonomous/trigger.py
new file mode 100644
index 0000000..eb03a99
--- /dev/null
+++ b/braggtrack/autonomous/trigger.py
@@ -0,0 +1,615 @@
+"""Autonomous DFXM "Scout-and-Zoom" trigger policy, GP setpoint forecasting, and BLISS output.
+
+Evaluates active tracked reflections at scan ``t`` using strictly causal kinematics,
+heteroscedastic Gaussian Process (GP) trajectory forecasting (:mod:`braggtrack.autonomous.gp`),
+and standard Bayesian acquisition functions (``ucb``, ``ei``, ``max_variance``) to select
+high-priority candidate grains for zoomed Dark-Field X-ray Microscopy (DFXM) acquisitions.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Literal
+
+import numpy as np
+
+from .gp import AcquisitionKind, forecast_grain_trajectory_gp, score_candidates_gp
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.models import AxisCalibration
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.tracking.kinematics import GrainKinematics
+ from braggtrack.tracking.precursors import ObservationPrecursorScore
+
+ExecutionMode = Literal["replay", "shadow", "watch", "active"]
+
+
+def clamp_setpoint_bounds(
+ pred: tuple[float, float, float],
+ half_widths: tuple[float, float, float],
+ *,
+ mu_bounds: tuple[float, float] | None = None,
+ chi_bounds: tuple[float, float] | None = None,
+ d_bounds: tuple[float, float] | None = None,
+) -> tuple[tuple[float, float, float], tuple[float, float, float], bool]:
+ """Clamp ``(mu, chi, d)`` setpoints and scan intervals to diffractometer soft limits."""
+ pm, pc, pd = pred
+ hm, hc, hd = half_widths
+ clamped = False
+
+ def _clamp_axis(val: float, hw: float, bounds: tuple[float, float] | None) -> tuple[float, float]:
+ nonlocal clamped
+ if bounds is None:
+ return val, hw
+ lo, hi = min(bounds[0], bounds[1]), max(bounds[0], bounds[1])
+ new_val = min(max(val, lo), hi)
+ if abs(new_val - val) > 1e-12:
+ clamped = True
+ max_span_half = 0.5 * (hi - lo)
+ new_hw = min(hw, max_span_half)
+ if new_val - new_hw < lo - 1e-12:
+ new_val = lo + new_hw
+ clamped = True
+ if new_val + new_hw > hi + 1e-12:
+ new_val = hi - new_hw
+ clamped = True
+ new_val = min(max(new_val, lo), hi)
+ return new_val, max(0.0, new_hw)
+
+ pm, hm = _clamp_axis(pm, hm, mu_bounds)
+ pc, hc = _clamp_axis(pc, hc, chi_bounds)
+ pd, hd = _clamp_axis(pd, hd, d_bounds)
+ return (pm, pc, pd), (hm, hc, hd), clamped
+
+
+@dataclass(frozen=True)
+class ZoomTriggerPolicy:
+ """Bayesian acquisition settings, thresholds, and practical safety bounds for DFXM zoom."""
+
+ min_precursor_score: float = 0.40
+ min_persistence: int = 2
+ min_subpeaks: int = 1
+ min_strain_rate: float = 0.0
+ min_intensity: float = 20.0
+ min_track_age: int = 2
+ min_assignment_margin: float = 0.0
+ cooldown_scans: int = 3
+ max_triggers_per_scan: int = 2
+ lead_scans: float = 1.0
+ window_sigma: float = 3.5
+ exclude_post_split_daughters: bool = True
+ acquisition: AcquisitionKind = "ucb"
+ ucb_beta: float = 1.96
+ ei_xi: float = 0.01
+ use_gp_extrapolation: bool = True
+ execution_mode: ExecutionMode = "replay"
+ mu_bounds: tuple[float, float] | None = None
+ chi_bounds: tuple[float, float] | None = None
+ d_bounds: tuple[float, float] | None = None
+ voltage_bounds_v: tuple[float, float] | None = None
+ max_zooms_per_lineage: int | None = None
+ max_extrapolation_steps: float = 12.0
+ min_beam_fraction: float = 0.20
+ exclude_border_truncated: bool = False
+
+
+@dataclass(frozen=True)
+class ZoomTarget:
+ """Actionable DFXM zoom acquisition target emitted after causal scan ``scan_idx``."""
+
+ scan_idx: int
+ scan_name: str
+ track_id: int
+ root_track_id: int
+ priority_rank: int
+ priority_score: float
+ precursor_score: float
+ persistence_scans: int
+ roi_subpeak_count: int
+ anisotropy: float
+ lineage_strain: float
+ strain_err: float
+ strain_rate: float
+ assignment_margin: float
+ current_centroid: tuple[float, float, float]
+ predicted_setpoint: tuple[float, float, float]
+ scan_half_widths: tuple[float, float, float]
+ centroid_err: tuple[float, float, float]
+ lead_scans: float
+ trigger_reasons: tuple[str, ...]
+ scattering_angles: dict[str, float] | None = None
+ operando_state: dict[str, Any] | None = None
+ predictive_std: tuple[float, float, float] = (0.0, 0.0, 0.0)
+ acquisition_kind: AcquisitionKind = "ucb"
+ gp_posterior_mean: float = 0.0
+ gp_posterior_std: float = 0.0
+ execution_mode: ExecutionMode = "replay"
+ actuate: bool = False
+
+ def contains_point(self, mu: float, chi: float, d: float) -> bool:
+ """Return True if ``(mu, chi, d)`` lies inside ``predicted_setpoint ± scan_half_widths``."""
+ pm, pc, pd = self.predicted_setpoint
+ hm, hc, hd = self.scan_half_widths
+ return abs(mu - pm) <= hm and abs(chi - pc) <= hc and abs(d - pd) <= hd
+
+ def to_bliss_command(self) -> dict[str, Any]:
+ """Format as an ESRF BLISS / Ewoks / Bluesky machine-readable DFXM zoom command payload."""
+ pm, pc, pd = self.predicted_setpoint
+ hm, hc, hd = self.scan_half_widths
+ em, ec, ed = self.centroid_err
+ sm, sc, sd = self.predictive_std
+ cmd: dict[str, Any] = {
+ "schema_version": "bliss_dfxm_zoom.v1",
+ "action": "dfxm_high_res_zoom",
+ "actuate": bool(self.actuate),
+ "execution_mode": self.execution_mode,
+ "acquisition": self.acquisition_kind,
+ "source_scan_idx": int(self.scan_idx),
+ "source_scan_name": self.scan_name,
+ "track_id": int(self.track_id),
+ "root_track_id": int(self.root_track_id),
+ "priority_rank": int(self.priority_rank),
+ "priority_score": round(float(self.priority_score), 4),
+ "lead_scans": round(float(self.lead_scans), 3),
+ "target_motors": {
+ "mu": round(float(pm), 6),
+ "chi": round(float(pc), 6),
+ "obpitch_or_2theta": round(float(pd), 6),
+ },
+ "scan_intervals": {
+ "mu": [round(float(pm - hm), 6), round(float(pm + hm), 6)],
+ "chi": [round(float(pc - hc), 6), round(float(pc + hc), 6)],
+ "obpitch_or_2theta": [round(float(pd - hd), 6), round(float(pd + hd), 6)],
+ },
+ "scan_half_widths": {
+ "mu": round(float(hm), 6),
+ "chi": round(float(hc), 6),
+ "obpitch_or_2theta": round(float(hd), 6),
+ },
+ "uncertainty_1sigma": {
+ "centroid_mu_err": round(float(em), 6),
+ "centroid_chi_err": round(float(ec), 6),
+ "centroid_d_err": round(float(ed), 6),
+ "gp_predictive_mu_std": round(float(sm), 6),
+ "gp_predictive_chi_std": round(float(sc), 6),
+ "gp_predictive_d_std": round(float(sd), 6),
+ "strain_err": round(float(self.strain_err), 7),
+ },
+ "diagnostics": {
+ "current_centroid": [round(float(x), 6) for x in self.current_centroid],
+ "gp_posterior_mean": round(float(self.gp_posterior_mean), 4),
+ "gp_posterior_std": round(float(self.gp_posterior_std), 4),
+ "precursor_score": round(float(self.precursor_score), 4),
+ "persistence_scans": int(self.persistence_scans),
+ "roi_subpeak_count": int(self.roi_subpeak_count),
+ "anisotropy": round(float(self.anisotropy), 4),
+ "lineage_strain": round(float(self.lineage_strain), 6),
+ "strain_rate": round(float(self.strain_rate), 6),
+ "assignment_margin": (
+ round(float(self.assignment_margin), 4) if math.isfinite(self.assignment_margin) else "inf"
+ ),
+ },
+ "trigger_reasons": list(self.trigger_reasons),
+ }
+ if self.scattering_angles is not None:
+ cmd["crystallographic_setpoint"] = dict(self.scattering_angles)
+ if self.operando_state is not None:
+ cmd["operando_state"] = dict(self.operando_state)
+ return cmd
+
+ def to_dict(self) -> dict[str, Any]:
+ return self.to_bliss_command()
+
+
+def _trailing_persistence(
+ track_obs_scores: list[ObservationPrecursorScore],
+ current_scan_idx: int,
+ threshold: float,
+) -> int:
+ """Return contiguous causal run length ending at ``current_scan_idx`` with score >= threshold."""
+ relevant = sorted(
+ (s for s in track_obs_scores if s.scan_idx <= current_scan_idx),
+ key=lambda s: s.scan_idx,
+ )
+ if not relevant or relevant[-1].scan_idx != current_scan_idx:
+ return 0
+ run = 0
+ expected_scan = current_scan_idx
+ for sc in reversed(relevant):
+ if sc.scan_idx != expected_scan or sc.precursor_score < threshold:
+ break
+ run += 1
+ expected_scan -= 1
+ return run
+
+
+def compute_target_geometry(
+ obs: dict[str, Any],
+ kin: GrainKinematics | None,
+ obs_pos_in_kin: int,
+ *,
+ lead_scans: float = 1.0,
+ window_sigma: float = 3.5,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ max_extrapolation_steps: float | None = None,
+ scan_elapsed_s: dict[int, float] | None = None,
+ use_gp: bool = True,
+) -> tuple[
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ dict[str, float] | None,
+]:
+ """Forecast ``(mu, chi, d)`` setpoint, scan half-widths, centroid errors, GP stds, and angles."""
+ cur_mu = float(obs["centroid_mu"])
+ cur_chi = float(obs["centroid_chi"])
+ cur_d = float(obs["centroid_d"])
+
+ step_mu = abs(float(calibration.mu_step)) if calibration is not None else float(obs.get("step_mu", 1.0))
+ step_chi = abs(float(calibration.chi_step)) if calibration is not None else float(obs.get("step_chi", 1.0))
+ step_d = abs(float(calibration.d_step)) if calibration is not None else float(obs.get("step_d", 1.0))
+
+ if kin is not None and 0 <= obs_pos_in_kin < len(kin.scan_indices):
+ end = obs_pos_in_kin + 1
+ s_indices = kin.scan_indices[:end]
+ err_mu = float(kin.centroid_mu_err[obs_pos_in_kin]) if len(kin.centroid_mu_err) > obs_pos_in_kin else 0.0
+ err_chi = float(kin.centroid_chi_err[obs_pos_in_kin]) if len(kin.centroid_chi_err) > obs_pos_in_kin else 0.0
+ err_d = float(kin.centroid_d_err[obs_pos_in_kin]) if len(kin.centroid_d_err) > obs_pos_in_kin else 0.0
+
+ if use_gp and len(s_indices) >= 2:
+ if scan_elapsed_s and all(int(si) in scan_elapsed_s for si in s_indices):
+ t_hist = np.asarray([float(scan_elapsed_s[int(si)]) for si in s_indices], dtype=np.float64)
+ diffs = np.diff(t_hist)
+ dt_step = float(np.median(diffs[diffs > 0])) if np.any(diffs > 0) else 1.0
+ else:
+ t_hist = np.asarray(s_indices, dtype=np.float64)
+ dt_step = 1.0
+
+ target_t = float(t_hist[-1]) + float(lead_scans) * dt_step
+ e_mu = (
+ np.asarray(kin.centroid_mu_err[:end], dtype=np.float64)
+ if len(kin.centroid_mu_err) >= end
+ else np.zeros(end, dtype=np.float64)
+ )
+ e_chi = (
+ np.asarray(kin.centroid_chi_err[:end], dtype=np.float64)
+ if len(kin.centroid_chi_err) >= end
+ else np.zeros(end, dtype=np.float64)
+ )
+ e_d = (
+ np.asarray(kin.centroid_d_err[:end], dtype=np.float64)
+ if len(kin.centroid_d_err) >= end
+ else np.zeros(end, dtype=np.float64)
+ )
+ gp_out = forecast_grain_trajectory_gp(
+ t_hist,
+ np.asarray(kin.centroid_mu[:end], dtype=np.float64),
+ np.asarray(kin.centroid_chi[:end], dtype=np.float64),
+ np.asarray(kin.centroid_d[:end], dtype=np.float64),
+ e_mu,
+ e_chi,
+ e_d,
+ target_t,
+ step_floors=(step_mu, step_chi, step_d),
+ )
+ dm = gp_out.mean[0] - cur_mu
+ dc = gp_out.mean[1] - cur_chi
+ dd = gp_out.mean[2] - cur_d
+ gp_std = gp_out.std
+ else:
+ v_mu = float(kin.rate_mu[obs_pos_in_kin]) if len(kin.rate_mu) > obs_pos_in_kin else 0.0
+ v_chi = float(kin.rate_chi[obs_pos_in_kin]) if len(kin.rate_chi) > obs_pos_in_kin else 0.0
+ v_d = float(kin.rate_d[obs_pos_in_kin]) if len(kin.rate_d) > obs_pos_in_kin else 0.0
+ dm = float(lead_scans) * v_mu
+ dc = float(lead_scans) * v_chi
+ dd = float(lead_scans) * v_d
+ gp_std = (
+ max(err_mu, 0.35 * step_mu),
+ max(err_chi, 0.35 * step_chi),
+ max(err_d, 0.35 * step_d),
+ )
+ else:
+ err_mu = float(obs.get("centroid_mu_err", 0.0))
+ err_chi = float(obs.get("centroid_chi_err", 0.0))
+ err_d = float(obs.get("centroid_d_err", 0.0))
+ dm = dc = dd = 0.0
+ gp_std = (
+ max(err_mu, 0.35 * step_mu),
+ max(err_chi, 0.35 * step_chi),
+ max(err_d, 0.35 * step_d),
+ )
+
+ if max_extrapolation_steps is not None and max_extrapolation_steps > 0.0:
+ cap_m = float(max_extrapolation_steps) * max(step_mu, 1e-12)
+ cap_c = float(max_extrapolation_steps) * max(step_chi, 1e-12)
+ cap_d = float(max_extrapolation_steps) * max(step_d, 1e-12)
+ dm = min(max(dm, -cap_m), cap_m)
+ dc = min(max(dc, -cap_c), cap_c)
+ dd = min(max(dd, -cap_d), cap_d)
+
+ pred_mu = cur_mu + dm
+ pred_chi = cur_chi + dc
+ pred_d = cur_d + dd
+
+ std_mu = math.sqrt(max(float(obs.get("cov_zz", obs.get("cov_mu_mu", 0.0))), 0.0))
+ std_chi = math.sqrt(max(float(obs.get("cov_xx", obs.get("cov_chi_chi", 0.0))), 0.0))
+ std_d = math.sqrt(max(float(obs.get("cov_yy", obs.get("cov_d_d", 0.0))), 0.0))
+
+ # Rocking window combines intrinsic reflection spread and GP posterior setpoint uncertainty
+ half_mu = max(window_sigma * math.sqrt(std_mu * std_mu + gp_std[0] * gp_std[0]), 1.5 * step_mu)
+ half_chi = max(window_sigma * math.sqrt(std_chi * std_chi + gp_std[1] * gp_std[1]), 1.5 * step_chi)
+ half_d = max(window_sigma * math.sqrt(std_d * std_d + gp_std[2] * gp_std[2]), 1.5 * step_d)
+
+ scattering: dict[str, float] | None = None
+ if detector is not None:
+ if calibration is not None and calibration.calibrated:
+ z_idx = (pred_mu - calibration.mu_origin) / (calibration.mu_step if calibration.mu_step != 0 else 1.0)
+ x_px = (pred_chi - calibration.chi_origin) / (calibration.chi_step if calibration.chi_step != 0 else 1.0)
+ y_px = (pred_d - calibration.d_origin) / (calibration.d_step if calibration.d_step != 0 else 1.0)
+ else:
+ z_idx, y_px, x_px = pred_mu, pred_d, pred_chi
+ scattering = detector.spot_to_scattering(z_idx, y_px, x_px)
+
+ return (
+ (cur_mu, cur_chi, cur_d),
+ (pred_mu, pred_chi, pred_d),
+ (half_mu, half_chi, half_d),
+ (err_mu, err_chi, err_d),
+ gp_std,
+ scattering,
+ )
+
+
+def select_zoom_targets(
+ *,
+ current_scan_idx: int,
+ scan_name: str,
+ track_table: list[dict[str, Any]],
+ kinematics: list[GrainKinematics],
+ obs_precursor_scores: list[ObservationPrecursorScore],
+ policy: ZoomTriggerPolicy | None = None,
+ acquisition_override: AcquisitionKind | None = None,
+ max_targets_override: int | None = None,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ stimulus: ScanStimulusPoint | None = None,
+ scan_elapsed_s: dict[int, float] | None = None,
+ last_triggered_scan_by_lineage: dict[int, int] | None = None,
+ zoom_counts_by_lineage: dict[int, int] | None = None,
+ zoom_outcomes: dict[int, tuple[np.ndarray, float]] | None = None,
+ beam_ok: bool = True,
+ commit: bool = True,
+) -> list[ZoomTarget]:
+ """Select, rank via Bayesian GP acquisition, and extrapolate DFXM zoom targets."""
+ pol = policy or ZoomTriggerPolicy()
+ if not beam_ok:
+ return []
+
+ if (
+ pol.voltage_bounds_v is not None
+ and stimulus is not None
+ and stimulus.voltage_v is not None
+ and math.isfinite(stimulus.voltage_v)
+ ):
+ v_lo, v_hi = min(pol.voltage_bounds_v), max(pol.voltage_bounds_v)
+ if not (v_lo <= float(stimulus.voltage_v) <= v_hi):
+ return []
+
+ acq_kind: AcquisitionKind = acquisition_override if acquisition_override is not None else pol.acquisition
+ limit = int(max_targets_override) if max_targets_override is not None else max(0, int(pol.max_triggers_per_scan))
+ if limit <= 0:
+ return []
+
+ kin_by_tid = {k.track_id: k for k in kinematics}
+ scores_by_tid: dict[int, list[ObservationPrecursorScore]] = {}
+ score_at_current: dict[int, ObservationPrecursorScore] = {}
+ for sc in obs_precursor_scores:
+ scores_by_tid.setdefault(sc.track_id, []).append(sc)
+ if sc.scan_idx == current_scan_idx:
+ score_at_current[sc.track_id] = sc
+
+ cooldown_map = last_triggered_scan_by_lineage if last_triggered_scan_by_lineage is not None else {}
+ dose_map = zoom_counts_by_lineage if zoom_counts_by_lineage is not None else {}
+ operando_dict = stimulus.to_dict() if stimulus is not None else None
+ should_actuate = bool(pol.execution_mode == "active")
+ has_supervised_outcomes = bool(zoom_outcomes and len(zoom_outcomes) > 0)
+
+ eligible: list[dict[str, Any]] = []
+ for obs in track_table:
+ if int(obs["scan_idx"]) != current_scan_idx:
+ continue
+ tid = int(obs["track_id"])
+ role = str(obs.get("lineage_role", ""))
+ if pol.exclude_post_split_daughters and role in ("split_child", "split_daughter"):
+ continue
+ if pol.exclude_border_truncated and bool(obs.get("touches_rocking_border", False)):
+ continue
+ if float(obs.get("integrated_intensity", 0.0)) < pol.min_intensity:
+ continue
+ margin = float(obs.get("assignment_margin", float("inf")))
+ if margin < pol.min_assignment_margin:
+ continue
+
+ kin = kin_by_tid.get(tid)
+ if kin is None or current_scan_idx not in kin.scan_indices:
+ continue
+ pos = kin.scan_indices.index(current_scan_idx)
+ if (pos + 1) < pol.min_track_age:
+ continue
+
+ root_tid = int(kin.root_track_id) if kin.root_track_id > 0 else tid
+ prev_trig = cooldown_map.get(root_tid)
+ if prev_trig is not None and (current_scan_idx - prev_trig) < pol.cooldown_scans:
+ continue
+ if pol.max_zooms_per_lineage is not None and dose_map.get(root_tid, 0) >= int(pol.max_zooms_per_lineage):
+ continue
+
+ sc_now = score_at_current.get(tid)
+ prec_score = float(sc_now.precursor_score) if sc_now is not None else 0.0
+ subpeaks = int(sc_now.roi_subpeak_count) if sc_now is not None else 1
+ aniso = float(sc_now.anisotropy) if sc_now is not None else float(kin.anisotropy[pos])
+ pers = _trailing_persistence(scores_by_tid.get(tid, []), current_scan_idx, pol.min_precursor_score)
+ s_rate = float(kin.strain_rate[pos]) if len(kin.strain_rate) > pos else 0.0
+
+ if abs(s_rate) < pol.min_strain_rate or subpeaks < pol.min_subpeaks:
+ continue
+
+ # In exploitation-aware modes without external DFXM zoom outcomes, gate on precursor persistence;
+ # in pure exploration ('max_variance') or with supervised zoom outcomes, score all eligible tracks.
+ if (
+ acq_kind in ("ucb", "ei")
+ and not has_supervised_outcomes
+ and (prec_score < pol.min_precursor_score or pers < pol.min_persistence)
+ ):
+ continue
+
+ l_strain = float(kin.lineage_strain[pos]) if len(kin.lineage_strain) > pos else float(kin.strain[pos])
+ s_err = float(kin.lineage_strain_err[pos]) if len(kin.lineage_strain_err) > pos else 0.0
+
+ track_scores = sorted(
+ (s for s in scores_by_tid.get(tid, []) if s.scan_idx <= current_scan_idx),
+ key=lambda s: s.scan_idx,
+ )
+ if scan_elapsed_s and all(int(s.scan_idx) in scan_elapsed_s for s in track_scores):
+ s_times = np.asarray([float(scan_elapsed_s[int(s.scan_idx)]) for s in track_scores], dtype=np.float64)
+ else:
+ s_times = np.asarray([float(s.scan_idx) for s in track_scores], dtype=np.float64)
+ s_hist = np.asarray([float(s.precursor_score) for s in track_scores], dtype=np.float64)
+
+ f_vec = np.array(
+ [
+ abs(l_strain),
+ abs(s_rate),
+ math.log1p(max(0.0, aniso - 1.0)),
+ float(max(0, subpeaks - 1)),
+ prec_score,
+ ],
+ dtype=np.float64,
+ )
+
+ eligible.append(
+ {
+ "obs": obs,
+ "kin": kin,
+ "pos": pos,
+ "tid": tid,
+ "root_tid": root_tid,
+ "prec_score": prec_score,
+ "subpeaks": subpeaks,
+ "aniso": aniso,
+ "pers": pers,
+ "s_rate": s_rate,
+ "l_strain": l_strain,
+ "s_err": s_err,
+ "margin": margin,
+ "score_times": s_times,
+ "score_history": s_hist,
+ "feature_vec": f_vec,
+ }
+ )
+
+ if not eligible:
+ return []
+
+ gp_mu, gp_std, acq_scores = score_candidates_gp(
+ eligible,
+ acquisition=acq_kind,
+ beta=pol.ucb_beta,
+ xi=pol.ei_xi,
+ lead_scans=pol.lead_scans,
+ zoom_outcomes=zoom_outcomes,
+ )
+
+ order = sorted(range(len(eligible)), key=lambda i: (-float(acq_scores[i]), int(eligible[i]["tid"])))
+ selected: list[ZoomTarget] = []
+ seen_roots: set[int] = set()
+
+ for idx in order:
+ if len(selected) >= limit:
+ break
+ rec = eligible[idx]
+ root_tid = int(rec["root_tid"])
+ if root_tid in seen_roots:
+ continue
+ seen_roots.add(root_tid)
+
+ cur_c, pred_c, half_w, c_err, pred_std, scat = compute_target_geometry(
+ rec["obs"],
+ rec["kin"],
+ int(rec["pos"]),
+ lead_scans=pol.lead_scans,
+ window_sigma=pol.window_sigma,
+ calibration=calibration,
+ detector=detector,
+ max_extrapolation_steps=pol.max_extrapolation_steps,
+ scan_elapsed_s=scan_elapsed_s,
+ use_gp=pol.use_gp_extrapolation,
+ )
+ pred_c, half_w, was_clamped = clamp_setpoint_bounds(
+ pred_c,
+ half_w,
+ mu_bounds=pol.mu_bounds,
+ chi_bounds=pol.chi_bounds,
+ d_bounds=pol.d_bounds,
+ )
+
+ reasons: list[str] = [
+ f"acquisition={acq_kind}({float(acq_scores[idx]):.3f})",
+ f"gp_posterior={float(gp_mu[idx]):.3f}+/-{float(gp_std[idx]):.3f}",
+ ]
+ if float(rec["prec_score"]) >= pol.min_precursor_score:
+ reasons.append(f"precursor_score={float(rec['prec_score']):.3f}")
+ if int(rec["pers"]) >= 1:
+ reasons.append(f"persistence={int(rec['pers'])}_scans")
+ if int(rec["subpeaks"]) >= 2:
+ reasons.append(f"bimodal_subpeaks={int(rec['subpeaks'])}")
+ if float(rec["aniso"]) >= 2.0:
+ reasons.append(f"mosaicity_anisotropy={float(rec['aniso']):.2f}")
+ if was_clamped:
+ reasons.append("clamped_to_motor_bounds")
+
+ if commit:
+ if last_triggered_scan_by_lineage is not None:
+ last_triggered_scan_by_lineage[root_tid] = current_scan_idx
+ if zoom_counts_by_lineage is not None:
+ zoom_counts_by_lineage[root_tid] = zoom_counts_by_lineage.get(root_tid, 0) + 1
+
+ selected.append(
+ ZoomTarget(
+ scan_idx=current_scan_idx,
+ scan_name=scan_name,
+ track_id=int(rec["tid"]),
+ root_track_id=root_tid,
+ priority_rank=len(selected) + 1,
+ priority_score=float(acq_scores[idx]),
+ precursor_score=float(rec["prec_score"]),
+ persistence_scans=int(rec["pers"]),
+ roi_subpeak_count=int(rec["subpeaks"]),
+ anisotropy=float(rec["aniso"]),
+ lineage_strain=float(rec["l_strain"]),
+ strain_err=float(rec["s_err"]),
+ strain_rate=float(rec["s_rate"]),
+ assignment_margin=float(rec["margin"]),
+ current_centroid=cur_c,
+ predicted_setpoint=pred_c,
+ scan_half_widths=half_w,
+ centroid_err=c_err,
+ lead_scans=pol.lead_scans,
+ trigger_reasons=tuple(reasons),
+ scattering_angles=scat,
+ operando_state=operando_dict,
+ predictive_std=pred_std,
+ acquisition_kind=acq_kind,
+ gp_posterior_mean=float(gp_mu[idx]),
+ gp_posterior_std=float(gp_std[idx]),
+ execution_mode=pol.execution_mode,
+ actuate=should_actuate,
+ )
+ )
+
+ return selected
diff --git a/braggtrack/cli/_utils.py b/braggtrack/cli/_utils.py
index 6e60042..04710db 100644
--- a/braggtrack/cli/_utils.py
+++ b/braggtrack/cli/_utils.py
@@ -55,7 +55,7 @@ def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w", newline="") as fh:
fh.write("")
return
- fieldnames = list(rows[0].keys())
+ fieldnames = list(dict.fromkeys(k for r in rows for k in r))
with path.open("w", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=fieldnames)
writer.writeheader()
diff --git a/braggtrack/cli/embed_dataset.py b/braggtrack/cli/embed_dataset.py
index f1baec7..b629819 100644
--- a/braggtrack/cli/embed_dataset.py
+++ b/braggtrack/cli/embed_dataset.py
@@ -40,6 +40,7 @@ def build_parser() -> argparse.ArgumentParser:
help="Embedding backend (mock needs no PyTorch; torch uses DINOv2 default or custom --model)",
)
p.add_argument("--model", default=DEFAULT_DINO_MODEL, help="HF model id when backend=torch")
+ p.add_argument("--batch-size", type=int, default=32, help="Batch size for ViT MIP forward passes")
p.add_argument(
"--allow-synthetic-fallback",
action="store_true",
@@ -107,23 +108,25 @@ def main(argv: list[str] | None = None) -> int:
return 1
label_ids: list[int] = []
- vectors: list[np.ndarray] = []
+ mips_batch: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = []
for row in rows:
lid = int(row["label"])
mip_mu, mip_chi, mip_d = orthogonal_mips(
crop_spot_cube(lazy_vol, labels_full, lid, row, margin=max(0, args.margin))[0]
)
- vec = enc.embed(mip_mu, mip_chi, mip_d)
label_ids.append(lid)
- vectors.append(vec)
-
- if not vectors:
- dim = 384
+ mips_batch.append((mip_mu, mip_chi, mip_d))
+
+ if mips_batch:
+ if hasattr(enc, "embed_batch"):
+ mat = enc.embed_batch(mips_batch, batch_size=max(1, args.batch_size))
+ else:
+ mat = np.stack([enc.embed(m0, m1, m2) for m0, m1, m2 in mips_batch], axis=0)
+ dim = int(mat.shape[1])
else:
- dim = int(vectors[0].shape[0])
-
- mat = np.stack(vectors, axis=0) if vectors else np.zeros((0, dim), dtype=np.float32)
+ dim = 1152 if resolved_backend == "torch" else 384
+ mat = np.zeros((0, dim), dtype=np.float32)
scan_out = outdir / name
scan_out.mkdir(parents=True, exist_ok=True)
np.savez_compressed(
diff --git a/braggtrack/cli/report_dataset.py b/braggtrack/cli/report_dataset.py
index f6ac4fc..65d0d6f 100644
--- a/braggtrack/cli/report_dataset.py
+++ b/braggtrack/cli/report_dataset.py
@@ -12,6 +12,7 @@
generate_html_report,
plot_kinematics_dashboard,
plot_lineage_dag,
+ plot_operando_strain_and_events,
plot_precursor_timeline,
)
@@ -29,6 +30,11 @@ def build_parser() -> argparse.ArgumentParser:
default=None,
help="Optional raw HDF5 dataset root for embedding 3-axis MIP spot thumbnails",
)
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional external operando CSV/TSV/Biologic .mpt log for synchronized stimulus plots",
+ )
parser.add_argument(
"--outdir",
default=None,
@@ -76,6 +82,7 @@ def main(argv: list[str] | None = None) -> int:
in_root,
out_root / "report.html",
dataset_root=args.dataset_root,
+ operando_log=args.operando_log,
max_mip_tracks=max(0, args.max_mip_tracks),
)
@@ -122,6 +129,32 @@ def main(argv: list[str] | None = None) -> int:
)
plt.close(fig_prec)
figures["precursor_timeline"] = str((fig_dir / "precursor_timeline.png").resolve())
+
+ # Synchronized operando stimulus + lineage strain figure if log or saved JSON exists
+ stim_rows: list[dict[str, Any]] = []
+ if args.operando_log:
+ from braggtrack.io import load_operando_log, synchronize_to_scans
+
+ n_scans_est = max((int(float(r["scan_idx"])) for r in track_rows), default=0) + 1
+ stim_rows = [
+ p.to_dict() for p in synchronize_to_scans(load_operando_log(args.operando_log), n_scans_est)
+ ]
+ elif (in_root / "operando_stimulus.json").exists():
+ raw_stim = json.loads((in_root / "operando_stimulus.json").read_text())
+ if isinstance(raw_stim, dict) and isinstance(raw_stim.get("scans"), list):
+ stim_rows = raw_stim["scans"]
+
+ if stim_rows:
+ fig_op, _ = plot_operando_strain_and_events(
+ track_rows,
+ stim_rows,
+ events=event_rows,
+ precursor_observations=prec_obs_rows,
+ calibrated=calibrated,
+ save_path=fig_dir / "operando_strain_and_events.png",
+ )
+ plt.close(fig_op)
+ figures["operando_strain_and_events"] = str((fig_dir / "operando_strain_and_events.png").resolve())
except ImportError:
pass
diff --git a/braggtrack/cli/run_pipeline.py b/braggtrack/cli/run_pipeline.py
index 756f5b1..bf192f5 100644
--- a/braggtrack/cli/run_pipeline.py
+++ b/braggtrack/cli/run_pipeline.py
@@ -51,6 +51,11 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--detector-json", default=None, help="Optional DetectorGeometry JSON path for (2theta, eta, Q)"
)
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional external operando CSV/TSV/Biologic .mpt log to synchronize with diffraction scans",
+ )
parser.add_argument("--cost-alpha", type=float, default=1.0, help="Geometry weight in semantic cost")
parser.add_argument("--cost-beta", type=float, default=0.25, help="Semantic embedding weight in tracking cost")
parser.add_argument("--max-gap", type=int, default=0, help="Max missing intermediate scans to bridge (0 disables)")
@@ -114,6 +119,7 @@ def _merge_config_overrides(args: argparse.Namespace) -> dict[str, Any]:
"smooth_window": args.smooth_window,
"calibration_json": args.calibration_json,
"detector_json": args.detector_json,
+ "operando_log": args.operando_log,
"cost_alpha": args.cost_alpha,
"cost_beta": args.cost_beta,
"max_gap": args.max_gap,
@@ -132,6 +138,13 @@ def _merge_config_overrides(args: argparse.Namespace) -> dict[str, Any]:
cfg_data = json.loads(Path(args.config).read_text())
if not isinstance(cfg_data, dict):
raise ValueError("--config must point to a JSON object")
+ allowed_keys = set(effective.keys()) | {"root"}
+ unknown_keys = sorted(set(cfg_data.keys()) - allowed_keys)
+ if unknown_keys:
+ raise ValueError(
+ f"Unknown --config override key(s): {', '.join(unknown_keys)}. "
+ f"Allowed keys: {', '.join(sorted(allowed_keys))}"
+ )
effective.update(cfg_data)
return effective
@@ -170,6 +183,23 @@ def main(argv: list[str] | None = None) -> int:
print(json.dumps({"error": "Dataset validation failed", "issues": val_payload["issues"]}))
return 1
+ # Optional external operando stimulus synchronization
+ stim_points = None
+ if cfg.get("operando_log"):
+ from braggtrack.io import load_operando_log, synchronize_to_scans
+
+ op_log = load_operando_log(str(cfg["operando_log"]))
+ stim_points = synchronize_to_scans(op_log, sequence)
+ (outdir / "operando_stimulus.json").write_text(
+ json.dumps(
+ {
+ "source_path": str(Path(str(cfg["operando_log"])).resolve()),
+ "scans": [pt.to_dict() for pt in stim_points],
+ },
+ indent=2,
+ )
+ )
+
# Compute input checksums
discovered = discover_operando_scans(root)
input_files = [
@@ -299,6 +329,15 @@ def main(argv: list[str] | None = None) -> int:
if rc_evt != 0:
return rc_evt
+ if stim_points is not None:
+ from braggtrack.cli._utils import load_feature_csv, write_csv
+ from braggtrack.io import annotate_tracks_with_stimulus
+
+ for target_csv in (trk_dir / "tracks.csv", evt_dir / "tracks.csv", trk_dir / "kinematics.csv"):
+ if target_csv.exists() and target_csv.stat().st_size > 0:
+ enriched_rows = annotate_tracks_with_stimulus(load_feature_csv(target_csv), stim_points)
+ write_csv(target_csv, enriched_rows)
+
if cfg.get("export_nexus") and cfg.get("detect_events"):
from braggtrack.io import DetectorGeometry, export_nexus_process
@@ -323,6 +362,9 @@ def _build_manifest(cur_total_sec: float) -> dict[str, Any]:
"total_duration_sec": cur_total_sec,
"artifacts": {
"validation_summary": str((outdir / "validation_summary.json").resolve()),
+ "operando_stimulus": str((outdir / "operando_stimulus.json").resolve())
+ if stim_points is not None
+ else None,
"segmentation_summary": str((seg_dir / "segmentation_summary.json").resolve()),
"embedding_summary": str((emb_dir / "embedding_summary.json").resolve()) if need_embed else None,
"tracking_summary": str((trk_dir / "tracking_summary.json").resolve()),
@@ -343,7 +385,10 @@ def _build_manifest(cur_total_sec: float) -> dict[str, Any]:
# 6. Optional interactive HTML & static figure report
if cfg.get("report"):
t0 = time.perf_counter()
- rc_rep = report_dataset.main([str(outdir), "--dataset-root", str(root), "--outdir", str(rep_dir)])
+ rep_argv = [str(outdir), "--dataset-root", str(root), "--outdir", str(rep_dir)]
+ if cfg.get("operando_log"):
+ rep_argv.extend(["--operando-log", str(cfg["operando_log"])])
+ rc_rep = report_dataset.main(rep_argv)
timings["report"] = round(time.perf_counter() - t0, 4)
if rc_rep != 0:
return rc_rep
diff --git a/braggtrack/cli/watch_autonomous.py b/braggtrack/cli/watch_autonomous.py
new file mode 100644
index 0000000..e5dab7f
--- /dev/null
+++ b/braggtrack/cli/watch_autonomous.py
@@ -0,0 +1,337 @@
+"""Autonomous DFXM "Scout-and-Zoom" beamline watcher and causal replay CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+from braggtrack.autonomous import (
+ CausalStreamingTracker,
+ ZoomTriggerPolicy,
+ evaluate_causal_replay,
+)
+from braggtrack.io import (
+ AxisCalibration,
+ BeamlineAdapter,
+ DetectorGeometry,
+ load_operando_log,
+ resolve_dataset_root,
+ synchronize_to_scans,
+)
+from braggtrack.io.nexus import _open_h5_readonly, _require_h5py, _resolve_volume_dataset
+from braggtrack.semantic import DEFAULT_DINO_MODEL
+from braggtrack.tracking.precursors import PrecursorConfig
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "root",
+ nargs="?",
+ default=None,
+ help="Dataset root containing scan subdirectories (default: data/sample_operando if present, else .)",
+ )
+ parser.add_argument(
+ "--mode",
+ choices=("replay", "shadow", "watch", "active"),
+ default="replay",
+ help="Execution mode: 'replay' (offline), 'shadow'/'watch' (live non-actuating), or 'active' (closed-loop)",
+ )
+ parser.add_argument(
+ "--acquisition",
+ choices=("ucb", "ei", "max_variance"),
+ default="ucb",
+ help="Bayesian GP acquisition function for candidate DFXM zoom ranking (default: ucb)",
+ )
+ parser.add_argument(
+ "--ucb-beta",
+ type=float,
+ default=1.96,
+ help="Exploration weight beta for Upper Confidence Bound (ucb) acquisition (default: 1.96)",
+ )
+ parser.add_argument("--outdir", default="artifacts/autonomous", help="Output directory for zoom triggers & logs")
+ parser.add_argument(
+ "--backend",
+ choices=("auto", "mock", "torch"),
+ default="auto",
+ help="Semantic/patch encoder backend (auto, mock, or torch)",
+ )
+ parser.add_argument("--model", default=DEFAULT_DINO_MODEL, help="HuggingFace model ID when backend=torch")
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional CSV/TSV/Biologic .mpt operando log to attach potential & dQ/dV to zoom triggers",
+ )
+ parser.add_argument("--calibration-json", default=None, help="Optional AxisCalibration JSON file")
+ parser.add_argument("--detector-json", default=None, help="Optional DetectorGeometry JSON file")
+ parser.add_argument("--min-precursor-score", type=float, default=0.40, help="Precursor trigger threshold")
+ parser.add_argument("--min-persistence", type=int, default=2, help="Required contiguous precursor scans")
+ parser.add_argument("--cooldown-scans", type=int, default=3, help="Per-lineage cooldown scans after firing")
+ parser.add_argument("--max-triggers-per-scan", type=int, default=2, help="Maximum zoom targets emitted per scan")
+ parser.add_argument("--lead-scans", type=float, default=1.0, help="GP forecast lead (in scan steps)")
+ parser.add_argument("--window-sigma", type=float, default=3.5, help="Scan half-width sigma multiplier")
+ parser.add_argument(
+ "--mu-bounds",
+ nargs=2,
+ type=float,
+ default=None,
+ metavar=("MIN", "MAX"),
+ help="Optional diffractometer soft-limits for rocking angle mu",
+ )
+ parser.add_argument(
+ "--chi-bounds",
+ nargs=2,
+ type=float,
+ default=None,
+ metavar=("MIN", "MAX"),
+ help="Optional diffractometer soft-limits for roll angle chi",
+ )
+ parser.add_argument(
+ "--d-bounds",
+ nargs=2,
+ type=float,
+ default=None,
+ metavar=("MIN", "MAX"),
+ help="Optional diffractometer soft-limits for d / 2theta / obpitch",
+ )
+ parser.add_argument(
+ "--voltage-bounds",
+ nargs=2,
+ type=float,
+ default=None,
+ metavar=("V_MIN", "V_MAX"),
+ help="Optional electrochemical voltage safety window (V); pauses zoom triggers outside range",
+ )
+ parser.add_argument(
+ "--max-zooms-per-lineage",
+ type=int,
+ default=None,
+ help="Optional per-grain maximum high-res zoom count to cap X-ray dose",
+ )
+ parser.add_argument(
+ "--sliding-window",
+ type=int,
+ default=24,
+ help="Bounded causal tracking window size in scans (default: 24 for O(1) latency)",
+ )
+ parser.add_argument(
+ "--zoom-budget",
+ type=int,
+ default=None,
+ help="Optional cap on total zoom acquisitions during --mode replay baseline comparison",
+ )
+ parser.add_argument("--poll-interval", type=float, default=0.5, help="Directory poll interval (s) in live modes")
+ parser.add_argument(
+ "--settle-sec",
+ type=float,
+ default=0.03,
+ help="File size/mtime quiescence window (s) before reading newly created HDF5 scans",
+ )
+ parser.add_argument(
+ "--max-scans",
+ type=int,
+ default=None,
+ help="Stop watching after ingesting N scans (useful for bounded live runs or tests)",
+ )
+ parser.add_argument(
+ "--max-idle-polls",
+ type=int,
+ default=20,
+ help="Stop live watcher after this many consecutive polls with no ready scans (0 = run indefinitely)",
+ )
+ return parser
+
+
+def _load_optional_calibration(path_str: str | None) -> AxisCalibration | None:
+ if not path_str:
+ return None
+ raw = json.loads(Path(path_str).read_text(encoding="utf-8"))
+ return AxisCalibration.from_dict(raw)
+
+
+def _is_hdf5_quiescent_and_readable(path: Path, *, settle_sec: float = 0.03) -> bool:
+ """Verify file size/mtime stability and SWMR HDF5 dataset readability before ingestion."""
+ try:
+ st0 = path.stat()
+ if st0.st_size <= 0:
+ return False
+ if settle_sec > 0.0:
+ time.sleep(settle_sec)
+ st1 = path.stat()
+ if st1.st_size != st0.st_size or st1.st_mtime_ns != st0.st_mtime_ns:
+ return False
+ h5py = _require_h5py()
+ with _open_h5_readonly(h5py, path, swmr=True) as handle:
+ ds_path = _resolve_volume_dataset(handle, h5py, auto_discover=True)
+ ds = handle[ds_path]
+ if ds.ndim != 3 or any(int(dim) <= 0 for dim in ds.shape):
+ return False
+ # Probe trailing frame voxel to catch incomplete LIMA chunk flushes
+ _ = ds[-1, 0:1, 0:1]
+ return True
+ except (OSError, KeyError, ValueError, RuntimeError):
+ return False
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = build_parser().parse_args(argv)
+ root = resolve_dataset_root(args.root)
+ outdir = Path(args.outdir)
+ outdir.mkdir(parents=True, exist_ok=True)
+
+ calibration = _load_optional_calibration(args.calibration_json)
+ detector = DetectorGeometry.from_json(args.detector_json) if args.detector_json else None
+ prec_cfg = PrecursorConfig(
+ score_threshold=float(args.min_precursor_score),
+ min_persistence=int(args.min_persistence),
+ )
+ policy = ZoomTriggerPolicy(
+ min_precursor_score=float(args.min_precursor_score),
+ min_persistence=int(args.min_persistence),
+ cooldown_scans=int(args.cooldown_scans),
+ max_triggers_per_scan=int(args.max_triggers_per_scan),
+ lead_scans=float(args.lead_scans),
+ window_sigma=float(args.window_sigma),
+ acquisition=args.acquisition,
+ ucb_beta=float(args.ucb_beta),
+ execution_mode=args.mode,
+ mu_bounds=(float(args.mu_bounds[0]), float(args.mu_bounds[1])) if args.mu_bounds else None,
+ chi_bounds=(float(args.chi_bounds[0]), float(args.chi_bounds[1])) if args.chi_bounds else None,
+ d_bounds=(float(args.d_bounds[0]), float(args.d_bounds[1])) if args.d_bounds else None,
+ voltage_bounds_v=(float(args.voltage_bounds[0]), float(args.voltage_bounds[1]))
+ if args.voltage_bounds
+ else None,
+ max_zooms_per_lineage=args.max_zooms_per_lineage,
+ )
+
+ triggers_jsonl = outdir / "zoom_triggers.jsonl"
+ steps_json = outdir / "autonomous_steps.json"
+
+ if args.mode == "replay":
+ seq = BeamlineAdapter(root).build_sequence()
+ if not seq.scans:
+ raise FileNotFoundError(f"No valid scan folders discovered under {root}")
+ scan_files = [sc.file_path for sc in seq.scans]
+ scan_names = [sc.scan_name for sc in seq.scans]
+ if args.max_scans is not None:
+ scan_files = scan_files[: int(args.max_scans)]
+ scan_names = scan_names[: int(args.max_scans)]
+
+ stimuli = None
+ if args.operando_log:
+ op_log = load_operando_log(args.operando_log)
+ stimuli = synchronize_to_scans(op_log, seq)[: len(scan_files)]
+
+ bench = evaluate_causal_replay(
+ scan_files,
+ scan_names=scan_names,
+ stimuli=stimuli,
+ calibration=calibration,
+ detector=detector,
+ backend=args.backend,
+ precursor_config=prec_cfg,
+ trigger_policy=policy,
+ zoom_budget=args.zoom_budget,
+ )
+
+ with triggers_jsonl.open("w", encoding="utf-8") as fh:
+ for tgt in bench.zoom_targets:
+ fh.write(json.dumps(tgt.to_bliss_command()) + "\n")
+
+ steps_json.write_text(json.dumps([s.to_dict() for s in bench.steps], indent=2), encoding="utf-8")
+ summary_path = outdir / "replay_summary.json"
+ summary_path.write_text(json.dumps(bench.to_dict(), indent=2), encoding="utf-8")
+
+ bt_m = bench.policy_comparison["braggtrack_autonomous"]
+ print(
+ f"[braggtrack-watch:replay] Processed {bench.n_scans} scans "
+ f"(mean {bench.mean_step_latency_sec:.3f}s/scan, acq={args.acquisition}) -> "
+ f"{len(bench.zoom_targets)} zoom trigger(s)"
+ )
+ print(
+ f" Fracture capture recall={bt_m.fracture_capture_recall:.2f}, "
+ f"precision={bt_m.fracture_capture_precision:.2f}, "
+ f"mean lead={bt_m.mean_lead_scans_before_split:.2f} scans, "
+ f"next-scan hit rate={bt_m.setpoint_hit_rate_next_scan:.2f}"
+ )
+ return 0
+
+ # Live directory polling modes: "shadow", "watch" (non-actuating), or "active" (closed-loop)
+ tracker = CausalStreamingTracker(
+ backend=args.backend,
+ model_name=args.model,
+ calibration=calibration,
+ detector=detector,
+ precursor_config=prec_cfg,
+ trigger_policy=policy,
+ sliding_window_scans=int(args.sliding_window) if args.sliding_window and args.sliding_window > 0 else None,
+ swmr=True,
+ )
+ seen_files: set[Path] = set()
+ step_dicts: list[dict] = []
+ idle_polls = 0
+
+ triggers_jsonl.write_text("", encoding="utf-8")
+ while True:
+ try:
+ seq = BeamlineAdapter(root).build_sequence()
+ except OSError:
+ seq = None
+
+ stim_by_name = {}
+ if seq is not None and args.operando_log and seq.scans:
+ try:
+ op_log = load_operando_log(args.operando_log)
+ for sp in synchronize_to_scans(op_log, seq):
+ stim_by_name[sp.scan_name] = sp
+ except (OSError, ValueError):
+ pass
+
+ candidates = [sc for sc in (seq.scans if seq is not None else []) if sc.file_path.resolve() not in seen_files]
+ ingested_any = False
+ for sc in candidates:
+ if not _is_hdf5_quiescent_and_readable(sc.file_path, settle_sec=max(0.0, float(args.settle_sec))):
+ continue
+ ts_iso = sc.start_time.isoformat() if sc.start_time is not None else None
+ try:
+ res = tracker.step(
+ sc.file_path,
+ scan_name=sc.scan_name,
+ timestamp=ts_iso,
+ stimulus=stim_by_name.get(sc.scan_name),
+ )
+ except (OSError, KeyError, ValueError):
+ # Transient partial HDF5 write or lock — retry cleanly on next poll without losing the scan
+ continue
+
+ seen_files.add(sc.file_path.resolve())
+ ingested_any = True
+ step_dicts.append(res.to_dict())
+ if res.zoom_targets:
+ with triggers_jsonl.open("a", encoding="utf-8") as fh:
+ for tgt in res.zoom_targets:
+ fh.write(json.dumps(tgt.to_bliss_command()) + "\n")
+ fh.flush()
+ steps_json.write_text(json.dumps(step_dicts, indent=2), encoding="utf-8")
+ print(
+ f"[braggtrack-watch:{args.mode}] scan={res.scan_name} spots={res.n_spots} "
+ f"beam_ok={res.beam_ok} zoom_triggers={len(res.zoom_targets)} ({res.step_latency_sec:.3f}s)"
+ )
+ if args.max_scans is not None and tracker.n_scans_processed >= int(args.max_scans):
+ return 0
+
+ if ingested_any:
+ idle_polls = 0
+ else:
+ idle_polls += 1
+ if args.max_idle_polls > 0 and idle_polls >= int(args.max_idle_polls):
+ break
+ time.sleep(max(0.01, float(args.poll_interval)))
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/braggtrack/io/__init__.py b/braggtrack/io/__init__.py
index 9a37a54..3b062f3 100644
--- a/braggtrack/io/__init__.py
+++ b/braggtrack/io/__init__.py
@@ -14,6 +14,13 @@
load_primary_volume,
summarize_hdf5_tree,
)
+from .operando_log import (
+ OperandoStimulusLog,
+ ScanStimulusPoint,
+ annotate_tracks_with_stimulus,
+ load_operando_log,
+ synchronize_to_scans,
+)
from .paths import default_dataset_root, resolve_dataset_root, sample_operando_root
from .validation import ValidationIssue, validate_sequence
@@ -26,9 +33,12 @@
"IMAGED11_COLUMNS",
"LazyVolume",
"MissingH5DependencyError",
+ "OperandoStimulusLog",
+ "ScanStimulusPoint",
"ScanVolumeMeta",
"ValidationIssue",
"annotate_crystallographic_columns",
+ "annotate_tracks_with_stimulus",
"default_dataset_root",
"discover_operando_scans",
"discover_primary_volume_path",
@@ -36,6 +46,7 @@
"export_nexus_process",
"extract_axis_calibration",
"extract_scan_metadata",
+ "load_operando_log",
"load_primary_volume",
"open_lazy_volume",
"otsu_threshold_chunked",
@@ -43,5 +54,6 @@
"resolve_dataset_root",
"sample_operando_root",
"summarize_hdf5_tree",
+ "synchronize_to_scans",
"validate_sequence",
]
diff --git a/braggtrack/io/beamline.py b/braggtrack/io/beamline.py
index 4762131..2b4bbcc 100644
--- a/braggtrack/io/beamline.py
+++ b/braggtrack/io/beamline.py
@@ -58,7 +58,7 @@ def _scan_to_meta(self, scan_file: ScanFile) -> ScanVolumeMeta:
volume_path = discover_primary_volume_path(scan_file.path)
except (KeyError, ValueError):
volume_path = None
- except MissingH5DependencyError as exc:
+ except (MissingH5DependencyError, OSError) as exc:
metadata_error = str(exc)
extras: dict[str, Any] = {}
diff --git a/braggtrack/io/lazy.py b/braggtrack/io/lazy.py
index 968a26e..ea5dc86 100644
--- a/braggtrack/io/lazy.py
+++ b/braggtrack/io/lazy.py
@@ -96,10 +96,13 @@ def open_lazy_volume(
candidates: list[str] | None = None,
*,
auto_discover: bool = True,
+ swmr: bool = False,
) -> LazyVolume:
"""Open an HDF5 file and return a :class:`LazyVolume` bound to its primary 3D dataset."""
+ from .nexus import _open_h5_readonly
+
h5py = _require_h5py()
- handle = h5py.File(path, "r")
+ handle = _open_h5_readonly(h5py, path, swmr=swmr)
try:
ds_path = _resolve_volume_dataset(handle, h5py, candidates=candidates, auto_discover=auto_discover)
ds = handle[ds_path]
diff --git a/braggtrack/io/nexus.py b/braggtrack/io/nexus.py
index e85f604..ad9c6a2 100644
--- a/braggtrack/io/nexus.py
+++ b/braggtrack/io/nexus.py
@@ -15,6 +15,9 @@
"/entry1/data/data",
"/entry_0000/ESRF-ID03/pco_nf/data",
"/entry_0000/data/data",
+ "/entry/darfix/data",
+ "/entry_0000/darfix/data",
+ "/entry/process/darfix/results/data",
"/entry/instrument/detector/data",
"/entry1/instrument/detector/data",
)
@@ -33,6 +36,17 @@ def _require_h5py() -> Any:
raise MissingH5DependencyError("h5py is required for HDF5/NeXus parsing. Install dependencies first.") from exc
+def _open_h5_readonly(h5py: Any, path: str | Path, *, swmr: bool = False) -> Any:
+ """Open an HDF5 file read-only, enabling SWMR when supported by the file superblock."""
+ if not swmr:
+ return h5py.File(path, "r")
+ try:
+ return h5py.File(path, "r", swmr=True)
+ except ValueError:
+ # Older HDF5 v0/v2 superblocks reject swmr=True with ValueError; fall back cleanly while keeping OSError intact.
+ return h5py.File(path, "r")
+
+
def summarize_hdf5_tree(path: str | Path) -> list[tuple[str, str, tuple[int, ...] | None, str | None]]:
"""Return a flat summary of groups/datasets in an HDF5 file.
@@ -66,6 +80,22 @@ def _decode_value(value: Any) -> Any:
return value
+def _decode_str_list(value: Any) -> list[str]:
+ if value is None:
+ return []
+ if isinstance(value, (bytes, str)):
+ s = _decode_value(value)
+ return [s] if isinstance(s, str) and s.strip() else []
+ if hasattr(value, "__iter__"):
+ out: list[str] = []
+ for item in value:
+ s = _decode_value(item)
+ if isinstance(s, str) and s.strip():
+ out.append(s)
+ return out
+ return []
+
+
def _candidate_entries(handle: Any, h5py: Any) -> list[str]:
seen: list[str] = []
@@ -101,7 +131,7 @@ def _read_first(handle: Any, candidates: list[str], h5py: Any | None = None) ->
return None
-def extract_scan_metadata(path: str | Path) -> dict[str, Any]:
+def extract_scan_metadata(path: str | Path, *, swmr: bool = False) -> dict[str, Any]:
"""Extract common beamline metadata from a NeXus/HDF5 scan file.
Probes standard NeXus ``/entry*`` groups (including ``/entry_0000`` used by
@@ -111,7 +141,7 @@ def extract_scan_metadata(path: str | Path) -> dict[str, Any]:
h5py = _require_h5py()
path = Path(path)
- with h5py.File(path, "r") as handle:
+ with _open_h5_readonly(h5py, path, swmr=swmr) as handle:
entries = _candidate_entries(handle, h5py) or ["/entry", "/entry1", "/entry_0000"]
start_candidates = [f"{e}/start_time" for e in entries]
end_candidates = [f"{e}/end_time" for e in entries]
@@ -133,19 +163,28 @@ def extract_scan_metadata(path: str | Path) -> dict[str, Any]:
return metadata
-def extract_axis_calibration(path: str | Path) -> AxisCalibration:
+_MOTOR_AXIS_SYNONYMS: dict[str, tuple[str, ...]] = {
+ "mu": ("mu", "omega", "obx", "rocking"),
+ "chi": ("chi", "phi", "oby"),
+ "d": ("d", "obpitch", "twotheta", "two_theta", "2theta", "tth"),
+}
+
+
+def extract_axis_calibration(path: str | Path, *, swmr: bool = False) -> AxisCalibration:
"""Extract linear ``(mu, chi, d)`` calibration from a NeXus/HDF5 file if present.
- Checks ``/entry*/calibration`` attributes/datasets or 1D coordinate arrays
- ``mu``, ``chi``, and ``d``. Returns an uncalibrated identity
- :class:`AxisCalibration` when physical axis scales are absent.
+ Probes:
+ 1. Explicit ``/entry*/calibration`` groups.
+ 2. NeXus ``NXdata`` ``@axes`` attributes on the primary 3D dataset's parent group (e.g. ``darfix``).
+ 3. 1D coordinate arrays under ``NXentry``, ``data``, ``measurement``, or ESRF BLISS
+ ``instrument/positioners/{mu, chi, obpitch, twotheta, d}``.
"""
import numpy as np
h5py = _require_h5py()
path = Path(path)
- with h5py.File(path, "r") as handle:
+ with _open_h5_readonly(h5py, path, swmr=swmr) as handle:
vol_parent: str | None = None
try:
ds_path = _resolve_volume_dataset(handle, h5py, auto_discover=True)
@@ -165,16 +204,50 @@ def extract_axis_calibration(path: str | Path) -> AxisCalibration:
if raw:
return AxisCalibration.from_dict(raw)
- # Check 1D axis arrays under NXentry, NXdata, or the primary 3D dataset's parent group.
+ # Check NeXus NXdata @axes attribute on parent group (e.g. darfix NXdata cubes: (z=mu, y=d/2theta, x=chi))
+ if vol_parent and vol_parent != "/" and vol_parent in handle:
+ pgrp = handle[vol_parent]
+ nx_axes = _decode_str_list(pgrp.attrs.get("axes"))
+ if len(nx_axes) == 3:
+ nx_found: dict[str, tuple[float, float]] = {}
+ # Voxel order (z, y, x) -> BraggTrack (mu, d, chi)
+ for dim_idx, target_axis in ((0, "mu"), (1, "d"), (2, "chi")):
+ ax_ds_name = nx_axes[dim_idx]
+ if ax_ds_name != "." and ax_ds_name in pgrp and isinstance(pgrp[ax_ds_name], h5py.Dataset):
+ arr = np.asarray(pgrp[ax_ds_name][()], dtype=np.float64).ravel()
+ if arr.size >= 2 and float(arr[-1] - arr[0]) != 0.0:
+ nx_found[target_axis] = (float(arr[0]), float((arr[-1] - arr[0]) / (arr.size - 1)))
+ if nx_found:
+ return AxisCalibration(
+ mu_origin=nx_found.get("mu", (0.0, 1.0))[0],
+ mu_step=nx_found.get("mu", (0.0, 1.0))[1],
+ chi_origin=nx_found.get("chi", (0.0, 1.0))[0],
+ chi_step=nx_found.get("chi", (0.0, 1.0))[1],
+ d_origin=nx_found.get("d", (0.0, 1.0))[0],
+ d_step=nx_found.get("d", (0.0, 1.0))[1],
+ calibrated=True,
+ )
+
+ for e in entries:
+ # Check 1D axis/motor arrays under NXentry, NXdata, measurement, or ESRF BLISS instrument/positioners
axes_found: dict[str, tuple[float, float]] = {}
- for axis_name in ("mu", "chi", "d"):
- search_paths = [f"{e}/{axis_name}", f"{e}/data/{axis_name}", f"{e}/measurement/{axis_name}"]
- if vol_parent and vol_parent != "/":
- search_paths.append(f"{vol_parent.rstrip('/')}/{axis_name}")
+ for axis_name, synonyms in _MOTOR_AXIS_SYNONYMS.items():
+ search_paths: list[str] = []
+ for syn in synonyms:
+ search_paths.extend(
+ [
+ f"{e}/{syn}",
+ f"{e}/data/{syn}",
+ f"{e}/measurement/{syn}",
+ f"{e}/instrument/positioners/{syn}",
+ ]
+ )
+ if vol_parent and vol_parent != "/":
+ search_paths.append(f"{vol_parent.rstrip('/')}/{syn}")
for sub in search_paths:
if sub in handle and isinstance(handle[sub], h5py.Dataset):
arr = np.asarray(handle[sub][()], dtype=np.float64).ravel()
- if arr.size >= 2:
+ if arr.size >= 2 and float(arr[-1] - arr[0]) != 0.0:
axes_found[axis_name] = (float(arr[0]), float((arr[-1] - arr[0]) / (arr.size - 1)))
break
if axes_found:
@@ -209,6 +282,19 @@ def _resolve_volume_dataset(
return "/" + key.lstrip("/")
if candidates is None and auto_discover:
+ # Check NeXus @default -> NXdata @signal chain first (used by darfix / silx / Ewoks)
+ for e in _candidate_entries(handle, h5py):
+ eg = handle[e]
+ def_data = _decode_value(eg.attrs.get("default"))
+ if isinstance(def_data, str) and def_data.strip():
+ dgrp_path = f"{e.rstrip('/')}/{def_data.strip('/')}"
+ if dgrp_path in handle and isinstance(handle[dgrp_path], h5py.Group):
+ sig = _decode_value(handle[dgrp_path].attrs.get("signal"))
+ if isinstance(sig, str) and sig in handle[dgrp_path]:
+ sig_ds = handle[dgrp_path][sig]
+ if isinstance(sig_ds, h5py.Dataset) and sig_ds.ndim == 3:
+ return f"{dgrp_path}/{sig}"
+
discovered: list[tuple[str, tuple[int, ...]]] = []
def _visit(name: str, obj: Any) -> None:
@@ -227,10 +313,11 @@ def discover_primary_volume_path(
candidates: list[str] | None = None,
*,
auto_discover: bool = True,
+ swmr: bool = False,
) -> str:
"""Return the HDF5 dataset path of the primary 3D detector volume."""
h5py = _require_h5py()
- with h5py.File(path, "r") as handle:
+ with _open_h5_readonly(h5py, path, swmr=swmr) as handle:
return _resolve_volume_dataset(handle, h5py, candidates=candidates, auto_discover=auto_discover)
@@ -239,11 +326,12 @@ def load_primary_volume(
candidates: list[str] | None = None,
*,
auto_discover: bool = True,
+ swmr: bool = False,
) -> np.ndarray:
"""Load a primary 3D detector volume from NeXus paths or largest-3D auto-discovery."""
import numpy as np
h5py = _require_h5py()
- with h5py.File(path, "r") as handle:
+ with _open_h5_readonly(h5py, path, swmr=swmr) as handle:
ds_path = _resolve_volume_dataset(handle, h5py, candidates=candidates, auto_discover=auto_discover)
return np.asarray(handle[ds_path][()], dtype=np.float64)
diff --git a/braggtrack/io/operando_log.py b/braggtrack/io/operando_log.py
new file mode 100644
index 0000000..4034070
--- /dev/null
+++ b/braggtrack/io/operando_log.py
@@ -0,0 +1,453 @@
+"""Operando external stimulus log ingestion and scan synchronization.
+
+Supports generic CSV/TSV potentiostat/load/thermal logs as well as Biologic
+EC-Lab ``.mpt`` ASCII exports (``Nb header lines : N``, ``time/s``, ``Ewe/V``,
+``/mA``, ``(Q-Qo)/mA.h``, ``cycle number``).
+"""
+
+from __future__ import annotations
+
+import csv
+import math
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+if TYPE_CHECKING:
+ from .models import ExperimentSequence
+
+
+_COL_ALIASES: dict[str, tuple[str, ...]] = {
+ "scan_idx": ("scan_idx", "scan", "scan_index", "frame", "frame_idx"),
+ "timestamp": ("timestamp", "iso_time", "datetime", "date_time", "utc_time", "abs_time"),
+ "uts": ("uts", "epoch", "epoch_s", "unix_time"),
+ "time_s": ("time_s", "elapsed_s", "time/s", "time", "t_s", "seconds"),
+ "voltage_v": ("voltage_v", "voltage", "ewe/v", "/v", "ecell/v", "v_cell", "potential_v"),
+ "current_ma": ("current_ma", "current", "i/ma", "/ma", "current/ma"),
+ "capacity_mah_g": (
+ "capacity_mah_g",
+ "specific_capacity",
+ "capacity",
+ "q_mah_g",
+ "(q-qo)/ma.h",
+ "q charge/ma.h",
+ "q discharge/ma.h",
+ "dq/ma.h",
+ ),
+ "dq_dv": ("dq_dv", "dq/dv", "dqdv"),
+ "cycle": ("cycle", "cycle_number", "cycle number", "half_cycle"),
+ "stress_mpa": ("stress_mpa", "stress", "load_mpa", "sigma_mpa"),
+ "temperature_c": ("temperature_c", "temperature", "temp_c", "t_degc", "temperature/°c", "temperature/c"),
+}
+
+
+@dataclass(frozen=True)
+class OperandoStimulusLog:
+ """Time-series external operando stimulus channels (electrochemistry, load, temperature)."""
+
+ time_s: np.ndarray
+ scan_idx: np.ndarray | None = None
+ epoch_s: np.ndarray | None = None
+ voltage_v: np.ndarray | None = None
+ current_ma: np.ndarray | None = None
+ capacity_mah_g: np.ndarray | None = None
+ dq_dv: np.ndarray | None = None
+ cycle: np.ndarray | None = None
+ stress_mpa: np.ndarray | None = None
+ temperature_c: np.ndarray | None = None
+ source_path: str | None = None
+
+ @property
+ def n_points(self) -> int:
+ return int(len(self.time_s))
+
+
+@dataclass(frozen=True)
+class ScanStimulusPoint:
+ """Synchronized external stimulus state evaluated at one diffraction scan."""
+
+ scan_idx: int
+ scan_name: str
+ elapsed_s: float
+ timestamp: str | None = None
+ voltage_v: float | None = None
+ current_ma: float | None = None
+ capacity_mah_g: float | None = None
+ dq_dv: float | None = None
+ cycle: int | None = None
+ stress_mpa: float | None = None
+ temperature_c: float | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "scan_idx": self.scan_idx,
+ "scan_name": self.scan_name,
+ "elapsed_s": round(float(self.elapsed_s), 4),
+ "timestamp": self.timestamp,
+ "voltage_v": round(float(self.voltage_v), 5) if self.voltage_v is not None else None,
+ "current_ma": round(float(self.current_ma), 5) if self.current_ma is not None else None,
+ "capacity_mah_g": round(float(self.capacity_mah_g), 4) if self.capacity_mah_g is not None else None,
+ "dq_dv": round(float(self.dq_dv), 4) if self.dq_dv is not None else None,
+ "cycle": int(self.cycle) if self.cycle is not None else None,
+ "stress_mpa": round(float(self.stress_mpa), 4) if self.stress_mpa is not None else None,
+ "temperature_c": round(float(self.temperature_c), 3) if self.temperature_c is not None else None,
+ }
+
+
+def _norm_col(name: str) -> str:
+ return re.sub(r"\s+", " ", name.strip().lower())
+
+
+def _match_columns(headers: list[str]) -> dict[str, str]:
+ norm_to_orig = {_norm_col(h): h for h in headers if h is not None}
+ matched: dict[str, str] = {}
+ for canonical, aliases in _COL_ALIASES.items():
+ for alias in aliases:
+ if alias in norm_to_orig:
+ matched[canonical] = norm_to_orig[alias]
+ break
+ return matched
+
+
+def _parse_iso_to_epoch(val: str) -> float | None:
+ s = val.strip()
+ if not s:
+ return None
+ if s.endswith("Z"):
+ s = s[:-1] + "+00:00"
+ try:
+ dt = datetime.fromisoformat(s)
+ except ValueError:
+ for fmt in ("%m/%d/%Y %H:%M:%S.%f", "%m/%d/%Y %H:%M:%S", "%Y/%m/%d %H:%M:%S"):
+ try:
+ dt = datetime.strptime(s, fmt)
+ break
+ except ValueError:
+ continue
+ else:
+ return None
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return float(dt.timestamp())
+
+
+def _parse_float(val: str | None) -> float:
+ if val is None:
+ return math.nan
+ s = val.strip().replace(",", ".") if "," in val and "." not in val else val.strip()
+ if not s:
+ return math.nan
+ try:
+ return float(s)
+ except ValueError:
+ return math.nan
+
+
+def _compute_regularized_dq_dv(
+ voltage_v: np.ndarray,
+ capacity: np.ndarray,
+ *,
+ min_dv: float = 1.5e-3,
+) -> np.ndarray:
+ """Compute causal backward-difference ``dQ/dV`` with mild voltage-step regularization."""
+ n = len(voltage_v)
+ out = np.zeros(n, dtype=np.float64)
+ if n < 2:
+ return out
+
+ # Smooth voltage slightly with a 3-point causal moving average to suppress potentiostat bit noise
+ v_smooth = np.empty(n, dtype=np.float64)
+ q_smooth = np.empty(n, dtype=np.float64)
+ for i in range(n):
+ lo = max(0, i - 2)
+ v_smooth[i] = float(np.nanmean(voltage_v[lo : i + 1]))
+ q_smooth[i] = float(np.nanmean(capacity[lo : i + 1]))
+
+ for i in range(1, n):
+ dv = v_smooth[i] - v_smooth[i - 1]
+ dq = abs(q_smooth[i] - q_smooth[i - 1])
+ if abs(dv) >= min_dv and math.isfinite(dv) and math.isfinite(dq):
+ out[i] = math.copysign(dq / abs(dv), dv)
+ else:
+ out[i] = out[i - 1]
+ out[0] = out[1]
+ return out
+
+
+def load_operando_log(path: str | Path) -> OperandoStimulusLog:
+ """Load an external operando CSV, TSV, or Biologic EC-Lab ``.mpt`` file."""
+ file_path = Path(path)
+ text = file_path.read_text(encoding="utf-8", errors="replace")
+ lines = text.splitlines()
+ if not lines:
+ raise ValueError(f"Operando stimulus log '{file_path}' is empty.")
+
+ # Detect Biologic EC-Lab ASCII header ("Nb header lines : N")
+ header_idx = 0
+ for line in lines[:30]:
+ m = re.match(r"^\s*Nb\s+header\s+lines\s*:\s*(\d+)", line, flags=re.IGNORECASE)
+ if m:
+ header_idx = max(0, int(m.group(1)) - 1)
+ break
+ else:
+ # Skip leading comment lines starting with '#'
+ while header_idx < len(lines) and (not lines[header_idx].strip() or lines[header_idx].lstrip().startswith("#")):
+ header_idx += 1
+
+ if header_idx >= len(lines):
+ raise ValueError(f"Could not locate table header in operando log '{file_path}'.")
+
+ body_lines = [ln for ln in lines[header_idx:] if ln.strip()]
+ header_line = body_lines[0]
+ delimiter = "\t" if "\t" in header_line else (";" if ";" in header_line and "," not in header_line else ",")
+
+ reader = csv.DictReader(body_lines, delimiter=delimiter)
+ if not reader.fieldnames:
+ raise ValueError(f"No column headers found in operando log '{file_path}'.")
+
+ col_map = _match_columns(list(reader.fieldnames))
+ raw_rows = list(reader)
+ if not raw_rows:
+ raise ValueError(f"Operando stimulus log '{file_path}' contains zero data rows.")
+
+ n = len(raw_rows)
+
+ def _extract_num(canon: str) -> np.ndarray | None:
+ col = col_map.get(canon)
+ if col is None:
+ return None
+ arr = np.array([_parse_float(r.get(col)) for r in raw_rows], dtype=np.float64)
+ if np.all(np.isnan(arr)):
+ return None
+ return arr
+
+ scan_arr = _extract_num("scan_idx")
+ time_arr = _extract_num("time_s")
+ uts_arr = _extract_num("uts")
+
+ epoch_arr: np.ndarray | None = uts_arr
+ if epoch_arr is None and "timestamp" in col_map:
+ ts_col = col_map["timestamp"]
+ parsed_ep = [_parse_iso_to_epoch(str(r.get(ts_col, ""))) for r in raw_rows]
+ if any(x is not None for x in parsed_ep):
+ epoch_arr = np.array([x if x is not None else math.nan for x in parsed_ep], dtype=np.float64)
+
+ if time_arr is None:
+ if epoch_arr is not None:
+ valid0 = float(epoch_arr[np.isfinite(epoch_arr)][0])
+ time_arr = epoch_arr - valid0
+ elif scan_arr is not None:
+ time_arr = scan_arr.astype(np.float64)
+ else:
+ time_arr = np.arange(n, dtype=np.float64)
+
+ voltage_v = _extract_num("voltage_v")
+ current_ma = _extract_num("current_ma")
+ capacity = _extract_num("capacity_mah_g")
+ dq_dv = _extract_num("dq_dv")
+ cycle = _extract_num("cycle")
+ stress_mpa = _extract_num("stress_mpa")
+ temperature_c = _extract_num("temperature_c")
+
+ # Derive Coulombic capacity (mAh) when current_mA & time_s exist without explicit capacity column
+ if capacity is None and current_ma is not None and len(time_arr) >= 2:
+ dt_h = np.diff(time_arr, prepend=time_arr[0]) / 3600.0
+ dt_h = np.where(np.isfinite(dt_h) & (dt_h > 0.0), dt_h, 0.0)
+ i_clean = np.where(np.isfinite(current_ma), np.abs(current_ma), 0.0)
+ capacity = np.cumsum(i_clean * dt_h)
+
+ if dq_dv is None and voltage_v is not None and capacity is not None:
+ dq_dv = _compute_regularized_dq_dv(voltage_v, capacity)
+
+ return OperandoStimulusLog(
+ time_s=time_arr,
+ scan_idx=scan_arr,
+ epoch_s=epoch_arr,
+ voltage_v=voltage_v,
+ current_ma=current_ma,
+ capacity_mah_g=capacity,
+ dq_dv=dq_dv,
+ cycle=cycle,
+ stress_mpa=stress_mpa,
+ temperature_c=temperature_c,
+ source_path=str(file_path),
+ )
+
+
+def _interp_optional(x_target: np.ndarray, x_src: np.ndarray, y_src: np.ndarray | None) -> np.ndarray | None:
+ if y_src is None:
+ return None
+ mask = np.isfinite(x_src) & np.isfinite(y_src)
+ if int(np.sum(mask)) == 0:
+ return None
+ xs = x_src[mask]
+ ys = y_src[mask]
+ order = np.argsort(xs, kind="mergesort")
+ xs_sorted = xs[order]
+ ys_sorted = ys[order]
+ return np.interp(x_target, xs_sorted, ys_sorted)
+
+
+def synchronize_to_scans(
+ log: OperandoStimulusLog,
+ sequence_or_count: ExperimentSequence | int,
+ *,
+ start_offset_s: float = 0.0,
+ scan_cadence_s: float | None = None,
+) -> list[ScanStimulusPoint]:
+ """Interpolate external stimulus channels onto each diffraction scan's midpoint time or index."""
+ if isinstance(sequence_or_count, int):
+ n_scans = max(0, sequence_or_count)
+ scan_names = [f"scan{i + 1:04d}" for i in range(n_scans)]
+ scan_epochs: list[float | None] = [None] * n_scans
+ scan_isos: list[str | None] = [None] * n_scans
+ else:
+ scans = sequence_or_count.scans
+ n_scans = len(scans)
+ scan_names = [s.scan_name for s in scans]
+ scan_epochs = []
+ scan_isos = []
+ for s in scans:
+ if s.start_time is not None and s.end_time is not None:
+ st = s.start_time if s.start_time.tzinfo else s.start_time.replace(tzinfo=timezone.utc)
+ et = s.end_time if s.end_time.tzinfo else s.end_time.replace(tzinfo=timezone.utc)
+ mid_ep = 0.5 * (st.timestamp() + et.timestamp())
+ scan_epochs.append(float(mid_ep))
+ scan_isos.append(st.isoformat())
+ elif s.start_time is not None:
+ st = s.start_time if s.start_time.tzinfo else s.start_time.replace(tzinfo=timezone.utc)
+ scan_epochs.append(float(st.timestamp()))
+ scan_isos.append(st.isoformat())
+ else:
+ scan_epochs.append(None)
+ scan_isos.append(None)
+
+ if n_scans == 0:
+ return []
+
+ idx_vec = np.arange(n_scans, dtype=np.float64)
+
+ # Choose synchronization coordinate: explicit scan_idx > absolute epoch > elapsed seconds
+ if log.scan_idx is not None and np.any(np.isfinite(log.scan_idx)):
+ x_src = log.scan_idx
+ x_query = idx_vec
+ elapsed_s = _interp_optional(x_query, x_src, log.time_s)
+ if elapsed_s is None:
+ elapsed_s = idx_vec * (scan_cadence_s or 60.0)
+ elif log.epoch_s is not None and all(ep is not None for ep in scan_epochs):
+ x_src = log.epoch_s
+ x_query = np.array([float(ep) for ep in scan_epochs if ep is not None], dtype=np.float64)
+ elapsed_s = x_query - x_query[0]
+ elif all(ep is not None for ep in scan_epochs):
+ ep_arr = np.array([float(ep) for ep in scan_epochs if ep is not None], dtype=np.float64)
+ elapsed_s = (ep_arr - ep_arr[0]) + start_offset_s
+ x_src = log.time_s
+ x_query = elapsed_s
+ else:
+ if scan_cadence_s is not None:
+ elapsed_s = start_offset_s + idx_vec * scan_cadence_s
+ else:
+ finite_t = log.time_s[np.isfinite(log.time_s)]
+ t_min = float(finite_t[0]) if len(finite_t) else 0.0
+ t_max = float(finite_t[-1]) if len(finite_t) else float(max(0, n_scans - 1) * 60.0)
+ elapsed_s = np.linspace(t_min, t_max, n_scans) if n_scans > 1 else np.array([t_min])
+ x_src = log.time_s
+ x_query = elapsed_s
+
+ v_interp = _interp_optional(x_query, x_src, log.voltage_v)
+ i_interp = _interp_optional(x_query, x_src, log.current_ma)
+ q_interp = _interp_optional(x_query, x_src, log.capacity_mah_g)
+ dqdv_interp = _interp_optional(x_query, x_src, log.dq_dv)
+ cyc_interp = _interp_optional(x_query, x_src, log.cycle)
+ sig_interp = _interp_optional(x_query, x_src, log.stress_mpa)
+ tmp_interp = _interp_optional(x_query, x_src, log.temperature_c)
+
+ points: list[ScanStimulusPoint] = []
+ for k in range(n_scans):
+ points.append(
+ ScanStimulusPoint(
+ scan_idx=k,
+ scan_name=scan_names[k],
+ elapsed_s=float(elapsed_s[k]),
+ timestamp=scan_isos[k],
+ voltage_v=float(v_interp[k]) if v_interp is not None else None,
+ current_ma=float(i_interp[k]) if i_interp is not None else None,
+ capacity_mah_g=float(q_interp[k]) if q_interp is not None else None,
+ dq_dv=float(dqdv_interp[k]) if dqdv_interp is not None else None,
+ cycle=int(round(float(cyc_interp[k]))) if cyc_interp is not None else None,
+ stress_mpa=float(sig_interp[k]) if sig_interp is not None else None,
+ temperature_c=float(tmp_interp[k]) if tmp_interp is not None else None,
+ )
+ )
+ return points
+
+
+def annotate_tracks_with_stimulus(
+ track_rows: list[dict[str, Any]],
+ scan_stimuli: list[ScanStimulusPoint],
+) -> list[dict[str, Any]]:
+ """Return shallow-copied track rows enriched with synchronized stimulus and ``d_strain_d_v``."""
+ if not track_rows:
+ return []
+
+ stim_by_scan = {pt.scan_idx: pt for pt in scan_stimuli}
+ enriched: list[dict[str, Any]] = []
+ prev_by_tid: dict[int, tuple[float | None, float | None]] = {}
+
+ computed_strain_by_key: dict[tuple[int, int], float] = {}
+ if "lineage_strain" not in track_rows[0] and "strain" not in track_rows[0] and "centroid_d" in track_rows[0]:
+ from braggtrack.tracking.kinematics import compute_grain_kinematics
+
+ for gk in compute_grain_kinematics(track_rows):
+ lin = gk.lineage_strain if len(gk.lineage_strain) else gk.strain
+ for idx_s, s_i in enumerate(gk.scan_indices):
+ computed_strain_by_key[(gk.track_id, s_i)] = float(lin[idx_s])
+
+ # Process in chronological order per track so d_strain_d_v is strictly causal
+ indexed = sorted(
+ enumerate(track_rows),
+ key=lambda item: (int(float(item[1]["track_id"])), int(float(item[1]["scan_idx"]))),
+ )
+ out_by_orig_idx: dict[int, dict[str, Any]] = {}
+
+ for orig_idx, row in indexed:
+ r = dict(row)
+ tid = int(float(r["track_id"]))
+ s_idx = int(float(r["scan_idx"]))
+ pt = stim_by_scan.get(s_idx)
+
+ v_val = pt.voltage_v if pt is not None else None
+ r["elapsed_s"] = round(pt.elapsed_s, 4) if pt is not None else ""
+ r["voltage_v"] = round(v_val, 5) if v_val is not None else ""
+ r["current_ma"] = round(pt.current_ma, 5) if (pt is not None and pt.current_ma is not None) else ""
+ r["capacity_mah_g"] = round(pt.capacity_mah_g, 4) if (pt is not None and pt.capacity_mah_g is not None) else ""
+ r["dq_dv"] = round(pt.dq_dv, 4) if (pt is not None and pt.dq_dv is not None) else ""
+ r["stress_mpa"] = round(pt.stress_mpa, 4) if (pt is not None and pt.stress_mpa is not None) else ""
+ r["temperature_c"] = round(pt.temperature_c, 3) if (pt is not None and pt.temperature_c is not None) else ""
+
+ strain_val: float | None = computed_strain_by_key.get((tid, s_idx))
+ if strain_val is None:
+ for key in ("lineage_strain", "strain"):
+ if key in r and r[key] != "":
+ try:
+ strain_val = float(r[key])
+ break
+ except (TypeError, ValueError):
+ pass
+
+ d_eps_dv: float | str = ""
+ if tid in prev_by_tid and v_val is not None and strain_val is not None:
+ prev_v, prev_eps = prev_by_tid[tid]
+ if prev_v is not None and prev_eps is not None and abs(v_val - prev_v) >= 2e-3:
+ d_eps_dv = round((strain_val - prev_eps) / (v_val - prev_v), 6)
+ r["d_strain_d_v"] = d_eps_dv
+ prev_by_tid[tid] = (v_val, strain_val)
+ out_by_orig_idx[orig_idx] = r
+
+ for i in range(len(track_rows)):
+ enriched.append(out_by_orig_idx[i])
+ return enriched
diff --git a/braggtrack/segmentation/classical.py b/braggtrack/segmentation/classical.py
index 878921b..79a31cd 100644
--- a/braggtrack/segmentation/classical.py
+++ b/braggtrack/segmentation/classical.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import math
from dataclasses import dataclass
import numpy as np
@@ -19,9 +20,18 @@ class ClassicalSegmentationResult:
response: np.ndarray
+def _float_work_dtype(arr: np.ndarray) -> np.dtype:
+ """Use float32 for <=32-bit detector arrays (50% RAM cut) and float64 when explicitly float64."""
+ dt = np.asarray(arr).dtype
+ if dt == np.float64:
+ return np.dtype(np.float64)
+ return np.dtype(np.float32)
+
+
def gaussian_blur_3d(volume: np.ndarray, passes: int = 1, sigma: float = 1.0) -> np.ndarray:
"""Apply Gaussian blur along each axis."""
- result = np.asarray(volume, dtype=np.float64)
+ work_dt = _float_work_dtype(volume)
+ result = np.asarray(volume, dtype=work_dt)
for _ in range(max(1, passes)):
result = gaussian_filter(result, sigma=sigma)
return result
@@ -29,7 +39,8 @@ def gaussian_blur_3d(volume: np.ndarray, passes: int = 1, sigma: float = 1.0) ->
def laplacian_3d(volume: np.ndarray) -> np.ndarray:
"""6-neighbor discrete Laplacian."""
- return laplace(np.asarray(volume, dtype=np.float64)).astype(np.float64)
+ work_dt = _float_work_dtype(volume)
+ return laplace(np.asarray(volume, dtype=work_dt)).astype(work_dt, copy=False)
def log_enhance_3d(
@@ -54,11 +65,13 @@ def h_maxima_seeds(
Each seed must be a local maximum whose value exceeds the highest
neighbour by at least *h*, and must be >= *min_value*.
"""
- vol = np.asarray(volume, dtype=np.float64)
+ vol = np.asarray(volume)
+ if not np.issubdtype(vol.dtype, np.floating):
+ vol = vol.astype(np.float32)
coords = peak_local_max(
vol,
min_distance=max(1, min_separation),
- threshold_abs=min_value,
+ threshold_abs=float(min_value),
)
if len(coords) == 0:
return []
@@ -66,15 +79,30 @@ def h_maxima_seeds(
seeds: list[tuple[int, int, int]] = []
for coord in coords:
z, y, x = int(coord[0]), int(coord[1]), int(coord[2])
- center_val = vol[z, y, x]
+ center_val = float(vol[z, y, x])
z_lo, z_hi = max(0, z - 1), min(vol.shape[0], z + 2)
y_lo, y_hi = max(0, y - 1), min(vol.shape[1], y + 2)
x_lo, x_hi = max(0, x - 1), min(vol.shape[2], x + 2)
- neighborhood = vol[z_lo:z_hi, y_lo:y_hi, x_lo:x_hi].copy()
+ neighborhood = vol[z_lo:z_hi, y_lo:y_hi, x_lo:x_hi].astype(np.float64, copy=True)
neighborhood[z - z_lo, y - y_lo, x - x_lo] = -np.inf
- max_neighbor = neighborhood.max()
+ max_neighbor = float(neighborhood.max())
if center_val - max_neighbor >= h:
seeds.append((z, y, x))
+ continue
+ # Handle symmetric sub-voxel peaks (e.g. center at y=15.5) where 2..8 apex voxels tie:
+ # verify prominence >= h against the 5x5x5 shell outside tied immediate apex neighbors.
+ tie_tol = 1e-4 * max(1.0, abs(center_val))
+ if h > 0.0 and abs(center_val - max_neighbor) <= tie_tol:
+ z2_lo, z2_hi = max(0, z - 2), min(vol.shape[0], z + 3)
+ y2_lo, y2_hi = max(0, y - 2), min(vol.shape[1], y + 3)
+ x2_lo, x2_hi = max(0, x - 2), min(vol.shape[2], x + 3)
+ box5 = vol[z2_lo:z2_hi, y2_lo:y2_hi, x2_lo:x2_hi].astype(np.float64, copy=True)
+ cz, cy, cx = z - z2_lo, y - y2_lo, x - x2_lo
+ core = box5[max(0, cz - 1) : cz + 2, max(0, cy - 1) : cy + 2, max(0, cx - 1) : cx + 2]
+ core[np.abs(core - center_val) <= tie_tol] = -np.inf
+ rem_max = float(box5.max())
+ if math.isfinite(rem_max) and (center_val - rem_max) >= h:
+ seeds.append((z, y, x))
return seeds
@@ -85,11 +113,13 @@ def local_maxima_seeds(
min_separation: int = 1,
) -> list[tuple[int, int, int]]:
"""Find local maxima seeds above a minimum value."""
- vol = np.asarray(volume, dtype=np.float64)
+ vol = np.asarray(volume)
+ if not np.issubdtype(vol.dtype, np.floating):
+ vol = vol.astype(np.float32)
coords = peak_local_max(
vol,
min_distance=max(1, min_separation),
- threshold_abs=min_value,
+ threshold_abs=float(min_value),
)
return [(int(c[0]), int(c[1]), int(c[2])) for c in coords]
@@ -105,12 +135,13 @@ def watershed_from_seeds(
Passing a mask derived from the response itself under-shoots catastrophically
because the response and volume distributions are on different scales.
"""
- response = np.asarray(response, dtype=np.float64)
- markers = np.zeros(response.shape, dtype=np.int32)
+ work_dt = _float_work_dtype(response)
+ response_arr = np.asarray(response, dtype=work_dt)
+ markers = np.zeros(response_arr.shape, dtype=np.int32)
for label_id, (z, y, x) in enumerate(seeds, start=1):
markers[z, y, x] = label_id
- labeled = watershed(-response, markers=markers, mask=np.asarray(mask, dtype=bool))
+ labeled = watershed(-response_arr, markers=markers, mask=np.asarray(mask, dtype=bool))
return labeled.astype(np.int32)
@@ -162,6 +193,9 @@ def segment_classical(
min_seed_separation: int = 2,
seed_peak_fraction: float = 0.2,
seed_response_percentile: float = 99.95,
+ *,
+ working_dtype: type | np.dtype = np.float32,
+ log_scale: bool = False,
) -> ClassicalSegmentationResult:
"""Run classical LoG + h-maxima + seeded watershed pipeline.
@@ -177,10 +211,17 @@ def segment_classical(
foreground mask*. The fraction term uses a robust peak reference
(99.99th percentile) rather than the absolute max, so isolated
bright voxels don't inflate the floor.
+ working_dtype
+ Floating-point dtype used for 3D blur, Laplacian, and watershed buffers
+ (default ``np.float32``, reducing peak 3D buffer RAM by 50% vs ``float64``).
+ log_scale
+ When ``True``, compute LoG enhancement and seed peaks on ``log1p(max(volume, 0))``
+ so weak satellite reflections are not suppressed by intense primary Bragg peaks.
"""
- volume = np.asarray(volume, dtype=np.float64)
- response = log_enhance_3d(volume, blur_passes=blur_passes, sigma=sigma)
- foreground = volume >= threshold
+ vol_arr = np.asarray(volume, dtype=working_dtype)
+ foreground = vol_arr >= np.asarray(threshold, dtype=working_dtype)
+ enh_input = np.log1p(np.maximum(vol_arr, 0.0)) if log_scale else vol_arr
+ response = log_enhance_3d(enh_input, blur_passes=blur_passes, sigma=sigma)
seed_floor = _seed_floor_from_response(
response[foreground],
seed_peak_fraction=seed_peak_fraction,
@@ -194,7 +235,7 @@ def segment_classical(
)
labels = watershed_from_seeds(response, seeds=seeds, mask=foreground)
return ClassicalSegmentationResult(
- threshold=threshold,
+ threshold=float(threshold),
seed_count=len(seeds),
component_count=_count_labels(labels),
labeled_volume=labels,
diff --git a/braggtrack/segmentation/dino_segment.py b/braggtrack/segmentation/dino_segment.py
index 1250b23..8cc4b01 100644
--- a/braggtrack/segmentation/dino_segment.py
+++ b/braggtrack/segmentation/dino_segment.py
@@ -41,11 +41,11 @@ def _extract_slice_features(
pixel-level ``(H, W)`` of each slice.
"""
n_slices = volume.shape[axis]
- feature_maps: list[np.ndarray] = []
- for i in range(n_slices):
- slc = np.take(volume, i, axis=axis)
- fmap = encoder.extract_patch_features(slc)
- feature_maps.append(fmap)
+ slices = [np.take(volume, i, axis=axis) for i in range(n_slices)]
+ if hasattr(encoder, "extract_patch_features_batch"):
+ feature_maps = encoder.extract_patch_features_batch(slices)
+ else:
+ feature_maps = [encoder.extract_patch_features(slc) for slc in slices]
if axis == 0:
slice_hw = (volume.shape[1], volume.shape[2])
diff --git a/braggtrack/segmentation/features.py b/braggtrack/segmentation/features.py
index 32dcc3d..bcf8dcf 100644
--- a/braggtrack/segmentation/features.py
+++ b/braggtrack/segmentation/features.py
@@ -11,26 +11,45 @@
from braggtrack.io.models import AxisCalibration
+def _excess_kurtosis(delta: np.ndarray, weights: np.ndarray) -> float:
+ """Standardised fourth central moment (Pearson excess kurtosis ``g_2 = mu_4 / mu_2^2 - 3``).
+
+ Zero for an ideal Gaussian rocking profile; positive (leptokurtic) for heavy-tailed
+ dislocation pile-ups/notches and negative (platykurtic) for incipient bimodal splitting.
+ """
+ m2 = float(np.dot(weights, delta * delta))
+ if m2 <= 1e-12:
+ return 0.0
+ m4 = float(np.dot(weights, (delta * delta) ** 2))
+ return float(m4 / (m2 * m2) - 3.0)
+
+
def extract_instance_table(
labels: np.ndarray,
intensity: np.ndarray,
*,
calibration: AxisCalibration | None = None,
+ subtract_background: bool = True,
+ detector_gain: float = 1.0,
) -> list[dict[str, float | int]]:
- """Compute centroid, bbox, integrated intensity, covariance and eigenvalues.
+ """Compute centroid, bbox, integrated intensity, covariance, kurtosis, and eigenvalues.
Voxel array axes are ``(z, y, x)`` mapping to reciprocal-space coordinates
as **μ → z**, **d → y**, **χ → x** (STFC / operando convention).
When *calibration* is provided, centroids, covariance components, and
eigenvalues are transformed into physical units while ``bbox_*`` bounds
- remain integer voxel indices for 3D array slicing.
+ remain integer voxel indices for 3D array slicing and ``vox_eig_*`` / ``step_*``
+ preserve dimensionless step-normalised geometry for unit-invariant tracking.
"""
labels = np.asarray(labels)
intensity = np.asarray(intensity, dtype=np.float64)
if labels.size == 0 or int(labels.max()) <= 0:
return []
+ nz, ny, nx = labels.shape
+ gain = max(float(detector_gain), 1e-6)
+ is_calibrated = int(calibration is not None and calibration.calibrated)
slices = find_objects(np.where(labels > 0, labels, 0))
rows: list[dict[str, float | int]] = []
for idx, slc in enumerate(slices):
@@ -47,38 +66,66 @@ def extract_instance_table(
count = len(vals)
sum_i = float(vals.sum())
+ # Estimate local detector pedestal from outer perimeter of 4-voxel padded window (labels == 0),
+ # well outside the immediate Bragg flank so sub-Otsu rocking-curve wings are not subtracted.
+ pz0, pz1 = max(0, sz.start - 4), min(nz, sz.stop + 4)
+ py0, py1 = max(0, sy.start - 4), min(ny, sy.stop + 4)
+ px0, px1 = max(0, sx.start - 4), min(nx, sx.stop + 4)
+ lbl_pad = labels[pz0:pz1, py0:py1, px0:px1]
+ vol_pad = intensity[pz0:pz1, py0:py1, px0:px1]
+ outer_ring = np.ones(lbl_pad.shape, dtype=bool)
+ if lbl_pad.shape[0] > 2 and lbl_pad.shape[1] > 2 and lbl_pad.shape[2] > 2:
+ outer_ring[1:-1, 1:-1, 1:-1] = False
+ bg_candidates = (lbl_pad == 0) & outer_ring
+ if not np.any(bg_candidates):
+ bg_candidates = lbl_pad == 0
+ if np.any(bg_candidates):
+ bg_level = max(0.0, float(np.median(vol_pad[bg_candidates])))
+ else:
+ bg_level = 0.0
+
+ vals_net = np.maximum(vals - bg_level, 0.0)
+ sum_net = float(vals_net.sum())
+ w_vals = vals_net if (subtract_background and sum_net > 0.0) else vals
+ w_sum = sum_net if (subtract_background and sum_net > 0.0) else sum_i
+
z_coords = local_coords[:, 0].astype(np.float64) + float(sz.start)
y_coords = local_coords[:, 1].astype(np.float64) + float(sy.start)
x_coords = local_coords[:, 2].astype(np.float64) + float(sx.start)
# Intensity-weighted centroid; fall back to geometric centroid
# if total intensity is non-positive (avoids sign-flip).
- if sum_i > 0:
- weights = vals / sum_i
- cz = float(np.dot(weights, z_coords))
- cy = float(np.dot(weights, y_coords))
- cx = float(np.dot(weights, x_coords))
+ if w_sum > 0.0:
+ w = w_vals / w_sum
+ cz = float(np.dot(w, z_coords))
+ cy = float(np.dot(w, y_coords))
+ cx = float(np.dot(w, x_coords))
else:
+ w = np.ones(count, dtype=np.float64) / count
cz = float(z_coords.mean())
cy = float(y_coords.mean())
cx = float(x_coords.mean())
+ dz_vox = z_coords - cz
+ dy_vox = y_coords - cy
+ dx_vox = x_coords - cx
+
if calibration is not None:
c_mu, c_chi, c_d = calibration.voxel_to_physical(cz, cy, cx)
- dz = (z_coords - cz) * calibration.mu_step
- dy = (y_coords - cy) * calibration.d_step
- dx = (x_coords - cx) * calibration.chi_step
+ dz = dz_vox * calibration.mu_step
+ dy = dy_vox * calibration.d_step
+ dx = dx_vox * calibration.chi_step
+ step_mu = abs(calibration.mu_step)
+ step_d = abs(calibration.d_step)
+ step_chi = abs(calibration.chi_step)
else:
c_mu, c_chi, c_d = cz, cx, cy
- dz = z_coords - cz
- dy = y_coords - cy
- dx = x_coords - cx
-
- # Weighted covariance (same weight logic)
- if sum_i > 0:
- w = vals / sum_i
- else:
- w = np.ones(count, dtype=np.float64) / count
+ dz = dz_vox
+ dy = dy_vox
+ dx = dx_vox
+ step_mu = 1.0
+ step_d = 1.0
+ step_chi = 1.0
czz = float(np.dot(w, dz * dz))
cyy = float(np.dot(w, dy * dy))
@@ -87,6 +134,17 @@ def extract_instance_table(
czx = float(np.dot(w, dz * dx))
cyx = float(np.dot(w, dy * dx))
+ # Analytical 1-sigma centroid standard error: under Poisson photon statistics
+ # Var(c) = cov / N_photons (N_photons = w_sum / detector_gain); when intensities are
+ # normalized (< n_eff), fall back to Kish effective sample size n_eff = 1 / sum(w_k^2),
+ # floored by uniform voxel quantization variance step^2 / 12.
+ w_sq = float(np.dot(w, w))
+ n_eff = (1.0 / w_sq) if w_sq > 0.0 else float(max(1, count))
+ eff_denom = max(w_sum / gain, n_eff, 1.0)
+ c_mu_err = float(np.sqrt(max(czz, (step_mu**2) / 12.0) / eff_denom))
+ c_chi_err = float(np.sqrt(max(cxx, (step_chi**2) / 12.0) / eff_denom))
+ c_d_err = float(np.sqrt(max(cyy, (step_d**2) / 12.0) / eff_denom))
+
cov = np.array(
[
[czz, czy, czx],
@@ -97,20 +155,74 @@ def extract_instance_table(
eigvals = np.linalg.eigvalsh(cov)
eigvals = np.maximum(0.0, np.sort(eigvals)[::-1]) # non-negative descending
+ if is_calibrated:
+ v_czz = float(np.dot(w, dz_vox * dz_vox))
+ v_cyy = float(np.dot(w, dy_vox * dy_vox))
+ v_cxx = float(np.dot(w, dx_vox * dx_vox))
+ v_czy = float(np.dot(w, dz_vox * dy_vox))
+ v_czx = float(np.dot(w, dz_vox * dx_vox))
+ v_cyx = float(np.dot(w, dy_vox * dx_vox))
+ vox_cov = np.array(
+ [
+ [v_czz, v_czy, v_czx],
+ [v_czy, v_cyy, v_cyx],
+ [v_czx, v_cyx, v_cxx],
+ ]
+ )
+ vox_eigs = np.maximum(0.0, np.sort(np.linalg.eigvalsh(vox_cov))[::-1])
+ else:
+ vox_eigs = eigvals
+
+ min_z, max_z = int(z_coords.min()), int(z_coords.max())
+ min_y, max_y = int(y_coords.min()), int(y_coords.max())
+ min_x, max_x = int(x_coords.min()), int(x_coords.max())
+ touches_rocking = bool(min_z == 0 or max_z >= nz - 1)
+ touches_any = bool(touches_rocking or min_y == 0 or max_y >= ny - 1 or min_x == 0 or max_x >= nx - 1)
+
+ # Evaluate rocking-curve Pearson excess kurtosis g_2 = mu_4 / sigma^4 - 3 over a 4-voxel
+ # padded local ROI (excluding foreign spot labels) so sub-Otsu diffuse scattering wings
+ # (LeHoux2026) are integrated rather than hard-clipped at the watershed boundary.
+ kz0, kz1 = max(0, min_z - 4), min(nz, max_z + 5)
+ ky0, ky1 = max(0, min_y - 4), min(ny, max_y + 5)
+ kx0, kx1 = max(0, min_x - 4), min(nx, max_x + 5)
+ lbl_win = labels[kz0:kz1, ky0:ky1, kx0:kx1]
+ vol_win = intensity[kz0:kz1, ky0:ky1, kx0:kx1]
+ own_or_bg = (lbl_win == 0) | (lbl_win == lbl)
+ net_win = np.where(own_or_bg, np.maximum(0.0, vol_win - bg_level), 0.0)
+ net_win_sum = float(net_win.sum())
+ if net_win_sum > 0.0:
+ rz, ry, rx = np.nonzero(net_win > 0.0)
+ w_kurt = net_win[rz, ry, rx] / net_win_sum
+ cz_k = float(np.dot(w_kurt, rz + kz0))
+ cy_k = float(np.dot(w_kurt, ry + ky0))
+ cx_k = float(np.dot(w_kurt, rx + kx0))
+ k_mu = _excess_kurtosis((rz + kz0) - cz_k, w_kurt)
+ k_chi = _excess_kurtosis((rx + kx0) - cx_k, w_kurt)
+ k_d = _excess_kurtosis((ry + ky0) - cy_k, w_kurt)
+ else:
+ k_mu = _excess_kurtosis(dz_vox, w)
+ k_chi = _excess_kurtosis(dx_vox, w)
+ k_d = _excess_kurtosis(dy_vox, w)
+
rows.append(
{
"label": int(lbl),
"voxel_count": count,
"integrated_intensity": sum_i,
+ "net_intensity": sum_net,
+ "background_level": bg_level,
"centroid_mu": c_mu,
"centroid_chi": c_chi,
"centroid_d": c_d,
- "bbox_min_z": int(z_coords.min()),
- "bbox_max_z": int(z_coords.max()),
- "bbox_min_y": int(y_coords.min()),
- "bbox_max_y": int(y_coords.max()),
- "bbox_min_x": int(x_coords.min()),
- "bbox_max_x": int(x_coords.max()),
+ "centroid_mu_err": c_mu_err,
+ "centroid_chi_err": c_chi_err,
+ "centroid_d_err": c_d_err,
+ "bbox_min_z": min_z,
+ "bbox_max_z": max_z,
+ "bbox_min_y": min_y,
+ "bbox_max_y": max_y,
+ "bbox_min_x": min_x,
+ "bbox_max_x": max_x,
"cov_zz": czz,
"cov_yy": cyy,
"cov_xx": cxx,
@@ -120,6 +232,18 @@ def extract_instance_table(
"eig_1": float(eigvals[0]),
"eig_2": float(eigvals[1]),
"eig_3": float(eigvals[2]),
+ "vox_eig_1": float(vox_eigs[0]),
+ "vox_eig_2": float(vox_eigs[1]),
+ "vox_eig_3": float(vox_eigs[2]),
+ "kurtosis_mu": k_mu,
+ "kurtosis_chi": k_chi,
+ "kurtosis_d": k_d,
+ "step_mu": float(step_mu),
+ "step_chi": float(step_chi),
+ "step_d": float(step_d),
+ "calibrated": is_calibrated,
+ "touches_border": touches_any,
+ "touches_rocking_border": touches_rocking,
}
)
diff --git a/braggtrack/semantic/dino.py b/braggtrack/semantic/dino.py
index c636052..0d6770f 100644
--- a/braggtrack/semantic/dino.py
+++ b/braggtrack/semantic/dino.py
@@ -12,6 +12,7 @@
import hashlib
import os
+from collections.abc import Sequence
from typing import Any, Literal, Protocol
import numpy as np
@@ -74,27 +75,55 @@ def _mock_embedding_from_mips(
return v
-def _mips_to_rgb_uint8(mip: np.ndarray) -> np.ndarray:
+def _mips_to_rgb_uint8(mip: np.ndarray, *, preserve_aspect: bool = True) -> np.ndarray:
m = np.asarray(mip, dtype=np.float64)
+ if m.size == 0:
+ return np.zeros((1, 1, 3), dtype=np.uint8)
lo, hi = float(np.percentile(m, 1.0)), float(np.percentile(m, 99.0))
if hi <= lo:
hi = lo + 1.0
u = np.clip((m - lo) / (hi - lo), 0.0, 1.0)
g = (u * 255.0).astype(np.uint8)
+ if preserve_aspect and g.ndim == 2 and g.shape[0] != g.shape[1]:
+ h, w = g.shape
+ # Pad onto a centered square canvas (with 8% margin for DINOv2's 256->224 center-crop)
+ side = max(h, w, int(np.ceil(max(h, w) / 0.875)))
+ canvas = np.zeros((side, side), dtype=np.uint8)
+ y0 = (side - h) // 2
+ x0 = (side - w) // 2
+ canvas[y0 : y0 + h, x0 : x0 + w] = g
+ g = canvas
return np.stack([g, g, g], axis=-1)
class MultiviewEncoder(Protocol):
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray: ...
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray: ...
+
class MockMultiviewEncoder:
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray:
return _mock_embedding_from_mips(mip_mu, mip_chi, mip_d)
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray:
+ if not mips_list:
+ return np.zeros((0, 384), dtype=np.float32)
+ return np.stack([self.embed(m0, m1, m2) for m0, m1, m2 in mips_list], axis=0)
+
class TorchDinoMultiviewEncoder:
- """Loads DINOv2/DINOv3 once; call :meth:`embed` per spot."""
+ """Loads DINOv2/DINOv3 once; supports both single-spot :meth:`embed` and batched :meth:`embed_batch`."""
def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = None) -> None:
import torch
@@ -104,26 +133,45 @@ def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = No
self._proc, self._model = _load_hf_dino(model_name)
self._model.eval()
self._model.to(self._device)
+ self._hidden = int(getattr(self._model.config, "hidden_size", 384))
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray:
- vecs: list[np.ndarray] = []
+ return self.embed_batch([(mip_mu, mip_chi, mip_d)], batch_size=3)[0]
+
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray:
+ if not mips_list:
+ return np.zeros((0, 3 * self._hidden), dtype=np.float32)
+
+ all_rgbs: list[np.ndarray] = []
+ for mip_mu, mip_chi, mip_d in mips_list:
+ all_rgbs.append(_mips_to_rgb_uint8(mip_mu))
+ all_rgbs.append(_mips_to_rgb_uint8(mip_chi))
+ all_rgbs.append(_mips_to_rgb_uint8(mip_d))
+
+ step = max(1, int(batch_size))
+ if str(self._device) == "cpu":
+ step = min(step, 16)
+ cls_chunks: list[np.ndarray] = []
with self._torch.no_grad():
- for mip in (mip_mu, mip_chi, mip_d):
- rgb = _mips_to_rgb_uint8(mip)
- inputs = self._proc(images=[rgb], return_tensors="pt")
+ for start in range(0, len(all_rgbs), step):
+ chunk = all_rgbs[start : start + step]
+ inputs = self._proc(images=chunk, return_tensors="pt")
inputs = {k: v.to(self._device) for k, v in inputs.items()}
- out = self._model(**inputs).last_hidden_state[:, 0, :].squeeze(0)
- x = out.float().cpu().numpy()
- n = float(np.linalg.norm(x))
- if n > 0:
- x = x / n
- vecs.append(x.astype(np.float32))
+ cls_tok = self._model(**inputs).last_hidden_state[:, 0, :].float().cpu().numpy()
+ norms = np.linalg.norm(cls_tok, axis=-1, keepdims=True)
+ norms = np.where(norms > 0.0, norms, 1.0)
+ cls_chunks.append((cls_tok / norms).astype(np.float32))
- fused = np.concatenate(vecs, axis=0)
- nf = float(np.linalg.norm(fused))
- if nf > 0:
- fused = fused / nf
- return fused.astype(np.float32)
+ view_vecs = np.concatenate(cls_chunks, axis=0) # (3 * N, D)
+ fused = view_vecs.reshape(len(mips_list), 3 * view_vecs.shape[-1])
+ fnorms = np.linalg.norm(fused, axis=-1, keepdims=True)
+ fnorms = np.where(fnorms > 0.0, fnorms, 1.0)
+ return (fused / fnorms).astype(np.float32)
def _requested_backend(explicit: BackendName | None) -> BackendName:
@@ -161,6 +209,15 @@ def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
"""Return (H_patches, W_patches, D_features) feature map from a 2D grayscale image."""
...
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ """Return list of (H_patches, W_patches, D_features) maps for a batch of 2D images."""
+ ...
+
@property
def patch_size(self) -> int: ...
@@ -194,9 +251,17 @@ def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
norms = np.where(norms > 0, norms, 1.0)
return (features / norms).astype(np.float32)
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ return [self.extract_patch_features(img) for img in images_2d]
+
class TorchDinoPatchEncoder:
- """Extracts DINOv2/DINOv3 patch tokens as a spatial feature map."""
+ """Extracts DINOv2/DINOv3 patch tokens as spatial feature maps (single or batched)."""
def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = None) -> None:
import torch
@@ -218,24 +283,43 @@ def feature_dim(self) -> int:
return self._feature_dim
def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
- rgb = _mips_to_rgb_uint8(image_2d)
+ return self.extract_patch_features_batch([image_2d], batch_size=1)[0]
+
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ if not images_2d:
+ return []
+ rgbs = [_mips_to_rgb_uint8(img, preserve_aspect=False) for img in images_2d]
+ step = max(1, int(batch_size))
+ if str(self._device) == "cpu":
+ step = min(step, 16)
+ results: list[np.ndarray] = []
with self._torch.no_grad():
- inputs = self._proc(images=[rgb], return_tensors="pt")
- inputs = {k: v.to(self._device) for k, v in inputs.items()}
- out = self._model(**inputs)
- h_img = int(inputs["pixel_values"].shape[2])
- w_img = int(inputs["pixel_values"].shape[3])
- h_p = h_img // self._patch_size
- w_p = w_img // self._patch_size
- expected_patches = h_p * w_p
- seq_len = int(out.last_hidden_state.shape[1])
- skip = max(0, seq_len - expected_patches)
- patch_tokens = out.last_hidden_state[:, skip : skip + expected_patches, :].squeeze(0)
- features = patch_tokens.reshape(h_p, w_p, -1)
- x = features.float().cpu().numpy()
- norms = np.linalg.norm(x, axis=-1, keepdims=True)
- norms = np.where(norms > 0, norms, 1.0)
- return (x / norms).astype(np.float32)
+ for start in range(0, len(rgbs), step):
+ chunk = rgbs[start : start + step]
+ inputs = self._proc(images=chunk, return_tensors="pt")
+ inputs = {k: v.to(self._device) for k, v in inputs.items()}
+ out = self._model(**inputs)
+ bsz = int(inputs["pixel_values"].shape[0])
+ h_img = int(inputs["pixel_values"].shape[2])
+ w_img = int(inputs["pixel_values"].shape[3])
+ h_p = h_img // self._patch_size
+ w_p = w_img // self._patch_size
+ expected_patches = h_p * w_p
+ seq_len = int(out.last_hidden_state.shape[1])
+ skip = max(0, seq_len - expected_patches)
+ patch_tokens = out.last_hidden_state[:, skip : skip + expected_patches, :]
+ fmaps = patch_tokens.reshape(bsz, h_p, w_p, -1).float().cpu().numpy()
+ norms = np.linalg.norm(fmaps, axis=-1, keepdims=True)
+ norms = np.where(norms > 0.0, norms, 1.0)
+ normed = (fmaps / norms).astype(np.float32)
+ for i in range(bsz):
+ results.append(normed[i])
+ return results
def make_patch_encoder(
diff --git a/braggtrack/tracking/__init__.py b/braggtrack/tracking/__init__.py
index 68a1f62..cd4713b 100644
--- a/braggtrack/tracking/__init__.py
+++ b/braggtrack/tracking/__init__.py
@@ -1,6 +1,6 @@
"""Multi-scan grain tracking with physics and semantic cost functions."""
-from .assignment import associate_frames
+from .assignment import associate_frames, compute_assignment_margins
from .cost import CostFunction, GeometrySemanticCost, PositionShapeCost
from .events import LineageEvent, SplitMergeConfig, events_to_table, resolve_splits_and_merges
from .gap_closing import GapCloseRecord, GapClosingConfig, resolve_track_gaps
@@ -39,6 +39,7 @@
"TrackEvent",
"associate_frames",
"build_tracks",
+ "compute_assignment_margins",
"compute_fracture_precursors",
"compute_grain_kinematics",
"compute_tracking_metrics",
diff --git a/braggtrack/tracking/assignment.py b/braggtrack/tracking/assignment.py
index a860cbb..b6af028 100644
--- a/braggtrack/tracking/assignment.py
+++ b/braggtrack/tracking/assignment.py
@@ -69,3 +69,34 @@ def associate_frames(
unmatched_t1 = [j for j in range(n_t1) if j not in matched_t1]
return matches, unmatched_t, unmatched_t1
+
+
+def compute_assignment_margins(
+ spots_t: list[dict],
+ spots_t1: list[dict],
+ matches: list[tuple[int, int]],
+ cost_fn: CostFunction,
+) -> dict[tuple[int, int], tuple[float, float]]:
+ """Compute ``(assigned_cost, margin)`` for each accepted Hungarian match.
+
+ The margin is ``min(second_best_row - c_ij, second_best_col - c_ij)``,
+ measuring how close a competing spot in frame *t* or *t+1* came to
+ claiming the link (``math.inf`` when no finite competitor exists).
+ """
+ if not matches or not spots_t or not spots_t1:
+ return {}
+
+ raw = np.asarray(cost_fn.pairwise_cost_matrix(spots_t, spots_t1), dtype=np.float64)
+ n_t, n_t1 = raw.shape
+ out: dict[tuple[int, int], tuple[float, float]] = {}
+ for r, c in matches:
+ c_ij = float(raw[r, c])
+ if not math.isfinite(c_ij):
+ out[(r, c)] = (math.inf, 0.0)
+ continue
+ row_others = [float(raw[r, k]) for k in range(n_t1) if k != c and math.isfinite(raw[r, k])]
+ col_others = [float(raw[k, c]) for k in range(n_t) if k != r and math.isfinite(raw[k, c])]
+ alt_costs = row_others + col_others
+ margin = (min(alt_costs) - c_ij) if alt_costs else math.inf
+ out[(r, c)] = (c_ij, float(margin))
+ return out
diff --git a/braggtrack/tracking/cost.py b/braggtrack/tracking/cost.py
index 45906b5..0b426e5 100644
--- a/braggtrack/tracking/cost.py
+++ b/braggtrack/tracking/cost.py
@@ -27,8 +27,39 @@ def __call__(self, spot_i: dict, spot_j: dict) -> float:
...
+def _spot_scales(spot: dict) -> tuple[float, float, float]:
+ """Return positive per-axis voxel step scales ``(step_mu, step_chi, step_d)``."""
+ sm = abs(float(spot.get("step_mu", 1.0)))
+ sc = abs(float(spot.get("step_chi", 1.0)))
+ sd = abs(float(spot.get("step_d", 1.0)))
+ return (
+ sm if sm > 1e-12 else 1.0,
+ sc if sc > 1e-12 else 1.0,
+ sd if sd > 1e-12 else 1.0,
+ )
+
+
+def _spot_eigs(spot: dict, *, normalize_by_step: bool) -> tuple[float, float, float]:
+ """Return shape eigenvalues (dimensionless voxel-space when calibrated & normalized)."""
+ if normalize_by_step and "vox_eig_1" in spot:
+ return (
+ float(spot.get("vox_eig_1", 0.0)),
+ float(spot.get("vox_eig_2", 0.0)),
+ float(spot.get("vox_eig_3", 0.0)),
+ )
+ e1 = float(spot.get("eig_1", 0.0))
+ e2 = float(spot.get("eig_2", 0.0))
+ e3 = float(spot.get("eig_3", 0.0))
+ if normalize_by_step and bool(spot.get("calibrated", 0)):
+ sm, sc, sd = _spot_scales(spot)
+ s2 = max(sm, sc, sd) ** 2
+ if s2 > 0.0:
+ return (e1 / s2, e2 / s2, e3 / s2)
+ return (e1, e2, e3)
+
+
class PositionShapeCost:
- """Weighted position + shape cost with per-axis gating.
+ """Weighted position + shape cost with per-axis gating and step-scale invariance.
Parameters
----------
@@ -37,10 +68,14 @@ class PositionShapeCost:
shape_weight : float
Multiplier for the shape (eigenvalue) distance term.
gate_mu, gate_chi, gate_d : float
- Maximum allowed displacement along each reciprocal-space axis.
- Pairs exceeding *any* gate are assigned ``inf`` cost. Defaults
- are deliberately loose; tighten ``gate_d`` less aggressively
- than mu/chi for operando data where d-spacing shifts are large.
+ Maximum allowed displacement along each reciprocal-space axis (evaluated
+ in the spot's native coordinate units). Pairs exceeding *any* gate are
+ assigned ``inf`` cost.
+ normalize_by_step : bool
+ When ``True`` (default), calibrated physical displacements ``(deg, deg, Å)``
+ are divided by ``(step_mu, step_chi, step_d)`` and ``vox_eig_*`` are used
+ so Euclidean association costs are dimensionless and invariant to choice
+ of physical units. Uncalibrated spots (``step_* = 1``) are unchanged.
"""
def __init__(
@@ -50,12 +85,15 @@ def __init__(
gate_mu: float = math.inf,
gate_chi: float = math.inf,
gate_d: float = math.inf,
+ *,
+ normalize_by_step: bool = True,
) -> None:
self.position_weight = position_weight
self.shape_weight = shape_weight
self.gate_mu = gate_mu
self.gate_chi = gate_chi
self.gate_d = gate_d
+ self.normalize_by_step = bool(normalize_by_step)
def pairwise_cost_matrix(self, spots_t: list[dict], spots_t1: list[dict]) -> np.ndarray:
n, m = len(spots_t), len(spots_t1)
@@ -71,40 +109,55 @@ def pairwise_cost_matrix(self, spots_t: list[dict], spots_t1: list[dict]) -> np.
dtype=np.float64,
)
e = np.array(
- [[s.get("eig_1", 0.0), s.get("eig_2", 0.0), s.get("eig_3", 0.0)] for s in spots_t],
+ [_spot_eigs(s, normalize_by_step=self.normalize_by_step) for s in spots_t],
dtype=np.float64,
)
f = np.array(
- [[s.get("eig_1", 0.0), s.get("eig_2", 0.0), s.get("eig_3", 0.0)] for s in spots_t1],
+ [_spot_eigs(s, normalize_by_step=self.normalize_by_step) for s in spots_t1],
dtype=np.float64,
)
- pos_d2 = cdist(x, y, metric="sqeuclidean")
- shape_d2 = cdist(e, f, metric="sqeuclidean")
-
dmu = np.abs(x[:, 0][:, np.newaxis] - y[:, 0][np.newaxis, :])
dchi = np.abs(x[:, 1][:, np.newaxis] - y[:, 1][np.newaxis, :])
dd = np.abs(x[:, 2][:, np.newaxis] - y[:, 2][np.newaxis, :])
bad = (dmu > self.gate_mu) | (dchi > self.gate_chi) | (dd > self.gate_d)
+ if self.normalize_by_step:
+ sx = np.array([_spot_scales(s) for s in spots_t], dtype=np.float64)
+ sy = np.array([_spot_scales(s) for s in spots_t1], dtype=np.float64)
+ smu = 0.5 * (sx[:, 0][:, np.newaxis] + sy[:, 0][np.newaxis, :])
+ schi = 0.5 * (sx[:, 1][:, np.newaxis] + sy[:, 1][np.newaxis, :])
+ sd = 0.5 * (sx[:, 2][:, np.newaxis] + sy[:, 2][np.newaxis, :])
+ pos_d2 = (dmu / smu) ** 2 + (dchi / schi) ** 2 + (dd / sd) ** 2
+ else:
+ pos_d2 = cdist(x, y, metric="sqeuclidean")
+
+ shape_d2 = cdist(e, f, metric="sqeuclidean")
+
cost = self.position_weight * pos_d2 + self.shape_weight * shape_d2
cost[bad] = np.inf
return cost
def __call__(self, spot_i: dict, spot_j: dict) -> float:
- dmu = abs(spot_i["centroid_mu"] - spot_j["centroid_mu"])
- dchi = abs(spot_i["centroid_chi"] - spot_j["centroid_chi"])
- dd = abs(spot_i["centroid_d"] - spot_j["centroid_d"])
+ dmu = abs(float(spot_i["centroid_mu"]) - float(spot_j["centroid_mu"]))
+ dchi = abs(float(spot_i["centroid_chi"]) - float(spot_j["centroid_chi"]))
+ dd = abs(float(spot_i["centroid_d"]) - float(spot_j["centroid_d"]))
if dmu > self.gate_mu or dchi > self.gate_chi or dd > self.gate_d:
return math.inf
+ if self.normalize_by_step:
+ smi, sci, sdi = _spot_scales(spot_i)
+ smj, scj, sdj = _spot_scales(spot_j)
+ dmu /= 0.5 * (smi + smj)
+ dchi /= 0.5 * (sci + scj)
+ dd /= 0.5 * (sdi + sdj)
+
pos_dist2 = dmu**2 + dchi**2 + dd**2
- deig1 = spot_i.get("eig_1", 0.0) - spot_j.get("eig_1", 0.0)
- deig2 = spot_i.get("eig_2", 0.0) - spot_j.get("eig_2", 0.0)
- deig3 = spot_i.get("eig_3", 0.0) - spot_j.get("eig_3", 0.0)
- shape_dist2 = deig1**2 + deig2**2 + deig3**2
+ ei = _spot_eigs(spot_i, normalize_by_step=self.normalize_by_step)
+ ej = _spot_eigs(spot_j, normalize_by_step=self.normalize_by_step)
+ shape_dist2 = (ei[0] - ej[0]) ** 2 + (ei[1] - ej[1]) ** 2 + (ei[2] - ej[2]) ** 2
return self.position_weight * pos_dist2 + self.shape_weight * shape_dist2
diff --git a/braggtrack/tracking/events.py b/braggtrack/tracking/events.py
index 595eea5..3ec8a7d 100644
--- a/braggtrack/tracking/events.py
+++ b/braggtrack/tracking/events.py
@@ -9,6 +9,10 @@
import networkx as nx
import numpy as np
+from .cost import _spot_scales
+
+SplitSubtype = Literal["mechanical_split", "phase_split", "mixed_split"]
+
@dataclass(frozen=True)
class SplitMergeConfig:
@@ -22,6 +26,8 @@ class SplitMergeConfig:
max_intensity_ratio: float = 1.6
min_fragment_share: float = 0.1
min_semantic_similarity: float | None = None
+ normalize_by_step: bool = True
+ phase_split_ratio: float = 2.0
@dataclass(frozen=True)
@@ -44,6 +50,9 @@ class LineageEvent:
semantic_similarity: float | None = None
confidence: float = 1.0
timestamp: str | None = None
+ split_subtype: SplitSubtype | None = None
+ delta_orientation_norm: float = 0.0
+ delta_d_norm: float = 0.0
def _spot_coords(node_data: dict[str, Any]) -> tuple[float, float, float]:
@@ -54,6 +63,40 @@ def _spot_coords(node_data: dict[str, Any]) -> tuple[float, float, float]:
)
+def _classify_split_subtype(
+ child_datas: list[dict[str, Any]],
+ *,
+ phase_split_ratio: float = 2.0,
+) -> tuple[SplitSubtype, float, float]:
+ """Disambiguate orientation cleavage (``mechanical_split``) vs two-phase ``d``-jump (``phase_split``)."""
+ max_orient = 0.0
+ max_d = 0.0
+ n_c = len(child_datas)
+ for i in range(n_c):
+ mu_i, chi_i, d_i = _spot_coords(child_datas[i])
+ smi, sci, sdi = _spot_scales(child_datas[i])
+ for j in range(i + 1, n_c):
+ mu_j, chi_j, d_j = _spot_coords(child_datas[j])
+ smj, scj, sdj = _spot_scales(child_datas[j])
+ dmu_n = abs(mu_i - mu_j) / (0.5 * (smi + smj))
+ dchi_n = abs(chi_i - chi_j) / (0.5 * (sci + scj))
+ dd_n = abs(d_i - d_j) / (0.5 * (sdi + sdj))
+ orient_sep = math.hypot(dmu_n, dchi_n)
+ if orient_sep > max_orient:
+ max_orient = orient_sep
+ if dd_n > max_d:
+ max_d = dd_n
+
+ ratio = max(1.05, float(phase_split_ratio))
+ if max_d >= ratio * max(max_orient, 1e-6):
+ subtype: SplitSubtype = "phase_split"
+ elif max_orient >= ratio * max(max_d, 1e-6):
+ subtype = "mechanical_split"
+ else:
+ subtype = "mixed_split"
+ return subtype, max_orient, max_d
+
+
def _pair_distance_and_sim(
a: dict[str, Any],
b: dict[str, Any],
@@ -66,6 +109,12 @@ def _pair_distance_and_sim(
dd = abs(d_a - d_b)
if dmu > config.gate_mu or dchi > config.gate_chi or dd > config.gate_d:
return None
+ if config.normalize_by_step:
+ sma, sca, sda = _spot_scales(a)
+ smb, scb, sdb = _spot_scales(b)
+ dmu /= 0.5 * (sma + smb)
+ dchi /= 0.5 * (sca + scb)
+ dd /= 0.5 * (sda + sdb)
dist = math.sqrt(dmu * dmu + dchi * dchi + dd * dd)
if dist > config.max_distance:
return None
@@ -255,14 +304,22 @@ def resolve_splits_and_merges(
if not valid_all:
continue
+ sorted_children = sorted(all_children)
+ subtype, d_orient_n, d_d_n = _classify_split_subtype(
+ [G.nodes[c] for c in sorted_children],
+ phase_split_ratio=cfg.phase_split_ratio,
+ )
+
parent_tid = int(G.nodes[p_nid]["track_id"])
G.nodes[p_nid]["lineage_role"] = "split_parent"
+ G.nodes[p_nid]["split_subtype"] = subtype
child_tids: list[int] = []
- for c_nid in sorted(all_children):
+ for c_nid in sorted_children:
if not G.has_edge(p_nid, c_nid):
- G.add_edge(p_nid, c_nid, edge_type="split")
+ G.add_edge(p_nid, c_nid, edge_type="split", split_subtype=subtype)
else:
G.edges[p_nid, c_nid]["edge_type"] = "split"
+ G.edges[p_nid, c_nid]["split_subtype"] = subtype
old_c_tid = int(G.nodes[c_nid]["track_id"])
if old_c_tid == parent_tid:
@@ -273,6 +330,7 @@ def resolve_splits_and_merges(
new_c_tid = old_c_tid
G.nodes[c_nid]["lineage_role"] = "split_child"
+ G.nodes[c_nid]["split_subtype"] = subtype
G.nodes[c_nid]["parent_track_ids"] = [parent_tid]
child_tids.append(new_c_tid)
@@ -288,7 +346,7 @@ def resolve_splits_and_merges(
parent_track_ids=(parent_tid,),
child_track_ids=tuple(child_tids),
parent_nodes=(p_nid,),
- child_nodes=tuple(sorted(all_children)),
+ child_nodes=tuple(sorted_children),
centroid_mu=p_mu,
centroid_chi=p_chi,
centroid_d=p_d,
@@ -297,6 +355,9 @@ def resolve_splits_and_merges(
semantic_similarity=round(mean_sim, 4) if mean_sim is not None else None,
confidence=_event_confidence(mean_dist, cfg.max_distance, ratio, mean_sim),
timestamp=ts,
+ split_subtype=subtype,
+ delta_orientation_norm=round(d_orient_n, 4),
+ delta_d_norm=round(d_d_n, 4),
)
)
next_event_id += 1
@@ -404,6 +465,7 @@ def events_to_table(events: list[LineageEvent]) -> list[dict[str, Any]]:
{
"event_id": ev.event_id,
"event_type": ev.event_type,
+ "split_subtype": ev.split_subtype or "",
"scan_idx_from": ev.scan_idx_from,
"scan_idx_to": ev.scan_idx_to,
"timestamp": ev.timestamp or "",
@@ -416,6 +478,8 @@ def events_to_table(events: list[LineageEvent]) -> list[dict[str, Any]]:
"centroid_d": ev.centroid_d,
"intensity_ratio": ev.intensity_ratio,
"mean_distance": ev.mean_distance,
+ "delta_orientation_norm": ev.delta_orientation_norm,
+ "delta_d_norm": ev.delta_d_norm,
"semantic_similarity": ev.semantic_similarity if ev.semantic_similarity is not None else "",
"confidence": ev.confidence,
}
diff --git a/braggtrack/tracking/gap_closing.py b/braggtrack/tracking/gap_closing.py
index c40ce8f..c436522 100644
--- a/braggtrack/tracking/gap_closing.py
+++ b/braggtrack/tracking/gap_closing.py
@@ -10,7 +10,7 @@
import numpy as np
from scipy.optimize import linear_sum_assignment
-from .cost import CostFunction, GeometrySemanticCost, PositionShapeCost
+from .cost import CostFunction, GeometrySemanticCost, PositionShapeCost, _spot_eigs, _spot_scales
from .lifecycle import TrackEvent
@@ -94,12 +94,18 @@ def _eval_gap_cost(
if (dmu / step_scale) > geo.gate_mu or (dchi / step_scale) > geo.gate_chi or (dd / step_scale) > geo.gate_d:
return math.inf
+ if geo.normalize_by_step:
+ smu, schi, sd = _spot_scales(u_data)
+ smv, schiv, sdv = _spot_scales(v_data)
+ dmu /= 0.5 * (smu + smv)
+ dchi /= 0.5 * (schi + schiv)
+ dd /= 0.5 * (sd + sdv)
+
# Diffusive scaling (dx^2 / delta_t) keeps gap costs comparable to 1-step costs.
pos_dist2 = (dmu**2 + dchi**2 + dd**2) / step_scale
- deig1 = float(u_data.get("eig_1", 0.0)) - float(v_data.get("eig_1", 0.0))
- deig2 = float(u_data.get("eig_2", 0.0)) - float(v_data.get("eig_2", 0.0))
- deig3 = float(u_data.get("eig_3", 0.0)) - float(v_data.get("eig_3", 0.0))
- shape_dist2 = deig1**2 + deig2**2 + deig3**2
+ eu = _spot_eigs(u_data, normalize_by_step=geo.normalize_by_step)
+ ev = _spot_eigs(v_data, normalize_by_step=geo.normalize_by_step)
+ shape_dist2 = (eu[0] - ev[0]) ** 2 + (eu[1] - ev[1]) ** 2 + (eu[2] - ev[2]) ** 2
g_cost = geo.position_weight * pos_dist2 + geo.shape_weight * shape_dist2
if beta == 0.0:
return alpha * g_cost
@@ -206,6 +212,7 @@ def _edge_is_jump(a: str, b: str) -> bool:
n_born = len(born_nodes)
if n_term == 0 or n_born == 0:
G.graph["gap_closures"] = []
+ G.graph["gap_closed_count"] = 0
return []
inadmissible = 1e12
@@ -272,4 +279,5 @@ def _edge_is_jump(a: str, b: str) -> bool:
)
G.graph["gap_closures"] = records
+ G.graph["gap_closed_count"] = len(records)
return records
diff --git a/braggtrack/tracking/kinematics.py b/braggtrack/tracking/kinematics.py
index 8221e28..c38825b 100644
--- a/braggtrack/tracking/kinematics.py
+++ b/braggtrack/tracking/kinematics.py
@@ -48,6 +48,24 @@ class GrainKinematics:
# Raw shift in d-axis (always meaningful even when uncalibrated)
delta_d: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ # Analytical 1-sigma uncertainties (propagated from centroid_*_err)
+ centroid_mu_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ centroid_chi_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ centroid_d_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ strain_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
+ # Ancestor-referenced lineage strain across 1->2+ splits and 2+->1 merges
+ root_track_id: int = 0
+ lineage_strain: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ lineage_strain_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
+ # Strictly causal backward finite-difference rates (per scan step, plus per-second when elapsed_s known)
+ rate_mu: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ rate_chi: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ rate_d: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ strain_rate: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ strain_rate_per_s: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
@dataclass
class KinematicsSummary:
@@ -66,11 +84,75 @@ class KinematicsSummary:
max_anisotropy: list[float]
calibrated_axes: bool = False
+ max_lineage_strain: list[float] = field(default_factory=list)
grain_kinematics: list[GrainKinematics] = field(default_factory=list, repr=False)
+def _parse_parent_ids(raw: Any) -> list[int]:
+ if raw is None or raw == "":
+ return []
+ if isinstance(raw, (list, tuple)):
+ out: list[int] = []
+ for item in raw:
+ try:
+ out.append(int(item))
+ except (TypeError, ValueError):
+ continue
+ return out
+ if isinstance(raw, (int, np.integer)):
+ return [int(raw)]
+ if isinstance(raw, str):
+ parts = [p.strip() for p in raw.split(";") if p.strip()]
+ parsed: list[int] = []
+ for p in parts:
+ try:
+ parsed.append(int(p))
+ except ValueError:
+ continue
+ return parsed
+ return []
+
+
+def _propagate_strain_err(
+ d: np.ndarray,
+ d_err: np.ndarray,
+ d_ref: float,
+ d_ref_err: float,
+ *,
+ zero_first: bool,
+) -> np.ndarray:
+ """Propagate 1-sigma uncertainty on ``(d - d_ref) / d_ref``."""
+ n = len(d)
+ if n == 0 or d_ref == 0.0:
+ return np.zeros(n, dtype=np.float64)
+ ratio = d / d_ref
+ err = np.sqrt(d_err**2 + (ratio * d_ref_err) ** 2) / abs(d_ref)
+ if zero_first and n > 0:
+ err[0] = 0.0
+ return err
+
+
+def _obs_elapsed_s(row: dict[str, Any], scan_elapsed_s: dict[int, float] | None) -> float | None:
+ s_idx = int(row["scan_idx"])
+ if scan_elapsed_s is not None and s_idx in scan_elapsed_s:
+ val = scan_elapsed_s[s_idx]
+ if val is not None and np.isfinite(val):
+ return float(val)
+ raw = row.get("elapsed_s")
+ if raw is not None and raw != "":
+ try:
+ f = float(raw)
+ if np.isfinite(f):
+ return f
+ except (TypeError, ValueError):
+ return None
+ return None
+
+
def compute_grain_kinematics(
track_table: list[dict[str, Any]],
+ *,
+ scan_elapsed_s: dict[int, float] | None = None,
) -> list[GrainKinematics]:
"""Compute kinematic time-series for each tracked grain.
@@ -80,48 +162,147 @@ def compute_grain_kinematics(
Output of :func:`~braggtrack.tracking.lifecycle.tracks_to_table`:
list of dicts with ``track_id``, ``scan_idx``, ``centroid_mu``,
``centroid_chi``, ``centroid_d``, ``eig_1``, ``eig_2``, ``eig_3``,
- ``integrated_intensity``, ``voxel_count``.
+ ``integrated_intensity``, ``voxel_count``, plus optional
+ ``centroid_*_err`` and ``parent_track_ids``.
+ scan_elapsed_s
+ Optional mapping ``scan_idx -> elapsed_seconds`` for non-uniform scan cadences.
Returns
-------
list[GrainKinematics]
One entry per track, sorted by track_id.
"""
- tracks: dict[int, list[dict]] = {}
+ tracks: dict[int, list[dict[str, Any]]] = {}
+ parent_map: dict[int, list[int]] = {}
for row in track_table:
- tracks.setdefault(int(row["track_id"]), []).append(row)
+ tid = int(row["track_id"])
+ tracks.setdefault(tid, []).append(row)
+ pids = _parse_parent_ids(row.get("parent_track_ids"))
+ if pids and tid not in parent_map:
+ parent_map[tid] = pids
+
+ sorted_tracks: dict[int, list[dict[str, Any]]] = {
+ tid: sorted(rows, key=lambda r: int(r["scan_idx"])) for tid, rows in tracks.items()
+ }
+
+ def _resolve_root_tid(start_tid: int) -> int:
+ curr = start_tid
+ visited: set[int] = set()
+ while curr not in visited:
+ visited.add(curr)
+ parents = [p for p in parent_map.get(curr, []) if p in sorted_tracks and p != curr]
+ if not parents:
+ break
+ # Prefer earliest-onset parent (tie-break by highest initial intensity, then lowest ID)
+ parents.sort(
+ key=lambda p: (
+ int(sorted_tracks[p][0]["scan_idx"]),
+ -float(sorted_tracks[p][0].get("integrated_intensity", 0.0)),
+ p,
+ )
+ )
+ curr = parents[0]
+ return curr
results: list[GrainKinematics] = []
- for tid in sorted(tracks):
- obs = sorted(tracks[tid], key=lambda r: r["scan_idx"])
+ for tid in sorted(sorted_tracks):
+ obs = sorted_tracks[tid]
n = len(obs)
scan_indices = [int(r["scan_idx"]) for r in obs]
- mu = np.array([float(r["centroid_mu"]) for r in obs])
- chi = np.array([float(r["centroid_chi"]) for r in obs])
- d = np.array([float(r["centroid_d"]) for r in obs])
+ mu = np.array([float(r["centroid_mu"]) for r in obs], dtype=np.float64)
+ chi = np.array([float(r["centroid_chi"]) for r in obs], dtype=np.float64)
+ d = np.array([float(r["centroid_d"]) for r in obs], dtype=np.float64)
+
+ mu_err = np.array([float(r.get("centroid_mu_err", 0.0)) for r in obs], dtype=np.float64)
+ chi_err = np.array([float(r.get("centroid_chi_err", 0.0)) for r in obs], dtype=np.float64)
+ d_err = np.array([float(r.get("centroid_d_err", 0.0)) for r in obs], dtype=np.float64)
- # Strain: (d - d₀) / d₀ and raw shift Δd = d - d₀
- d0 = d[0]
+ # Segment-local strain: (d - d₀) / d₀ and raw shift Δd = d - d₀
+ d0 = float(d[0])
+ d0_err = float(d_err[0])
delta_d = d - d0
- strain = delta_d / d0 if d0 != 0 else np.zeros(n)
+ strain = delta_d / d0 if d0 != 0.0 else np.zeros(n, dtype=np.float64)
+ strain_err = _propagate_strain_err(d, d_err, d0, d0_err, zero_first=True)
+
+ # Ancestor-referenced lineage strain across splits/merges
+ root_tid = _resolve_root_tid(tid)
+ root_obs0 = sorted_tracks[root_tid][0]
+ d_root0 = float(root_obs0["centroid_d"])
+ d_root0_err = float(root_obs0.get("centroid_d_err", 0.0))
+ lineage_strain = (d - d_root0) / d_root0 if d_root0 != 0.0 else np.zeros(n, dtype=np.float64)
+ lineage_strain_err = _propagate_strain_err(
+ d,
+ d_err,
+ d_root0,
+ d_root0_err,
+ zero_first=(root_tid == tid),
+ )
+
+ # Strictly causal backward finite-difference rates (per scan step & wall-clock /s)
+ rate_mu = np.zeros(n, dtype=np.float64)
+ rate_chi = np.zeros(n, dtype=np.float64)
+ rate_d = np.zeros(n, dtype=np.float64)
+ strain_rate = np.zeros(n, dtype=np.float64)
+ strain_rate_per_s = np.zeros(n, dtype=np.float64)
+
+ # Seed initial rate at i=0 from immediate parent's last observation if split/merge child
+ imm_parents = [p for p in parent_map.get(tid, []) if p in sorted_tracks]
+ if imm_parents and n > 0:
+ p_last = sorted_tracks[imm_parents[0]][-1]
+ dt0 = max(1, scan_indices[0] - int(p_last["scan_idx"]))
+ rate_mu[0] = (mu[0] - float(p_last["centroid_mu"])) / dt0
+ rate_chi[0] = (chi[0] - float(p_last["centroid_chi"])) / dt0
+ rate_d[0] = (d[0] - float(p_last["centroid_d"])) / dt0
+ if d_root0 != 0.0:
+ p_lin_strain = (float(p_last["centroid_d"]) - d_root0) / d_root0
+ strain_rate[0] = (lineage_strain[0] - p_lin_strain) / dt0
+ t_now0 = _obs_elapsed_s(obs[0], scan_elapsed_s)
+ t_prev0 = _obs_elapsed_s(p_last, scan_elapsed_s)
+ if t_now0 is not None and t_prev0 is not None and (t_now0 - t_prev0) > 1e-6:
+ strain_rate_per_s[0] = (lineage_strain[0] - p_lin_strain) / (t_now0 - t_prev0)
+
+ for i in range(1, n):
+ dt = max(1, scan_indices[i] - scan_indices[i - 1])
+ rate_mu[i] = (mu[i] - mu[i - 1]) / dt
+ rate_chi[i] = (chi[i] - chi[i - 1]) / dt
+ rate_d[i] = (d[i] - d[i - 1]) / dt
+ strain_rate[i] = (lineage_strain[i] - lineage_strain[i - 1]) / dt
+ t_now = _obs_elapsed_s(obs[i], scan_elapsed_s)
+ t_prev = _obs_elapsed_s(obs[i - 1], scan_elapsed_s)
+ if t_now is not None and t_prev is not None and (t_now - t_prev) > 1e-6:
+ strain_rate_per_s[i] = (lineage_strain[i] - lineage_strain[i - 1]) / (t_now - t_prev)
# Misorientation: drift from initial position
dmu = mu - mu[0]
dchi = chi - chi[0]
misorientation_total = np.sqrt(dmu**2 + dchi**2)
- intensity = np.array([float(r["integrated_intensity"]) for r in obs])
- voxels = np.array([float(r["voxel_count"]) for r in obs])
+ intensity = np.array([float(r["integrated_intensity"]) for r in obs], dtype=np.float64)
+ voxels = np.array([float(r["voxel_count"]) for r in obs], dtype=np.float64)
i0 = intensity[0]
v0 = voxels[0]
- rel_intensity = (intensity - i0) / i0 if i0 != 0 else np.zeros(n)
- rel_volume = (voxels - v0) / v0 if v0 != 0 else np.zeros(n)
+ rel_intensity = (intensity - i0) / i0 if i0 != 0 else np.zeros(n, dtype=np.float64)
+ rel_volume = (voxels - v0) / v0 if v0 != 0 else np.zeros(n, dtype=np.float64)
- eigs = np.array([[float(r["eig_1"]), float(r["eig_2"]), float(r["eig_3"])] for r in obs])
- e3 = eigs[:, 2]
- anisotropy = np.where(e3 > 0, eigs[:, 0] / e3, 1.0)
+ eigs = np.array(
+ [[float(r.get("eig_1", 1.0)), float(r.get("eig_2", 1.0)), float(r.get("eig_3", 1.0))] for r in obs],
+ dtype=np.float64,
+ )
+ shape_eigs = np.array(
+ [
+ [
+ float(r.get("vox_eig_1", r.get("eig_1", 1.0))),
+ float(r.get("vox_eig_2", r.get("eig_2", 1.0))),
+ float(r.get("vox_eig_3", r.get("eig_3", 1.0))),
+ ]
+ for r in obs
+ ],
+ dtype=np.float64,
+ )
+ e3 = shape_eigs[:, 2]
+ anisotropy = np.where(e3 > 0, shape_eigs[:, 0] / e3, 1.0)
cov_trace = eigs.sum(axis=1)
results.append(
@@ -143,6 +324,18 @@ def compute_grain_kinematics(
anisotropy=anisotropy,
covariance_trace=cov_trace,
delta_d=delta_d,
+ centroid_mu_err=mu_err,
+ centroid_chi_err=chi_err,
+ centroid_d_err=d_err,
+ strain_err=strain_err,
+ root_track_id=root_tid,
+ lineage_strain=lineage_strain,
+ lineage_strain_err=lineage_strain_err,
+ rate_mu=rate_mu,
+ rate_chi=rate_chi,
+ rate_d=rate_d,
+ strain_rate=strain_rate,
+ strain_rate_per_s=strain_rate_per_s,
)
)
return results
@@ -154,19 +347,10 @@ def summarize_kinematics(
*,
calibrated_axes: bool = False,
) -> KinematicsSummary:
- """Aggregate per-grain kinematics into a summary table.
-
- Parameters
- ----------
- grain_kinematics
- Output of :func:`compute_grain_kinematics`.
- n_scans
- Total number of scans in the sequence.
- calibrated_axes
- Whether centroids/eigenvalues were converted to physical ``(mu, chi, d)`` units.
- """
+ """Aggregate per-grain kinematics into a summary table."""
track_ids: list[int] = []
max_strain: list[float] = []
+ max_lineage_strain: list[float] = []
total_misorientation: list[float] = []
intensity_change: list[float] = []
volume_change: list[float] = []
@@ -175,6 +359,8 @@ def summarize_kinematics(
for gk in grain_kinematics:
track_ids.append(gk.track_id)
max_strain.append(float(np.max(np.abs(gk.strain))) if len(gk.strain) > 0 else 0.0)
+ lin_s = gk.lineage_strain if len(gk.lineage_strain) > 0 else gk.strain
+ max_lineage_strain.append(float(np.max(np.abs(lin_s))) if len(lin_s) > 0 else 0.0)
total_misorientation.append(float(gk.misorientation_total[-1]) if len(gk.misorientation_total) > 0 else 0.0)
intensity_change.append(float(gk.relative_intensity[-1]) if len(gk.relative_intensity) > 0 else 0.0)
volume_change.append(float(gk.relative_volume[-1]) if len(gk.relative_volume) > 0 else 0.0)
@@ -193,6 +379,7 @@ def summarize_kinematics(
volume_change_frac=volume_change,
max_anisotropy=max_anisotropy,
calibrated_axes=calibrated_axes,
+ max_lineage_strain=max_lineage_strain,
grain_kinematics=grain_kinematics,
)
@@ -206,6 +393,7 @@ def summary_to_dict(summary: KinematicsSummary) -> dict[str, Any]:
"calibrated_axes": summary.calibrated_axes,
"track_ids": summary.track_ids,
"max_strain": summary.max_strain,
+ "max_lineage_strain": summary.max_lineage_strain,
"total_misorientation": summary.total_misorientation,
"intensity_change_frac": summary.intensity_change_frac,
"volume_change_frac": summary.volume_change_frac,
@@ -218,22 +406,29 @@ def kinematics_to_table(
*,
include_delta_d: bool = False,
) -> list[dict[str, Any]]:
- """Flatten per-grain kinematics into a row-per-observation table.
-
- Each row contains the track_id, scan_idx, and all computed kinematic
- quantities for that observation. Suitable for CSV export or DataFrame
- construction.
- """
+ """Flatten per-grain kinematics into a row-per-observation table."""
rows: list[dict[str, Any]] = []
for gk in grain_kinematics:
+ root_tid = gk.root_track_id if gk.root_track_id > 0 else gk.track_id
for i, scan_idx in enumerate(gk.scan_indices):
row: dict[str, Any] = {
"track_id": gk.track_id,
+ "root_track_id": root_tid,
"scan_idx": scan_idx,
"centroid_mu": float(gk.centroid_mu[i]),
"centroid_chi": float(gk.centroid_chi[i]),
"centroid_d": float(gk.centroid_d[i]),
+ "centroid_mu_err": float(gk.centroid_mu_err[i]) if len(gk.centroid_mu_err) > i else 0.0,
+ "centroid_chi_err": float(gk.centroid_chi_err[i]) if len(gk.centroid_chi_err) > i else 0.0,
+ "centroid_d_err": float(gk.centroid_d_err[i]) if len(gk.centroid_d_err) > i else 0.0,
"strain": float(gk.strain[i]),
+ "strain_err": float(gk.strain_err[i]) if len(gk.strain_err) > i else 0.0,
+ "lineage_strain": float(gk.lineage_strain[i]) if len(gk.lineage_strain) > i else float(gk.strain[i]),
+ "lineage_strain_err": (float(gk.lineage_strain_err[i]) if len(gk.lineage_strain_err) > i else 0.0),
+ "strain_rate": float(gk.strain_rate[i]) if len(gk.strain_rate) > i else 0.0,
+ "rate_mu": float(gk.rate_mu[i]) if len(gk.rate_mu) > i else 0.0,
+ "rate_chi": float(gk.rate_chi[i]) if len(gk.rate_chi) > i else 0.0,
+ "rate_d": float(gk.rate_d[i]) if len(gk.rate_d) > i else 0.0,
"misorientation_mu": float(gk.misorientation_mu[i]),
"misorientation_chi": float(gk.misorientation_chi[i]),
"misorientation_total": float(gk.misorientation_total[i]),
diff --git a/braggtrack/tracking/lifecycle.py b/braggtrack/tracking/lifecycle.py
index 54b8c8c..9feb96c 100644
--- a/braggtrack/tracking/lifecycle.py
+++ b/braggtrack/tracking/lifecycle.py
@@ -7,7 +7,7 @@
import networkx as nx
-from .assignment import associate_frames
+from .assignment import associate_frames, compute_assignment_margins
from .cost import CostFunction, PositionShapeCost
if TYPE_CHECKING:
@@ -56,7 +56,15 @@ def build_tracks(
track_map: dict[int, int] = {} # spot_idx -> track_id for current frame
for idx, spot in enumerate(scan_tables[0]):
nid = _node_id(0, idx)
- G.add_node(nid, scan_idx=0, spot_idx=idx, track_id=next_track_id, event=TrackEvent.BORN, **spot)
+ G.add_node(
+ nid,
+ scan_idx=0,
+ spot_idx=idx,
+ track_id=next_track_id,
+ event=TrackEvent.BORN,
+ assignment_margin=float("inf"),
+ **spot,
+ )
track_map[idx] = next_track_id
next_track_id += 1
@@ -71,6 +79,7 @@ def build_tracks(
cost_fn=active_cost,
max_cost=max_cost,
)
+ margins = compute_assignment_margins(spots_t, spots_t1, matches, active_cost)
new_track_map: dict[int, int] = {}
@@ -79,10 +88,23 @@ def build_tracks(
tid = track_map[i_t]
nid_prev = _node_id(t, i_t)
nid_next = _node_id(t + 1, i_t1)
+ c_ij, margin = margins.get((i_t, i_t1), (0.0, float("inf")))
G.add_node(
- nid_next, scan_idx=t + 1, spot_idx=i_t1, track_id=tid, event=TrackEvent.CONTINUED, **spots_t1[i_t1]
+ nid_next,
+ scan_idx=t + 1,
+ spot_idx=i_t1,
+ track_id=tid,
+ event=TrackEvent.CONTINUED,
+ assignment_margin=margin,
+ **spots_t1[i_t1],
+ )
+ G.add_edge(
+ nid_prev,
+ nid_next,
+ edge_type="continuation",
+ cost=c_ij,
+ assignment_margin=margin,
)
- G.add_edge(nid_prev, nid_next, edge_type="continuation")
new_track_map[i_t1] = tid
# Unmatched in t → TERMINATED (mark on existing node)
@@ -94,7 +116,13 @@ def build_tracks(
for i_t1 in unmatched_t1:
nid = _node_id(t + 1, i_t1)
G.add_node(
- nid, scan_idx=t + 1, spot_idx=i_t1, track_id=next_track_id, event=TrackEvent.BORN, **spots_t1[i_t1]
+ nid,
+ scan_idx=t + 1,
+ spot_idx=i_t1,
+ track_id=next_track_id,
+ event=TrackEvent.BORN,
+ assignment_margin=float("inf"),
+ **spots_t1[i_t1],
)
new_track_map[i_t1] = next_track_id
next_track_id += 1
diff --git a/braggtrack/tracking/precursors.py b/braggtrack/tracking/precursors.py
index 053903a..980e885 100644
--- a/braggtrack/tracking/precursors.py
+++ b/braggtrack/tracking/precursors.py
@@ -103,17 +103,38 @@ def _count_internal_subpeaks(masked_cube: np.ndarray, mask: np.ndarray) -> tuple
return n_peaks, subpeak_score
-def _patch_dispersion_score(
+def _dispersion_from_fmap(fmap: np.ndarray, mm_up: np.ndarray) -> float | None:
+ """Compute foreground patch-token cosine dispersion for one upsampled MIP feature map."""
+ hp, wp, dim = fmap.shape
+ if hp * wp < 2:
+ return None
+
+ bh = max(1, mm_up.shape[0] // hp)
+ bw = max(1, mm_up.shape[1] // wp)
+ cropped_fg = (mm_up[: hp * bh, : wp * bw] > 0).reshape(hp, bh, wp, bw).max(axis=(1, 3))
+ fg_vecs = fmap[cropped_fg] if int(cropped_fg.sum()) >= 2 else fmap.reshape(-1, dim)
+ if fg_vecs.shape[0] < 2:
+ return None
+
+ mean_vec = fg_vecs.mean(axis=0)
+ nrm = float(np.linalg.norm(mean_vec))
+ if nrm <= 0.0:
+ return None
+ mean_vec = mean_vec / nrm
+ cos_sims = np.clip(fg_vecs @ mean_vec, -1.0, 1.0)
+ cos_disp = float(1.0 - np.mean(cos_sims))
+ return float(np.clip(cos_disp / 0.6, 0.0, 1.0))
+
+
+def _prepare_upsampled_roi_mips(
masked_cube: np.ndarray,
mask: np.ndarray,
- patch_encoder: PatchFeatureEncoder,
upsample_size: int,
-) -> float:
- """Compute mean foreground patch-token cosine dispersion across orthogonal MIPs."""
+) -> list[tuple[np.ndarray, np.ndarray]]:
+ """Return upsampled ``(mip_up, mask_up)`` pairs for non-empty orthogonal views."""
mip_mu, mip_chi, mip_d = orthogonal_mips(masked_cube)
mask_mu, mask_chi, mask_d = orthogonal_mips(mask.astype(np.float64))
-
- dispersions: list[float] = []
+ pairs: list[tuple[np.ndarray, np.ndarray]] = []
for mip, mm in ((mip_mu, mask_mu), (mip_chi, mask_chi), (mip_d, mask_d)):
if mip.size == 0 or float(mm.max()) <= 0.0:
continue
@@ -122,29 +143,28 @@ def _patch_dispersion_score(
zw = max(1.0, upsample_size / max(1, w))
mip_up = zoom(mip.astype(np.float64), (zh, zw), order=1) if (zh > 1.0 or zw > 1.0) else mip.astype(np.float64)
mm_up = zoom(mm.astype(np.float64), (zh, zw), order=0) if (zh > 1.0 or zw > 1.0) else mm.astype(np.float64)
+ pairs.append((mip_up, mm_up))
+ return pairs
- fmap = patch_encoder.extract_patch_features(mip_up)
- hp, wp, dim = fmap.shape
- if hp * wp < 2:
- continue
-
- # Map foreground mask onto (hp, wp) patch grid.
- bh = max(1, mm_up.shape[0] // hp)
- bw = max(1, mm_up.shape[1] // wp)
- cropped_fg = (mm_up[: hp * bh, : wp * bw] > 0).reshape(hp, bh, wp, bw).max(axis=(1, 3))
- fg_vecs = fmap[cropped_fg] if int(cropped_fg.sum()) >= 2 else fmap.reshape(-1, dim)
- if fg_vecs.shape[0] < 2:
- continue
-
- mean_vec = fg_vecs.mean(axis=0)
- nrm = float(np.linalg.norm(mean_vec))
- if nrm <= 0.0:
- continue
- mean_vec = mean_vec / nrm
- cos_sims = np.clip(fg_vecs @ mean_vec, -1.0, 1.0)
- cos_disp = float(1.0 - np.mean(cos_sims))
- dispersions.append(float(np.clip(cos_disp / 0.6, 0.0, 1.0)))
+def _patch_dispersion_score(
+ masked_cube: np.ndarray,
+ mask: np.ndarray,
+ patch_encoder: PatchFeatureEncoder,
+ upsample_size: int,
+) -> float:
+ """Compute mean foreground patch-token cosine dispersion across orthogonal MIPs."""
+ pairs = _prepare_upsampled_roi_mips(masked_cube, mask, upsample_size)
+ if not pairs:
+ return 0.0
+ mips = [p[0] for p in pairs]
+ batch_fn = getattr(patch_encoder, "extract_patch_features_batch", None)
+ fmaps = batch_fn(mips) if callable(batch_fn) else [patch_encoder.extract_patch_features(m) for m in mips]
+ dispersions: list[float] = []
+ for fmap, (_, mm_up) in zip(fmaps, pairs, strict=True):
+ val = _dispersion_from_fmap(fmap, mm_up)
+ if val is not None:
+ dispersions.append(val)
return float(np.mean(dispersions)) if dispersions else 0.0
@@ -156,6 +176,8 @@ def compute_fracture_precursors(
label_volumes: dict[int, np.ndarray] | list[np.ndarray] | None = None,
patch_encoder: PatchFeatureEncoder | None = None,
lineage_events: list[LineageEvent] | None = None,
+ roi_cache: dict[tuple[int, int], tuple[int, float]] | None = None,
+ batch_size: int = 32,
) -> tuple[list[PrecursorCandidate], list[ObservationPrecursorScore]]:
"""Score per-grain fracture precursors and return persistent ranked candidates.
@@ -173,6 +195,11 @@ def compute_fracture_precursors(
lineage_events
Optional list of :class:`LineageEvent` records used to annotate whether a
precursor candidate subsequently fractured (``subsequent_split``).
+ roi_cache
+ Optional mutable ``(scan_idx, label) -> (n_subpeaks, roi_term)`` cache
+ for incremental online streaming so historical scans are not re-cropped.
+ batch_size
+ Mini-batch size when running ``patch_encoder.extract_patch_features_batch``.
"""
cfg = config or PrecursorConfig()
if not track_table:
@@ -191,6 +218,65 @@ def compute_fracture_precursors(
lab_map = dict(enumerate(label_volumes))
has_roi = bool(vol_map and lab_map)
+ effective_cache: dict[tuple[int, int], tuple[int, float]] = roi_cache if roi_cache is not None else {}
+
+ # Pre-compute uncached ROI sub-peaks and batch all DINO patch-feature extractions together.
+ if has_roi:
+ pending_keys: list[tuple[int, int]] = []
+ pending_subpeaks: list[tuple[int, float]] = []
+ pending_view_slices: list[tuple[int, int]] = []
+ all_mips: list[np.ndarray] = []
+ all_masks: list[np.ndarray] = []
+
+ for row in track_table:
+ s_idx = int(row["scan_idx"])
+ lbl_i = int(row.get("label", 0))
+ key = (s_idx, lbl_i)
+ if lbl_i <= 0 or "bbox_min_z" not in row or key in effective_cache:
+ continue
+ if s_idx not in vol_map or s_idx not in lab_map:
+ continue
+ # Mark placeholder immediately in case duplicate rows exist
+ effective_cache[key] = (1, 0.0)
+ masked_cube, mask = crop_spot_cube(
+ vol_map[s_idx],
+ lab_map[s_idx],
+ lbl_i,
+ row,
+ margin=cfg.roi_margin,
+ )
+ n_sub, sub_term = _count_internal_subpeaks(masked_cube, mask)
+ if patch_encoder is None:
+ effective_cache[key] = (n_sub, sub_term)
+ else:
+ pairs = _prepare_upsampled_roi_mips(masked_cube, mask, cfg.roi_upsample_size)
+ v_start = len(all_mips)
+ for mip_up, mm_up in pairs:
+ all_mips.append(mip_up)
+ all_masks.append(mm_up)
+ v_end = len(all_mips)
+ pending_keys.append(key)
+ pending_subpeaks.append((n_sub, sub_term))
+ pending_view_slices.append((v_start, v_end))
+
+ if patch_encoder is not None and pending_keys:
+ batch_fn = getattr(patch_encoder, "extract_patch_features_batch", None)
+ if callable(batch_fn):
+ all_fmaps = batch_fn(all_mips, batch_size=batch_size) if all_mips else []
+ else:
+ all_fmaps = [patch_encoder.extract_patch_features(m) for m in all_mips]
+
+ for key, (n_sub, sub_term), (v_start, v_end) in zip(
+ pending_keys, pending_subpeaks, pending_view_slices, strict=True
+ ):
+ dispersions: list[float] = []
+ for v_idx in range(v_start, v_end):
+ val = _dispersion_from_fmap(all_fmaps[v_idx], all_masks[v_idx])
+ if val is not None:
+ dispersions.append(val)
+ p_disp = float(np.mean(dispersions)) if dispersions else 0.0
+ roi_val = float(np.clip(0.65 * sub_term + 0.35 * p_disp, 0.0, 1.0))
+ effective_cache[key] = (n_sub, roi_val)
split_parent_tids: set[int] = set()
if lineage_events:
@@ -207,11 +293,27 @@ def compute_fracture_precursors(
for tid in sorted(by_track):
obs_list = sorted(by_track[tid], key=lambda r: int(r["scan_idx"]))
- e0 = (
- float(obs_list[0].get("eig_1", 0.0))
- + float(obs_list[0].get("eig_2", 0.0))
- + float(obs_list[0].get("eig_3", 0.0))
+ eig1_0 = max(0.0, float(obs_list[0].get("eig_1", 0.0)))
+ eig2_0 = max(0.0, float(obs_list[0].get("eig_2", 0.0)))
+ eig3_0 = max(0.0, float(obs_list[0].get("eig_3", 0.0)))
+ e0 = eig1_0 + eig2_0 + eig3_0
+ aniso_0 = eig1_0 / eig3_0 if eig3_0 > 1e-12 else 1.0
+ c0_mu = max(1e-12, float(obs_list[0].get("cov_zz", 1.0)))
+ c0_chi = max(1e-12, float(obs_list[0].get("cov_xx", 1.0)))
+ c0_d = max(1e-12, float(obs_list[0].get("cov_yy", 1.0)))
+ has_cov = all(k in obs_list[0] for k in ("cov_zz", "cov_xx", "cov_yy"))
+ has_bbox = all(
+ k in obs_list[0]
+ for k in ("bbox_min_z", "bbox_max_z", "bbox_min_x", "bbox_max_x", "bbox_min_y", "bbox_max_y")
)
+ if has_cov and has_bbox:
+ dz0 = max(float(obs_list[0]["bbox_max_z"]) - float(obs_list[0]["bbox_min_z"]), 1.0)
+ dx0 = max(float(obs_list[0]["bbox_max_x"]) - float(obs_list[0]["bbox_min_x"]), 1.0)
+ dy0 = max(float(obs_list[0]["bbox_max_y"]) - float(obs_list[0]["bbox_min_y"]), 1.0)
+ scales2 = [c0_mu / (dz0 * dz0), c0_chi / (dx0 * dx0), c0_d / (dy0 * dy0)]
+ mixed_units = (max(scales2) / max(min(scales2), 1e-18)) > 25.0
+ else:
+ mixed_units = has_cov and aniso_0 > 50.0
track_scores: list[ObservationPrecursorScore] = []
for obs in obs_list:
@@ -221,32 +323,29 @@ def compute_fracture_precursors(
eig2 = max(0.0, float(obs.get("eig_2", 0.0)))
eig3 = max(0.0, float(obs.get("eig_3", 0.0)))
- aniso = eig1 / eig3 if eig3 > 1e-9 else (1.0 if eig1 <= 1e-9 else 10.0)
- aniso_term = float(np.clip((aniso - 1.0) / max(cfg.anisotropy_scale, 1e-6), 0.0, 1.0))
+ if mixed_units:
+ r_mu = max(0.0, float(obs.get("cov_zz", 0.0))) / c0_mu
+ r_chi = max(0.0, float(obs.get("cov_xx", 0.0))) / c0_chi
+ r_d = max(0.0, float(obs.get("cov_yy", 0.0))) / c0_d
+ r_max = max(r_mu, r_chi, r_d)
+ r_min = max(min(r_mu, r_chi, r_d), 1e-9)
+ aniso = r_max / r_min
+ spread_growth = (r_mu + r_chi + r_d - 3.0) / 3.0
+ else:
+ aniso = eig1 / eig3 if eig3 > 1e-9 else (1.0 if eig1 <= 1e-9 else 10.0)
+ trace = eig1 + eig2 + eig3
+ spread_growth = (trace - e0) / e0 if e0 > 1e-9 else 0.0
- trace = eig1 + eig2 + eig3
- spread_growth = (trace - e0) / e0 if e0 > 1e-9 else 0.0
+ aniso_term = float(np.clip((aniso - 1.0) / max(cfg.anisotropy_scale, 1e-6), 0.0, 1.0))
spread_term = float(np.clip(max(0.0, spread_growth) / max(cfg.spread_growth_scale, 1e-6), 0.0, 1.0))
n_subpeaks = 1
roi_term = 0.0
- use_roi_here = has_roi and scan_idx in vol_map and scan_idx in lab_map and lbl > 0
- if use_roi_here and "bbox_min_z" in obs:
- masked_cube, mask = crop_spot_cube(
- vol_map[scan_idx],
- lab_map[scan_idx],
- lbl,
- obs,
- margin=cfg.roi_margin,
- )
- n_subpeaks, subpeak_term = _count_internal_subpeaks(masked_cube, mask)
- if patch_encoder is not None:
- p_disp = _patch_dispersion_score(masked_cube, mask, patch_encoder, cfg.roi_upsample_size)
- roi_term = float(np.clip(0.65 * subpeak_term + 0.35 * p_disp, 0.0, 1.0))
- else:
- roi_term = subpeak_term
-
+ use_roi_here = lbl > 0 and (
+ (scan_idx, lbl) in effective_cache or (has_roi and scan_idx in vol_map and scan_idx in lab_map)
+ )
if use_roi_here and "bbox_min_z" in obs:
+ n_subpeaks, roi_term = effective_cache.get((scan_idx, lbl), (1, 0.0))
w_sum = cfg.weight_anisotropy + cfg.weight_spread + cfg.weight_roi
raw_score = (
cfg.weight_anisotropy * aniso_term + cfg.weight_spread * spread_term + cfg.weight_roi * roi_term
diff --git a/braggtrack/types.py b/braggtrack/types.py
index 67e631e..f498f8c 100644
--- a/braggtrack/types.py
+++ b/braggtrack/types.py
@@ -18,9 +18,14 @@ class SpotRecord(TypedDict, total=False):
label: int
voxel_count: int
integrated_intensity: float
+ net_intensity: float
+ background_level: float
centroid_mu: float
centroid_chi: float
centroid_d: float
+ centroid_mu_err: float
+ centroid_chi_err: float
+ centroid_d_err: float
bbox_min_z: int
bbox_max_z: int
bbox_min_y: int
@@ -36,6 +41,18 @@ class SpotRecord(TypedDict, total=False):
eig_1: float
eig_2: float
eig_3: float
+ vox_eig_1: float
+ vox_eig_2: float
+ vox_eig_3: float
+ kurtosis_mu: float
+ kurtosis_chi: float
+ kurtosis_d: float
+ step_mu: float
+ step_chi: float
+ step_d: float
+ calibrated: int
+ touches_border: bool
+ touches_rocking_border: bool
# Optional — semantic stage
embedding: np.ndarray
diff --git a/braggtrack/viz/__init__.py b/braggtrack/viz/__init__.py
index 52705e6..4c50333 100644
--- a/braggtrack/viz/__init__.py
+++ b/braggtrack/viz/__init__.py
@@ -4,6 +4,7 @@
from .kinematics import plot_kinematics_dashboard
from .lineage import plot_lineage_dag
from .mips import plot_scan_projection_overlay, plot_spot_mip_card
+from .operando import plot_operando_strain_and_events
from .precursors import plot_precursor_timeline
__all__ = [
@@ -11,6 +12,7 @@
"generate_html_report",
"plot_kinematics_dashboard",
"plot_lineage_dag",
+ "plot_operando_strain_and_events",
"plot_precursor_timeline",
"plot_scan_projection_overlay",
"plot_spot_mip_card",
diff --git a/braggtrack/viz/html_report.py b/braggtrack/viz/html_report.py
index 49518ef..d938cde 100644
--- a/braggtrack/viz/html_report.py
+++ b/braggtrack/viz/html_report.py
@@ -148,6 +148,7 @@ def build_explorer_payload(
artifact_dir: str | Path,
*,
dataset_root: str | Path | None = None,
+ operando_log: str | Path | None = None,
max_mip_tracks: int = 16,
) -> dict[str, Any]:
"""Assemble structured JSON payload for the interactive 4D explorer from pipeline/tracking outputs."""
@@ -218,6 +219,7 @@ def build_explorer_payload(
events_list.append(
{
"event_type": etype,
+ "split_subtype": str(r.get("split_subtype", "")),
"parent_scan": int(float(r.get("scan_idx_from", r.get("parent_scan", 0)))),
"child_scan": int(float(r.get("scan_idx_to", r.get("child_scan", 1)))),
"parent_track_ids": ptids,
@@ -256,6 +258,30 @@ def build_explorer_payload(
if n_scans == 0 and track_rows:
n_scans = max(int(r["scan_idx"]) for r in track_rows) + 1
+ # Load synchronized operando stimulus from explicit log path or saved artifact JSON
+ stimuli_list: list[dict[str, Any]] = []
+ saved_stim = _read_json(root / "operando_stimulus.json")
+ if operando_log is not None:
+ from braggtrack.io import BeamlineAdapter, load_operando_log, synchronize_to_scans
+
+ op_log = load_operando_log(operando_log)
+ if ds_root is not None and ds_root.is_dir():
+ try:
+ seq = BeamlineAdapter(ds_root).build_sequence()
+ pts = (
+ synchronize_to_scans(op_log, seq)
+ if len(seq.scans) == n_scans
+ else synchronize_to_scans(op_log, n_scans)
+ )
+ except Exception:
+ pts = synchronize_to_scans(op_log, n_scans)
+ else:
+ pts = synchronize_to_scans(op_log, n_scans)
+ stimuli_list = [p.to_dict() for p in pts]
+ elif isinstance(saved_stim, dict) and isinstance(saved_stim.get("scans"), list):
+ stimuli_list = list(saved_stim["scans"])
+ stim_by_scan = {int(s["scan_idx"]): s for s in stimuli_list if "scan_idx" in s}
+
priority_tids = sorted(
by_tid.keys(),
key=lambda tid: (
@@ -279,6 +305,16 @@ def build_explorer_payload(
for idx_o, o in enumerate(obs_sorted):
s_i = int(o["scan_idx"])
strain_pct = float(gk.strain[idx_o] * 100.0) if gk is not None and idx_o < len(gk.strain) else 0.0
+ lin_strain_pct = (
+ float(gk.lineage_strain[idx_o] * 100.0)
+ if gk is not None and idx_o < len(gk.lineage_strain)
+ else strain_pct
+ )
+ strain_err_pct = (
+ float(gk.lineage_strain_err[idx_o] * 100.0)
+ if gk is not None and idx_o < len(gk.lineage_strain_err)
+ else 0.0
+ )
miso = (
float(gk.misorientation_total[idx_o])
if gk is not None and idx_o < len(gk.misorientation_total)
@@ -286,6 +322,7 @@ def build_explorer_payload(
)
aniso = float(gk.anisotropy[idx_o]) if gk is not None and idx_o < len(gk.anisotropy) else 1.0
po = prec_obs_by_key.get((tid, s_i), {})
+ stim = stim_by_scan.get(s_i, {})
obs_items.append(
{
"scan_idx": s_i,
@@ -296,6 +333,12 @@ def build_explorer_payload(
"intensity": round(float(o.get("integrated_intensity", 0.0)), 2),
"voxels": int(o.get("voxel_count", 0)),
"strain_pct": round(strain_pct, 4),
+ "lineage_strain_pct": round(lin_strain_pct, 4),
+ "strain_err_pct": round(strain_err_pct, 4),
+ "voltage_v": stim.get("voltage_v"),
+ "dq_dv": stim.get("dq_dv"),
+ "stress_mpa": stim.get("stress_mpa"),
+ "temperature_c": stim.get("temperature_c"),
"misorientation": round(miso, 4),
"anisotropy": round(aniso, 3),
"precursor_score": round(float(po.get("score", 0.0)), 4),
@@ -308,12 +351,13 @@ def build_explorer_payload(
tracks_payload.append(
{
"track_id": tid,
+ "root_track_id": gk.root_track_id if gk is not None and gk.root_track_id > 0 else tid,
"is_full": is_full,
"has_gap": has_gap,
"is_split": tid in split_tids,
"is_merge": tid in merge_tids,
"is_precursor": tid in prec_tids,
- "max_abs_strain_pct": round(max((abs(x["strain_pct"]) for x in obs_items), default=0.0), 4),
+ "max_abs_strain_pct": round(max((abs(x["lineage_strain_pct"]) for x in obs_items), default=0.0), 4),
"max_precursor": round(max((x["precursor_score"] for x in obs_items), default=0.0), 4),
"observations": obs_items,
}
@@ -330,7 +374,7 @@ def build_explorer_payload(
"calibrated_axes": bool(trk_summary.get("calibrated_axes", False)),
"backend": evt_summary.get("backend") or manifest.get("config", {}).get("backend") or "n/a",
"model": evt_summary.get("model") or manifest.get("config", {}).get("model") or "n/a",
- "braggtrack_version": manifest.get("braggtrack_version", "0.3.0"),
+ "braggtrack_version": manifest.get("braggtrack_version", "0.4.0"),
"total_duration_sec": manifest.get("total_duration_sec"),
"kpis": {
"total_tracks": len(tracks_payload),
@@ -340,6 +384,7 @@ def build_explorer_payload(
"merge_count": sum(1 for e in events_list if e["event_type"] == "merge"),
"precursor_count": len(precursors_list),
},
+ "stimuli": stimuli_list,
"tracks": tracks_payload,
"events": events_list,
"precursors": precursors_list,
@@ -351,6 +396,7 @@ def build_explorer_payload(
+
__TITLE__