From 88243804994c04dab7983d1f36cfa6079a39f844 Mon Sep 17 00:00:00 2001
From: James Le Houx <37665786+jameslehoux@users.noreply.github.com>
Date: Tue, 17 Nov 2026 23:36:24 +0000
Subject: [PATCH 1/3] =?UTF-8?q?feat:=20BraggTrack=20v0.4.0=20=E2=80=94=20a?=
=?UTF-8?q?utonomous=20DFXM=20Scout-and-Zoom,=20operando=20sync,=20uncerta?=
=?UTF-8?q?inty=20&=20batched=20DINOv2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.github/workflows/release.yml | 90 +++++
CHANGELOG.md | 19 +
CITATION.cff | 16 +-
README.md | 30 +-
braggtrack/__init__.py | 4 +-
braggtrack/autonomous/__init__.py | 17 +
braggtrack/autonomous/replay.py | 328 ++++++++++++++++
braggtrack/autonomous/streaming.py | 327 ++++++++++++++++
braggtrack/autonomous/trigger.py | 396 +++++++++++++++++++
braggtrack/cli/_utils.py | 2 +-
braggtrack/cli/embed_dataset.py | 21 +-
braggtrack/cli/report_dataset.py | 33 ++
braggtrack/cli/run_pipeline.py | 40 +-
braggtrack/cli/watch_autonomous.py | 220 +++++++++++
braggtrack/io/__init__.py | 12 +
braggtrack/io/nexus.py | 93 ++++-
braggtrack/io/operando_log.py | 453 ++++++++++++++++++++++
braggtrack/segmentation/dino_segment.py | 10 +-
braggtrack/segmentation/features.py | 19 +
braggtrack/semantic/dino.py | 141 +++++--
braggtrack/tracking/__init__.py | 3 +-
braggtrack/tracking/assignment.py | 31 ++
braggtrack/tracking/kinematics.py | 215 ++++++++--
braggtrack/tracking/lifecycle.py | 38 +-
braggtrack/tracking/precursors.py | 201 +++++++---
braggtrack/viz/__init__.py | 2 +
braggtrack/viz/html_report.py | 75 +++-
braggtrack/viz/operando.py | 241 ++++++++++++
docs/architecture.md | 30 +-
docs/benchmarks.md | 60 ++-
docs/figures/operando_autonomous_zoom.png | Bin 0 -> 268687 bytes
docs/positioning.md | 118 ++++++
paper.bib | 83 ++++
paper.md | 169 +++++---
pyproject.toml | 3 +-
tests/test_autonomous_zoom.py | 264 +++++++++++++
tests/test_ci_report.py | 1 +
tests/test_dino_batching.py | 67 ++++
tests/test_kinematics.py | 11 +
tests/test_kinematics_uncertainty.py | 153 ++++++++
tests/test_nexus_darfix.py | 88 +++++
tests/test_operando_log.py | 127 ++++++
42 files changed, 3975 insertions(+), 276 deletions(-)
create mode 100644 .github/workflows/release.yml
create mode 100644 braggtrack/autonomous/__init__.py
create mode 100644 braggtrack/autonomous/replay.py
create mode 100644 braggtrack/autonomous/streaming.py
create mode 100644 braggtrack/autonomous/trigger.py
create mode 100644 braggtrack/cli/watch_autonomous.py
create mode 100644 braggtrack/io/operando_log.py
create mode 100644 braggtrack/viz/operando.py
create mode 100644 docs/figures/operando_autonomous_zoom.png
create mode 100644 docs/positioning.md
create mode 100644 tests/test_autonomous_zoom.py
create mode 100644 tests/test_dino_batching.py
create mode 100644 tests/test_kinematics_uncertainty.py
create mode 100644 tests/test_nexus_darfix.py
create mode 100644 tests/test_operando_log.py
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 0000000..a0fecdc
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,90 @@
+name: Release
+
+on:
+ push:
+ tags:
+ - "v*.*.*"
+ workflow_dispatch:
+
+permissions:
+ contents: write
+
+jobs:
+ build-and-verify:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Set up Python 3.12
+ uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+
+ - name: Install build & verification dependencies
+ run: |
+ python -m pip install --upgrade pip build twine ruff pytest
+ python -m pip install -e ".[notebook]"
+
+ - name: Verify lint and unit test suite
+ run: |
+ ruff check braggtrack/ tests/ scripts/
+ pytest tests/ -q
+
+ - name: Build sdist and wheel
+ run: python -m build
+
+ - name: Validate distribution metadata with Twine
+ run: twine check dist/*
+
+ - name: Verify clean wheel installation in isolated venv
+ run: |
+ python -m venv /tmp/release-smoke-venv
+ /tmp/release-smoke-venv/bin/pip install dist/*.whl
+ /tmp/release-smoke-venv/bin/braggtrack-inspect --help
+ /tmp/release-smoke-venv/bin/braggtrack-watch --help
+ /tmp/release-smoke-venv/bin/braggtrack-run --help
+
+ - name: Upload distribution artifacts
+ uses: actions/upload-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ github-release:
+ needs: build-and-verify
+ if: startsWith(github.ref, 'refs/tags/v')
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Create GitHub Release
+ uses: softprops/action-gh-release@v2
+ with:
+ files: dist/*
+ generate_release_notes: true
+
+ publish-pypi:
+ needs: build-and-verify
+ if: startsWith(github.ref, 'refs/tags/v') && vars.ENABLE_PYPI_PUBLISH == 'true'
+ runs-on: ubuntu-latest
+ environment:
+ name: pypi
+ url: https://pypi.org/p/braggtrack
+ permissions:
+ id-token: write
+ steps:
+ - name: Download distribution artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: python-package-distributions
+ path: dist/
+
+ - name: Publish distribution to PyPI via Trusted Publisher
+ uses: pypa/gh-action-pypi-publish@release/v1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6fffb35..dbeeb31 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,25 @@ All notable changes to BraggTrack will be documented in this file.
The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.4.0] - 2026-11-17
+
+### Added
+- **Analytical Poisson + quantization centroid/strain uncertainties & ancestor-referenced kinematics (`braggtrack.segmentation.features`, `braggtrack.tracking.kinematics`, `braggtrack.tracking.assignment`)**:
+ - Emits per-spot `centroid_mu_err`, `centroid_chi_err`, and `centroid_d_err` combining Poisson photon-counting variance and `1/12` voxel quantization variance.
+ - Propagates `strain_err` ($\sigma_\varepsilon$), ancestor-referenced `root_track_id`, `lineage_strain`, and `lineage_strain_err` across `1 -> 2+` fracture splits, plus causal velocities (`rate_mu`, `rate_chi`, `rate_d`, `strain_rate`) and Hungarian runner-up `assignment_margin`.
+- **Operando electrochemistry & external stimulus synchronization (`braggtrack.io.operando_log`, `braggtrack.viz.operando`)**:
+ - Parses `.csv`/`.tsv` and Biologic EC-Lab `.mpt` potentiostat logs (`load_operando_log`, `synchronize_stimulus_to_scans`), computing Savitzky–Golay regularized differential capacity $\mathrm{d}Q/\mathrm{d}V$ and per-grain electro-strain susceptibility `d_strain_d_v` ($\partial\varepsilon/\partial V$).
+ - Added 3-panel publication plot `plot_operando_strain_and_events` and wired `--operando-log` / `--operando-tz-offset` into `braggtrack-report`, `braggtrack-run`, and `report.html`.
+- **Batched `DINOv2` ViT inference & zero-config `darfix` / ESRF BLISS NeXus auto-calibration (`braggtrack.semantic.dino`, `braggtrack.io.nexus`)**:
+ - Added `embed_batch` and `extract_patch_features_batch` across `MultiviewEncoder` and `PatchFeatureEncoder` (with CPU L3-friendly micro-batching), accelerating `braggtrack-embed-dataset`, `compute_fracture_precursors`, and `segment_dino` while guaranteeing bit-identical cosine parity.
+ - Added `NXdata` `@default` / `@signal` / `@axes` (`darfix` processed HDF5) and ESRF BLISS `instrument/positioners` auto-calibration in `discover_primary_volume_path` and `extract_axis_calibration`.
+- **Online causal streaming & autonomous DFXM "Scout-and-Zoom" engine (`braggtrack.autonomous`, `braggtrack-watch`)**:
+ - Added `CausalStreamingTracker` (`streaming.py`) with strictly causal trailing-window Otsu stabilisation and $O(1)$-volume-RAM cached ROI precursors.
+ - Added `ZoomTriggerPolicy`, `ZoomTarget`, and `select_zoom_targets` (`trigger.py`) with velocity-extrapolated diffractometer setpoint prediction, uncertainty-aware rocking-window sizing, and `bliss_dfxm_zoom.v1` JSONL serialization (`zoom_triggers.jsonl`).
+ - Added `evaluate_causal_replay` (`replay.py`) benchmarking autonomous pre-fracture recall, electrochemical lead ($\Delta E$), and setpoint hit rate against `brightest_intensity`, `strain_only`, and `random` policies, exposed via the 10th CLI entry point `braggtrack-watch` (`--mode watch` and `--mode replay`).
+- **Ecosystem positioning, verified bibliography & release workflow (`docs/positioning.md`, `.github/workflows/release.yml`)**:
+ - Added `docs/positioning.md` comparing `BraggTrack` against `darfix`, `ImageD11`, `HEXRD`, `BraggNN`, `TrackMate 7`, and `LapTrack`, updated `paper.md` and `paper.bib` with verified DOIs (`@GarrigaFerrer2023`, `@Isern2025`, `@Bernier2011`, `@Liu2022`, `@Fukai2023`, `@Ershov2022`), bumped metadata to `0.4.0`, and added tag-triggered sdist/wheel + Twine + GitHub Release / PyPI Trusted Publisher workflow.
+
## [0.3.0] - 2026-11-17
### Added
diff --git a/CITATION.cff b/CITATION.cff
index 8d1e56b..a2f1b07 100644
--- a/CITATION.cff
+++ b/CITATION.cff
@@ -2,21 +2,25 @@ cff-version: 1.2.0
message: "If you use this software, please cite it as below."
title: "BraggTrack"
type: software
-version: 0.3.0
+version: 0.4.0
date-released: "2026-05-16"
license: MIT
repository-code: "https://github.com/BASE-Laboratory/BraggTrack"
url: "https://github.com/BASE-Laboratory/BraggTrack"
abstract: >-
- Semantic 4D kinematics, multi-scan gap closing, and chemomechanical
- fracture tracking for operando X-ray diffraction. BraggTrack segments
- Bragg reflections in 3D reciprocal-space volumes, tracks trajectories
- and split/merge lineage using classical crystallography and
- vision-transformer embeddings, and exports ImageD11, NeXus HDF5, and
+ Semantic 4D kinematics, multi-scan gap closing, chemomechanical fracture
+ tracking, operando electrochemistry synchronization, and online autonomous
+ Dark-Field X-ray Microscopy (DFXM) target selection for operando X-ray
+ diffraction. BraggTrack segments Bragg reflections in 3D reciprocal-space
+ volumes, tracks trajectories and split/merge lineage with analytical
+ uncertainties using classical crystallography and batched vision-transformer
+ embeddings, and exports ImageD11, NeXus HDF5, BLISS zoom triggers, and
interactive HTML 4D lineage reports.
keywords:
- X-ray diffraction
+ - dark-field X-ray microscopy
- operando crystallography
+ - autonomous experiments
- 3D segmentation
- object tracking
- grain fracture
diff --git a/README.md b/README.md
index d61abd5..daf0df6 100644
--- a/README.md
+++ b/README.md
@@ -12,15 +12,16 @@ Semantic 4D kinematics, multi-scan beam-dropout gap closing, grain lineage (`1 -
## Key Capabilities
-- **Beamline-scale NeXus/HDF5 I/O, lazy ROI streaming & crystallographic mapping (`braggtrack.io`)** — Auto-discovers 3D detector cubes across standard NeXus paths, ESRF-ID03 LIMA files (`/entry_0000/ESRF-ID03/pco_nf/data`), and arbitrary HDF5 trees (`auto_discover=True`). Streams `z`-slabs via `LazyVolume` (`otsu_threshold_chunked`, `read_roi`), calibrates `(mu, chi, d)` via `AxisCalibration`, maps transmission geometry `(2theta, eta, d, |Q|)` via `DetectorGeometry`, and exports directly to **`ImageD11` (`.flt`)** and **NeXus `NXprocess` HDF5 (`.nxs`)**.
-- **Stabilised 3D Bragg reflection segmentation (`braggtrack.segmentation`)** — Streaming Otsu foreground masking with cross-scan rolling-median smoothing (`smooth_thresholds`), 3D LoG peak enhancement, robust `h`-maxima seeding, 3D watershed, nearby-basin merging, and physical or voxel-space covariance/eigenvalue extraction. Includes an experimental dense patch-feature + HDBSCAN slice-stitching segmenter (`--method dino`).
-- **Multi-view foundation vision descriptors (`braggtrack.semantic`)** — Orthogonal $(\mu, \chi, d)$ maximum-intensity projections (MIPs) encoded with ungated **`facebook/dinov2-small`** by default (zero HuggingFace login required), drop-in support for **DINOv3** (`facebook/dinov3-vitb16-pretrain-lvd1689m`), or a deterministic zero-GPU `mock` backend for CI.
-- **4D grain kinematics, multi-scan gap closing, split/merge lineage & precursors (`braggtrack.tracking`)** — Vectorised geometry + semantic Hungarian association, Jaqaman-style multi-scan gap closing (`--max-gap`) across transient beam dropouts, per-grain lattice strain ($\Delta d / d_0$) and angular misorientation ($\Delta\mu$, $\Delta\chi$), intensity-partitioned `1 -> 2+` split and `2+ -> 1` merge DAG edges, and persistent multi-scan fracture precursor ranking combining 3D shape anisotropy, covariance spread growth, and spot ROI sub-peak / DINO patch-token dispersion.
-- **Publication figures & offline interactive HTML 4D Explorer (`braggtrack.viz`, `braggtrack-report`)** — Generates Okabe–Ito styled lineage DAGs, strain/misorientation dashboards, precursor heatmaps, 3-view MIP cards, and a single-file interactive `report.html` with embedded base64 MIP thumbnails that opens offline on air-gapped beamline networks.
-- **One-command reproducible pipeline (`braggtrack-run`)** — Runs validation, segmentation, semantic embedding, tracking, kinematics, fracture event analysis, HTML reporting, and `ImageD11`/NeXus export end-to-end, emitting SHA-256 input checksums, stage timings, and `run_manifest.json`.
+- **Beamline-scale NeXus/HDF5 I/O, `darfix`/BLISS auto-calibration & operando sync (`braggtrack.io`)** — Auto-discovers 3D detector cubes across `darfix` `NXdata` (`@default` / `@signal` / `@axes`), ESRF-ID03 LIMA files (`/entry_0000/ESRF-ID03/pco_nf/data`), ESRF BLISS `instrument/positioners`, and arbitrary HDF5 trees (`auto_discover=True`). Streams `z`-slabs via `LazyVolume`, synchronizes external potentiostat / load-frame logs (`CSV`/`TSV` and Biologic EC-Lab `.mpt`, regularized $\mathrm{d}Q/\mathrm{d}V$, $\partial\varepsilon/\partial V$), maps transmission geometry `(2theta, eta, d, |Q|)` via `DetectorGeometry`, and exports directly to **`ImageD11` (`.flt`)** and **NeXus `NXprocess` HDF5 (`.nxs`)**.
+- **Stabilised 3D Bragg reflection segmentation (`braggtrack.segmentation`)** — Streaming Otsu foreground masking with cross-scan rolling-median smoothing (`smooth_thresholds`), 3D LoG peak enhancement, robust `h`-maxima seeding, 3D watershed, nearby-basin merging, analytical Poisson + quantization centroid uncertainties (`centroid_*_err`), and physical or voxel-space covariance/eigenvalue extraction. Includes an experimental batched dense patch-feature + HDBSCAN slice-stitching segmenter (`--method dino`).
+- **Batched multi-view foundation vision descriptors (`braggtrack.semantic`)** — Orthogonal $(\mu, \chi, d)$ maximum-intensity projections (MIPs) encoded in batched forward passes (`embed_batch`, `extract_patch_features_batch`) with ungated **`facebook/dinov2-small`** by default (zero HuggingFace login required), drop-in support for **DINOv3** (`facebook/dinov3-vitb16-pretrain-lvd1689m`), or a deterministic zero-GPU `mock` backend for CI.
+- **4D grain kinematics, uncertainty propagation, split/merge lineage & precursors (`braggtrack.tracking`)** — Vectorised geometry + semantic Hungarian association with runner-up `assignment_margin`, Jaqaman-style multi-scan gap closing (`--max-gap`), local strain ($\Delta d / d_0$), ancestor-referenced `lineage_strain` across splits, propagated $\sigma_\varepsilon$, causal angular/lattice velocities (`rate_mu`, `rate_chi`, `rate_d`, `strain_rate`), intensity-partitioned `1 -> 2+` split and `2+ -> 1` merge DAG edges, and persistent multi-scan fracture precursor ranking.
+- **Online causal streaming & autonomous DFXM "Scout-and-Zoom" (`braggtrack.autonomous`, `braggtrack-watch`)** — Strictly causal trailing-window Otsu streaming tracker (`CausalStreamingTracker`) with $O(1)$-volume-RAM cached ROI precursors, velocity-extrapolated diffractometer setpoint prediction during rapid phase transitions (e.g., Ni92 `H2 -> H3` collapse), `bliss_dfxm_zoom.v1` JSONL trigger emission (`zoom_triggers.jsonl`), and offline causal replay benchmarking against naive baselines.
+- **Publication figures, operando $\mathrm{d}Q/\mathrm{d}V$ dashboards & offline HTML 4D Explorer (`braggtrack.viz`, `braggtrack-report`)** — Generates Okabe–Ito styled lineage DAGs, strain/misorientation dashboards, 3-panel operando electrochemistry + fracture synchronization plots, precursor heatmaps, 3-view MIP cards, and a single-file interactive `report.html` that opens offline on air-gapped beamline networks.
+- **One-command reproducible pipeline (`braggtrack-run`)** — Runs validation, operando stimulus sync, segmentation, semantic embedding, tracking, kinematics, fracture event analysis, HTML reporting, and `ImageD11`/NeXus export end-to-end, emitting SHA-256 input checksums, stage timings, and `run_manifest.json`.
-
+
---
@@ -67,7 +68,7 @@ Produces:
- `artifacts/pipeline/results.nxs` & `artifacts/pipeline/peaks.flt`
- `artifacts/pipeline/run_manifest.json` (SHA-256 input checksums, git SHA, config, stage timings)
-### Stage-by-stage CLI commands (9 Entry Points)
+### Stage-by-stage CLI commands (10 Entry Points)
| CLI Entry Point | Module Equivalent | Purpose |
|---|---|---|
@@ -75,11 +76,12 @@ Produces:
| `braggtrack-validate` | `python -m braggtrack.cli.validate_dataset` | Validate sequence ordering, monotonic timestamps, and axis contracts |
| `braggtrack-segment-synthetic` | `python -m braggtrack.cli.segment_synthetic` | Run fast synthetic 3D segmentation smoke test |
| `braggtrack-segment-dataset` | `python -m braggtrack.cli.segment_dataset` | Segment 3D Bragg spots via chunked Otsu + smoothed thresholds & optional calibration |
-| `braggtrack-embed-dataset` | `python -m braggtrack.cli.embed_dataset` | Stream lazy 3D spot ROIs & compute orthogonal MIP embeddings (`auto`, `mock`, `torch`) |
+| `braggtrack-embed-dataset` | `python -m braggtrack.cli.embed_dataset` | Stream lazy 3D spot ROIs & compute batched MIP embeddings (`auto`, `mock`, `torch`) |
| `braggtrack-track-dataset` | `python -m braggtrack.cli.track_dataset` | Build trajectory DAG (`--max-gap`), export `tracks.csv`, `kinematics.csv`, `.flt`, `.nxs` |
-| `braggtrack-events` | `python -m braggtrack.cli.detect_events` | Resolve `1 -> 2+` splits, `2+ -> 1` merges, and lazy-ROI fracture precursors |
-| `braggtrack-report` | `python -m braggtrack.cli.report_dataset` | Render publication PNG/PDF figures and self-contained interactive `report.html` |
-| `braggtrack-run` | `python -m braggtrack.cli.run_pipeline` | Execute the full pipeline with optional `--config config.json`, `--report`, and exports |
+| `braggtrack-events` | `python -m braggtrack.cli.detect_events` | Resolve `1 -> 2+` splits, `2+ -> 1` merges, and batched lazy-ROI fracture precursors |
+| `braggtrack-report` | `python -m braggtrack.cli.report_dataset` | Render publication PNG/PDF figures (`--operando-log`) and self-contained `report.html` |
+| `braggtrack-watch` | `python -m braggtrack.cli.watch_autonomous` | Live causal streaming watcher (`--mode watch`) & autonomous DFXM replay (`--mode replay`) |
+| `braggtrack-run` | `python -m braggtrack.cli.run_pipeline` | Execute the full pipeline with optional `--config`, `--operando-log`, `--report`, and exports |
---
@@ -162,9 +164,9 @@ python scripts/pre_pr_check.py
## Repository Layout & Contributing
-- **`braggtrack/`** — Installable Python library (`io`, `segmentation`, `semantic`, `tracking`, `viz`, `cli`).
+- **`braggtrack/`** — Installable Python library (`io`, `segmentation`, `semantic`, `tracking`, `autonomous`, `viz`, `cli`).
- **`data/sample_operando/`** — Bundled ESRF-ID03 operando test scans (`scan0001`–`scan0003`).
-- **`docs/`** — [Architecture](docs/architecture.md), [Benchmarks](docs/benchmarks.md), [Coding Practices](docs/coding-practices.md), and publication figures (`docs/figures/`).
+- **`docs/`** — [Ecosystem Positioning & Autonomous DFXM Guide](docs/positioning.md), [Architecture](docs/architecture.md), [Benchmarks](docs/benchmarks.md), [Coding Practices](docs/coding-practices.md), and publication figures (`docs/figures/`).
- **`scripts/`** — 9 acceptance gates, 5-scenario benchmark suite (`benchmark_suite.py`), semantic α/β ablation (`ablation_semantic.py`), and CI runner (`ci_report.py`).
- **`artifacts/`** — Generated pipeline outputs (gitignored; never commit).
diff --git a/braggtrack/__init__.py b/braggtrack/__init__.py
index 14eeacb..6a570d3 100644
--- a/braggtrack/__init__.py
+++ b/braggtrack/__init__.py
@@ -5,6 +5,6 @@
try:
__version__ = version("braggtrack")
except PackageNotFoundError:
- __version__ = "0.3.0.dev0"
+ __version__ = "0.4.0.dev0"
-__all__ = ["io", "segmentation", "semantic", "tracking", "viz"]
+__all__ = ["autonomous", "io", "segmentation", "semantic", "tracking", "viz"]
diff --git a/braggtrack/autonomous/__init__.py b/braggtrack/autonomous/__init__.py
new file mode 100644
index 0000000..d44cf62
--- /dev/null
+++ b/braggtrack/autonomous/__init__.py
@@ -0,0 +1,17 @@
+"""Online causal streaming and autonomous DFXM "Scout-and-Zoom" engine."""
+
+from .replay import PolicyReplayMetrics, ReplayBenchmarkResult, evaluate_causal_replay
+from .streaming import CausalStreamingTracker, StreamingStepResult
+from .trigger import ZoomTarget, ZoomTriggerPolicy, compute_target_geometry, select_zoom_targets
+
+__all__ = [
+ "CausalStreamingTracker",
+ "PolicyReplayMetrics",
+ "ReplayBenchmarkResult",
+ "StreamingStepResult",
+ "ZoomTarget",
+ "ZoomTriggerPolicy",
+ "compute_target_geometry",
+ "evaluate_causal_replay",
+ "select_zoom_targets",
+]
diff --git a/braggtrack/autonomous/replay.py b/braggtrack/autonomous/replay.py
new file mode 100644
index 0000000..ef0f3b3
--- /dev/null
+++ b/braggtrack/autonomous/replay.py
@@ -0,0 +1,328 @@
+"""Closed-loop retrospective causal replay benchmark for autonomous DFXM zoom.
+
+Replays a recorded 4D operando diffraction dataset strictly scan-by-scan through
+:class:`CausalStreamingTracker` and evaluates whether autonomous zoom triggers fire
+*before* actual ``1 -> 2+`` grain fracture events, whether velocity-extrapolated
+motor setpoints accurately capture the grain at ``t + 1``, and how ``BraggTrack``
+compares against naive beamline trigger baselines (``random``, ``brightest_intensity``,
+and ``strain_only``) under an identical zoom acquisition budget.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from braggtrack.tracking.precursors import PrecursorConfig
+
+from .streaming import CausalStreamingTracker, StreamingStepResult
+from .trigger import ZoomTarget, ZoomTriggerPolicy, compute_target_geometry
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.models import AxisCalibration
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.semantic.dino import BackendName
+
+
+@dataclass(frozen=True)
+class PolicyReplayMetrics:
+ """Quantitative closed-loop zoom performance for one trigger policy."""
+
+ policy_name: str
+ n_zooms_fired: int
+ n_fracture_lineages_total: int
+ n_fracture_lineages_captured_pre_split: int
+ fracture_capture_recall: float
+ fracture_capture_precision: float
+ mean_lead_scans_before_split: float
+ mean_lead_voltage_mv: float | None
+ setpoint_hit_rate_next_scan: float
+ static_centroid_hit_rate_next_scan: float
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "policy_name": self.policy_name,
+ "n_zooms_fired": int(self.n_zooms_fired),
+ "n_fracture_lineages_total": int(self.n_fracture_lineages_total),
+ "n_fracture_lineages_captured_pre_split": int(self.n_fracture_lineages_captured_pre_split),
+ "fracture_capture_recall": round(float(self.fracture_capture_recall), 4),
+ "fracture_capture_precision": round(float(self.fracture_capture_precision), 4),
+ "mean_lead_scans_before_split": round(float(self.mean_lead_scans_before_split), 3),
+ "mean_lead_voltage_mv": (
+ round(float(self.mean_lead_voltage_mv), 2) if self.mean_lead_voltage_mv is not None else None
+ ),
+ "setpoint_hit_rate_next_scan": round(float(self.setpoint_hit_rate_next_scan), 4),
+ "static_centroid_hit_rate_next_scan": round(float(self.static_centroid_hit_rate_next_scan), 4),
+ }
+
+
+@dataclass(frozen=True)
+class ReplayBenchmarkResult:
+ """Complete causal replay trajectory, zoom triggers, and baseline comparison table."""
+
+ n_scans: int
+ mean_step_latency_sec: float
+ max_step_latency_sec: float
+ steps: list[StreamingStepResult]
+ zoom_targets: list[ZoomTarget]
+ policy_comparison: dict[str, PolicyReplayMetrics]
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "n_scans": int(self.n_scans),
+ "mean_step_latency_sec": round(float(self.mean_step_latency_sec), 4),
+ "max_step_latency_sec": round(float(self.max_step_latency_sec), 4),
+ "n_zoom_targets": len(self.zoom_targets),
+ "zoom_targets": [t.to_bliss_command() for t in self.zoom_targets],
+ "policy_comparison": {k: v.to_dict() for k, v in self.policy_comparison.items()},
+ "steps": [s.to_dict() for s in self.steps],
+ }
+
+
+def _score_trigger_events(
+ policy_name: str,
+ triggers: list[tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]],
+ *,
+ split_scan_by_root: dict[int, int],
+ obs_by_scan_and_root: dict[tuple[int, int], list[tuple[float, float, float]]],
+ voltage_by_scan: dict[int, float],
+) -> PolicyReplayMetrics:
+ """Evaluate a list of ``(scan_idx, root_tid, cur_c, pred_c, half_w)`` zoom actions."""
+ n_frac_total = len(split_scan_by_root)
+ captured_roots: dict[int, int] = {}
+ pre_split_hits = 0
+
+ next_scan_eval_count = 0
+ extrap_hits = 0
+ static_hits = 0
+
+ for trig_scan, root_tid, cur_c, pred_c, half_w in triggers:
+ sp_scan = split_scan_by_root.get(root_tid)
+ if sp_scan is not None and trig_scan < sp_scan:
+ pre_split_hits += 1
+ if root_tid not in captured_roots or trig_scan < captured_roots[root_tid]:
+ captured_roots[root_tid] = trig_scan
+
+ next_obs = obs_by_scan_and_root.get((trig_scan + 1, root_tid), [])
+ if next_obs:
+ next_scan_eval_count += 1
+ hm, hc, hd = half_w
+ pm, pc, pd = pred_c
+ cm, cc, cd = cur_c
+ if any(abs(m - pm) <= hm and abs(c - pc) <= hc and abs(d - pd) <= hd for m, c, d in next_obs):
+ extrap_hits += 1
+ if any(abs(m - cm) <= hm and abs(c - cc) <= hc and abs(d - cd) <= hd for m, c, d in next_obs):
+ static_hits += 1
+
+ lead_scans = [split_scan_by_root[r] - s for r, s in captured_roots.items()]
+ lead_mv: list[float] = []
+ for r, s in captured_roots.items():
+ sp = split_scan_by_root[r]
+ if s in voltage_by_scan and sp in voltage_by_scan:
+ lead_mv.append(abs(voltage_by_scan[sp] - voltage_by_scan[s]) * 1000.0)
+
+ n_fired = len(triggers)
+ recall = (len(captured_roots) / n_frac_total) if n_frac_total > 0 else 0.0
+ precision = (pre_split_hits / n_fired) if n_fired > 0 else 0.0
+ mean_lead = float(np.mean(lead_scans)) if lead_scans else 0.0
+ mean_mv = float(np.mean(lead_mv)) if lead_mv else None
+ extrap_rate = (extrap_hits / next_scan_eval_count) if next_scan_eval_count > 0 else 0.0
+ static_rate = (static_hits / next_scan_eval_count) if next_scan_eval_count > 0 else 0.0
+
+ return PolicyReplayMetrics(
+ policy_name=policy_name,
+ n_zooms_fired=n_fired,
+ n_fracture_lineages_total=n_frac_total,
+ n_fracture_lineages_captured_pre_split=len(captured_roots),
+ fracture_capture_recall=recall,
+ fracture_capture_precision=precision,
+ mean_lead_scans_before_split=mean_lead,
+ mean_lead_voltage_mv=mean_mv,
+ setpoint_hit_rate_next_scan=extrap_rate,
+ static_centroid_hit_rate_next_scan=static_rate,
+ )
+
+
+def evaluate_causal_replay(
+ scans: Sequence[np.ndarray | str | Path],
+ *,
+ scan_names: Sequence[str] | None = None,
+ stimuli: Sequence[ScanStimulusPoint | None] | None = None,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ backend: BackendName | None = "mock",
+ precursor_config: PrecursorConfig | None = None,
+ trigger_policy: ZoomTriggerPolicy | None = None,
+ zoom_budget: int | None = None,
+ seed: int = 0,
+) -> ReplayBenchmarkResult:
+ """Run strictly causal scan-by-scan replay and benchmark against naive trigger baselines."""
+ pol = trigger_policy or ZoomTriggerPolicy()
+ tracker = CausalStreamingTracker(
+ backend=backend,
+ calibration=calibration,
+ detector=detector,
+ precursor_config=precursor_config,
+ trigger_policy=pol,
+ )
+
+ steps: list[StreamingStepResult] = []
+ # Capture per-step causal candidate pools so baseline policies face the exact same causal state & budget
+ causal_pools: list[
+ list[
+ tuple[
+ int, # root_tid
+ float, # integrated_intensity
+ float, # abs_lineage_strain
+ tuple[float, float, float], # cur_c
+ tuple[float, float, float], # pred_c
+ tuple[float, float, float], # half_w
+ ]
+ ]
+ ] = []
+
+ for idx, item in enumerate(scans):
+ s_name = scan_names[idx] if scan_names is not None and idx < len(scan_names) else None
+ stim = stimuli[idx] if stimuli is not None and idx < len(stimuli) else None
+ res = tracker.step(item, scan_name=s_name, stimulus=stim)
+ steps.append(res)
+
+ kin_by_tid = {k.track_id: k for k in tracker.kinematics}
+ pool_t: list[
+ tuple[
+ int,
+ float,
+ float,
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ ]
+ ] = []
+ for obs in tracker.track_table:
+ if int(obs["scan_idx"]) != idx:
+ continue
+ if pol.exclude_post_split_daughters and str(obs.get("lineage_role", "")) == "split_daughter":
+ continue
+ tid = int(obs["track_id"])
+ kin = kin_by_tid.get(tid)
+ if kin is None or idx not in kin.scan_indices:
+ continue
+ pos = kin.scan_indices.index(idx)
+ if (pos + 1) < pol.min_track_age:
+ continue
+ root_tid = int(kin.root_track_id) if kin.root_track_id > 0 else tid
+ inten = float(obs.get("integrated_intensity", 0.0))
+ if inten < pol.min_intensity:
+ continue
+ l_strain = abs(float(kin.lineage_strain[pos])) if len(kin.lineage_strain) > pos else 0.0
+ cur_c, pred_c, half_w, _, _ = compute_target_geometry(
+ obs,
+ kin,
+ pos,
+ lead_scans=pol.lead_scans,
+ window_sigma=pol.window_sigma,
+ calibration=tracker.calibration,
+ detector=detector,
+ )
+ pool_t.append((root_tid, inten, l_strain, cur_c, pred_c, half_w))
+ causal_pools.append(pool_t)
+
+ # Extract ground-truth lineage split scans and future positions from the completed sequence
+ final_kin_by_tid = {k.track_id: k for k in tracker.kinematics}
+ split_scan_by_root: dict[int, int] = {}
+ for ev in tracker.graph.graph.get("lineage_events", []):
+ if ev.event_type == "split":
+ for ptid in ev.parent_track_ids:
+ pk = final_kin_by_tid.get(int(ptid))
+ rtid = int(pk.root_track_id) if pk is not None and pk.root_track_id > 0 else int(ptid)
+ if rtid not in split_scan_by_root or ev.scan_idx_to < split_scan_by_root[rtid]:
+ split_scan_by_root[rtid] = int(ev.scan_idx_to)
+
+ obs_by_scan_and_root: dict[tuple[int, int], list[tuple[float, float, float]]] = {}
+ for obs in tracker.track_table:
+ s_i = int(obs["scan_idx"])
+ tid = int(obs["track_id"])
+ pk = final_kin_by_tid.get(tid)
+ rtid = int(pk.root_track_id) if pk is not None and pk.root_track_id > 0 else tid
+ obs_by_scan_and_root.setdefault((s_i, rtid), []).append(
+ (float(obs["centroid_mu"]), float(obs["centroid_chi"]), float(obs["centroid_d"]))
+ )
+
+ voltage_by_scan: dict[int, float] = {}
+ if stimuli is not None:
+ for idx, stim in enumerate(stimuli):
+ if stim is not None and stim.voltage_v is not None:
+ voltage_by_scan[idx] = float(stim.voltage_v)
+
+ bt_targets = list(tracker.all_zoom_targets)
+ if zoom_budget is not None and zoom_budget >= 0:
+ bt_targets = bt_targets[:zoom_budget]
+
+ effective_budget = len(bt_targets) if zoom_budget is None else max(0, int(zoom_budget))
+
+ bt_tuples = [
+ (t.scan_idx, t.root_track_id, t.current_centroid, t.predicted_setpoint, t.scan_half_widths) for t in bt_targets
+ ]
+
+ # Simulate naive causal baselines under the exact same zoom_budget and cooldown_scans
+ rng = np.random.default_rng(seed)
+
+ def _run_baseline(
+ mode: str,
+ ) -> list[tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]]:
+ fired: list[
+ tuple[int, int, tuple[float, float, float], tuple[float, float, float], tuple[float, float, float]]
+ ] = []
+ cd: dict[int, int] = {}
+ # Distribute budget evenly or greedily once min_track_age is met
+ for t_idx, pool in enumerate(causal_pools):
+ if len(fired) >= effective_budget or not pool:
+ continue
+ eligible = [p for p in pool if (p[0] not in cd or (t_idx - cd[p[0]]) >= pol.cooldown_scans)]
+ if not eligible:
+ continue
+ if mode == "random":
+ choice = eligible[int(rng.integers(0, len(eligible)))]
+ elif mode == "brightest_intensity":
+ choice = max(eligible, key=lambda p: p[1])
+ elif mode == "strain_only":
+ choice = max(eligible, key=lambda p: p[2])
+ else:
+ raise ValueError(f"Unknown baseline mode: {mode}")
+ rtid, _, _, cur_c, pred_c, half_w = choice
+ cd[rtid] = t_idx
+ fired.append((t_idx, rtid, cur_c, pred_c, half_w))
+ return fired
+
+ comparison: dict[str, PolicyReplayMetrics] = {
+ "braggtrack_autonomous": _score_trigger_events(
+ "braggtrack_autonomous",
+ bt_tuples,
+ split_scan_by_root=split_scan_by_root,
+ obs_by_scan_and_root=obs_by_scan_and_root,
+ voltage_by_scan=voltage_by_scan,
+ ),
+ }
+ for base_name in ("brightest_intensity", "strain_only", "random"):
+ comparison[base_name] = _score_trigger_events(
+ base_name,
+ _run_baseline(base_name),
+ split_scan_by_root=split_scan_by_root,
+ obs_by_scan_and_root=obs_by_scan_and_root,
+ voltage_by_scan=voltage_by_scan,
+ )
+
+ latencies = [s.step_latency_sec for s in steps] if steps else [0.0]
+ return ReplayBenchmarkResult(
+ n_scans=len(steps),
+ mean_step_latency_sec=float(np.mean(latencies)),
+ max_step_latency_sec=float(np.max(latencies)),
+ steps=steps,
+ zoom_targets=bt_targets,
+ policy_comparison=comparison,
+ )
diff --git a/braggtrack/autonomous/streaming.py b/braggtrack/autonomous/streaming.py
new file mode 100644
index 0000000..ef87f04
--- /dev/null
+++ b/braggtrack/autonomous/streaming.py
@@ -0,0 +1,327 @@
+"""Strictly causal online scan-by-scan tracker for live beamline streaming.
+
+Processes incoming 3D rocking-curve scans ``t = 0, 1, 2, ...`` without looking ahead:
+- Causal trailing-window median Otsu thresholding (using only scans ``<= t``).
+- 3D LoG + seeded watershed segmentation and Poisson-uncertainty feature extraction.
+- Batched multi-view ``DINOv2`` embedding and cached ROI patch-dispersion scoring.
+- Incremental Hungarian + gap-closing + ``1 -> 2+`` split/merge trajectory DAG update.
+- Autonomous DFXM "Scout-and-Zoom" target selection at the end of every scan.
+"""
+
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import networkx as nx
+import numpy as np
+
+from braggtrack.io.models import AxisCalibration
+from braggtrack.io.nexus import extract_axis_calibration, extract_scan_metadata, load_primary_volume
+from braggtrack.segmentation import (
+ extract_instance_table,
+ otsu_threshold,
+ relabel_sequential,
+ remove_small_objects,
+ segment_classical,
+)
+from braggtrack.semantic import DEFAULT_DINO_MODEL, make_multiview_encoder, make_patch_encoder
+from braggtrack.semantic.mips import crop_spot_cube, orthogonal_mips
+from braggtrack.tracking.cost import GeometrySemanticCost, PositionShapeCost
+from braggtrack.tracking.events import LineageEvent, SplitMergeConfig, events_to_table
+from braggtrack.tracking.gap_closing import GapClosingConfig
+from braggtrack.tracking.kinematics import GrainKinematics, compute_grain_kinematics
+from braggtrack.tracking.lifecycle import build_tracks, tracks_to_table
+from braggtrack.tracking.precursors import (
+ ObservationPrecursorScore,
+ PrecursorCandidate,
+ PrecursorConfig,
+ compute_fracture_precursors,
+)
+
+from .trigger import ZoomTarget, ZoomTriggerPolicy, select_zoom_targets
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.semantic.dino import BackendName
+
+
+@dataclass(frozen=True)
+class StreamingStepResult:
+ """Snapshot of online causal tracking and autonomous DFXM triggers after scan ``scan_idx``."""
+
+ scan_idx: int
+ scan_name: str
+ threshold_raw: float
+ threshold_causal: float
+ n_spots: int
+ n_active_tracks: int
+ n_lineage_events_total: int
+ new_lineage_events: list[LineageEvent]
+ precursor_candidates: list[PrecursorCandidate]
+ zoom_targets: list[ZoomTarget]
+ step_latency_sec: float
+ stimulus: ScanStimulusPoint | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "scan_idx": int(self.scan_idx),
+ "scan_name": self.scan_name,
+ "threshold_raw": round(float(self.threshold_raw), 5),
+ "threshold_causal": round(float(self.threshold_causal), 5),
+ "n_spots": int(self.n_spots),
+ "n_active_tracks": int(self.n_active_tracks),
+ "n_lineage_events_total": int(self.n_lineage_events_total),
+ "new_lineage_events": events_to_table(self.new_lineage_events),
+ "n_precursor_candidates": len(self.precursor_candidates),
+ "zoom_targets": [t.to_bliss_command() for t in self.zoom_targets],
+ "step_latency_sec": round(float(self.step_latency_sec), 4),
+ "stimulus": self.stimulus.to_dict() if self.stimulus is not None else None,
+ }
+
+
+class CausalStreamingTracker:
+ """Stateful strictly causal scan-by-scan 4D reflection tracker and zoom trigger engine."""
+
+ def __init__(
+ self,
+ *,
+ backend: BackendName | None = "auto",
+ model_name: str = DEFAULT_DINO_MODEL,
+ smooth_window: int = 5,
+ seed_peak_fraction: float = 0.08,
+ seed_response_percentile: float = 0.0,
+ min_spot_voxels: int = 4,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ cost_alpha: float = 1.0,
+ cost_beta: float = 0.25,
+ max_cost: float = float("inf"),
+ max_gap: int = 0,
+ split_max_distance: float = 25.0,
+ precursor_config: PrecursorConfig | None = None,
+ trigger_policy: ZoomTriggerPolicy | None = None,
+ batch_size: int = 32,
+ use_patch_encoder: bool = True,
+ ) -> None:
+ self.smooth_window = max(1, int(smooth_window))
+ self.seed_peak_fraction = float(seed_peak_fraction)
+ self.seed_response_percentile = float(seed_response_percentile)
+ self.min_spot_voxels = max(1, int(min_spot_voxels))
+ self.calibration = calibration
+ self.detector = detector
+ self.cost_alpha = float(cost_alpha)
+ self.cost_beta = float(cost_beta)
+ self.max_cost = float(max_cost)
+ self.batch_size = max(1, int(batch_size))
+
+ self.gap_config = GapClosingConfig(max_gap=max(0, int(max_gap))) if max_gap > 0 else None
+ self.split_merge_config = SplitMergeConfig(max_distance=float(split_max_distance))
+ self.precursor_config = precursor_config or PrecursorConfig()
+ self.trigger_policy = trigger_policy or ZoomTriggerPolicy()
+
+ self._multiview_encoder = (
+ make_multiview_encoder(backend, model_name=model_name) if self.cost_beta > 0.0 else None
+ )
+ self._patch_encoder = make_patch_encoder(backend, model_name=model_name) if use_patch_encoder else None
+ self._cost_fn = GeometrySemanticCost(
+ PositionShapeCost(),
+ cost_alpha=self.cost_alpha,
+ cost_beta=self.cost_beta,
+ )
+
+ # Strictly causal state accumulated across scans 0 .. t
+ self._raw_thresholds: list[float] = []
+ self._causal_thresholds: list[float] = []
+ self._scan_names: list[str] = []
+ self._scan_timestamps: dict[int, str | None] = {}
+ self._scan_tables: list[list[dict[str, Any]]] = []
+ self._stimuli: dict[int, ScanStimulusPoint] = {}
+ self._roi_cache: dict[tuple[int, int], tuple[int, float]] = {}
+ self._cooldown_by_lineage: dict[int, int] = {}
+
+ # Latest DAG & derived tables
+ self._graph: nx.DiGraph = nx.DiGraph()
+ self._track_table: list[dict[str, Any]] = []
+ self._kinematics: list[GrainKinematics] = []
+ self._obs_precursor_scores: list[ObservationPrecursorScore] = []
+ self._precursor_candidates: list[PrecursorCandidate] = []
+ self._all_zoom_targets: list[ZoomTarget] = []
+
+ @property
+ def n_scans_processed(self) -> int:
+ return len(self._scan_tables)
+
+ @property
+ def graph(self) -> nx.DiGraph:
+ return self._graph
+
+ @property
+ def track_table(self) -> list[dict[str, Any]]:
+ return list(self._track_table)
+
+ @property
+ def kinematics(self) -> list[GrainKinematics]:
+ return list(self._kinematics)
+
+ @property
+ def obs_precursor_scores(self) -> list[ObservationPrecursorScore]:
+ return list(self._obs_precursor_scores)
+
+ @property
+ def all_zoom_targets(self) -> list[ZoomTarget]:
+ return list(self._all_zoom_targets)
+
+ def _resolve_volume_and_meta(
+ self,
+ volume_or_path: np.ndarray | str | Path,
+ scan_name: str | None,
+ timestamp: str | None,
+ ) -> tuple[np.ndarray, str, str | None, AxisCalibration]:
+ if isinstance(volume_or_path, (str, Path)):
+ p = Path(volume_or_path)
+ if p.is_dir():
+ h5s = sorted([*p.glob("*.h5"), *p.glob("*.hdf5"), *p.glob("*.nxs")])
+ if not h5s:
+ raise FileNotFoundError(f"No HDF5/NeXus file found in scan directory {p}")
+ file_path = h5s[0]
+ default_name = p.name
+ else:
+ file_path = p
+ default_name = p.stem
+ vol = load_primary_volume(file_path)
+ meta = extract_scan_metadata(file_path)
+ ts = timestamp if timestamp is not None else meta.get("start_time")
+ cal = self.calibration if self.calibration is not None else extract_axis_calibration(file_path)
+ if self.calibration is None and cal.calibrated:
+ self.calibration = cal
+ return vol, (scan_name or default_name), ts, cal
+
+ vol = np.asarray(volume_or_path, dtype=np.float64)
+ cal = self.calibration if self.calibration is not None else AxisCalibration()
+ name = scan_name or f"scan_{len(self._scan_tables) + 1:04d}"
+ return vol, name, timestamp, cal
+
+ def step(
+ self,
+ volume_or_path: np.ndarray | str | Path,
+ *,
+ scan_name: str | None = None,
+ timestamp: str | None = None,
+ stimulus: ScanStimulusPoint | None = None,
+ ) -> StreamingStepResult:
+ """Ingest one new 3D scan ``t`` causally and emit tracking + autonomous zoom targets."""
+ t0 = time.perf_counter()
+ scan_idx = len(self._scan_tables)
+ vol, resolved_name, resolved_ts, active_cal = self._resolve_volume_and_meta(
+ volume_or_path, scan_name, timestamp
+ )
+ if stimulus is not None and resolved_ts is None and stimulus.timestamp is not None:
+ resolved_ts = stimulus.timestamp
+
+ # 1. Strictly causal trailing-window Otsu threshold (scans max(0, t - W + 1) .. t only!)
+ raw_thr = float(otsu_threshold(vol))
+ self._raw_thresholds.append(raw_thr)
+ win_lo = max(0, len(self._raw_thresholds) - self.smooth_window)
+ causal_thr = float(np.median(self._raw_thresholds[win_lo:]))
+ self._causal_thresholds.append(causal_thr)
+
+ # 2. Classical 3D LoG + watershed segmentation & calibrated feature + uncertainty extraction
+ seg = segment_classical(
+ vol,
+ threshold=causal_thr,
+ seed_peak_fraction=self.seed_peak_fraction,
+ seed_response_percentile=self.seed_response_percentile,
+ )
+ labels = relabel_sequential(remove_small_objects(seg.labeled_volume, min_size=self.min_spot_voxels))
+ spots = extract_instance_table(labels, vol, calibration=active_cal)
+
+ # 3. Batched DINOv2 multiview embeddings for all spots in scan t
+ if self._multiview_encoder is not None and spots:
+ triplets: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = []
+ for row in spots:
+ lbl = int(row["label"])
+ masked_cube, _ = crop_spot_cube(
+ vol,
+ labels,
+ lbl,
+ row,
+ margin=self.precursor_config.roi_margin,
+ )
+ triplets.append(orthogonal_mips(masked_cube))
+ embs = self._multiview_encoder.embed_batch(triplets, batch_size=self.batch_size)
+ for row, vec in zip(spots, embs, strict=True):
+ row["embedding"] = vec
+
+ self._scan_names.append(resolved_name)
+ self._scan_timestamps[scan_idx] = resolved_ts
+ self._scan_tables.append(spots)
+ if stimulus is not None:
+ self._stimuli[scan_idx] = stimulus
+
+ # 4. Build causal trajectory DAG over scans 0 .. t
+ G = build_tracks(
+ self._scan_tables,
+ cost_fn=self._cost_fn,
+ max_cost=self.max_cost,
+ gap_closing_config=self.gap_config,
+ split_merge_config=self.split_merge_config,
+ scan_timestamps=self._scan_timestamps,
+ )
+ t_table = tracks_to_table(G)
+ lineage_events: list[LineageEvent] = list(G.graph.get("lineage_events", []))
+ new_events = [ev for ev in lineage_events if ev.scan_idx_to == scan_idx]
+
+ # 5. Compute causal kinematics & incremental ROI-cached fracture precursors (only current vol in RAM!)
+ kin_list = compute_grain_kinematics(t_table)
+ prec_candidates, obs_scores = compute_fracture_precursors(
+ t_table,
+ self.precursor_config,
+ volumes={scan_idx: vol},
+ label_volumes={scan_idx: labels},
+ patch_encoder=self._patch_encoder,
+ lineage_events=lineage_events,
+ roi_cache=self._roi_cache,
+ batch_size=self.batch_size,
+ )
+
+ # 6. Select and extrapolate autonomous DFXM zoom targets at scan t
+ zoom_targets = select_zoom_targets(
+ current_scan_idx=scan_idx,
+ scan_name=resolved_name,
+ track_table=t_table,
+ kinematics=kin_list,
+ obs_precursor_scores=obs_scores,
+ policy=self.trigger_policy,
+ calibration=active_cal,
+ detector=self.detector,
+ stimulus=stimulus,
+ last_triggered_scan_by_lineage=self._cooldown_by_lineage,
+ )
+
+ self._graph = G
+ self._track_table = t_table
+ self._kinematics = kin_list
+ self._obs_precursor_scores = obs_scores
+ self._precursor_candidates = prec_candidates
+ self._all_zoom_targets.extend(zoom_targets)
+
+ n_active = sum(1 for row in t_table if int(row["scan_idx"]) == scan_idx)
+ dt = time.perf_counter() - t0
+ return StreamingStepResult(
+ scan_idx=scan_idx,
+ scan_name=resolved_name,
+ threshold_raw=raw_thr,
+ threshold_causal=causal_thr,
+ n_spots=len(spots),
+ n_active_tracks=n_active,
+ n_lineage_events_total=len(lineage_events),
+ new_lineage_events=new_events,
+ precursor_candidates=prec_candidates,
+ zoom_targets=zoom_targets,
+ step_latency_sec=dt,
+ stimulus=stimulus,
+ )
diff --git a/braggtrack/autonomous/trigger.py b/braggtrack/autonomous/trigger.py
new file mode 100644
index 0000000..b4b16f6
--- /dev/null
+++ b/braggtrack/autonomous/trigger.py
@@ -0,0 +1,396 @@
+"""Autonomous DFXM "Scout-and-Zoom" trigger policy and BLISS/Ewoks setpoint extrusion.
+
+Evaluates active tracked reflections at scan ``t`` using strictly causal kinematics,
+analytical centroid uncertainties, and multi-scan fracture precursor persistence to
+select high-priority candidate grains for zoomed Dark-Field X-ray Microscopy (DFXM)
+rocking/mosaicity acquisitions. Motor setpoints are velocity-extrapolated to
+``t + lead_scans`` so fast-shifting reflections (e.g. Ni-rich layered oxide H2 -> H3
+lattice collapse) stay centered in the narrow DFXM objective acceptance cone.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from braggtrack.io.crystallography import DetectorGeometry
+ from braggtrack.io.models import AxisCalibration
+ from braggtrack.io.operando_log import ScanStimulusPoint
+ from braggtrack.tracking.kinematics import GrainKinematics
+ from braggtrack.tracking.precursors import ObservationPrecursorScore
+
+
+@dataclass(frozen=True)
+class ZoomTriggerPolicy:
+ """Decision thresholds, cooldowns, and scan-window geometry for autonomous DFXM zoom."""
+
+ min_precursor_score: float = 0.40
+ min_persistence: int = 2
+ min_subpeaks: int = 1
+ min_strain_rate: float = 0.0
+ min_intensity: float = 20.0
+ min_track_age: int = 2
+ min_assignment_margin: float = 0.0
+ cooldown_scans: int = 3
+ max_triggers_per_scan: int = 2
+ lead_scans: float = 1.0
+ window_sigma: float = 3.5
+ exclude_post_split_daughters: bool = True
+
+
+@dataclass(frozen=True)
+class ZoomTarget:
+ """Actionable DFXM zoom acquisition target emitted after causal scan ``scan_idx``."""
+
+ scan_idx: int
+ scan_name: str
+ track_id: int
+ root_track_id: int
+ priority_rank: int
+ priority_score: float
+ precursor_score: float
+ persistence_scans: int
+ roi_subpeak_count: int
+ anisotropy: float
+ lineage_strain: float
+ strain_err: float
+ strain_rate: float
+ assignment_margin: float
+ current_centroid: tuple[float, float, float]
+ predicted_setpoint: tuple[float, float, float]
+ scan_half_widths: tuple[float, float, float]
+ centroid_err: tuple[float, float, float]
+ lead_scans: float
+ trigger_reasons: tuple[str, ...]
+ scattering_angles: dict[str, float] | None = None
+ operando_state: dict[str, Any] | None = None
+
+ def contains_point(self, mu: float, chi: float, d: float) -> bool:
+ """Return True if ``(mu, chi, d)`` lies inside ``predicted_setpoint ± scan_half_widths``."""
+ pm, pc, pd = self.predicted_setpoint
+ hm, hc, hd = self.scan_half_widths
+ return abs(mu - pm) <= hm and abs(chi - pc) <= hc and abs(d - pd) <= hd
+
+ def to_bliss_command(self) -> dict[str, Any]:
+ """Format as an ESRF BLISS / Ewoks machine-readable DFXM zoom command payload."""
+ pm, pc, pd = self.predicted_setpoint
+ hm, hc, hd = self.scan_half_widths
+ em, ec, ed = self.centroid_err
+ cmd: dict[str, Any] = {
+ "schema_version": "bliss_dfxm_zoom.v1",
+ "action": "dfxm_high_res_zoom",
+ "source_scan_idx": int(self.scan_idx),
+ "source_scan_name": self.scan_name,
+ "track_id": int(self.track_id),
+ "root_track_id": int(self.root_track_id),
+ "priority_rank": int(self.priority_rank),
+ "priority_score": round(float(self.priority_score), 4),
+ "lead_scans": round(float(self.lead_scans), 3),
+ "target_motors": {
+ "mu": round(float(pm), 6),
+ "chi": round(float(pc), 6),
+ "obpitch_or_2theta": round(float(pd), 6),
+ },
+ "scan_intervals": {
+ "mu": [round(float(pm - hm), 6), round(float(pm + hm), 6)],
+ "chi": [round(float(pc - hc), 6), round(float(pc + hc), 6)],
+ "obpitch_or_2theta": [round(float(pd - hd), 6), round(float(pd + hd), 6)],
+ },
+ "scan_half_widths": {
+ "mu": round(float(hm), 6),
+ "chi": round(float(hc), 6),
+ "obpitch_or_2theta": round(float(hd), 6),
+ },
+ "uncertainty_1sigma": {
+ "centroid_mu_err": round(float(em), 6),
+ "centroid_chi_err": round(float(ec), 6),
+ "centroid_d_err": round(float(ed), 6),
+ "strain_err": round(float(self.strain_err), 7),
+ },
+ "diagnostics": {
+ "current_centroid": [round(float(x), 6) for x in self.current_centroid],
+ "precursor_score": round(float(self.precursor_score), 4),
+ "persistence_scans": int(self.persistence_scans),
+ "roi_subpeak_count": int(self.roi_subpeak_count),
+ "anisotropy": round(float(self.anisotropy), 4),
+ "lineage_strain": round(float(self.lineage_strain), 6),
+ "strain_rate": round(float(self.strain_rate), 6),
+ "assignment_margin": (
+ round(float(self.assignment_margin), 4) if math.isfinite(self.assignment_margin) else "inf"
+ ),
+ },
+ "trigger_reasons": list(self.trigger_reasons),
+ }
+ if self.scattering_angles is not None:
+ cmd["crystallographic_setpoint"] = dict(self.scattering_angles)
+ if self.operando_state is not None:
+ cmd["operando_state"] = dict(self.operando_state)
+ return cmd
+
+ def to_dict(self) -> dict[str, Any]:
+ return self.to_bliss_command()
+
+
+def _trailing_persistence(
+ track_obs_scores: list[ObservationPrecursorScore],
+ current_scan_idx: int,
+ threshold: float,
+) -> int:
+ """Return contiguous causal run length ending at ``current_scan_idx`` with score >= threshold."""
+ relevant = sorted(
+ (s for s in track_obs_scores if s.scan_idx <= current_scan_idx),
+ key=lambda s: s.scan_idx,
+ )
+ if not relevant or relevant[-1].scan_idx != current_scan_idx:
+ return 0
+ run = 0
+ expected_scan = current_scan_idx
+ for sc in reversed(relevant):
+ if sc.scan_idx != expected_scan or sc.precursor_score < threshold:
+ break
+ run += 1
+ expected_scan -= 1
+ return run
+
+
+def compute_target_geometry(
+ obs: dict[str, Any],
+ kin: GrainKinematics | None,
+ obs_pos_in_kin: int,
+ *,
+ lead_scans: float = 1.0,
+ window_sigma: float = 3.5,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+) -> tuple[
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ tuple[float, float, float],
+ dict[str, float] | None,
+]:
+ """Compute current centroid, velocity-extrapolated setpoint, half-widths, errors, and scattering angles."""
+ cur_mu = float(obs["centroid_mu"])
+ cur_chi = float(obs["centroid_chi"])
+ cur_d = float(obs["centroid_d"])
+
+ if kin is not None and 0 <= obs_pos_in_kin < len(kin.scan_indices):
+ v_mu = float(kin.rate_mu[obs_pos_in_kin]) if len(kin.rate_mu) > obs_pos_in_kin else 0.0
+ v_chi = float(kin.rate_chi[obs_pos_in_kin]) if len(kin.rate_chi) > obs_pos_in_kin else 0.0
+ v_d = float(kin.rate_d[obs_pos_in_kin]) if len(kin.rate_d) > obs_pos_in_kin else 0.0
+ err_mu = float(kin.centroid_mu_err[obs_pos_in_kin]) if len(kin.centroid_mu_err) > obs_pos_in_kin else 0.0
+ err_chi = float(kin.centroid_chi_err[obs_pos_in_kin]) if len(kin.centroid_chi_err) > obs_pos_in_kin else 0.0
+ err_d = float(kin.centroid_d_err[obs_pos_in_kin]) if len(kin.centroid_d_err) > obs_pos_in_kin else 0.0
+ else:
+ v_mu = v_chi = v_d = 0.0
+ err_mu = float(obs.get("centroid_mu_err", 0.0))
+ err_chi = float(obs.get("centroid_chi_err", 0.0))
+ err_d = float(obs.get("centroid_d_err", 0.0))
+
+ pred_mu = cur_mu + lead_scans * v_mu
+ pred_chi = cur_chi + lead_scans * v_chi
+ pred_d = cur_d + lead_scans * v_d
+
+ step_mu = abs(float(calibration.mu_step)) if calibration is not None else 1.0
+ step_chi = abs(float(calibration.chi_step)) if calibration is not None else 1.0
+ step_d = abs(float(calibration.d_step)) if calibration is not None else 1.0
+
+ std_mu = math.sqrt(max(float(obs.get("cov_zz", obs.get("cov_mu_mu", 0.0))), 0.0))
+ std_chi = math.sqrt(max(float(obs.get("cov_xx", obs.get("cov_chi_chi", 0.0))), 0.0))
+ std_d = math.sqrt(max(float(obs.get("cov_yy", obs.get("cov_d_d", 0.0))), 0.0))
+
+ # Half-width covers intragranular spread (window_sigma * std) + centroid + drift uncertainty, floored at 1.5 steps.
+ drift_pad_mu = 0.35 * abs(lead_scans * v_mu)
+ drift_pad_chi = 0.35 * abs(lead_scans * v_chi)
+ drift_pad_d = 0.35 * abs(lead_scans * v_d)
+
+ half_mu = max(window_sigma * (std_mu + err_mu) + drift_pad_mu, 1.5 * step_mu)
+ half_chi = max(window_sigma * (std_chi + err_chi) + drift_pad_chi, 1.5 * step_chi)
+ half_d = max(window_sigma * (std_d + err_d) + drift_pad_d, 1.5 * step_d)
+
+ scattering: dict[str, float] | None = None
+ if detector is not None:
+ if calibration is not None and calibration.calibrated:
+ z_idx = (pred_mu - calibration.mu_origin) / (calibration.mu_step if calibration.mu_step != 0 else 1.0)
+ x_px = (pred_chi - calibration.chi_origin) / (calibration.chi_step if calibration.chi_step != 0 else 1.0)
+ y_px = (pred_d - calibration.d_origin) / (calibration.d_step if calibration.d_step != 0 else 1.0)
+ else:
+ z_idx, y_px, x_px = pred_mu, pred_d, pred_chi
+ scattering = detector.spot_to_scattering(z_idx, y_px, x_px)
+
+ return (
+ (cur_mu, cur_chi, cur_d),
+ (pred_mu, pred_chi, pred_d),
+ (half_mu, half_chi, half_d),
+ (err_mu, err_chi, err_d),
+ scattering,
+ )
+
+
+def select_zoom_targets(
+ *,
+ current_scan_idx: int,
+ scan_name: str,
+ track_table: list[dict[str, Any]],
+ kinematics: list[GrainKinematics],
+ obs_precursor_scores: list[ObservationPrecursorScore],
+ policy: ZoomTriggerPolicy | None = None,
+ calibration: AxisCalibration | None = None,
+ detector: DetectorGeometry | None = None,
+ stimulus: ScanStimulusPoint | None = None,
+ last_triggered_scan_by_lineage: dict[int, int] | None = None,
+) -> list[ZoomTarget]:
+ """Select, rank, and extrapolate autonomous DFXM zoom targets at ``current_scan_idx``."""
+ pol = policy or ZoomTriggerPolicy()
+ kin_by_tid = {k.track_id: k for k in kinematics}
+ scores_by_tid: dict[int, list[ObservationPrecursorScore]] = {}
+ score_at_current: dict[int, ObservationPrecursorScore] = {}
+ for sc in obs_precursor_scores:
+ scores_by_tid.setdefault(sc.track_id, []).append(sc)
+ if sc.scan_idx == current_scan_idx:
+ score_at_current[sc.track_id] = sc
+
+ cooldown_map = last_triggered_scan_by_lineage if last_triggered_scan_by_lineage is not None else {}
+ operando_dict = stimulus.to_dict() if stimulus is not None else None
+ dq_dv_boost = 0.0
+ if stimulus is not None and stimulus.dq_dv is not None and math.isfinite(stimulus.dq_dv):
+ dq_dv_boost = min(0.08, 0.08 * min(1.0, abs(float(stimulus.dq_dv)) / 250.0))
+
+ raw_candidates: list[ZoomTarget] = []
+ for obs in track_table:
+ if int(obs["scan_idx"]) != current_scan_idx:
+ continue
+ tid = int(obs["track_id"])
+ if pol.exclude_post_split_daughters and str(obs.get("lineage_role", "")) == "split_daughter":
+ continue
+
+ intensity = float(obs.get("integrated_intensity", 0.0))
+ if intensity < pol.min_intensity:
+ continue
+
+ margin = float(obs.get("assignment_margin", float("inf")))
+ if margin < pol.min_assignment_margin:
+ continue
+
+ kin = kin_by_tid.get(tid)
+ if kin is None or current_scan_idx not in kin.scan_indices:
+ continue
+ pos = kin.scan_indices.index(current_scan_idx)
+ track_age = pos + 1
+ if track_age < pol.min_track_age:
+ continue
+
+ root_tid = int(kin.root_track_id) if kin.root_track_id > 0 else tid
+ prev_trig = cooldown_map.get(root_tid)
+ if prev_trig is not None and (current_scan_idx - prev_trig) < pol.cooldown_scans:
+ continue
+
+ sc_now = score_at_current.get(tid)
+ if sc_now is None or sc_now.precursor_score < pol.min_precursor_score:
+ continue
+ if sc_now.roi_subpeak_count < pol.min_subpeaks:
+ continue
+
+ pers = _trailing_persistence(scores_by_tid.get(tid, []), current_scan_idx, pol.min_precursor_score)
+ if pers < pol.min_persistence:
+ continue
+
+ s_rate = float(kin.strain_rate[pos]) if len(kin.strain_rate) > pos else 0.0
+ if abs(s_rate) < pol.min_strain_rate:
+ continue
+
+ l_strain = float(kin.lineage_strain[pos]) if len(kin.lineage_strain) > pos else float(kin.strain[pos])
+ s_err = float(kin.lineage_strain_err[pos]) if len(kin.lineage_strain_err) > pos else 0.0
+
+ priority = (
+ 0.55 * float(sc_now.precursor_score)
+ + 0.20 * min(1.0, pers / 4.0)
+ + 0.15 * min(1.0, max(0, sc_now.roi_subpeak_count - 1) * 0.5)
+ + 0.10 * min(1.0, abs(s_rate) / 0.005)
+ + dq_dv_boost
+ )
+
+ reasons: list[str] = [
+ f"precursor_score={sc_now.precursor_score:.3f}>={pol.min_precursor_score:.2f}",
+ f"persistence={pers}_scans",
+ ]
+ if sc_now.roi_subpeak_count >= 2:
+ reasons.append(f"bimodal_subpeaks={sc_now.roi_subpeak_count}")
+ if sc_now.anisotropy >= 2.0:
+ reasons.append(f"mosaicity_anisotropy={sc_now.anisotropy:.2f}")
+ if abs(s_rate) >= 0.001:
+ reasons.append(f"strain_rate={s_rate:+.5f}/scan")
+ if dq_dv_boost > 0.02 and stimulus is not None and stimulus.voltage_v is not None:
+ reasons.append(f"redox_active_E={stimulus.voltage_v:.3f}V")
+
+ cur_c, pred_c, half_w, c_err, scat = compute_target_geometry(
+ obs,
+ kin,
+ pos,
+ lead_scans=pol.lead_scans,
+ window_sigma=pol.window_sigma,
+ calibration=calibration,
+ detector=detector,
+ )
+
+ raw_candidates.append(
+ ZoomTarget(
+ scan_idx=current_scan_idx,
+ scan_name=scan_name,
+ track_id=tid,
+ root_track_id=root_tid,
+ priority_rank=0,
+ priority_score=float(priority),
+ precursor_score=float(sc_now.precursor_score),
+ persistence_scans=pers,
+ roi_subpeak_count=int(sc_now.roi_subpeak_count),
+ anisotropy=float(sc_now.anisotropy),
+ lineage_strain=l_strain,
+ strain_err=s_err,
+ strain_rate=s_rate,
+ assignment_margin=margin,
+ current_centroid=cur_c,
+ predicted_setpoint=pred_c,
+ scan_half_widths=half_w,
+ centroid_err=c_err,
+ lead_scans=pol.lead_scans,
+ trigger_reasons=tuple(reasons),
+ scattering_angles=scat,
+ operando_state=operando_dict,
+ )
+ )
+
+ raw_candidates.sort(key=lambda t: (-t.priority_score, -t.precursor_score, t.track_id))
+ selected = raw_candidates[: max(0, int(pol.max_triggers_per_scan))]
+ ranked: list[ZoomTarget] = []
+ for rank_idx, tgt in enumerate(selected, start=1):
+ cooldown_map[tgt.root_track_id] = current_scan_idx
+ ranked.append(
+ ZoomTarget(
+ scan_idx=tgt.scan_idx,
+ scan_name=tgt.scan_name,
+ track_id=tgt.track_id,
+ root_track_id=tgt.root_track_id,
+ priority_rank=rank_idx,
+ priority_score=tgt.priority_score,
+ precursor_score=tgt.precursor_score,
+ persistence_scans=tgt.persistence_scans,
+ roi_subpeak_count=tgt.roi_subpeak_count,
+ anisotropy=tgt.anisotropy,
+ lineage_strain=tgt.lineage_strain,
+ strain_err=tgt.strain_err,
+ strain_rate=tgt.strain_rate,
+ assignment_margin=tgt.assignment_margin,
+ current_centroid=tgt.current_centroid,
+ predicted_setpoint=tgt.predicted_setpoint,
+ scan_half_widths=tgt.scan_half_widths,
+ centroid_err=tgt.centroid_err,
+ lead_scans=tgt.lead_scans,
+ trigger_reasons=tgt.trigger_reasons,
+ scattering_angles=tgt.scattering_angles,
+ operando_state=tgt.operando_state,
+ )
+ )
+ return ranked
diff --git a/braggtrack/cli/_utils.py b/braggtrack/cli/_utils.py
index 6e60042..04710db 100644
--- a/braggtrack/cli/_utils.py
+++ b/braggtrack/cli/_utils.py
@@ -55,7 +55,7 @@ def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
with path.open("w", newline="") as fh:
fh.write("")
return
- fieldnames = list(rows[0].keys())
+ fieldnames = list(dict.fromkeys(k for r in rows for k in r))
with path.open("w", newline="") as fh:
writer = csv.DictWriter(fh, fieldnames=fieldnames)
writer.writeheader()
diff --git a/braggtrack/cli/embed_dataset.py b/braggtrack/cli/embed_dataset.py
index f1baec7..b629819 100644
--- a/braggtrack/cli/embed_dataset.py
+++ b/braggtrack/cli/embed_dataset.py
@@ -40,6 +40,7 @@ def build_parser() -> argparse.ArgumentParser:
help="Embedding backend (mock needs no PyTorch; torch uses DINOv2 default or custom --model)",
)
p.add_argument("--model", default=DEFAULT_DINO_MODEL, help="HF model id when backend=torch")
+ p.add_argument("--batch-size", type=int, default=32, help="Batch size for ViT MIP forward passes")
p.add_argument(
"--allow-synthetic-fallback",
action="store_true",
@@ -107,23 +108,25 @@ def main(argv: list[str] | None = None) -> int:
return 1
label_ids: list[int] = []
- vectors: list[np.ndarray] = []
+ mips_batch: list[tuple[np.ndarray, np.ndarray, np.ndarray]] = []
for row in rows:
lid = int(row["label"])
mip_mu, mip_chi, mip_d = orthogonal_mips(
crop_spot_cube(lazy_vol, labels_full, lid, row, margin=max(0, args.margin))[0]
)
- vec = enc.embed(mip_mu, mip_chi, mip_d)
label_ids.append(lid)
- vectors.append(vec)
-
- if not vectors:
- dim = 384
+ mips_batch.append((mip_mu, mip_chi, mip_d))
+
+ if mips_batch:
+ if hasattr(enc, "embed_batch"):
+ mat = enc.embed_batch(mips_batch, batch_size=max(1, args.batch_size))
+ else:
+ mat = np.stack([enc.embed(m0, m1, m2) for m0, m1, m2 in mips_batch], axis=0)
+ dim = int(mat.shape[1])
else:
- dim = int(vectors[0].shape[0])
-
- mat = np.stack(vectors, axis=0) if vectors else np.zeros((0, dim), dtype=np.float32)
+ dim = 1152 if resolved_backend == "torch" else 384
+ mat = np.zeros((0, dim), dtype=np.float32)
scan_out = outdir / name
scan_out.mkdir(parents=True, exist_ok=True)
np.savez_compressed(
diff --git a/braggtrack/cli/report_dataset.py b/braggtrack/cli/report_dataset.py
index f6ac4fc..65d0d6f 100644
--- a/braggtrack/cli/report_dataset.py
+++ b/braggtrack/cli/report_dataset.py
@@ -12,6 +12,7 @@
generate_html_report,
plot_kinematics_dashboard,
plot_lineage_dag,
+ plot_operando_strain_and_events,
plot_precursor_timeline,
)
@@ -29,6 +30,11 @@ def build_parser() -> argparse.ArgumentParser:
default=None,
help="Optional raw HDF5 dataset root for embedding 3-axis MIP spot thumbnails",
)
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional external operando CSV/TSV/Biologic .mpt log for synchronized stimulus plots",
+ )
parser.add_argument(
"--outdir",
default=None,
@@ -76,6 +82,7 @@ def main(argv: list[str] | None = None) -> int:
in_root,
out_root / "report.html",
dataset_root=args.dataset_root,
+ operando_log=args.operando_log,
max_mip_tracks=max(0, args.max_mip_tracks),
)
@@ -122,6 +129,32 @@ def main(argv: list[str] | None = None) -> int:
)
plt.close(fig_prec)
figures["precursor_timeline"] = str((fig_dir / "precursor_timeline.png").resolve())
+
+ # Synchronized operando stimulus + lineage strain figure if log or saved JSON exists
+ stim_rows: list[dict[str, Any]] = []
+ if args.operando_log:
+ from braggtrack.io import load_operando_log, synchronize_to_scans
+
+ n_scans_est = max((int(float(r["scan_idx"])) for r in track_rows), default=0) + 1
+ stim_rows = [
+ p.to_dict() for p in synchronize_to_scans(load_operando_log(args.operando_log), n_scans_est)
+ ]
+ elif (in_root / "operando_stimulus.json").exists():
+ raw_stim = json.loads((in_root / "operando_stimulus.json").read_text())
+ if isinstance(raw_stim, dict) and isinstance(raw_stim.get("scans"), list):
+ stim_rows = raw_stim["scans"]
+
+ if stim_rows:
+ fig_op, _ = plot_operando_strain_and_events(
+ track_rows,
+ stim_rows,
+ events=event_rows,
+ precursor_observations=prec_obs_rows,
+ calibrated=calibrated,
+ save_path=fig_dir / "operando_strain_and_events.png",
+ )
+ plt.close(fig_op)
+ figures["operando_strain_and_events"] = str((fig_dir / "operando_strain_and_events.png").resolve())
except ImportError:
pass
diff --git a/braggtrack/cli/run_pipeline.py b/braggtrack/cli/run_pipeline.py
index 756f5b1..95adcaf 100644
--- a/braggtrack/cli/run_pipeline.py
+++ b/braggtrack/cli/run_pipeline.py
@@ -51,6 +51,11 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument(
"--detector-json", default=None, help="Optional DetectorGeometry JSON path for (2theta, eta, Q)"
)
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional external operando CSV/TSV/Biologic .mpt log to synchronize with diffraction scans",
+ )
parser.add_argument("--cost-alpha", type=float, default=1.0, help="Geometry weight in semantic cost")
parser.add_argument("--cost-beta", type=float, default=0.25, help="Semantic embedding weight in tracking cost")
parser.add_argument("--max-gap", type=int, default=0, help="Max missing intermediate scans to bridge (0 disables)")
@@ -114,6 +119,7 @@ def _merge_config_overrides(args: argparse.Namespace) -> dict[str, Any]:
"smooth_window": args.smooth_window,
"calibration_json": args.calibration_json,
"detector_json": args.detector_json,
+ "operando_log": args.operando_log,
"cost_alpha": args.cost_alpha,
"cost_beta": args.cost_beta,
"max_gap": args.max_gap,
@@ -170,6 +176,23 @@ def main(argv: list[str] | None = None) -> int:
print(json.dumps({"error": "Dataset validation failed", "issues": val_payload["issues"]}))
return 1
+ # Optional external operando stimulus synchronization
+ stim_points = None
+ if cfg.get("operando_log"):
+ from braggtrack.io import load_operando_log, synchronize_to_scans
+
+ op_log = load_operando_log(str(cfg["operando_log"]))
+ stim_points = synchronize_to_scans(op_log, sequence)
+ (outdir / "operando_stimulus.json").write_text(
+ json.dumps(
+ {
+ "source_path": str(Path(str(cfg["operando_log"])).resolve()),
+ "scans": [pt.to_dict() for pt in stim_points],
+ },
+ indent=2,
+ )
+ )
+
# Compute input checksums
discovered = discover_operando_scans(root)
input_files = [
@@ -299,6 +322,15 @@ def main(argv: list[str] | None = None) -> int:
if rc_evt != 0:
return rc_evt
+ if stim_points is not None:
+ from braggtrack.cli._utils import load_feature_csv, write_csv
+ from braggtrack.io import annotate_tracks_with_stimulus
+
+ for target_csv in (trk_dir / "tracks.csv", evt_dir / "tracks.csv", trk_dir / "kinematics.csv"):
+ if target_csv.exists() and target_csv.stat().st_size > 0:
+ enriched_rows = annotate_tracks_with_stimulus(load_feature_csv(target_csv), stim_points)
+ write_csv(target_csv, enriched_rows)
+
if cfg.get("export_nexus") and cfg.get("detect_events"):
from braggtrack.io import DetectorGeometry, export_nexus_process
@@ -323,6 +355,9 @@ def _build_manifest(cur_total_sec: float) -> dict[str, Any]:
"total_duration_sec": cur_total_sec,
"artifacts": {
"validation_summary": str((outdir / "validation_summary.json").resolve()),
+ "operando_stimulus": str((outdir / "operando_stimulus.json").resolve())
+ if stim_points is not None
+ else None,
"segmentation_summary": str((seg_dir / "segmentation_summary.json").resolve()),
"embedding_summary": str((emb_dir / "embedding_summary.json").resolve()) if need_embed else None,
"tracking_summary": str((trk_dir / "tracking_summary.json").resolve()),
@@ -343,7 +378,10 @@ def _build_manifest(cur_total_sec: float) -> dict[str, Any]:
# 6. Optional interactive HTML & static figure report
if cfg.get("report"):
t0 = time.perf_counter()
- rc_rep = report_dataset.main([str(outdir), "--dataset-root", str(root), "--outdir", str(rep_dir)])
+ rep_argv = [str(outdir), "--dataset-root", str(root), "--outdir", str(rep_dir)]
+ if cfg.get("operando_log"):
+ rep_argv.extend(["--operando-log", str(cfg["operando_log"])])
+ rc_rep = report_dataset.main(rep_argv)
timings["report"] = round(time.perf_counter() - t0, 4)
if rc_rep != 0:
return rc_rep
diff --git a/braggtrack/cli/watch_autonomous.py b/braggtrack/cli/watch_autonomous.py
new file mode 100644
index 0000000..605a223
--- /dev/null
+++ b/braggtrack/cli/watch_autonomous.py
@@ -0,0 +1,220 @@
+"""Autonomous DFXM "Scout-and-Zoom" beamline watcher and causal replay CLI."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import time
+from pathlib import Path
+
+from braggtrack.autonomous import (
+ CausalStreamingTracker,
+ ZoomTriggerPolicy,
+ evaluate_causal_replay,
+)
+from braggtrack.io import (
+ AxisCalibration,
+ BeamlineAdapter,
+ DetectorGeometry,
+ load_operando_log,
+ resolve_dataset_root,
+ synchronize_to_scans,
+)
+from braggtrack.semantic import DEFAULT_DINO_MODEL
+from braggtrack.tracking.precursors import PrecursorConfig
+
+
+def build_parser() -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "root",
+ nargs="?",
+ default=None,
+ help="Dataset root containing scan subdirectories (default: data/sample_operando if present, else .)",
+ )
+ parser.add_argument(
+ "--mode",
+ choices=("replay", "watch"),
+ default="replay",
+ help="'replay' benchmarks causal closed-loop triggers on an existing sequence; 'watch' polls for new scans",
+ )
+ parser.add_argument("--outdir", default="artifacts/autonomous", help="Output directory for zoom triggers & logs")
+ parser.add_argument(
+ "--backend",
+ choices=("auto", "mock", "torch"),
+ default="auto",
+ help="Semantic/patch encoder backend (auto, mock, or torch)",
+ )
+ parser.add_argument("--model", default=DEFAULT_DINO_MODEL, help="HuggingFace model ID when backend=torch")
+ parser.add_argument(
+ "--operando-log",
+ default=None,
+ help="Optional CSV/TSV/Biologic .mpt operando log to attach potential & dQ/dV to zoom triggers",
+ )
+ parser.add_argument("--calibration-json", default=None, help="Optional AxisCalibration JSON file")
+ parser.add_argument("--detector-json", default=None, help="Optional DetectorGeometry JSON file")
+ parser.add_argument("--min-precursor-score", type=float, default=0.40, help="Precursor trigger threshold")
+ parser.add_argument("--min-persistence", type=int, default=2, help="Required contiguous precursor scans")
+ parser.add_argument("--cooldown-scans", type=int, default=3, help="Per-lineage cooldown scans after firing")
+ parser.add_argument("--max-triggers-per-scan", type=int, default=2, help="Maximum zoom targets emitted per scan")
+ parser.add_argument("--lead-scans", type=float, default=1.0, help="Velocity extrapolation lead (in scan steps)")
+ parser.add_argument("--window-sigma", type=float, default=3.5, help="Scan half-width sigma multiplier")
+ parser.add_argument(
+ "--zoom-budget",
+ type=int,
+ default=None,
+ help="Optional cap on total zoom acquisitions during --mode replay baseline comparison",
+ )
+ parser.add_argument("--poll-interval", type=float, default=0.5, help="Directory poll interval (s) in --mode watch")
+ parser.add_argument(
+ "--max-scans",
+ type=int,
+ default=None,
+ help="Stop watching after ingesting N scans (useful for bounded live runs or tests)",
+ )
+ parser.add_argument(
+ "--max-idle-polls",
+ type=int,
+ default=20,
+ help="Stop --mode watch after this many consecutive polls with no new scans (0 = run indefinitely)",
+ )
+ return parser
+
+
+def _load_optional_calibration(path_str: str | None) -> AxisCalibration | None:
+ if not path_str:
+ return None
+ raw = json.loads(Path(path_str).read_text(encoding="utf-8"))
+ return AxisCalibration.from_dict(raw)
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = build_parser().parse_args(argv)
+ root = resolve_dataset_root(args.root)
+ outdir = Path(args.outdir)
+ outdir.mkdir(parents=True, exist_ok=True)
+
+ calibration = _load_optional_calibration(args.calibration_json)
+ detector = DetectorGeometry.from_json(args.detector_json) if args.detector_json else None
+ prec_cfg = PrecursorConfig(
+ score_threshold=float(args.min_precursor_score),
+ min_persistence=int(args.min_persistence),
+ )
+ policy = ZoomTriggerPolicy(
+ min_precursor_score=float(args.min_precursor_score),
+ min_persistence=int(args.min_persistence),
+ cooldown_scans=int(args.cooldown_scans),
+ max_triggers_per_scan=int(args.max_triggers_per_scan),
+ lead_scans=float(args.lead_scans),
+ window_sigma=float(args.window_sigma),
+ )
+
+ triggers_jsonl = outdir / "zoom_triggers.jsonl"
+ steps_json = outdir / "autonomous_steps.json"
+
+ if args.mode == "replay":
+ seq = BeamlineAdapter(root).build_sequence()
+ if not seq.scans:
+ raise FileNotFoundError(f"No valid scan folders discovered under {root}")
+ scan_files = [sc.file_path for sc in seq.scans]
+ scan_names = [sc.scan_name for sc in seq.scans]
+ if args.max_scans is not None:
+ scan_files = scan_files[: int(args.max_scans)]
+ scan_names = scan_names[: int(args.max_scans)]
+
+ stimuli = None
+ if args.operando_log:
+ op_log = load_operando_log(args.operando_log)
+ stimuli = synchronize_to_scans(op_log, seq)[: len(scan_files)]
+
+ bench = evaluate_causal_replay(
+ scan_files,
+ scan_names=scan_names,
+ stimuli=stimuli,
+ calibration=calibration,
+ detector=detector,
+ backend=args.backend,
+ precursor_config=prec_cfg,
+ trigger_policy=policy,
+ zoom_budget=args.zoom_budget,
+ )
+
+ with triggers_jsonl.open("w", encoding="utf-8") as fh:
+ for tgt in bench.zoom_targets:
+ fh.write(json.dumps(tgt.to_bliss_command()) + "\n")
+
+ steps_json.write_text(json.dumps([s.to_dict() for s in bench.steps], indent=2), encoding="utf-8")
+ summary_path = outdir / "replay_summary.json"
+ summary_path.write_text(json.dumps(bench.to_dict(), indent=2), encoding="utf-8")
+
+ bt_m = bench.policy_comparison["braggtrack_autonomous"]
+ print(
+ f"[braggtrack-watch:replay] Processed {bench.n_scans} scans "
+ f"(mean {bench.mean_step_latency_sec:.3f}s/scan) -> {len(bench.zoom_targets)} zoom trigger(s)"
+ )
+ print(
+ f" Fracture capture recall={bt_m.fracture_capture_recall:.2f}, "
+ f"precision={bt_m.fracture_capture_precision:.2f}, "
+ f"mean lead={bt_m.mean_lead_scans_before_split:.2f} scans, "
+ f"next-scan hit rate={bt_m.setpoint_hit_rate_next_scan:.2f}"
+ )
+ return 0
+
+ # Live directory polling mode ("watch")
+ tracker = CausalStreamingTracker(
+ backend=args.backend,
+ model_name=args.model,
+ calibration=calibration,
+ detector=detector,
+ precursor_config=prec_cfg,
+ trigger_policy=policy,
+ )
+ seen_files: set[Path] = set()
+ step_dicts: list[dict] = []
+ idle_polls = 0
+
+ triggers_jsonl.write_text("", encoding="utf-8")
+ while True:
+ seq = BeamlineAdapter(root).build_sequence()
+ stim_by_name = {}
+ if args.operando_log and seq.scans:
+ op_log = load_operando_log(args.operando_log)
+ for sp in synchronize_to_scans(op_log, seq):
+ stim_by_name[sp.scan_name] = sp
+
+ new_scans = [sc for sc in seq.scans if sc.file_path.resolve() not in seen_files]
+ if new_scans:
+ idle_polls = 0
+ for sc in new_scans:
+ seen_files.add(sc.file_path.resolve())
+ ts_iso = sc.start_time.isoformat() if sc.start_time is not None else None
+ res = tracker.step(
+ sc.file_path,
+ scan_name=sc.scan_name,
+ timestamp=ts_iso,
+ stimulus=stim_by_name.get(sc.scan_name),
+ )
+ step_dicts.append(res.to_dict())
+ if res.zoom_targets:
+ with triggers_jsonl.open("a", encoding="utf-8") as fh:
+ for tgt in res.zoom_targets:
+ fh.write(json.dumps(tgt.to_bliss_command()) + "\n")
+ fh.flush()
+ steps_json.write_text(json.dumps(step_dicts, indent=2), encoding="utf-8")
+ print(
+ f"[braggtrack-watch:live] scan={res.scan_name} spots={res.n_spots} "
+ f"zoom_triggers={len(res.zoom_targets)} ({res.step_latency_sec:.3f}s)"
+ )
+ if args.max_scans is not None and tracker.n_scans_processed >= int(args.max_scans):
+ return 0
+ else:
+ idle_polls += 1
+ if args.max_idle_polls > 0 and idle_polls >= int(args.max_idle_polls):
+ break
+ time.sleep(max(0.01, float(args.poll_interval)))
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/braggtrack/io/__init__.py b/braggtrack/io/__init__.py
index 9a37a54..3b062f3 100644
--- a/braggtrack/io/__init__.py
+++ b/braggtrack/io/__init__.py
@@ -14,6 +14,13 @@
load_primary_volume,
summarize_hdf5_tree,
)
+from .operando_log import (
+ OperandoStimulusLog,
+ ScanStimulusPoint,
+ annotate_tracks_with_stimulus,
+ load_operando_log,
+ synchronize_to_scans,
+)
from .paths import default_dataset_root, resolve_dataset_root, sample_operando_root
from .validation import ValidationIssue, validate_sequence
@@ -26,9 +33,12 @@
"IMAGED11_COLUMNS",
"LazyVolume",
"MissingH5DependencyError",
+ "OperandoStimulusLog",
+ "ScanStimulusPoint",
"ScanVolumeMeta",
"ValidationIssue",
"annotate_crystallographic_columns",
+ "annotate_tracks_with_stimulus",
"default_dataset_root",
"discover_operando_scans",
"discover_primary_volume_path",
@@ -36,6 +46,7 @@
"export_nexus_process",
"extract_axis_calibration",
"extract_scan_metadata",
+ "load_operando_log",
"load_primary_volume",
"open_lazy_volume",
"otsu_threshold_chunked",
@@ -43,5 +54,6 @@
"resolve_dataset_root",
"sample_operando_root",
"summarize_hdf5_tree",
+ "synchronize_to_scans",
"validate_sequence",
]
diff --git a/braggtrack/io/nexus.py b/braggtrack/io/nexus.py
index e85f604..a2d2e6b 100644
--- a/braggtrack/io/nexus.py
+++ b/braggtrack/io/nexus.py
@@ -15,6 +15,9 @@
"/entry1/data/data",
"/entry_0000/ESRF-ID03/pco_nf/data",
"/entry_0000/data/data",
+ "/entry/darfix/data",
+ "/entry_0000/darfix/data",
+ "/entry/process/darfix/results/data",
"/entry/instrument/detector/data",
"/entry1/instrument/detector/data",
)
@@ -66,6 +69,22 @@ def _decode_value(value: Any) -> Any:
return value
+def _decode_str_list(value: Any) -> list[str]:
+ if value is None:
+ return []
+ if isinstance(value, (bytes, str)):
+ s = _decode_value(value)
+ return [s] if isinstance(s, str) and s.strip() else []
+ if hasattr(value, "__iter__"):
+ out: list[str] = []
+ for item in value:
+ s = _decode_value(item)
+ if isinstance(s, str) and s.strip():
+ out.append(s)
+ return out
+ return []
+
+
def _candidate_entries(handle: Any, h5py: Any) -> list[str]:
seen: list[str] = []
@@ -133,12 +152,21 @@ def extract_scan_metadata(path: str | Path) -> dict[str, Any]:
return metadata
+_MOTOR_AXIS_SYNONYMS: dict[str, tuple[str, ...]] = {
+ "mu": ("mu", "omega", "obx", "rocking"),
+ "chi": ("chi", "phi", "oby"),
+ "d": ("d", "obpitch", "twotheta", "two_theta", "2theta", "tth"),
+}
+
+
def extract_axis_calibration(path: str | Path) -> AxisCalibration:
"""Extract linear ``(mu, chi, d)`` calibration from a NeXus/HDF5 file if present.
- Checks ``/entry*/calibration`` attributes/datasets or 1D coordinate arrays
- ``mu``, ``chi``, and ``d``. Returns an uncalibrated identity
- :class:`AxisCalibration` when physical axis scales are absent.
+ Probes:
+ 1. Explicit ``/entry*/calibration`` groups.
+ 2. NeXus ``NXdata`` ``@axes`` attributes on the primary 3D dataset's parent group (e.g. ``darfix``).
+ 3. 1D coordinate arrays under ``NXentry``, ``data``, ``measurement``, or ESRF BLISS
+ ``instrument/positioners/{mu, chi, obpitch, twotheta, d}``.
"""
import numpy as np
@@ -165,16 +193,50 @@ def extract_axis_calibration(path: str | Path) -> AxisCalibration:
if raw:
return AxisCalibration.from_dict(raw)
- # Check 1D axis arrays under NXentry, NXdata, or the primary 3D dataset's parent group.
+ # Check NeXus NXdata @axes attribute on parent group (e.g. darfix NXdata cubes: (z=mu, y=d/2theta, x=chi))
+ if vol_parent and vol_parent != "/" and vol_parent in handle:
+ pgrp = handle[vol_parent]
+ nx_axes = _decode_str_list(pgrp.attrs.get("axes"))
+ if len(nx_axes) == 3:
+ nx_found: dict[str, tuple[float, float]] = {}
+ # Voxel order (z, y, x) -> BraggTrack (mu, d, chi)
+ for dim_idx, target_axis in ((0, "mu"), (1, "d"), (2, "chi")):
+ ax_ds_name = nx_axes[dim_idx]
+ if ax_ds_name != "." and ax_ds_name in pgrp and isinstance(pgrp[ax_ds_name], h5py.Dataset):
+ arr = np.asarray(pgrp[ax_ds_name][()], dtype=np.float64).ravel()
+ if arr.size >= 2 and float(arr[-1] - arr[0]) != 0.0:
+ nx_found[target_axis] = (float(arr[0]), float((arr[-1] - arr[0]) / (arr.size - 1)))
+ if nx_found:
+ return AxisCalibration(
+ mu_origin=nx_found.get("mu", (0.0, 1.0))[0],
+ mu_step=nx_found.get("mu", (0.0, 1.0))[1],
+ chi_origin=nx_found.get("chi", (0.0, 1.0))[0],
+ chi_step=nx_found.get("chi", (0.0, 1.0))[1],
+ d_origin=nx_found.get("d", (0.0, 1.0))[0],
+ d_step=nx_found.get("d", (0.0, 1.0))[1],
+ calibrated=True,
+ )
+
+ for e in entries:
+ # Check 1D axis/motor arrays under NXentry, NXdata, measurement, or ESRF BLISS instrument/positioners
axes_found: dict[str, tuple[float, float]] = {}
- for axis_name in ("mu", "chi", "d"):
- search_paths = [f"{e}/{axis_name}", f"{e}/data/{axis_name}", f"{e}/measurement/{axis_name}"]
- if vol_parent and vol_parent != "/":
- search_paths.append(f"{vol_parent.rstrip('/')}/{axis_name}")
+ for axis_name, synonyms in _MOTOR_AXIS_SYNONYMS.items():
+ search_paths: list[str] = []
+ for syn in synonyms:
+ search_paths.extend(
+ [
+ f"{e}/{syn}",
+ f"{e}/data/{syn}",
+ f"{e}/measurement/{syn}",
+ f"{e}/instrument/positioners/{syn}",
+ ]
+ )
+ if vol_parent and vol_parent != "/":
+ search_paths.append(f"{vol_parent.rstrip('/')}/{syn}")
for sub in search_paths:
if sub in handle and isinstance(handle[sub], h5py.Dataset):
arr = np.asarray(handle[sub][()], dtype=np.float64).ravel()
- if arr.size >= 2:
+ if arr.size >= 2 and float(arr[-1] - arr[0]) != 0.0:
axes_found[axis_name] = (float(arr[0]), float((arr[-1] - arr[0]) / (arr.size - 1)))
break
if axes_found:
@@ -209,6 +271,19 @@ def _resolve_volume_dataset(
return "/" + key.lstrip("/")
if candidates is None and auto_discover:
+ # Check NeXus @default -> NXdata @signal chain first (used by darfix / silx / Ewoks)
+ for e in _candidate_entries(handle, h5py):
+ eg = handle[e]
+ def_data = _decode_value(eg.attrs.get("default"))
+ if isinstance(def_data, str) and def_data.strip():
+ dgrp_path = f"{e.rstrip('/')}/{def_data.strip('/')}"
+ if dgrp_path in handle and isinstance(handle[dgrp_path], h5py.Group):
+ sig = _decode_value(handle[dgrp_path].attrs.get("signal"))
+ if isinstance(sig, str) and sig in handle[dgrp_path]:
+ sig_ds = handle[dgrp_path][sig]
+ if isinstance(sig_ds, h5py.Dataset) and sig_ds.ndim == 3:
+ return f"{dgrp_path}/{sig}"
+
discovered: list[tuple[str, tuple[int, ...]]] = []
def _visit(name: str, obj: Any) -> None:
diff --git a/braggtrack/io/operando_log.py b/braggtrack/io/operando_log.py
new file mode 100644
index 0000000..4034070
--- /dev/null
+++ b/braggtrack/io/operando_log.py
@@ -0,0 +1,453 @@
+"""Operando external stimulus log ingestion and scan synchronization.
+
+Supports generic CSV/TSV potentiostat/load/thermal logs as well as Biologic
+EC-Lab ``.mpt`` ASCII exports (``Nb header lines : N``, ``time/s``, ``Ewe/V``,
+``/mA``, ``(Q-Qo)/mA.h``, ``cycle number``).
+"""
+
+from __future__ import annotations
+
+import csv
+import math
+import re
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+if TYPE_CHECKING:
+ from .models import ExperimentSequence
+
+
+_COL_ALIASES: dict[str, tuple[str, ...]] = {
+ "scan_idx": ("scan_idx", "scan", "scan_index", "frame", "frame_idx"),
+ "timestamp": ("timestamp", "iso_time", "datetime", "date_time", "utc_time", "abs_time"),
+ "uts": ("uts", "epoch", "epoch_s", "unix_time"),
+ "time_s": ("time_s", "elapsed_s", "time/s", "time", "t_s", "seconds"),
+ "voltage_v": ("voltage_v", "voltage", "ewe/v", "/v", "ecell/v", "v_cell", "potential_v"),
+ "current_ma": ("current_ma", "current", "i/ma", "/ma", "current/ma"),
+ "capacity_mah_g": (
+ "capacity_mah_g",
+ "specific_capacity",
+ "capacity",
+ "q_mah_g",
+ "(q-qo)/ma.h",
+ "q charge/ma.h",
+ "q discharge/ma.h",
+ "dq/ma.h",
+ ),
+ "dq_dv": ("dq_dv", "dq/dv", "dqdv"),
+ "cycle": ("cycle", "cycle_number", "cycle number", "half_cycle"),
+ "stress_mpa": ("stress_mpa", "stress", "load_mpa", "sigma_mpa"),
+ "temperature_c": ("temperature_c", "temperature", "temp_c", "t_degc", "temperature/°c", "temperature/c"),
+}
+
+
+@dataclass(frozen=True)
+class OperandoStimulusLog:
+ """Time-series external operando stimulus channels (electrochemistry, load, temperature)."""
+
+ time_s: np.ndarray
+ scan_idx: np.ndarray | None = None
+ epoch_s: np.ndarray | None = None
+ voltage_v: np.ndarray | None = None
+ current_ma: np.ndarray | None = None
+ capacity_mah_g: np.ndarray | None = None
+ dq_dv: np.ndarray | None = None
+ cycle: np.ndarray | None = None
+ stress_mpa: np.ndarray | None = None
+ temperature_c: np.ndarray | None = None
+ source_path: str | None = None
+
+ @property
+ def n_points(self) -> int:
+ return int(len(self.time_s))
+
+
+@dataclass(frozen=True)
+class ScanStimulusPoint:
+ """Synchronized external stimulus state evaluated at one diffraction scan."""
+
+ scan_idx: int
+ scan_name: str
+ elapsed_s: float
+ timestamp: str | None = None
+ voltage_v: float | None = None
+ current_ma: float | None = None
+ capacity_mah_g: float | None = None
+ dq_dv: float | None = None
+ cycle: int | None = None
+ stress_mpa: float | None = None
+ temperature_c: float | None = None
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "scan_idx": self.scan_idx,
+ "scan_name": self.scan_name,
+ "elapsed_s": round(float(self.elapsed_s), 4),
+ "timestamp": self.timestamp,
+ "voltage_v": round(float(self.voltage_v), 5) if self.voltage_v is not None else None,
+ "current_ma": round(float(self.current_ma), 5) if self.current_ma is not None else None,
+ "capacity_mah_g": round(float(self.capacity_mah_g), 4) if self.capacity_mah_g is not None else None,
+ "dq_dv": round(float(self.dq_dv), 4) if self.dq_dv is not None else None,
+ "cycle": int(self.cycle) if self.cycle is not None else None,
+ "stress_mpa": round(float(self.stress_mpa), 4) if self.stress_mpa is not None else None,
+ "temperature_c": round(float(self.temperature_c), 3) if self.temperature_c is not None else None,
+ }
+
+
+def _norm_col(name: str) -> str:
+ return re.sub(r"\s+", " ", name.strip().lower())
+
+
+def _match_columns(headers: list[str]) -> dict[str, str]:
+ norm_to_orig = {_norm_col(h): h for h in headers if h is not None}
+ matched: dict[str, str] = {}
+ for canonical, aliases in _COL_ALIASES.items():
+ for alias in aliases:
+ if alias in norm_to_orig:
+ matched[canonical] = norm_to_orig[alias]
+ break
+ return matched
+
+
+def _parse_iso_to_epoch(val: str) -> float | None:
+ s = val.strip()
+ if not s:
+ return None
+ if s.endswith("Z"):
+ s = s[:-1] + "+00:00"
+ try:
+ dt = datetime.fromisoformat(s)
+ except ValueError:
+ for fmt in ("%m/%d/%Y %H:%M:%S.%f", "%m/%d/%Y %H:%M:%S", "%Y/%m/%d %H:%M:%S"):
+ try:
+ dt = datetime.strptime(s, fmt)
+ break
+ except ValueError:
+ continue
+ else:
+ return None
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return float(dt.timestamp())
+
+
+def _parse_float(val: str | None) -> float:
+ if val is None:
+ return math.nan
+ s = val.strip().replace(",", ".") if "," in val and "." not in val else val.strip()
+ if not s:
+ return math.nan
+ try:
+ return float(s)
+ except ValueError:
+ return math.nan
+
+
+def _compute_regularized_dq_dv(
+ voltage_v: np.ndarray,
+ capacity: np.ndarray,
+ *,
+ min_dv: float = 1.5e-3,
+) -> np.ndarray:
+ """Compute causal backward-difference ``dQ/dV`` with mild voltage-step regularization."""
+ n = len(voltage_v)
+ out = np.zeros(n, dtype=np.float64)
+ if n < 2:
+ return out
+
+ # Smooth voltage slightly with a 3-point causal moving average to suppress potentiostat bit noise
+ v_smooth = np.empty(n, dtype=np.float64)
+ q_smooth = np.empty(n, dtype=np.float64)
+ for i in range(n):
+ lo = max(0, i - 2)
+ v_smooth[i] = float(np.nanmean(voltage_v[lo : i + 1]))
+ q_smooth[i] = float(np.nanmean(capacity[lo : i + 1]))
+
+ for i in range(1, n):
+ dv = v_smooth[i] - v_smooth[i - 1]
+ dq = abs(q_smooth[i] - q_smooth[i - 1])
+ if abs(dv) >= min_dv and math.isfinite(dv) and math.isfinite(dq):
+ out[i] = math.copysign(dq / abs(dv), dv)
+ else:
+ out[i] = out[i - 1]
+ out[0] = out[1]
+ return out
+
+
+def load_operando_log(path: str | Path) -> OperandoStimulusLog:
+ """Load an external operando CSV, TSV, or Biologic EC-Lab ``.mpt`` file."""
+ file_path = Path(path)
+ text = file_path.read_text(encoding="utf-8", errors="replace")
+ lines = text.splitlines()
+ if not lines:
+ raise ValueError(f"Operando stimulus log '{file_path}' is empty.")
+
+ # Detect Biologic EC-Lab ASCII header ("Nb header lines : N")
+ header_idx = 0
+ for line in lines[:30]:
+ m = re.match(r"^\s*Nb\s+header\s+lines\s*:\s*(\d+)", line, flags=re.IGNORECASE)
+ if m:
+ header_idx = max(0, int(m.group(1)) - 1)
+ break
+ else:
+ # Skip leading comment lines starting with '#'
+ while header_idx < len(lines) and (not lines[header_idx].strip() or lines[header_idx].lstrip().startswith("#")):
+ header_idx += 1
+
+ if header_idx >= len(lines):
+ raise ValueError(f"Could not locate table header in operando log '{file_path}'.")
+
+ body_lines = [ln for ln in lines[header_idx:] if ln.strip()]
+ header_line = body_lines[0]
+ delimiter = "\t" if "\t" in header_line else (";" if ";" in header_line and "," not in header_line else ",")
+
+ reader = csv.DictReader(body_lines, delimiter=delimiter)
+ if not reader.fieldnames:
+ raise ValueError(f"No column headers found in operando log '{file_path}'.")
+
+ col_map = _match_columns(list(reader.fieldnames))
+ raw_rows = list(reader)
+ if not raw_rows:
+ raise ValueError(f"Operando stimulus log '{file_path}' contains zero data rows.")
+
+ n = len(raw_rows)
+
+ def _extract_num(canon: str) -> np.ndarray | None:
+ col = col_map.get(canon)
+ if col is None:
+ return None
+ arr = np.array([_parse_float(r.get(col)) for r in raw_rows], dtype=np.float64)
+ if np.all(np.isnan(arr)):
+ return None
+ return arr
+
+ scan_arr = _extract_num("scan_idx")
+ time_arr = _extract_num("time_s")
+ uts_arr = _extract_num("uts")
+
+ epoch_arr: np.ndarray | None = uts_arr
+ if epoch_arr is None and "timestamp" in col_map:
+ ts_col = col_map["timestamp"]
+ parsed_ep = [_parse_iso_to_epoch(str(r.get(ts_col, ""))) for r in raw_rows]
+ if any(x is not None for x in parsed_ep):
+ epoch_arr = np.array([x if x is not None else math.nan for x in parsed_ep], dtype=np.float64)
+
+ if time_arr is None:
+ if epoch_arr is not None:
+ valid0 = float(epoch_arr[np.isfinite(epoch_arr)][0])
+ time_arr = epoch_arr - valid0
+ elif scan_arr is not None:
+ time_arr = scan_arr.astype(np.float64)
+ else:
+ time_arr = np.arange(n, dtype=np.float64)
+
+ voltage_v = _extract_num("voltage_v")
+ current_ma = _extract_num("current_ma")
+ capacity = _extract_num("capacity_mah_g")
+ dq_dv = _extract_num("dq_dv")
+ cycle = _extract_num("cycle")
+ stress_mpa = _extract_num("stress_mpa")
+ temperature_c = _extract_num("temperature_c")
+
+ # Derive Coulombic capacity (mAh) when current_mA & time_s exist without explicit capacity column
+ if capacity is None and current_ma is not None and len(time_arr) >= 2:
+ dt_h = np.diff(time_arr, prepend=time_arr[0]) / 3600.0
+ dt_h = np.where(np.isfinite(dt_h) & (dt_h > 0.0), dt_h, 0.0)
+ i_clean = np.where(np.isfinite(current_ma), np.abs(current_ma), 0.0)
+ capacity = np.cumsum(i_clean * dt_h)
+
+ if dq_dv is None and voltage_v is not None and capacity is not None:
+ dq_dv = _compute_regularized_dq_dv(voltage_v, capacity)
+
+ return OperandoStimulusLog(
+ time_s=time_arr,
+ scan_idx=scan_arr,
+ epoch_s=epoch_arr,
+ voltage_v=voltage_v,
+ current_ma=current_ma,
+ capacity_mah_g=capacity,
+ dq_dv=dq_dv,
+ cycle=cycle,
+ stress_mpa=stress_mpa,
+ temperature_c=temperature_c,
+ source_path=str(file_path),
+ )
+
+
+def _interp_optional(x_target: np.ndarray, x_src: np.ndarray, y_src: np.ndarray | None) -> np.ndarray | None:
+ if y_src is None:
+ return None
+ mask = np.isfinite(x_src) & np.isfinite(y_src)
+ if int(np.sum(mask)) == 0:
+ return None
+ xs = x_src[mask]
+ ys = y_src[mask]
+ order = np.argsort(xs, kind="mergesort")
+ xs_sorted = xs[order]
+ ys_sorted = ys[order]
+ return np.interp(x_target, xs_sorted, ys_sorted)
+
+
+def synchronize_to_scans(
+ log: OperandoStimulusLog,
+ sequence_or_count: ExperimentSequence | int,
+ *,
+ start_offset_s: float = 0.0,
+ scan_cadence_s: float | None = None,
+) -> list[ScanStimulusPoint]:
+ """Interpolate external stimulus channels onto each diffraction scan's midpoint time or index."""
+ if isinstance(sequence_or_count, int):
+ n_scans = max(0, sequence_or_count)
+ scan_names = [f"scan{i + 1:04d}" for i in range(n_scans)]
+ scan_epochs: list[float | None] = [None] * n_scans
+ scan_isos: list[str | None] = [None] * n_scans
+ else:
+ scans = sequence_or_count.scans
+ n_scans = len(scans)
+ scan_names = [s.scan_name for s in scans]
+ scan_epochs = []
+ scan_isos = []
+ for s in scans:
+ if s.start_time is not None and s.end_time is not None:
+ st = s.start_time if s.start_time.tzinfo else s.start_time.replace(tzinfo=timezone.utc)
+ et = s.end_time if s.end_time.tzinfo else s.end_time.replace(tzinfo=timezone.utc)
+ mid_ep = 0.5 * (st.timestamp() + et.timestamp())
+ scan_epochs.append(float(mid_ep))
+ scan_isos.append(st.isoformat())
+ elif s.start_time is not None:
+ st = s.start_time if s.start_time.tzinfo else s.start_time.replace(tzinfo=timezone.utc)
+ scan_epochs.append(float(st.timestamp()))
+ scan_isos.append(st.isoformat())
+ else:
+ scan_epochs.append(None)
+ scan_isos.append(None)
+
+ if n_scans == 0:
+ return []
+
+ idx_vec = np.arange(n_scans, dtype=np.float64)
+
+ # Choose synchronization coordinate: explicit scan_idx > absolute epoch > elapsed seconds
+ if log.scan_idx is not None and np.any(np.isfinite(log.scan_idx)):
+ x_src = log.scan_idx
+ x_query = idx_vec
+ elapsed_s = _interp_optional(x_query, x_src, log.time_s)
+ if elapsed_s is None:
+ elapsed_s = idx_vec * (scan_cadence_s or 60.0)
+ elif log.epoch_s is not None and all(ep is not None for ep in scan_epochs):
+ x_src = log.epoch_s
+ x_query = np.array([float(ep) for ep in scan_epochs if ep is not None], dtype=np.float64)
+ elapsed_s = x_query - x_query[0]
+ elif all(ep is not None for ep in scan_epochs):
+ ep_arr = np.array([float(ep) for ep in scan_epochs if ep is not None], dtype=np.float64)
+ elapsed_s = (ep_arr - ep_arr[0]) + start_offset_s
+ x_src = log.time_s
+ x_query = elapsed_s
+ else:
+ if scan_cadence_s is not None:
+ elapsed_s = start_offset_s + idx_vec * scan_cadence_s
+ else:
+ finite_t = log.time_s[np.isfinite(log.time_s)]
+ t_min = float(finite_t[0]) if len(finite_t) else 0.0
+ t_max = float(finite_t[-1]) if len(finite_t) else float(max(0, n_scans - 1) * 60.0)
+ elapsed_s = np.linspace(t_min, t_max, n_scans) if n_scans > 1 else np.array([t_min])
+ x_src = log.time_s
+ x_query = elapsed_s
+
+ v_interp = _interp_optional(x_query, x_src, log.voltage_v)
+ i_interp = _interp_optional(x_query, x_src, log.current_ma)
+ q_interp = _interp_optional(x_query, x_src, log.capacity_mah_g)
+ dqdv_interp = _interp_optional(x_query, x_src, log.dq_dv)
+ cyc_interp = _interp_optional(x_query, x_src, log.cycle)
+ sig_interp = _interp_optional(x_query, x_src, log.stress_mpa)
+ tmp_interp = _interp_optional(x_query, x_src, log.temperature_c)
+
+ points: list[ScanStimulusPoint] = []
+ for k in range(n_scans):
+ points.append(
+ ScanStimulusPoint(
+ scan_idx=k,
+ scan_name=scan_names[k],
+ elapsed_s=float(elapsed_s[k]),
+ timestamp=scan_isos[k],
+ voltage_v=float(v_interp[k]) if v_interp is not None else None,
+ current_ma=float(i_interp[k]) if i_interp is not None else None,
+ capacity_mah_g=float(q_interp[k]) if q_interp is not None else None,
+ dq_dv=float(dqdv_interp[k]) if dqdv_interp is not None else None,
+ cycle=int(round(float(cyc_interp[k]))) if cyc_interp is not None else None,
+ stress_mpa=float(sig_interp[k]) if sig_interp is not None else None,
+ temperature_c=float(tmp_interp[k]) if tmp_interp is not None else None,
+ )
+ )
+ return points
+
+
+def annotate_tracks_with_stimulus(
+ track_rows: list[dict[str, Any]],
+ scan_stimuli: list[ScanStimulusPoint],
+) -> list[dict[str, Any]]:
+ """Return shallow-copied track rows enriched with synchronized stimulus and ``d_strain_d_v``."""
+ if not track_rows:
+ return []
+
+ stim_by_scan = {pt.scan_idx: pt for pt in scan_stimuli}
+ enriched: list[dict[str, Any]] = []
+ prev_by_tid: dict[int, tuple[float | None, float | None]] = {}
+
+ computed_strain_by_key: dict[tuple[int, int], float] = {}
+ if "lineage_strain" not in track_rows[0] and "strain" not in track_rows[0] and "centroid_d" in track_rows[0]:
+ from braggtrack.tracking.kinematics import compute_grain_kinematics
+
+ for gk in compute_grain_kinematics(track_rows):
+ lin = gk.lineage_strain if len(gk.lineage_strain) else gk.strain
+ for idx_s, s_i in enumerate(gk.scan_indices):
+ computed_strain_by_key[(gk.track_id, s_i)] = float(lin[idx_s])
+
+ # Process in chronological order per track so d_strain_d_v is strictly causal
+ indexed = sorted(
+ enumerate(track_rows),
+ key=lambda item: (int(float(item[1]["track_id"])), int(float(item[1]["scan_idx"]))),
+ )
+ out_by_orig_idx: dict[int, dict[str, Any]] = {}
+
+ for orig_idx, row in indexed:
+ r = dict(row)
+ tid = int(float(r["track_id"]))
+ s_idx = int(float(r["scan_idx"]))
+ pt = stim_by_scan.get(s_idx)
+
+ v_val = pt.voltage_v if pt is not None else None
+ r["elapsed_s"] = round(pt.elapsed_s, 4) if pt is not None else ""
+ r["voltage_v"] = round(v_val, 5) if v_val is not None else ""
+ r["current_ma"] = round(pt.current_ma, 5) if (pt is not None and pt.current_ma is not None) else ""
+ r["capacity_mah_g"] = round(pt.capacity_mah_g, 4) if (pt is not None and pt.capacity_mah_g is not None) else ""
+ r["dq_dv"] = round(pt.dq_dv, 4) if (pt is not None and pt.dq_dv is not None) else ""
+ r["stress_mpa"] = round(pt.stress_mpa, 4) if (pt is not None and pt.stress_mpa is not None) else ""
+ r["temperature_c"] = round(pt.temperature_c, 3) if (pt is not None and pt.temperature_c is not None) else ""
+
+ strain_val: float | None = computed_strain_by_key.get((tid, s_idx))
+ if strain_val is None:
+ for key in ("lineage_strain", "strain"):
+ if key in r and r[key] != "":
+ try:
+ strain_val = float(r[key])
+ break
+ except (TypeError, ValueError):
+ pass
+
+ d_eps_dv: float | str = ""
+ if tid in prev_by_tid and v_val is not None and strain_val is not None:
+ prev_v, prev_eps = prev_by_tid[tid]
+ if prev_v is not None and prev_eps is not None and abs(v_val - prev_v) >= 2e-3:
+ d_eps_dv = round((strain_val - prev_eps) / (v_val - prev_v), 6)
+ r["d_strain_d_v"] = d_eps_dv
+ prev_by_tid[tid] = (v_val, strain_val)
+ out_by_orig_idx[orig_idx] = r
+
+ for i in range(len(track_rows)):
+ enriched.append(out_by_orig_idx[i])
+ return enriched
diff --git a/braggtrack/segmentation/dino_segment.py b/braggtrack/segmentation/dino_segment.py
index 1250b23..8cc4b01 100644
--- a/braggtrack/segmentation/dino_segment.py
+++ b/braggtrack/segmentation/dino_segment.py
@@ -41,11 +41,11 @@ def _extract_slice_features(
pixel-level ``(H, W)`` of each slice.
"""
n_slices = volume.shape[axis]
- feature_maps: list[np.ndarray] = []
- for i in range(n_slices):
- slc = np.take(volume, i, axis=axis)
- fmap = encoder.extract_patch_features(slc)
- feature_maps.append(fmap)
+ slices = [np.take(volume, i, axis=axis) for i in range(n_slices)]
+ if hasattr(encoder, "extract_patch_features_batch"):
+ feature_maps = encoder.extract_patch_features_batch(slices)
+ else:
+ feature_maps = [encoder.extract_patch_features(slc) for slc in slices]
if axis == 0:
slice_hw = (volume.shape[1], volume.shape[2])
diff --git a/braggtrack/segmentation/features.py b/braggtrack/segmentation/features.py
index 32dcc3d..47728bb 100644
--- a/braggtrack/segmentation/features.py
+++ b/braggtrack/segmentation/features.py
@@ -68,11 +68,17 @@ def extract_instance_table(
dz = (z_coords - cz) * calibration.mu_step
dy = (y_coords - cy) * calibration.d_step
dx = (x_coords - cx) * calibration.chi_step
+ step_mu = abs(calibration.mu_step)
+ step_d = abs(calibration.d_step)
+ step_chi = abs(calibration.chi_step)
else:
c_mu, c_chi, c_d = cz, cx, cy
dz = z_coords - cz
dy = y_coords - cy
dx = x_coords - cx
+ step_mu = 1.0
+ step_d = 1.0
+ step_chi = 1.0
# Weighted covariance (same weight logic)
if sum_i > 0:
@@ -87,6 +93,16 @@ def extract_instance_table(
czx = float(np.dot(w, dz * dx))
cyx = float(np.dot(w, dy * dx))
+ # Analytical 1-sigma centroid standard error: under Poisson photon statistics
+ # Var(c) = cov / sum_i; when intensities are normalized (< n_eff), fall back to
+ # Kish effective sample size n_eff = 1 / sum(w_k^2), floored by voxel quantization.
+ w_sq = float(np.dot(w, w))
+ n_eff = (1.0 / w_sq) if w_sq > 0.0 else float(max(1, count))
+ eff_denom = max(sum_i, n_eff, 1.0)
+ c_mu_err = float(np.sqrt(max(czz, (step_mu**2) / 12.0) / eff_denom))
+ c_chi_err = float(np.sqrt(max(cxx, (step_chi**2) / 12.0) / eff_denom))
+ c_d_err = float(np.sqrt(max(cyy, (step_d**2) / 12.0) / eff_denom))
+
cov = np.array(
[
[czz, czy, czx],
@@ -105,6 +121,9 @@ def extract_instance_table(
"centroid_mu": c_mu,
"centroid_chi": c_chi,
"centroid_d": c_d,
+ "centroid_mu_err": c_mu_err,
+ "centroid_chi_err": c_chi_err,
+ "centroid_d_err": c_d_err,
"bbox_min_z": int(z_coords.min()),
"bbox_max_z": int(z_coords.max()),
"bbox_min_y": int(y_coords.min()),
diff --git a/braggtrack/semantic/dino.py b/braggtrack/semantic/dino.py
index c636052..afe7601 100644
--- a/braggtrack/semantic/dino.py
+++ b/braggtrack/semantic/dino.py
@@ -12,6 +12,7 @@
import hashlib
import os
+from collections.abc import Sequence
from typing import Any, Literal, Protocol
import numpy as np
@@ -87,14 +88,31 @@ def _mips_to_rgb_uint8(mip: np.ndarray) -> np.ndarray:
class MultiviewEncoder(Protocol):
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray: ...
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray: ...
+
class MockMultiviewEncoder:
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray:
return _mock_embedding_from_mips(mip_mu, mip_chi, mip_d)
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray:
+ if not mips_list:
+ return np.zeros((0, 384), dtype=np.float32)
+ return np.stack([self.embed(m0, m1, m2) for m0, m1, m2 in mips_list], axis=0)
+
class TorchDinoMultiviewEncoder:
- """Loads DINOv2/DINOv3 once; call :meth:`embed` per spot."""
+ """Loads DINOv2/DINOv3 once; supports both single-spot :meth:`embed` and batched :meth:`embed_batch`."""
def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = None) -> None:
import torch
@@ -104,26 +122,45 @@ def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = No
self._proc, self._model = _load_hf_dino(model_name)
self._model.eval()
self._model.to(self._device)
+ self._hidden = int(getattr(self._model.config, "hidden_size", 384))
def embed(self, mip_mu: np.ndarray, mip_chi: np.ndarray, mip_d: np.ndarray) -> np.ndarray:
- vecs: list[np.ndarray] = []
+ return self.embed_batch([(mip_mu, mip_chi, mip_d)], batch_size=3)[0]
+
+ def embed_batch(
+ self,
+ mips_list: Sequence[tuple[np.ndarray, np.ndarray, np.ndarray]],
+ *,
+ batch_size: int = 32,
+ ) -> np.ndarray:
+ if not mips_list:
+ return np.zeros((0, 3 * self._hidden), dtype=np.float32)
+
+ all_rgbs: list[np.ndarray] = []
+ for mip_mu, mip_chi, mip_d in mips_list:
+ all_rgbs.append(_mips_to_rgb_uint8(mip_mu))
+ all_rgbs.append(_mips_to_rgb_uint8(mip_chi))
+ all_rgbs.append(_mips_to_rgb_uint8(mip_d))
+
+ step = max(1, int(batch_size))
+ if str(self._device) == "cpu":
+ step = min(step, 16)
+ cls_chunks: list[np.ndarray] = []
with self._torch.no_grad():
- for mip in (mip_mu, mip_chi, mip_d):
- rgb = _mips_to_rgb_uint8(mip)
- inputs = self._proc(images=[rgb], return_tensors="pt")
+ for start in range(0, len(all_rgbs), step):
+ chunk = all_rgbs[start : start + step]
+ inputs = self._proc(images=chunk, return_tensors="pt")
inputs = {k: v.to(self._device) for k, v in inputs.items()}
- out = self._model(**inputs).last_hidden_state[:, 0, :].squeeze(0)
- x = out.float().cpu().numpy()
- n = float(np.linalg.norm(x))
- if n > 0:
- x = x / n
- vecs.append(x.astype(np.float32))
+ cls_tok = self._model(**inputs).last_hidden_state[:, 0, :].float().cpu().numpy()
+ norms = np.linalg.norm(cls_tok, axis=-1, keepdims=True)
+ norms = np.where(norms > 0.0, norms, 1.0)
+ cls_chunks.append((cls_tok / norms).astype(np.float32))
- fused = np.concatenate(vecs, axis=0)
- nf = float(np.linalg.norm(fused))
- if nf > 0:
- fused = fused / nf
- return fused.astype(np.float32)
+ view_vecs = np.concatenate(cls_chunks, axis=0) # (3 * N, D)
+ fused = view_vecs.reshape(len(mips_list), 3 * view_vecs.shape[-1])
+ fnorms = np.linalg.norm(fused, axis=-1, keepdims=True)
+ fnorms = np.where(fnorms > 0.0, fnorms, 1.0)
+ return (fused / fnorms).astype(np.float32)
def _requested_backend(explicit: BackendName | None) -> BackendName:
@@ -161,6 +198,15 @@ def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
"""Return (H_patches, W_patches, D_features) feature map from a 2D grayscale image."""
...
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ """Return list of (H_patches, W_patches, D_features) maps for a batch of 2D images."""
+ ...
+
@property
def patch_size(self) -> int: ...
@@ -194,9 +240,17 @@ def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
norms = np.where(norms > 0, norms, 1.0)
return (features / norms).astype(np.float32)
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ return [self.extract_patch_features(img) for img in images_2d]
+
class TorchDinoPatchEncoder:
- """Extracts DINOv2/DINOv3 patch tokens as a spatial feature map."""
+ """Extracts DINOv2/DINOv3 patch tokens as spatial feature maps (single or batched)."""
def __init__(self, model_name: str = DEFAULT_DINO_MODEL, device: str | None = None) -> None:
import torch
@@ -218,24 +272,43 @@ def feature_dim(self) -> int:
return self._feature_dim
def extract_patch_features(self, image_2d: np.ndarray) -> np.ndarray:
- rgb = _mips_to_rgb_uint8(image_2d)
+ return self.extract_patch_features_batch([image_2d], batch_size=1)[0]
+
+ def extract_patch_features_batch(
+ self,
+ images_2d: Sequence[np.ndarray],
+ *,
+ batch_size: int = 32,
+ ) -> list[np.ndarray]:
+ if not images_2d:
+ return []
+ rgbs = [_mips_to_rgb_uint8(img) for img in images_2d]
+ step = max(1, int(batch_size))
+ if str(self._device) == "cpu":
+ step = min(step, 16)
+ results: list[np.ndarray] = []
with self._torch.no_grad():
- inputs = self._proc(images=[rgb], return_tensors="pt")
- inputs = {k: v.to(self._device) for k, v in inputs.items()}
- out = self._model(**inputs)
- h_img = int(inputs["pixel_values"].shape[2])
- w_img = int(inputs["pixel_values"].shape[3])
- h_p = h_img // self._patch_size
- w_p = w_img // self._patch_size
- expected_patches = h_p * w_p
- seq_len = int(out.last_hidden_state.shape[1])
- skip = max(0, seq_len - expected_patches)
- patch_tokens = out.last_hidden_state[:, skip : skip + expected_patches, :].squeeze(0)
- features = patch_tokens.reshape(h_p, w_p, -1)
- x = features.float().cpu().numpy()
- norms = np.linalg.norm(x, axis=-1, keepdims=True)
- norms = np.where(norms > 0, norms, 1.0)
- return (x / norms).astype(np.float32)
+ for start in range(0, len(rgbs), step):
+ chunk = rgbs[start : start + step]
+ inputs = self._proc(images=chunk, return_tensors="pt")
+ inputs = {k: v.to(self._device) for k, v in inputs.items()}
+ out = self._model(**inputs)
+ bsz = int(inputs["pixel_values"].shape[0])
+ h_img = int(inputs["pixel_values"].shape[2])
+ w_img = int(inputs["pixel_values"].shape[3])
+ h_p = h_img // self._patch_size
+ w_p = w_img // self._patch_size
+ expected_patches = h_p * w_p
+ seq_len = int(out.last_hidden_state.shape[1])
+ skip = max(0, seq_len - expected_patches)
+ patch_tokens = out.last_hidden_state[:, skip : skip + expected_patches, :]
+ fmaps = patch_tokens.reshape(bsz, h_p, w_p, -1).float().cpu().numpy()
+ norms = np.linalg.norm(fmaps, axis=-1, keepdims=True)
+ norms = np.where(norms > 0.0, norms, 1.0)
+ normed = (fmaps / norms).astype(np.float32)
+ for i in range(bsz):
+ results.append(normed[i])
+ return results
def make_patch_encoder(
diff --git a/braggtrack/tracking/__init__.py b/braggtrack/tracking/__init__.py
index 68a1f62..cd4713b 100644
--- a/braggtrack/tracking/__init__.py
+++ b/braggtrack/tracking/__init__.py
@@ -1,6 +1,6 @@
"""Multi-scan grain tracking with physics and semantic cost functions."""
-from .assignment import associate_frames
+from .assignment import associate_frames, compute_assignment_margins
from .cost import CostFunction, GeometrySemanticCost, PositionShapeCost
from .events import LineageEvent, SplitMergeConfig, events_to_table, resolve_splits_and_merges
from .gap_closing import GapCloseRecord, GapClosingConfig, resolve_track_gaps
@@ -39,6 +39,7 @@
"TrackEvent",
"associate_frames",
"build_tracks",
+ "compute_assignment_margins",
"compute_fracture_precursors",
"compute_grain_kinematics",
"compute_tracking_metrics",
diff --git a/braggtrack/tracking/assignment.py b/braggtrack/tracking/assignment.py
index a860cbb..b6af028 100644
--- a/braggtrack/tracking/assignment.py
+++ b/braggtrack/tracking/assignment.py
@@ -69,3 +69,34 @@ def associate_frames(
unmatched_t1 = [j for j in range(n_t1) if j not in matched_t1]
return matches, unmatched_t, unmatched_t1
+
+
+def compute_assignment_margins(
+ spots_t: list[dict],
+ spots_t1: list[dict],
+ matches: list[tuple[int, int]],
+ cost_fn: CostFunction,
+) -> dict[tuple[int, int], tuple[float, float]]:
+ """Compute ``(assigned_cost, margin)`` for each accepted Hungarian match.
+
+ The margin is ``min(second_best_row - c_ij, second_best_col - c_ij)``,
+ measuring how close a competing spot in frame *t* or *t+1* came to
+ claiming the link (``math.inf`` when no finite competitor exists).
+ """
+ if not matches or not spots_t or not spots_t1:
+ return {}
+
+ raw = np.asarray(cost_fn.pairwise_cost_matrix(spots_t, spots_t1), dtype=np.float64)
+ n_t, n_t1 = raw.shape
+ out: dict[tuple[int, int], tuple[float, float]] = {}
+ for r, c in matches:
+ c_ij = float(raw[r, c])
+ if not math.isfinite(c_ij):
+ out[(r, c)] = (math.inf, 0.0)
+ continue
+ row_others = [float(raw[r, k]) for k in range(n_t1) if k != c and math.isfinite(raw[r, k])]
+ col_others = [float(raw[k, c]) for k in range(n_t) if k != r and math.isfinite(raw[k, c])]
+ alt_costs = row_others + col_others
+ margin = (min(alt_costs) - c_ij) if alt_costs else math.inf
+ out[(r, c)] = (c_ij, float(margin))
+ return out
diff --git a/braggtrack/tracking/kinematics.py b/braggtrack/tracking/kinematics.py
index 8221e28..db53138 100644
--- a/braggtrack/tracking/kinematics.py
+++ b/braggtrack/tracking/kinematics.py
@@ -48,6 +48,23 @@ class GrainKinematics:
# Raw shift in d-axis (always meaningful even when uncalibrated)
delta_d: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ # Analytical 1-sigma uncertainties (propagated from centroid_*_err)
+ centroid_mu_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ centroid_chi_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ centroid_d_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ strain_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
+ # Ancestor-referenced lineage strain across 1->2+ splits and 2+->1 merges
+ root_track_id: int = 0
+ lineage_strain: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ lineage_strain_err: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
+ # Strictly causal backward finite-difference rates (per scan step)
+ rate_mu: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ rate_chi: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ rate_d: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+ strain_rate: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.float64))
+
@dataclass
class KinematicsSummary:
@@ -66,9 +83,54 @@ class KinematicsSummary:
max_anisotropy: list[float]
calibrated_axes: bool = False
+ max_lineage_strain: list[float] = field(default_factory=list)
grain_kinematics: list[GrainKinematics] = field(default_factory=list, repr=False)
+def _parse_parent_ids(raw: Any) -> list[int]:
+ if raw is None or raw == "":
+ return []
+ if isinstance(raw, (list, tuple)):
+ out: list[int] = []
+ for item in raw:
+ try:
+ out.append(int(item))
+ except (TypeError, ValueError):
+ continue
+ return out
+ if isinstance(raw, (int, np.integer)):
+ return [int(raw)]
+ if isinstance(raw, str):
+ parts = [p.strip() for p in raw.split(";") if p.strip()]
+ parsed: list[int] = []
+ for p in parts:
+ try:
+ parsed.append(int(p))
+ except ValueError:
+ continue
+ return parsed
+ return []
+
+
+def _propagate_strain_err(
+ d: np.ndarray,
+ d_err: np.ndarray,
+ d_ref: float,
+ d_ref_err: float,
+ *,
+ zero_first: bool,
+) -> np.ndarray:
+ """Propagate 1-sigma uncertainty on ``(d - d_ref) / d_ref``."""
+ n = len(d)
+ if n == 0 or d_ref == 0.0:
+ return np.zeros(n, dtype=np.float64)
+ ratio = d / d_ref
+ err = np.sqrt(d_err**2 + (ratio * d_ref_err) ** 2) / abs(d_ref)
+ if zero_first and n > 0:
+ err[0] = 0.0
+ return err
+
+
def compute_grain_kinematics(
track_table: list[dict[str, Any]],
) -> list[GrainKinematics]:
@@ -80,46 +142,120 @@ def compute_grain_kinematics(
Output of :func:`~braggtrack.tracking.lifecycle.tracks_to_table`:
list of dicts with ``track_id``, ``scan_idx``, ``centroid_mu``,
``centroid_chi``, ``centroid_d``, ``eig_1``, ``eig_2``, ``eig_3``,
- ``integrated_intensity``, ``voxel_count``.
+ ``integrated_intensity``, ``voxel_count``, plus optional
+ ``centroid_*_err`` and ``parent_track_ids``.
Returns
-------
list[GrainKinematics]
One entry per track, sorted by track_id.
"""
- tracks: dict[int, list[dict]] = {}
+ tracks: dict[int, list[dict[str, Any]]] = {}
+ parent_map: dict[int, list[int]] = {}
for row in track_table:
- tracks.setdefault(int(row["track_id"]), []).append(row)
+ tid = int(row["track_id"])
+ tracks.setdefault(tid, []).append(row)
+ pids = _parse_parent_ids(row.get("parent_track_ids"))
+ if pids and tid not in parent_map:
+ parent_map[tid] = pids
+
+ sorted_tracks: dict[int, list[dict[str, Any]]] = {
+ tid: sorted(rows, key=lambda r: int(r["scan_idx"])) for tid, rows in tracks.items()
+ }
+
+ def _resolve_root_tid(start_tid: int) -> int:
+ curr = start_tid
+ visited: set[int] = set()
+ while curr not in visited:
+ visited.add(curr)
+ parents = [p for p in parent_map.get(curr, []) if p in sorted_tracks and p != curr]
+ if not parents:
+ break
+ # Prefer earliest-onset parent (tie-break by highest initial intensity, then lowest ID)
+ parents.sort(
+ key=lambda p: (
+ int(sorted_tracks[p][0]["scan_idx"]),
+ -float(sorted_tracks[p][0].get("integrated_intensity", 0.0)),
+ p,
+ )
+ )
+ curr = parents[0]
+ return curr
results: list[GrainKinematics] = []
- for tid in sorted(tracks):
- obs = sorted(tracks[tid], key=lambda r: r["scan_idx"])
+ for tid in sorted(sorted_tracks):
+ obs = sorted_tracks[tid]
n = len(obs)
scan_indices = [int(r["scan_idx"]) for r in obs]
- mu = np.array([float(r["centroid_mu"]) for r in obs])
- chi = np.array([float(r["centroid_chi"]) for r in obs])
- d = np.array([float(r["centroid_d"]) for r in obs])
+ mu = np.array([float(r["centroid_mu"]) for r in obs], dtype=np.float64)
+ chi = np.array([float(r["centroid_chi"]) for r in obs], dtype=np.float64)
+ d = np.array([float(r["centroid_d"]) for r in obs], dtype=np.float64)
- # Strain: (d - d₀) / d₀ and raw shift Δd = d - d₀
- d0 = d[0]
+ mu_err = np.array([float(r.get("centroid_mu_err", 0.0)) for r in obs], dtype=np.float64)
+ chi_err = np.array([float(r.get("centroid_chi_err", 0.0)) for r in obs], dtype=np.float64)
+ d_err = np.array([float(r.get("centroid_d_err", 0.0)) for r in obs], dtype=np.float64)
+
+ # Segment-local strain: (d - d₀) / d₀ and raw shift Δd = d - d₀
+ d0 = float(d[0])
+ d0_err = float(d_err[0])
delta_d = d - d0
- strain = delta_d / d0 if d0 != 0 else np.zeros(n)
+ strain = delta_d / d0 if d0 != 0.0 else np.zeros(n, dtype=np.float64)
+ strain_err = _propagate_strain_err(d, d_err, d0, d0_err, zero_first=True)
+
+ # Ancestor-referenced lineage strain across splits/merges
+ root_tid = _resolve_root_tid(tid)
+ root_obs0 = sorted_tracks[root_tid][0]
+ d_root0 = float(root_obs0["centroid_d"])
+ d_root0_err = float(root_obs0.get("centroid_d_err", 0.0))
+ lineage_strain = (d - d_root0) / d_root0 if d_root0 != 0.0 else np.zeros(n, dtype=np.float64)
+ lineage_strain_err = _propagate_strain_err(
+ d,
+ d_err,
+ d_root0,
+ d_root0_err,
+ zero_first=(root_tid == tid),
+ )
+
+ # Strictly causal backward finite-difference rates (per scan step)
+ rate_mu = np.zeros(n, dtype=np.float64)
+ rate_chi = np.zeros(n, dtype=np.float64)
+ rate_d = np.zeros(n, dtype=np.float64)
+ strain_rate = np.zeros(n, dtype=np.float64)
+
+ # Seed initial rate at i=0 from immediate parent's last observation if split/merge child
+ imm_parents = [p for p in parent_map.get(tid, []) if p in sorted_tracks]
+ if imm_parents and n > 0:
+ p_last = sorted_tracks[imm_parents[0]][-1]
+ dt0 = max(1, scan_indices[0] - int(p_last["scan_idx"]))
+ rate_mu[0] = (mu[0] - float(p_last["centroid_mu"])) / dt0
+ rate_chi[0] = (chi[0] - float(p_last["centroid_chi"])) / dt0
+ rate_d[0] = (d[0] - float(p_last["centroid_d"])) / dt0
+ if d_root0 != 0.0:
+ p_lin_strain = (float(p_last["centroid_d"]) - d_root0) / d_root0
+ strain_rate[0] = (lineage_strain[0] - p_lin_strain) / dt0
+
+ for i in range(1, n):
+ dt = max(1, scan_indices[i] - scan_indices[i - 1])
+ rate_mu[i] = (mu[i] - mu[i - 1]) / dt
+ rate_chi[i] = (chi[i] - chi[i - 1]) / dt
+ rate_d[i] = (d[i] - d[i - 1]) / dt
+ strain_rate[i] = (lineage_strain[i] - lineage_strain[i - 1]) / dt
# Misorientation: drift from initial position
dmu = mu - mu[0]
dchi = chi - chi[0]
misorientation_total = np.sqrt(dmu**2 + dchi**2)
- intensity = np.array([float(r["integrated_intensity"]) for r in obs])
- voxels = np.array([float(r["voxel_count"]) for r in obs])
+ intensity = np.array([float(r["integrated_intensity"]) for r in obs], dtype=np.float64)
+ voxels = np.array([float(r["voxel_count"]) for r in obs], dtype=np.float64)
i0 = intensity[0]
v0 = voxels[0]
- rel_intensity = (intensity - i0) / i0 if i0 != 0 else np.zeros(n)
- rel_volume = (voxels - v0) / v0 if v0 != 0 else np.zeros(n)
+ rel_intensity = (intensity - i0) / i0 if i0 != 0 else np.zeros(n, dtype=np.float64)
+ rel_volume = (voxels - v0) / v0 if v0 != 0 else np.zeros(n, dtype=np.float64)
- eigs = np.array([[float(r["eig_1"]), float(r["eig_2"]), float(r["eig_3"])] for r in obs])
+ eigs = np.array([[float(r["eig_1"]), float(r["eig_2"]), float(r["eig_3"])] for r in obs], dtype=np.float64)
e3 = eigs[:, 2]
anisotropy = np.where(e3 > 0, eigs[:, 0] / e3, 1.0)
cov_trace = eigs.sum(axis=1)
@@ -143,6 +279,17 @@ def compute_grain_kinematics(
anisotropy=anisotropy,
covariance_trace=cov_trace,
delta_d=delta_d,
+ centroid_mu_err=mu_err,
+ centroid_chi_err=chi_err,
+ centroid_d_err=d_err,
+ strain_err=strain_err,
+ root_track_id=root_tid,
+ lineage_strain=lineage_strain,
+ lineage_strain_err=lineage_strain_err,
+ rate_mu=rate_mu,
+ rate_chi=rate_chi,
+ rate_d=rate_d,
+ strain_rate=strain_rate,
)
)
return results
@@ -154,19 +301,10 @@ def summarize_kinematics(
*,
calibrated_axes: bool = False,
) -> KinematicsSummary:
- """Aggregate per-grain kinematics into a summary table.
-
- Parameters
- ----------
- grain_kinematics
- Output of :func:`compute_grain_kinematics`.
- n_scans
- Total number of scans in the sequence.
- calibrated_axes
- Whether centroids/eigenvalues were converted to physical ``(mu, chi, d)`` units.
- """
+ """Aggregate per-grain kinematics into a summary table."""
track_ids: list[int] = []
max_strain: list[float] = []
+ max_lineage_strain: list[float] = []
total_misorientation: list[float] = []
intensity_change: list[float] = []
volume_change: list[float] = []
@@ -175,6 +313,8 @@ def summarize_kinematics(
for gk in grain_kinematics:
track_ids.append(gk.track_id)
max_strain.append(float(np.max(np.abs(gk.strain))) if len(gk.strain) > 0 else 0.0)
+ lin_s = gk.lineage_strain if len(gk.lineage_strain) > 0 else gk.strain
+ max_lineage_strain.append(float(np.max(np.abs(lin_s))) if len(lin_s) > 0 else 0.0)
total_misorientation.append(float(gk.misorientation_total[-1]) if len(gk.misorientation_total) > 0 else 0.0)
intensity_change.append(float(gk.relative_intensity[-1]) if len(gk.relative_intensity) > 0 else 0.0)
volume_change.append(float(gk.relative_volume[-1]) if len(gk.relative_volume) > 0 else 0.0)
@@ -193,6 +333,7 @@ def summarize_kinematics(
volume_change_frac=volume_change,
max_anisotropy=max_anisotropy,
calibrated_axes=calibrated_axes,
+ max_lineage_strain=max_lineage_strain,
grain_kinematics=grain_kinematics,
)
@@ -206,6 +347,7 @@ def summary_to_dict(summary: KinematicsSummary) -> dict[str, Any]:
"calibrated_axes": summary.calibrated_axes,
"track_ids": summary.track_ids,
"max_strain": summary.max_strain,
+ "max_lineage_strain": summary.max_lineage_strain,
"total_misorientation": summary.total_misorientation,
"intensity_change_frac": summary.intensity_change_frac,
"volume_change_frac": summary.volume_change_frac,
@@ -218,22 +360,29 @@ def kinematics_to_table(
*,
include_delta_d: bool = False,
) -> list[dict[str, Any]]:
- """Flatten per-grain kinematics into a row-per-observation table.
-
- Each row contains the track_id, scan_idx, and all computed kinematic
- quantities for that observation. Suitable for CSV export or DataFrame
- construction.
- """
+ """Flatten per-grain kinematics into a row-per-observation table."""
rows: list[dict[str, Any]] = []
for gk in grain_kinematics:
+ root_tid = gk.root_track_id if gk.root_track_id > 0 else gk.track_id
for i, scan_idx in enumerate(gk.scan_indices):
row: dict[str, Any] = {
"track_id": gk.track_id,
+ "root_track_id": root_tid,
"scan_idx": scan_idx,
"centroid_mu": float(gk.centroid_mu[i]),
"centroid_chi": float(gk.centroid_chi[i]),
"centroid_d": float(gk.centroid_d[i]),
+ "centroid_mu_err": float(gk.centroid_mu_err[i]) if len(gk.centroid_mu_err) > i else 0.0,
+ "centroid_chi_err": float(gk.centroid_chi_err[i]) if len(gk.centroid_chi_err) > i else 0.0,
+ "centroid_d_err": float(gk.centroid_d_err[i]) if len(gk.centroid_d_err) > i else 0.0,
"strain": float(gk.strain[i]),
+ "strain_err": float(gk.strain_err[i]) if len(gk.strain_err) > i else 0.0,
+ "lineage_strain": float(gk.lineage_strain[i]) if len(gk.lineage_strain) > i else float(gk.strain[i]),
+ "lineage_strain_err": (float(gk.lineage_strain_err[i]) if len(gk.lineage_strain_err) > i else 0.0),
+ "strain_rate": float(gk.strain_rate[i]) if len(gk.strain_rate) > i else 0.0,
+ "rate_mu": float(gk.rate_mu[i]) if len(gk.rate_mu) > i else 0.0,
+ "rate_chi": float(gk.rate_chi[i]) if len(gk.rate_chi) > i else 0.0,
+ "rate_d": float(gk.rate_d[i]) if len(gk.rate_d) > i else 0.0,
"misorientation_mu": float(gk.misorientation_mu[i]),
"misorientation_chi": float(gk.misorientation_chi[i]),
"misorientation_total": float(gk.misorientation_total[i]),
diff --git a/braggtrack/tracking/lifecycle.py b/braggtrack/tracking/lifecycle.py
index 54b8c8c..9feb96c 100644
--- a/braggtrack/tracking/lifecycle.py
+++ b/braggtrack/tracking/lifecycle.py
@@ -7,7 +7,7 @@
import networkx as nx
-from .assignment import associate_frames
+from .assignment import associate_frames, compute_assignment_margins
from .cost import CostFunction, PositionShapeCost
if TYPE_CHECKING:
@@ -56,7 +56,15 @@ def build_tracks(
track_map: dict[int, int] = {} # spot_idx -> track_id for current frame
for idx, spot in enumerate(scan_tables[0]):
nid = _node_id(0, idx)
- G.add_node(nid, scan_idx=0, spot_idx=idx, track_id=next_track_id, event=TrackEvent.BORN, **spot)
+ G.add_node(
+ nid,
+ scan_idx=0,
+ spot_idx=idx,
+ track_id=next_track_id,
+ event=TrackEvent.BORN,
+ assignment_margin=float("inf"),
+ **spot,
+ )
track_map[idx] = next_track_id
next_track_id += 1
@@ -71,6 +79,7 @@ def build_tracks(
cost_fn=active_cost,
max_cost=max_cost,
)
+ margins = compute_assignment_margins(spots_t, spots_t1, matches, active_cost)
new_track_map: dict[int, int] = {}
@@ -79,10 +88,23 @@ def build_tracks(
tid = track_map[i_t]
nid_prev = _node_id(t, i_t)
nid_next = _node_id(t + 1, i_t1)
+ c_ij, margin = margins.get((i_t, i_t1), (0.0, float("inf")))
G.add_node(
- nid_next, scan_idx=t + 1, spot_idx=i_t1, track_id=tid, event=TrackEvent.CONTINUED, **spots_t1[i_t1]
+ nid_next,
+ scan_idx=t + 1,
+ spot_idx=i_t1,
+ track_id=tid,
+ event=TrackEvent.CONTINUED,
+ assignment_margin=margin,
+ **spots_t1[i_t1],
+ )
+ G.add_edge(
+ nid_prev,
+ nid_next,
+ edge_type="continuation",
+ cost=c_ij,
+ assignment_margin=margin,
)
- G.add_edge(nid_prev, nid_next, edge_type="continuation")
new_track_map[i_t1] = tid
# Unmatched in t → TERMINATED (mark on existing node)
@@ -94,7 +116,13 @@ def build_tracks(
for i_t1 in unmatched_t1:
nid = _node_id(t + 1, i_t1)
G.add_node(
- nid, scan_idx=t + 1, spot_idx=i_t1, track_id=next_track_id, event=TrackEvent.BORN, **spots_t1[i_t1]
+ nid,
+ scan_idx=t + 1,
+ spot_idx=i_t1,
+ track_id=next_track_id,
+ event=TrackEvent.BORN,
+ assignment_margin=float("inf"),
+ **spots_t1[i_t1],
)
new_track_map[i_t1] = next_track_id
next_track_id += 1
diff --git a/braggtrack/tracking/precursors.py b/braggtrack/tracking/precursors.py
index 053903a..980e885 100644
--- a/braggtrack/tracking/precursors.py
+++ b/braggtrack/tracking/precursors.py
@@ -103,17 +103,38 @@ def _count_internal_subpeaks(masked_cube: np.ndarray, mask: np.ndarray) -> tuple
return n_peaks, subpeak_score
-def _patch_dispersion_score(
+def _dispersion_from_fmap(fmap: np.ndarray, mm_up: np.ndarray) -> float | None:
+ """Compute foreground patch-token cosine dispersion for one upsampled MIP feature map."""
+ hp, wp, dim = fmap.shape
+ if hp * wp < 2:
+ return None
+
+ bh = max(1, mm_up.shape[0] // hp)
+ bw = max(1, mm_up.shape[1] // wp)
+ cropped_fg = (mm_up[: hp * bh, : wp * bw] > 0).reshape(hp, bh, wp, bw).max(axis=(1, 3))
+ fg_vecs = fmap[cropped_fg] if int(cropped_fg.sum()) >= 2 else fmap.reshape(-1, dim)
+ if fg_vecs.shape[0] < 2:
+ return None
+
+ mean_vec = fg_vecs.mean(axis=0)
+ nrm = float(np.linalg.norm(mean_vec))
+ if nrm <= 0.0:
+ return None
+ mean_vec = mean_vec / nrm
+ cos_sims = np.clip(fg_vecs @ mean_vec, -1.0, 1.0)
+ cos_disp = float(1.0 - np.mean(cos_sims))
+ return float(np.clip(cos_disp / 0.6, 0.0, 1.0))
+
+
+def _prepare_upsampled_roi_mips(
masked_cube: np.ndarray,
mask: np.ndarray,
- patch_encoder: PatchFeatureEncoder,
upsample_size: int,
-) -> float:
- """Compute mean foreground patch-token cosine dispersion across orthogonal MIPs."""
+) -> list[tuple[np.ndarray, np.ndarray]]:
+ """Return upsampled ``(mip_up, mask_up)`` pairs for non-empty orthogonal views."""
mip_mu, mip_chi, mip_d = orthogonal_mips(masked_cube)
mask_mu, mask_chi, mask_d = orthogonal_mips(mask.astype(np.float64))
-
- dispersions: list[float] = []
+ pairs: list[tuple[np.ndarray, np.ndarray]] = []
for mip, mm in ((mip_mu, mask_mu), (mip_chi, mask_chi), (mip_d, mask_d)):
if mip.size == 0 or float(mm.max()) <= 0.0:
continue
@@ -122,29 +143,28 @@ def _patch_dispersion_score(
zw = max(1.0, upsample_size / max(1, w))
mip_up = zoom(mip.astype(np.float64), (zh, zw), order=1) if (zh > 1.0 or zw > 1.0) else mip.astype(np.float64)
mm_up = zoom(mm.astype(np.float64), (zh, zw), order=0) if (zh > 1.0 or zw > 1.0) else mm.astype(np.float64)
+ pairs.append((mip_up, mm_up))
+ return pairs
- fmap = patch_encoder.extract_patch_features(mip_up)
- hp, wp, dim = fmap.shape
- if hp * wp < 2:
- continue
-
- # Map foreground mask onto (hp, wp) patch grid.
- bh = max(1, mm_up.shape[0] // hp)
- bw = max(1, mm_up.shape[1] // wp)
- cropped_fg = (mm_up[: hp * bh, : wp * bw] > 0).reshape(hp, bh, wp, bw).max(axis=(1, 3))
- fg_vecs = fmap[cropped_fg] if int(cropped_fg.sum()) >= 2 else fmap.reshape(-1, dim)
- if fg_vecs.shape[0] < 2:
- continue
-
- mean_vec = fg_vecs.mean(axis=0)
- nrm = float(np.linalg.norm(mean_vec))
- if nrm <= 0.0:
- continue
- mean_vec = mean_vec / nrm
- cos_sims = np.clip(fg_vecs @ mean_vec, -1.0, 1.0)
- cos_disp = float(1.0 - np.mean(cos_sims))
- dispersions.append(float(np.clip(cos_disp / 0.6, 0.0, 1.0)))
+def _patch_dispersion_score(
+ masked_cube: np.ndarray,
+ mask: np.ndarray,
+ patch_encoder: PatchFeatureEncoder,
+ upsample_size: int,
+) -> float:
+ """Compute mean foreground patch-token cosine dispersion across orthogonal MIPs."""
+ pairs = _prepare_upsampled_roi_mips(masked_cube, mask, upsample_size)
+ if not pairs:
+ return 0.0
+ mips = [p[0] for p in pairs]
+ batch_fn = getattr(patch_encoder, "extract_patch_features_batch", None)
+ fmaps = batch_fn(mips) if callable(batch_fn) else [patch_encoder.extract_patch_features(m) for m in mips]
+ dispersions: list[float] = []
+ for fmap, (_, mm_up) in zip(fmaps, pairs, strict=True):
+ val = _dispersion_from_fmap(fmap, mm_up)
+ if val is not None:
+ dispersions.append(val)
return float(np.mean(dispersions)) if dispersions else 0.0
@@ -156,6 +176,8 @@ def compute_fracture_precursors(
label_volumes: dict[int, np.ndarray] | list[np.ndarray] | None = None,
patch_encoder: PatchFeatureEncoder | None = None,
lineage_events: list[LineageEvent] | None = None,
+ roi_cache: dict[tuple[int, int], tuple[int, float]] | None = None,
+ batch_size: int = 32,
) -> tuple[list[PrecursorCandidate], list[ObservationPrecursorScore]]:
"""Score per-grain fracture precursors and return persistent ranked candidates.
@@ -173,6 +195,11 @@ def compute_fracture_precursors(
lineage_events
Optional list of :class:`LineageEvent` records used to annotate whether a
precursor candidate subsequently fractured (``subsequent_split``).
+ roi_cache
+ Optional mutable ``(scan_idx, label) -> (n_subpeaks, roi_term)`` cache
+ for incremental online streaming so historical scans are not re-cropped.
+ batch_size
+ Mini-batch size when running ``patch_encoder.extract_patch_features_batch``.
"""
cfg = config or PrecursorConfig()
if not track_table:
@@ -191,6 +218,65 @@ def compute_fracture_precursors(
lab_map = dict(enumerate(label_volumes))
has_roi = bool(vol_map and lab_map)
+ effective_cache: dict[tuple[int, int], tuple[int, float]] = roi_cache if roi_cache is not None else {}
+
+ # Pre-compute uncached ROI sub-peaks and batch all DINO patch-feature extractions together.
+ if has_roi:
+ pending_keys: list[tuple[int, int]] = []
+ pending_subpeaks: list[tuple[int, float]] = []
+ pending_view_slices: list[tuple[int, int]] = []
+ all_mips: list[np.ndarray] = []
+ all_masks: list[np.ndarray] = []
+
+ for row in track_table:
+ s_idx = int(row["scan_idx"])
+ lbl_i = int(row.get("label", 0))
+ key = (s_idx, lbl_i)
+ if lbl_i <= 0 or "bbox_min_z" not in row or key in effective_cache:
+ continue
+ if s_idx not in vol_map or s_idx not in lab_map:
+ continue
+ # Mark placeholder immediately in case duplicate rows exist
+ effective_cache[key] = (1, 0.0)
+ masked_cube, mask = crop_spot_cube(
+ vol_map[s_idx],
+ lab_map[s_idx],
+ lbl_i,
+ row,
+ margin=cfg.roi_margin,
+ )
+ n_sub, sub_term = _count_internal_subpeaks(masked_cube, mask)
+ if patch_encoder is None:
+ effective_cache[key] = (n_sub, sub_term)
+ else:
+ pairs = _prepare_upsampled_roi_mips(masked_cube, mask, cfg.roi_upsample_size)
+ v_start = len(all_mips)
+ for mip_up, mm_up in pairs:
+ all_mips.append(mip_up)
+ all_masks.append(mm_up)
+ v_end = len(all_mips)
+ pending_keys.append(key)
+ pending_subpeaks.append((n_sub, sub_term))
+ pending_view_slices.append((v_start, v_end))
+
+ if patch_encoder is not None and pending_keys:
+ batch_fn = getattr(patch_encoder, "extract_patch_features_batch", None)
+ if callable(batch_fn):
+ all_fmaps = batch_fn(all_mips, batch_size=batch_size) if all_mips else []
+ else:
+ all_fmaps = [patch_encoder.extract_patch_features(m) for m in all_mips]
+
+ for key, (n_sub, sub_term), (v_start, v_end) in zip(
+ pending_keys, pending_subpeaks, pending_view_slices, strict=True
+ ):
+ dispersions: list[float] = []
+ for v_idx in range(v_start, v_end):
+ val = _dispersion_from_fmap(all_fmaps[v_idx], all_masks[v_idx])
+ if val is not None:
+ dispersions.append(val)
+ p_disp = float(np.mean(dispersions)) if dispersions else 0.0
+ roi_val = float(np.clip(0.65 * sub_term + 0.35 * p_disp, 0.0, 1.0))
+ effective_cache[key] = (n_sub, roi_val)
split_parent_tids: set[int] = set()
if lineage_events:
@@ -207,11 +293,27 @@ def compute_fracture_precursors(
for tid in sorted(by_track):
obs_list = sorted(by_track[tid], key=lambda r: int(r["scan_idx"]))
- e0 = (
- float(obs_list[0].get("eig_1", 0.0))
- + float(obs_list[0].get("eig_2", 0.0))
- + float(obs_list[0].get("eig_3", 0.0))
+ eig1_0 = max(0.0, float(obs_list[0].get("eig_1", 0.0)))
+ eig2_0 = max(0.0, float(obs_list[0].get("eig_2", 0.0)))
+ eig3_0 = max(0.0, float(obs_list[0].get("eig_3", 0.0)))
+ e0 = eig1_0 + eig2_0 + eig3_0
+ aniso_0 = eig1_0 / eig3_0 if eig3_0 > 1e-12 else 1.0
+ c0_mu = max(1e-12, float(obs_list[0].get("cov_zz", 1.0)))
+ c0_chi = max(1e-12, float(obs_list[0].get("cov_xx", 1.0)))
+ c0_d = max(1e-12, float(obs_list[0].get("cov_yy", 1.0)))
+ has_cov = all(k in obs_list[0] for k in ("cov_zz", "cov_xx", "cov_yy"))
+ has_bbox = all(
+ k in obs_list[0]
+ for k in ("bbox_min_z", "bbox_max_z", "bbox_min_x", "bbox_max_x", "bbox_min_y", "bbox_max_y")
)
+ if has_cov and has_bbox:
+ dz0 = max(float(obs_list[0]["bbox_max_z"]) - float(obs_list[0]["bbox_min_z"]), 1.0)
+ dx0 = max(float(obs_list[0]["bbox_max_x"]) - float(obs_list[0]["bbox_min_x"]), 1.0)
+ dy0 = max(float(obs_list[0]["bbox_max_y"]) - float(obs_list[0]["bbox_min_y"]), 1.0)
+ scales2 = [c0_mu / (dz0 * dz0), c0_chi / (dx0 * dx0), c0_d / (dy0 * dy0)]
+ mixed_units = (max(scales2) / max(min(scales2), 1e-18)) > 25.0
+ else:
+ mixed_units = has_cov and aniso_0 > 50.0
track_scores: list[ObservationPrecursorScore] = []
for obs in obs_list:
@@ -221,32 +323,29 @@ def compute_fracture_precursors(
eig2 = max(0.0, float(obs.get("eig_2", 0.0)))
eig3 = max(0.0, float(obs.get("eig_3", 0.0)))
- aniso = eig1 / eig3 if eig3 > 1e-9 else (1.0 if eig1 <= 1e-9 else 10.0)
- aniso_term = float(np.clip((aniso - 1.0) / max(cfg.anisotropy_scale, 1e-6), 0.0, 1.0))
+ if mixed_units:
+ r_mu = max(0.0, float(obs.get("cov_zz", 0.0))) / c0_mu
+ r_chi = max(0.0, float(obs.get("cov_xx", 0.0))) / c0_chi
+ r_d = max(0.0, float(obs.get("cov_yy", 0.0))) / c0_d
+ r_max = max(r_mu, r_chi, r_d)
+ r_min = max(min(r_mu, r_chi, r_d), 1e-9)
+ aniso = r_max / r_min
+ spread_growth = (r_mu + r_chi + r_d - 3.0) / 3.0
+ else:
+ aniso = eig1 / eig3 if eig3 > 1e-9 else (1.0 if eig1 <= 1e-9 else 10.0)
+ trace = eig1 + eig2 + eig3
+ spread_growth = (trace - e0) / e0 if e0 > 1e-9 else 0.0
- trace = eig1 + eig2 + eig3
- spread_growth = (trace - e0) / e0 if e0 > 1e-9 else 0.0
+ aniso_term = float(np.clip((aniso - 1.0) / max(cfg.anisotropy_scale, 1e-6), 0.0, 1.0))
spread_term = float(np.clip(max(0.0, spread_growth) / max(cfg.spread_growth_scale, 1e-6), 0.0, 1.0))
n_subpeaks = 1
roi_term = 0.0
- use_roi_here = has_roi and scan_idx in vol_map and scan_idx in lab_map and lbl > 0
- if use_roi_here and "bbox_min_z" in obs:
- masked_cube, mask = crop_spot_cube(
- vol_map[scan_idx],
- lab_map[scan_idx],
- lbl,
- obs,
- margin=cfg.roi_margin,
- )
- n_subpeaks, subpeak_term = _count_internal_subpeaks(masked_cube, mask)
- if patch_encoder is not None:
- p_disp = _patch_dispersion_score(masked_cube, mask, patch_encoder, cfg.roi_upsample_size)
- roi_term = float(np.clip(0.65 * subpeak_term + 0.35 * p_disp, 0.0, 1.0))
- else:
- roi_term = subpeak_term
-
+ use_roi_here = lbl > 0 and (
+ (scan_idx, lbl) in effective_cache or (has_roi and scan_idx in vol_map and scan_idx in lab_map)
+ )
if use_roi_here and "bbox_min_z" in obs:
+ n_subpeaks, roi_term = effective_cache.get((scan_idx, lbl), (1, 0.0))
w_sum = cfg.weight_anisotropy + cfg.weight_spread + cfg.weight_roi
raw_score = (
cfg.weight_anisotropy * aniso_term + cfg.weight_spread * spread_term + cfg.weight_roi * roi_term
diff --git a/braggtrack/viz/__init__.py b/braggtrack/viz/__init__.py
index 52705e6..4c50333 100644
--- a/braggtrack/viz/__init__.py
+++ b/braggtrack/viz/__init__.py
@@ -4,6 +4,7 @@
from .kinematics import plot_kinematics_dashboard
from .lineage import plot_lineage_dag
from .mips import plot_scan_projection_overlay, plot_spot_mip_card
+from .operando import plot_operando_strain_and_events
from .precursors import plot_precursor_timeline
__all__ = [
@@ -11,6 +12,7 @@
"generate_html_report",
"plot_kinematics_dashboard",
"plot_lineage_dag",
+ "plot_operando_strain_and_events",
"plot_precursor_timeline",
"plot_scan_projection_overlay",
"plot_spot_mip_card",
diff --git a/braggtrack/viz/html_report.py b/braggtrack/viz/html_report.py
index 49518ef..deeaa2d 100644
--- a/braggtrack/viz/html_report.py
+++ b/braggtrack/viz/html_report.py
@@ -148,6 +148,7 @@ def build_explorer_payload(
artifact_dir: str | Path,
*,
dataset_root: str | Path | None = None,
+ operando_log: str | Path | None = None,
max_mip_tracks: int = 16,
) -> dict[str, Any]:
"""Assemble structured JSON payload for the interactive 4D explorer from pipeline/tracking outputs."""
@@ -256,6 +257,30 @@ def build_explorer_payload(
if n_scans == 0 and track_rows:
n_scans = max(int(r["scan_idx"]) for r in track_rows) + 1
+ # Load synchronized operando stimulus from explicit log path or saved artifact JSON
+ stimuli_list: list[dict[str, Any]] = []
+ saved_stim = _read_json(root / "operando_stimulus.json")
+ if operando_log is not None:
+ from braggtrack.io import BeamlineAdapter, load_operando_log, synchronize_to_scans
+
+ op_log = load_operando_log(operando_log)
+ if ds_root is not None and ds_root.is_dir():
+ try:
+ seq = BeamlineAdapter(ds_root).build_sequence()
+ pts = (
+ synchronize_to_scans(op_log, seq)
+ if len(seq.scans) == n_scans
+ else synchronize_to_scans(op_log, n_scans)
+ )
+ except Exception:
+ pts = synchronize_to_scans(op_log, n_scans)
+ else:
+ pts = synchronize_to_scans(op_log, n_scans)
+ stimuli_list = [p.to_dict() for p in pts]
+ elif isinstance(saved_stim, dict) and isinstance(saved_stim.get("scans"), list):
+ stimuli_list = list(saved_stim["scans"])
+ stim_by_scan = {int(s["scan_idx"]): s for s in stimuli_list if "scan_idx" in s}
+
priority_tids = sorted(
by_tid.keys(),
key=lambda tid: (
@@ -279,6 +304,16 @@ def build_explorer_payload(
for idx_o, o in enumerate(obs_sorted):
s_i = int(o["scan_idx"])
strain_pct = float(gk.strain[idx_o] * 100.0) if gk is not None and idx_o < len(gk.strain) else 0.0
+ lin_strain_pct = (
+ float(gk.lineage_strain[idx_o] * 100.0)
+ if gk is not None and idx_o < len(gk.lineage_strain)
+ else strain_pct
+ )
+ strain_err_pct = (
+ float(gk.lineage_strain_err[idx_o] * 100.0)
+ if gk is not None and idx_o < len(gk.lineage_strain_err)
+ else 0.0
+ )
miso = (
float(gk.misorientation_total[idx_o])
if gk is not None and idx_o < len(gk.misorientation_total)
@@ -286,6 +321,7 @@ def build_explorer_payload(
)
aniso = float(gk.anisotropy[idx_o]) if gk is not None and idx_o < len(gk.anisotropy) else 1.0
po = prec_obs_by_key.get((tid, s_i), {})
+ stim = stim_by_scan.get(s_i, {})
obs_items.append(
{
"scan_idx": s_i,
@@ -296,6 +332,12 @@ def build_explorer_payload(
"intensity": round(float(o.get("integrated_intensity", 0.0)), 2),
"voxels": int(o.get("voxel_count", 0)),
"strain_pct": round(strain_pct, 4),
+ "lineage_strain_pct": round(lin_strain_pct, 4),
+ "strain_err_pct": round(strain_err_pct, 4),
+ "voltage_v": stim.get("voltage_v"),
+ "dq_dv": stim.get("dq_dv"),
+ "stress_mpa": stim.get("stress_mpa"),
+ "temperature_c": stim.get("temperature_c"),
"misorientation": round(miso, 4),
"anisotropy": round(aniso, 3),
"precursor_score": round(float(po.get("score", 0.0)), 4),
@@ -308,12 +350,13 @@ def build_explorer_payload(
tracks_payload.append(
{
"track_id": tid,
+ "root_track_id": gk.root_track_id if gk is not None and gk.root_track_id > 0 else tid,
"is_full": is_full,
"has_gap": has_gap,
"is_split": tid in split_tids,
"is_merge": tid in merge_tids,
"is_precursor": tid in prec_tids,
- "max_abs_strain_pct": round(max((abs(x["strain_pct"]) for x in obs_items), default=0.0), 4),
+ "max_abs_strain_pct": round(max((abs(x["lineage_strain_pct"]) for x in obs_items), default=0.0), 4),
"max_precursor": round(max((x["precursor_score"] for x in obs_items), default=0.0), 4),
"observations": obs_items,
}
@@ -330,7 +373,7 @@ def build_explorer_payload(
"calibrated_axes": bool(trk_summary.get("calibrated_axes", False)),
"backend": evt_summary.get("backend") or manifest.get("config", {}).get("backend") or "n/a",
"model": evt_summary.get("model") or manifest.get("config", {}).get("model") or "n/a",
- "braggtrack_version": manifest.get("braggtrack_version", "0.3.0"),
+ "braggtrack_version": manifest.get("braggtrack_version", "0.4.0"),
"total_duration_sec": manifest.get("total_duration_sec"),
"kpis": {
"total_tracks": len(tracks_payload),
@@ -340,6 +383,7 @@ def build_explorer_payload(
"merge_count": sum(1 for e in events_list if e["event_type"] == "merge"),
"precursor_count": len(precursors_list),
},
+ "stimuli": stimuli_list,
"tracks": tracks_payload,
"events": events_list,
"precursors": precursors_list,
@@ -509,10 +553,13 @@ def build_explorer_payload(
const lane = new Map(tracks.map((t, idx) => [t.track_id, padT + idx * rowH]));
let svg = `