diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index 668083d..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,45 +0,0 @@
-# Build artifacts
-build/
-dist/
-*.egg-info/
-*.egg
-__pycache__/
-*.py[cod]
-*$py.class
-.Python
-
-# Virtual environments
-venv/
-env/
-.venv/
-.env/
-ENV/
-
-# IDE
-.vscode/
-.idea/
-*.swp
-*.swo
-.DS_Store
-
-# Test / coverage
-.pytest_cache/
-.coverage
-htmlcov/
-.tox/
-.cache
-
-# Tools
-.mypy_cache/
-.ruff_cache/
-
-# Project-specific
-training_data/
-checkpoints_*/
-*.pt
-*.npz
-*.h5
-*.vti
-
-# Logs
-*.log
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 60be931..73e93e2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,122 +1,91 @@
# Changelog
-All notable changes to CDI-ST are documented here.
-The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
-and the project adheres to [Semantic Versioning](https://semver.org/).
+## [0.3.0] - 2026-11-08
-## [Unreleased]
+Faceted particles, photon-budget noise, detector/sample space, standalone phase
+retrieval and training diagnostics.
-## [0.2.2] — 2026-06-18
-
-### Fixed
-- **ID01 SPEC+EDF conversion now tolerates missing edge frames.**
- Acquisitions are sometimes started a frame late or stopped a frame
- early, so a SPEC scan can list frames 262–662 (401 points) while the
- EDF directory only has 263–661 (399 files). The converter now probes
- the disk first, loads only the EDF files that actually exist, and
- trims the per-frame motor arrays (`eta`, `phi`, `nu`, `delta`,
- `mpx4inr`) to stay in lock-step with the loaded data. A clear note
- identifies which leading / trailing / interior frames were skipped.
- New helper `_filter_existing_edf_frames()` exposes this filtering for
- programmatic use.
-
-## [0.2.1] — 2026-06-15
+With thanks to **Dr. Gerards Hinsley**, whose review of v0.2.2 prompted most of
+the changes in this release: the facet options, the photon-count control, the
+detector-space toggle, the standalone reconstruction path, the training log, and
+the corrections to the noise model and colour-scale controls.
### Added
-- **PETRA III P10 / DESY Eiger compressed-data support.** Auto-imports
- `hdf5plugin` at module load so bitshuffle + LZ4 compressed datasets
- (the Eiger default at DESY P10 and many other beamlines) read
- transparently. `hdf5plugin>=4.0` is now a runtime dependency.
-- **HDF5 external-link resolution.** The P10 loader now enumerates and
- resolves external links in master files instead of relying on h5py's
- default link-following, which was unreliable on Windows. Each chunk is
- opened directly to bypass any path-mangling. Master files with hundreds
- of external links to chunk files (the standard Eiger layout) work out
- of the box.
-- **Scan range selection for P10** (new in the P10 converter dialog) —
- similar to the ID01 SPEC+EDF scan browser. Avoids loading 49 GB of
- detector frames into RAM when you only need the rocking-curve maximum:
- - **Probe scan** button reads chunk metadata only (no decompression),
- instantly reporting total frame count, detector shape, and estimated
- full-load memory footprint.
- - **Frame range** (`from`, `to`) reads only the requested global frame
- range across all chunks. Loaded chunk-by-chunk so memory scales with
- the chosen range, not the total dataset size.
- - **Detector ROI** reads a centered square ROI from each frame instead
- of the full detector. For BCDI you typically only need ~256×256
- around the Bragg peak — cuts memory by `(2167*2070)/(roi²)` for
- Eiger 4M data.
- - **Memory budget** (default 8 GB) — the loader refuses reads that
- would exceed this and tells you which `frame_range` would fit,
- instead of crashing the Python process with an OOM error.
-- **Programmatic API additions** to `nn_experimental_loader`:
- `p10_scan_info()` for cheap metadata previews, plus new
- `frame_range`, `detector_roi`, `memory_budget_gb` parameters on
- `load_p10_from_external_links()` and `p10_h5_to_npz()`.
-### Fixed
-- Cryptic "Can't synchronously read data (can't open directory)" error
- on P10 / Eiger files. The message was HDF5 looking for the compression
- plugin DLL in a non-existent directory — now caught and converted to
- a clear "install hdf5plugin" instruction.
-- AutoPhase_NN tab missing `_on_epoch` slot (introduced in 0.2.0 during
- the live-curve refactor; AutoPhase_NN's `epoch_done` signal had no
- receiver and emitted unhandled errors). Restored.
-- CDI_NN `KeyError: 'phase_true'` — `BCDIDataset.__getitem__` now exposes
- the GUI's expected keys (`phase_true`, `amplitude`) alongside the legacy
- ones (`target_phase`, `input`).
-- `BCDIPhaseLoss` accepts both the GUI's descriptive kwargs (`alpha_amp`,
- `beta_phase`, `gamma_diff`, `diff_amp`) and the original short ones
- (`alpha`, `beta`, `gamma`, `amplitude`). Either spelling now works.
-- Training Stop button no longer crashes the program. `QThread.terminate()`
- was killing PyTorch mid-`backward()` and corrupting CUDA / autograd
- state — replaced with a cooperative `request_stop()` flag that the
- training loop checks at batch / epoch boundaries, then exits cleanly
- with a saved checkpoint.
+**Simulation**
+- Crystallographic facet shapes built from a Wulff half-space construction
+ (`facets.py`): octahedron, cuboctahedron, truncated octahedron, truncated cube,
+ rhombic dodecahedron, tunable Wulff, and hexagonal prism / bipyramid /
+ truncated bipyramid. Facet normals come from the reciprocal lattice, so the
+ construction is correct for every crystal system.
+- Facet multiplicities control: a ratio of facet distances, off by default,
+ enabled by a tick box, with a plain-language tooltip giving the shape each
+ value produces.
+- Particle size entered directly in nm; supercell multiples follow from the
+ lattice vectors. Cell limit raised from 2500 to 100000 per axis, with a size
+ notice above 2M atoms and a confirmation above 10M.
+- Photon budget: the pattern is scaled to a chosen number of photons before
+ Poisson sampling, so the counting statistics mean something. The field accepts
+ scientific notation (1e9, 2.5e12).
+- Detector / sample space toggle. `ortho.det_matrix` builds the real
+ (dq/dpx, dq/dpy, dq/drocking) transform; the Results tab switches between the
+ orthogonal lab q grid and the raw skewed detector grid and reports the skew.
+- Custom 2theta with a live readout: the offset from the nearest reflection, the
+ resulting dq, and the peak shift in both simulation voxels and real detector
+ pixels, with a warning when the peak leaves the frame.
+- Prismatic dislocation loop alongside the existing line dislocations.
+- Separate simulation grid control, independent of the detector panel size.
+- Visualization controls (bonds, strain, dislocation) grouped together.
-### Changed
-- Live training curve in AutoPhase_NN and CDI_NN — the dashed "running"
- line updates every ~2 s during training so users can see progress
- immediately on slow CPU runs (rather than staring at a blank plot
- for an entire epoch).
-- `inspect_h5()` now properly shows external links with their target
- files and `[OK]` / `[MISSING]` markers. The empty-looking `/entry/data/`
- group on Eiger masters is no longer misleading.
+**Data processing**
+- `recon.py` — standalone ER / HIO / RAAR phase retrieval with shrinkwrap,
+ bad-pixel masking, chi-squared history, centring, phase-ramp removal and PRTF.
+ No network needed.
+- `recon.load_recon` — reads reconstructions from `.npz`, `.npy`, `.h5`, `.cxi`,
+ `.nxs` and `.mat`, including PyNX `/entry_1/image_1/data` and amplitude/phase
+ pairs, so work reconstructed outside CDI-St can be loaded.
+- `trainlog.py` — per-step training log (`.log` + `.jsonl`) that flags NaN/Inf
+ losses, non-finite gradients and weights, and loss spikes, then dumps input
+ statistics and the likely cause. `scan_dataset()` pre-flights a training set.
+- `viz3d.py` — 14 colormaps including Turbo; clip plane with a filled cut face
+ showing the data on the cut layer; explicit colour-scale scope (whole volume /
+ rendered shell / current slice) with a label stating which was used.
-## [0.1.0] — Alpha — 2026-05-08
+### Changed
-### Added
-- **Initial public release** of CDI-ST as a packaged Python application.
-- BCDI Simulation module (Material → Beam → Results)
- - Material presets: Si, Ge, Pt, Au, SiC, GaAs, and more
- - CIF import for arbitrary crystals
- - Particle shapes: cube, sphere, cylinder, hexagonal prism, octahedron, dodecahedron
- - Strain fields: radial gradient, edge dislocation, random
- - Configurable line dislocations: edge / screw / mixed with arbitrary Burgers vector
- - 3D atomic lattice viewer (Plotly)
- - Detector presets: Maxipix 2×2, Eiger 2M, custom rectangular
- - Analytical and FFT simulation paths
-- BCDI Data Analysis module (Generate Data → AutoPhase_NN → CDI_NN → Reconstruction → 3D Viewer)
- - Training-data generator with random material/shape/size/strain/dislocation
- - Toggles for randomized strain and dislocations
- - AutoPhaseNet3D (unsupervised, dual-decoder, physics-driven loss)
- - PhaseUNet3D (supervised, BCDI loss with α/β/γ weights)
- - Inference: NN-only, refined (HIO+RAAR+ER), ensemble, compare
- - Iterative phase retrieval: HIO (β=0.9), RAAR (β=0.9), ER, with shrink-wrap
- - Experimental data loader: HDF5, ID01 SPEC+EDF, q-space orthogonalization
- - Detector chip-gap masking, hot pixel removal, beamstop streak removal
- - 3D viewer with point cloud / isosurface / surface-only modes
- - Per-voxel opacity weighted by intensity
- - VTI export for ParaView
-- Splash screen and theme-matched launcher
-- In-app Reports & Suggestions feature
-- Cross-platform support (Linux, macOS, Windows) via PyQt6
-- MIT license, full source on GitHub
+- Read noise is applied only to integrating detectors. Hybrid photon-counting
+ detectors (EIGER, PILATUS, MAXIPIX, MERLIN, TIMEPIX) have Poisson statistics
+ only, and the Readout option disables itself when one is selected. A
+ `CCD (integrating)` preset was added for the case where read noise applies.
+- Noise controls are named per effect (Poisson / Readout / Air / Dead px /
+ Hot px) instead of a single "Add noise", and all are off by default, so the
+ ideal pattern is what you see until you ask for noise.
+- Default simulation grid raised from 128 to 192, giving 2.25x the pixels in the
+ 2D slices for about 2 s per simulation.
+- The simulation panel is resizable; the fixed 520 px left column is a splitter.
+- Frame transform uses cubic interpolation (0.7% round-trip error, against 3.6%
+ for linear).
-### Known limitations (alpha)
-- Q-space orthogonalization is ID01-specific (can be extended)
-- Some checkpoints from earlier prototypes lack `grid_size` metadata
-- Mask-aware modulus constraint not yet implemented (gaps are filled instead)
+### Fixed
-[Unreleased]: https://github.com/SAIDIsoufiane/CDI-St/compare/v0.1.0...HEAD
-[0.1.0]: https://github.com/SAIDIsoufiane/CDI-St/releases/tag/v0.1.0
+- Poisson noise was on by default and applied the photon budget unconditionally,
+ which quantised 83% of the volume to zero counts and erased the weak fringes
+ in the 3D peak and the 2D slices. Noise is now opt-in.
+- The simulation grid was capped by the detector panel size, so a 128 x 128
+ panel forced 128^3 slices regardless of the grid setting. Panel geometry
+ (exported as metadata) and simulation grid are now independent.
+- Reflections are labelled by their canonical Friedel mate, so selecting (1 1 1)
+ no longer displays (-1 -1 -1). Both label the same physical peak; the hkl
+ search simply reached the negative representative first. The raw index is kept
+ as `hkl_raw`.
+- `select_reflection` returns full-precision d, 2theta, q, |F|^2 and LP. It
+ previously used the four-decimal values rounded for the table.
+- Half-voxel registration error in the detector/lab transform. An orthogonal
+ transform is now an exact no-op for any q-grid convention.
+- Shrinkwrap could inflate the support until the algorithm fit the noise (89% of
+ the array at 1e5 counts). An absolute cap keeps reconstructions stable from
+ 1e3 to 1e8 photons.
+- Selecting a large detector preset set the transform size to the full panel:
+ Eiger2 4M attempted a 2068^3 complex FFT, about 140 GB.
+- The facet ratio control was not read when building, and the large-supercell
+ notice was missing.
diff --git a/src/cdi_st/__main__.py b/src/cdi_st/__main__.py
index 509ccd8..7de9c83 100644
--- a/src/cdi_st/__main__.py
+++ b/src/cdi_st/__main__.py
@@ -1,4 +1,3 @@
-"""Entry point for `python -m cdi_st`."""
from cdi_st.bcdi_gui import main
if __name__ == "__main__":
diff --git a/src/cdi_st/bcdi_core.py b/src/cdi_st/bcdi_core.py
index eb6ed34..5212718 100644
--- a/src/cdi_st/bcdi_core.py
+++ b/src/cdi_st/bcdi_core.py
@@ -1,4 +1,3 @@
-"""bcdi_core.py v6.0 — dislocation loops, CIF import, rect detector, detector presets, export."""
from __future__ import annotations
import os,warnings,re,tempfile
from pathlib import Path
@@ -14,6 +13,7 @@
import xrayutilities as xu; _HAS_XU=True
except: xu=None; _HAS_XU=False
warnings.filterwarnings('ignore')
+import facets, ortho
def _6c_R3m(z):
return [[0,0,z%1],[2/3,1/3,(z+1/3)%1],[1/3,2/3,(z+2/3)%1],[0,0,(1-z)%1],[2/3,1/3,(1-z+1/3)%1],[1/3,2/3,(1-z+2/3)%1]]
@@ -22,17 +22,19 @@ def _3a_R3m():
def _3_from_6c_R3m(z):
return [[0,0,z%1],[2/3,1/3,(z+1/3)%1],[1/3,2/3,(z+2/3)%1]]
+# counting=True -> hybrid photon-counting pixel detector: Poisson only, no read noise
DETECTOR_PRESETS={
- 'Custom':{'nx':128,'ny':128,'pixel_um':55.0},
- 'Eiger2 500K':{'nx':1028,'ny':512,'pixel_um':75.0},
- 'Eiger2 1M':{'nx':1028,'ny':1062,'pixel_um':75.0},
- 'Eiger2 4M':{'nx':2068,'ny':2162,'pixel_um':75.0},
- 'Maxipix 1x1':{'nx':256,'ny':256,'pixel_um':55.0},
- 'Maxipix 2x2':{'nx':516,'ny':516,'pixel_um':55.0},
- 'Merlin Quad':{'nx':512,'ny':512,'pixel_um':55.0},
- 'Pilatus 100K':{'nx':487,'ny':195,'pixel_um':172.0},
- 'Pilatus 300K':{'nx':487,'ny':619,'pixel_um':172.0},
- 'Timepix3':{'nx':256,'ny':256,'pixel_um':55.0},
+ 'Custom':{'nx':128,'ny':128,'pixel_um':55.0,'counting':True,'dark_cps':0.0},
+ 'Eiger2 500K':{'nx':1028,'ny':512,'pixel_um':75.0,'counting':True,'dark_cps':0.0},
+ 'Eiger2 1M':{'nx':1028,'ny':1062,'pixel_um':75.0,'counting':True,'dark_cps':0.0},
+ 'Eiger2 4M':{'nx':2068,'ny':2162,'pixel_um':75.0,'counting':True,'dark_cps':0.0},
+ 'Maxipix 1x1':{'nx':256,'ny':256,'pixel_um':55.0,'counting':True,'dark_cps':0.0},
+ 'Maxipix 2x2':{'nx':516,'ny':516,'pixel_um':55.0,'counting':True,'dark_cps':0.0},
+ 'Merlin Quad':{'nx':512,'ny':512,'pixel_um':55.0,'counting':True,'dark_cps':0.0},
+ 'Pilatus 100K':{'nx':487,'ny':195,'pixel_um':172.0,'counting':True,'dark_cps':0.0},
+ 'Pilatus 300K':{'nx':487,'ny':619,'pixel_um':172.0,'counting':True,'dark_cps':0.0},
+ 'Timepix3':{'nx':256,'ny':256,'pixel_um':55.0,'counting':True,'dark_cps':0.0},
+ 'CCD (integrating)':{'nx':1024,'ny':1024,'pixel_um':13.5,'counting':False,'dark_cps':0.05},
}
MATERIAL_PRESETS={
@@ -113,11 +115,18 @@ def default_shape_for_material(n):
if ct in('hcp','wurtzite') or m['gamma']==120: return 'hexagonal'
return 'cube'
def compatible_shapes(n):
- if n not in MATERIAL_PRESETS: return ['cube','rectangle','cylinder','hexagonal','sphere']
+ base=['cube','rectangle','cylinder','sphere']
+ ded=lambda L:list(dict.fromkeys(L))
+ if n not in MATERIAL_PRESETS: return ded(base+['hexagonal']+facets.CUBIC_SHAPES+facets.HEX_SHAPES)
m=MATERIAL_PRESETS[n];ct=m['crystal_type']
- if ct in('fcc','bcc','diamond'): return ['cube','rectangle','cylinder','sphere']
- if ct in('hcp','wurtzite') or m['gamma']==120: return ['hexagonal','cylinder','sphere']
- return ['cube','rectangle','cylinder','hexagonal','sphere']
+ if ct in('fcc','bcc','diamond'): return ded(base+facets.CUBIC_SHAPES)
+ if ct in('hcp','wurtzite') or m['gamma']==120: return ['hexagonal','cylinder','sphere']+facets.HEX_SHAPES
+ return ded(base+['hexagonal']+facets.CUBIC_SHAPES+facets.HEX_SHAPES)
+
+def cells_for_size_nm(name,size_nm):
+ """Supercell multiples giving a particle of the requested size in nm."""
+ c=BCDIConfig(name);M=c.lattice_vectors_cartesian;t=float(size_nm)*10.
+ return tuple(max(1,int(round(t/np.linalg.norm(M[i])))) for i in range(3))
def find_preset(q):
q=q.strip()
if q in MATERIAL_PRESETS: return q
@@ -156,10 +165,16 @@ def __init__(self,material_name='Al'):
self.ROCKING_STEPS=128;self.ROCKING_STEP_DEG=.00315
self.SOURCE_SIZE_H_UM=300.;self.SOURCE_SIZE_V_UM=10.;self.SOURCE_DISTANCE_M=50.;self.MONOCHR_BANDWIDTH=1e-4
self.STRAIN_TYPE='none';self.STRAIN_MAGNITUDE=1e-4;self.TARGET_OVERSAMPLING=5.;self.ANGULAR_OFFSET_DEG=0.0
+ self.N_PHOTONS=1e9;self.PHOTON_MODE='total';self.SIM_GRID_MAX=192;self.WULFF_RATIO=1.15;self.Q_FRAME='lab';self.DET_COUNTING=True
self.HKL_MAX_SEARCH=5;self.Q_MAX_INV_ANG=8.;self.TWO_THETA_MIN=10.;self.TWO_THETA_MAX=90.
self.OUTPUT_DIR=Path('bcdi_output');self.OUTPUT_DIR.mkdir(exist_ok=True)
@property
- def DETECTOR_N_PIXELS(self): return max(self.DETECTOR_NX, self.DETECTOR_NY)
+ def DETECTOR_N_PIXELS(self):
+ """Side of the cubic FFT grid: the Bragg ROI cropped from the panel, as in a
+ real analysis. DETECTOR_NX/NY describe the physical panel and are exported
+ as metadata; they do not set the transform size (a 4 Mpixel panel would be
+ a 2068^3 complex FFT)."""
+ return int(self.SIM_GRID_MAX)
@property
def wavelength_angstrom(self): return self.HC_KEV_ANG/self.BEAM_ENERGY_KEV
@property
@@ -216,6 +231,9 @@ def apply_shape_filter(self):
elif s=='sphere': R=0.47*min(Lx,Ly,Lz);mask=(r**2).sum(1)<=R**2
elif s=='hexagonal':
s3=np.sqrt(3.)/2.;mask=np.maximum(np.abs(r[:,0]),np.maximum(np.abs(.5*r[:,0]+s3*r[:,1]),np.abs(.5*r[:,0]-s3*r[:,1])))<=R_xy
+ elif s in facets.SHAPES:
+ Nn,Dd=facets.normals(s,cfg.reciprocal_lattice_vectors,cfg.WULFF_RATIO)
+ R=facets.scale_to_size(Nn,Dd,0.94*min(Lx,Ly,Lz));mask=facets.mask(p,center,Nn,Dd,R)
else: return
self.supercell_positions_ang=p[mask];self.supercell_species=[sv for sv,m in zip(self.supercell_species,mask) if m]
if self._cell_indices: self._cell_indices=[c for c,m in zip(self._cell_indices,mask) if m]
@@ -351,7 +369,8 @@ def calculate(self):
fk=self._fk(np.array([h,k,l]),ops);iu=fk not in seen
if iu: seen.add(fk)
else: iu=(h>=0)and(h>=k)and(k>=l)and(l>=0)
- rows.append({'hkl':(h,k,l),'hkl_str':f'({h} {k} {l})','d_Ang':round(d,4),'2theta':round(tt,4),'theta_B':round(tt/2.,4),'q_Ang':round(q,4),'|F|':round(Fa,3),'F_sq':round(F2,2),'LP':round(LP,4),'unique':iu})
+ rows.append({'hkl':(h,k,l),'hkl_str':f'({h} {k} {l})','d_Ang':round(d,4),'2theta':round(tt,4),'theta_B':round(tt/2.,4),'q_Ang':round(q,4),'|F|':round(Fa,3),'F_sq':round(F2,2),'LP':round(LP,4),'unique':iu,
+ '_d':d,'_tt':tt,'_q':q,'_F2':F2,'_LP':LP})
df=pd.DataFrame(rows).sort_values('q_Ang').reset_index(drop=True)
df=df[df['unique']].copy().reset_index(drop=True)
df['I_rel']=df['F_sq']*df['LP'];mx=df['I_rel'].max() if len(df) else 1.
@@ -381,7 +400,11 @@ def select_reflection(self,hkl=None):
if len(m): break
if len(m)==0: raise ValueError(f'Reflection ({h} {k} {l}) not found')
sel=m.iloc[0]
- return{'hkl':sel['hkl'],'hkl_str':sel['hkl_str'],'d_hkl':sel['d_Ang'],'theta_B_rad':np.radians(sel['theta_B']),'two_theta_rad':np.radians(sel['2theta']),'q_magnitude':sel['q_Ang'],'F_squared':sel['F_sq'],'LP_factor':sel['LP'],'intensity_norm':sel['I_norm']}
+ # full-precision values; the rounded columns are for display only
+ return{'hkl':sel['hkl'],'hkl_str':sel.get('hkl_display',sel['hkl_str']),'hkl_raw':sel['hkl_str'],
+ 'd_hkl':float(sel['_d']),'theta_B_rad':np.radians(sel['_tt']/2.),'two_theta_rad':np.radians(sel['_tt']),
+ 'q_magnitude':float(sel['_q']),'F_squared':float(sel['_F2']),'LP_factor':float(sel['_LP']),
+ 'intensity_norm':float(sel['I_norm'])}
class BCDISimulator:
def __init__(self,config,reflection,crystal_builder=None):
@@ -437,6 +460,9 @@ def _sup(self,N,OS):
if s=='sphere': return np.sqrt(X**2+Y**2+Z**2)<=h
if s=='hexagonal':
s3=np.sqrt(3.)/2.;return(np.maximum(np.abs(X),np.maximum(np.abs(.5*X+s3*Y),np.abs(.5*X-s3*Y)))<=h)&(np.abs(Z)<=h)
+ if s in facets.SHAPES:
+ Nn,Dd=facets.normals(s,self.config.reciprocal_lattice_vectors,self.config.WULFF_RATIO)
+ return facets.grid_mask(X,Y,Z,Nn,Dd,facets.scale_to_size(Nn,Dd,2*h))
return(np.abs(X)<=h)&(np.abs(Y)<=h)&(np.abs(Z)<=h)
def simulate(self,progress_cb=None):
c=self.config;N=c.DETECTOR_N_PIXELS;tA=c.particle_size_angstrom;F2=self.reflection['F_squared'];Vc=c.unit_cell_volume_ang3;LP=self.reflection['LP_factor'];OS=c.TARGET_OVERSAMPLING
@@ -448,13 +474,11 @@ def simulate(self,progress_cb=None):
sup=self._sup(N,OS).astype(np.float32);ph=self._displacement(N,tA,OS);psi=sup*np.exp(1j*ph)
if progress_cb: progress_cb(40)
# Apply angular offset as phase ramp (shifts Bragg peak off-center)
- # AND attenuate intensity by the kinematic rocking curve envelope
if abs(c.ANGULAR_OFFSET_DEG)>1e-6:
delta_th=np.radians(c.ANGULAR_OFFSET_DEG/2.)
delta_q=(4*np.pi/c.wavelength_angstrom)*np.cos(self.reflection['theta_B_rad'])*delta_th
- # Phase ramp along first spatial axis (real-space extent = OS*tA)
- ext0=OS*tA[0]
- rx=np.linspace(-ext0/2,ext0/2,N,dtype=np.float32)
+ # Phase ramp along first spatial axis
+ rx=np.linspace(-ext[0]/2,ext[0]/2,N,dtype=np.float32)
phase_ramp=np.exp(1j*delta_q*rx)[:,None,None]
psi=psi*phase_ramp
pq=np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(psi)))
@@ -472,54 +496,64 @@ def simulate(self,progress_cb=None):
diff=pf*(sx**2)[:,None,None]*(sy**2)[None,:,None]*(sz**2)[None,None,:]
if progress_cb: progress_cb(70)
diff=self._pc(diff.astype(np.float32),dqx,dqy,dqz)
- # Angular offset: attenuate by kinematic rocking curve envelope
- # Away from exact Bragg condition, intensity drops as sinc^2
- if abs(c.ANGULAR_OFFSET_DEG)>1e-6:
- # Shape-aware physical particle size (not just supercell box)
- # For sphere/cylinder/hexagonal, the inscribed shape is smaller than the box
- shape=c.PARTICLE_SHAPE
- if shape=='sphere':
- D_phys=float(min(tA)) # diameter = smallest box dim
- elif shape=='cylinder':
- D_phys=float(min(tA[0],tA[1])) # diameter in basal plane
- elif shape=='hexagonal':
- D_phys=float(min(tA[0],tA[1])) # in-plane width
- else:
- D_phys=float(np.mean(tA)) # cube/rectangle: full extent
- th_B=self.reflection['theta_B_rad']
- # Kinematic rocking curve FWHM: dw ~ 2*lambda/(D*sin(2*theta_B))
- dw_fwhm=2.*c.wavelength_angstrom/(D_phys*np.sin(2.*th_B)+1e-12)
- delta_deg=abs(c.ANGULAR_OFFSET_DEG)
- # Number of FWHM widths away
- n_fwhm=delta_deg/max(np.degrees(dw_fwhm),1e-10)
- # Gaussian attenuation (matches kinematic rocking curve envelope)
- attenuation=np.exp(-2.*np.log(2)*(n_fwhm**2))
- diff*=attenuation
if progress_cb: progress_cb(95)
self.diff_volume=diff.astype(np.float32);self.q_grids={'qx':qx,'qy':qy,'qz':qz}
xh,xv,xl=self._coh();self.coherence={'xi_h_A':xh,'xi_v_A':xv,'xi_l_A':xl}
if progress_cb: progress_cb(100);return diff,self.q_grids
-def add_experimental_noise(vol,poisson=True,readout_noise=0,air_scatter=0,dead_pixels_frac=0,seed=42):
- rng=np.random.default_rng(seed);out=vol.copy().astype(np.float64)
+def add_experimental_noise(vol,n_photons=None,photon_mode='total',poisson=True,air_scatter=0,
+ readout_noise=0,dark_cps=0,exposure_s=1.,dead_pixels_frac=0,
+ hot_pixels_frac=0,counting=True,seed=42):
+ """Photon budget first, then instrument effects.
+
+ n_photons sets the counting statistics: everything else is relative to it.
+ readout_noise applies only to integrating detectors (counting=False); EIGER,
+ PILATUS, MAXIPIX, MERLIN and TIMEPIX are photon counting and have none.
+ """
+ rng=np.random.default_rng(seed);out=np.asarray(vol,np.float64)
if air_scatter>0:
- N=vol.shape[0];c=N//2;r0=max(1,N//4);x=np.arange(N)-c;QX,QY,QZ=np.meshgrid(x,x,x,indexing='ij')
- r_norm=(QX**2+QY**2+QZ**2)/(r0**2+1e-10);peak=max(1.0,vol.max());out+=peak*air_scatter*1e-6/(1.+r_norm**2)
- if poisson:
- scale=1e6/max(1,out.max());counts=np.clip(out*scale,0,1e12);out=rng.poisson(counts.astype(np.float64)).astype(np.float64)/scale
- if readout_noise>0: out+=rng.normal(0,readout_noise*out.max()*1e-6,out.shape);out=np.maximum(out,0)
- if dead_pixels_frac>0:
- n_dead=int(dead_pixels_frac*out.size);idx=rng.choice(out.size,n_dead,replace=False);out.flat[idx]=0
- return out.astype(np.float32)
+ N=out.shape[0];c=N//2;r0=max(1,N//4);x=np.arange(N)-c;QX,QY,QZ=np.meshgrid(x,x,x,indexing='ij')
+ rn=(QX**2+QY**2+QZ**2)/(r0**2+1e-10);out=out+max(1.,out.max())*air_scatter*1e-6/(1.+rn**2)
+ if n_photons and n_photons>0:
+ out=ortho.apply_photons(out,n_photons,photon_mode,poisson=poisson,seed=seed).astype(np.float64)
+ elif poisson:
+ sc=1e6/max(1,out.max());out=rng.poisson(np.clip(out*sc,0,1e12)).astype(np.float64)/sc
+ return ortho.detector_noise(out,dark_cps=dark_cps,exposure_s=exposure_s,
+ readout_e=0. if counting else readout_noise,
+ dead_frac=dead_pixels_frac,hot_frac=hot_pixels_frac,seed=seed)
+
+def q_sampling(cfg,theta_B_rad=None):
+ """Compare the simulation q voxel with a real detector pixel.
+
+ The simulated grid is set by the particle size and oversampling; the panel
+ grid by pixel size and distance. Matching them 1:1 needs N >> OS, so the
+ simulation is deliberately coarser. Returns (dq_sim, dq_pixel, implied_OS).
+ """
+ k=2*np.pi/cfg.wavelength_angstrom
+ dq_sim=(2*np.pi/cfg.particle_size_angstrom[0])/cfg.TARGET_OVERSAMPLING
+ dq_pix=k*(cfg.DETECTOR_PIXEL_UM*1e-6)/cfg.SAMPLE_DETECTOR_DISTANCE_M
+ return dq_sim,dq_pix,(2*np.pi/cfg.particle_size_angstrom[0])/dq_pix
+
+def offset_report(cfg,theta_B_rad,d2th_deg):
+ """What a 2theta offset does: dq, shift in sim voxels and in real pixels."""
+ dq=(4*np.pi/cfg.wavelength_angstrom)*np.cos(theta_B_rad)*np.radians(d2th_deg/2.)
+ ds,dp,_=q_sampling(cfg)
+ N=cfg.DETECTOR_N_PIXELS
+ return dq,dq/ds,dq/dp,abs(dq/ds)>N/2
+
+def to_frame(sim,frame='lab',vol=None):
+ """Return (volume, label). 'lab' = orthogonal q; 'det' = raw skewed detector grid."""
+ v=sim.diff_volume if vol is None else vol
+ if frame!='det': return v,'sample / lab frame (orthogonal q)'
+ c=sim.config;tt=np.degrees(sim.reflection['two_theta_rad'])
+ M,_=ortho.det_matrix(c.wavelength_angstrom,tt,c.DETECTOR_PIXEL_UM,
+ c.SAMPLE_DETECTOR_DISTANCE_M,c.ROCKING_STEP_DEG)
+ return ortho.lab_to_det(v,sim.q_grids,M,order=3),f'detector frame (raw grid, skew={ortho.skew(M):.3f})'
def export_measurement(sim,refl,cfg,filepath,noise_opts=None):
"""Export simulation as .npz (like real BCDI measurement data)."""
vol=sim.diff_volume.copy()
if noise_opts: vol=add_experimental_noise(vol,**noise_opts)
- # Real-space voxel pitch in nm = (object size * oversampling) / grid_size
- N=cfg.DETECTOR_N_PIXELS
- psize_arr=np.asarray(cfg.particle_size_nm,dtype=np.float32)
- voxel_nm=(psize_arr*float(cfg.TARGET_OVERSAMPLING)/float(N))
data={'diffraction_volume':vol,'qx':sim.q_grids['qx'],'qy':sim.q_grids['qy'],'qz':sim.q_grids['qz'],
'hkl':np.array(refl['hkl']),'d_hkl':refl['d_hkl'],'two_theta':np.degrees(refl['two_theta_rad']),
'energy_keV':cfg.BEAM_ENERGY_KEV,'wavelength_A':cfg.wavelength_angstrom,
@@ -529,7 +563,9 @@ def export_measurement(sim,refl,cfg,filepath,noise_opts=None):
'detector_nx':cfg.DETECTOR_NX,'detector_ny':cfg.DETECTOR_NY,
'rocking_step_deg':cfg.ROCKING_STEP_DEG,'rocking_steps':cfg.ROCKING_STEPS,
'beam_size_um':cfg.BEAM_SIZE_UM,'oversampling':cfg.TARGET_OVERSAMPLING,
- 'particle_size_nm':psize_arr,'voxel_size_nm':voxel_nm}
+ 'n_photons':getattr(cfg,'N_PHOTONS',0),'photon_mode':getattr(cfg,'PHOTON_MODE','total'),
+ 'q_frame':getattr(cfg,'Q_FRAME','lab'),'detector_counting':getattr(cfg,'DET_COUNTING',True),
+ 'particle_size_nm':cfg.particle_size_nm}
if filepath.endswith('.h5') or filepath.endswith('.hdf5'):
import h5py
with h5py.File(filepath,'w') as f:
@@ -621,11 +657,12 @@ def lattice_html_with_controls(builder,max_atoms=50000,atom_size=3,show_bonds=Fa
html=html.replace('
',''+_loading_html('Rendering 3D Lattice...'),1)
return html.replace('',PANEL+'',1)
-def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=None):
+def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=None,vol=None,frame_label=''):
import plotly.graph_objects as go
if sim.diff_volume is None: return None
- vol=sim.diff_volume.copy()
- if noise_opts: vol=add_experimental_noise(vol,**noise_opts)
+ if vol is None:
+ vol=sim.diff_volume.copy()
+ if noise_opts: vol=add_experimental_noise(vol,**noise_opts)
d3=vol[::downsample,::downsample,::downsample]
qx=sim.q_grids['qx'][::downsample];qy=sim.q_grids['qy'][::downsample];qz=sim.q_grids['qz'][::downsample]
vl=np.log10(d3+1.).astype(np.float32);vm=float(vl.max())
@@ -641,6 +678,7 @@ def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=Non
if noise_opts.get('air_scatter',0)>0: parts.append("air")
if noise_opts.get('dead_pixels_frac',0)>0: parts.append("dead")
if parts: noise_str=" | "+", ".join(parts)
+ if frame_label: noise_str+=" | "+frame_label
fig.update_layout(template='plotly_dark',paper_bgcolor='#0d0e12',scene=dict(xaxis=dict(**ax,title='qx(1/A)'),yaxis=dict(**ax,title='qy(1/A)'),zaxis=dict(**ax,title='qz(1/A)'),camera=dict(eye=dict(x=1.4,y=1.4,z=.7)),aspectmode='data'),font=dict(family='Consolas,monospace',size=11,color='#c0c4cc'),title=dict(text=f'{cfg.MATERIAL_NAME} {sim.reflection["hkl_str"]} | {sz[0]:.0f}x{sz[1]:.0f}x{sz[2]:.0f}nm{noise_str}',font=dict(size=12),x=.5,xanchor='center'),margin=dict(l=0,r=0,t=40,b=0))
# Fringe measurement JS
measure_js="""
diff --git a/src/cdi_st/bcdi_gui.py b/src/cdi_st/bcdi_gui.py
index 9b78eb0..28c14a7 100644
--- a/src/cdi_st/bcdi_gui.py
+++ b/src/cdi_st/bcdi_gui.py
@@ -1,13 +1,12 @@
-"""bcdi_gui.py v7.0 — dislocation loops, CIF import, rect detector, presets, export, measure, NN reconstruction."""
import sys,tempfile,os
import numpy as np
from collections import Counter
-from PyQt6.QtCore import Qt,QThread,pyqtSignal,QUrl,QSize,QTimer,QRect
-from PyQt6.QtGui import QPalette,QColor,QFont,QPixmap,QPainter,QFontMetrics,QBrush,QPen
+from PyQt6.QtCore import Qt,QThread,pyqtSignal,QUrl,QSize,QTimer
+from PyQt6.QtGui import QPalette,QColor,QFont
from PyQt6.QtWidgets import (QApplication,QMainWindow,QWidget,QTabWidget,QVBoxLayout,QHBoxLayout,
QFormLayout,QGridLayout,QGroupBox,QLabel,QPushButton,QComboBox,QSpinBox,QDoubleSpinBox,
QLineEdit,QTableWidget,QTableWidgetItem,QHeaderView,QProgressBar,QMessageBox,QStatusBar,
- QSplitter,QPlainTextEdit,QFrame,QCheckBox,QScrollArea,QFileDialog,QSplashScreen)
+ QSplitter,QPlainTextEdit,QFrame,QCheckBox,QScrollArea,QFileDialog)
try:
from PyQt6.QtWebEngineWidgets import QWebEngineView; _HAS_WEB=True
except: _HAS_WEB=False
@@ -16,12 +15,11 @@
from matplotlib.backends.backend_qtagg import NavigationToolbar2QT as NavigationToolbar
from matplotlib.figure import Figure
from mpl_toolkits.axes_grid1 import make_axes_locatable
-from .bcdi_core import (MATERIAL_PRESETS,DETECTOR_PRESETS,BCDIConfig,CrystalBuilder,ReflectionCalculator,
+from bcdi_core import (MATERIAL_PRESETS,DETECTOR_PRESETS,BCDIConfig,CrystalBuilder,ReflectionCalculator,
BCDISimulator,DislocationConfig,DislocationLoopConfig,lattice_html_with_controls,bragg_html_with_controls,
- add_experimental_noise,export_measurement,load_cif_as_preset,find_preset,default_shape_for_material,compatible_shapes)
-try:
- from cdi_st.nn_gui_tabs import T_Gen,T4,T4_Sup,T5,T6;_HAS_NN=True
-except ImportError: _HAS_NN=False
+ add_experimental_noise,export_measurement,load_cif_as_preset,find_preset,default_shape_for_material,
+ compatible_shapes,cells_for_size_nm,to_frame)
+import facets
QSS="""QMainWindow,QWidget{background:#0d1117;color:#e6edf3;font-family:'Segoe UI','Consolas',sans-serif;font-size:10pt}
QTabWidget::pane{border:1px solid #30363d;background:#0d1117;top:-1px}QTabBar::tab{background:#161b22;color:#8b949e;padding:10px 22px;margin-right:2px;border:1px solid #30363d;border-bottom:none;border-top-left-radius:6px;border-top-right-radius:6px;font-weight:600;min-width:140px}QTabBar::tab:selected{background:#0d1117;color:#4f98a3;border-bottom:2px solid #4f98a3}QTabBar::tab:disabled{color:#30363d}
@@ -39,6 +37,15 @@ def _dbl(lo,hi,v,d,suf=""):
s=QDoubleSpinBox();s.setRange(lo,hi);s.setDecimals(d);s.setValue(v);s.setMinimumWidth(100)
if suf: s.setSuffix(suf)
return s
+class _Sci(QLineEdit):
+ """Number field that accepts 1e9, 1.5e12 or a plain number."""
+ def __init__(self,v=1e9,w=110):
+ super().__init__(f'{v:g}');self.setMaximumWidth(w)
+ def value(self):
+ try: return float(self.text().strip().replace(' ',''))
+ except ValueError: return 0.0
+ def setValue(self,v): self.setText(f'{float(v):g}')
+
def _nb():
b=QPushButton("next >>");b.setEnabled(False);b.setFixedSize(QSize(80,28));b.setStyleSheet("background:#238636;font-size:9pt;padding:4px 10px;min-height:20px;font-weight:700");return b
@@ -68,148 +75,73 @@ class T1(QWidget):
def __init__(self): super().__init__();self.builder=None;self.config=None;self._w=None;self._ui()
def _ui(self):
root=QHBoxLayout(self);root.setContentsMargins(10,10,10,10);root.setSpacing(10)
- sc=QScrollArea();sc.setWidgetResizable(True);sc.setFixedWidth(520);inner=QWidget();ll=QVBoxLayout(inner);ll.setSpacing(6);ll.setContentsMargins(4,4,8,4)
+ split=QSplitter(Qt.Orientation.Horizontal)
+ sc=QScrollArea();sc.setWidgetResizable(True);sc.setMinimumWidth(340);inner=QWidget();ll=QVBoxLayout(inner);ll.setSpacing(6);ll.setContentsMargins(4,4,8,4)
h=QHBoxLayout();t=QLabel("Material & Lattice");t.setStyleSheet("color:#4f98a3;font-size:13pt;font-weight:700");h.addWidget(t,1);self.nb=_nb();h.addWidget(self.nb);ll.addLayout(h)
# Material with CIF import
mb=QGroupBox("Material");mv=QVBoxLayout(mb);mv.setSpacing(4)
r1=QHBoxLayout();r1.addWidget(QLabel("Preset:"));self.combo=QComboBox()
- self.combo.setToolTip(
- "Pre-configured materials. Each preset contains the lattice "
- "constants (a, b, c, angles), space group, atomic basis, and "
- "common BCDI reflections.\n"
- "Includes: Si, Ge, Pt, Au, Cu, Pd, GaAs, GaN, SiC, ZnO,\n"
- "BaTiO3, SrTiO3, Fe5GeTe2, V2O3, hexagonal/cubic polytypes,\n"
- "and any custom .cif files you've loaded."
- )
for k in MATERIAL_PRESETS: self.combo.addItem(k)
self.combo.currentTextChanged.connect(self._pc);r1.addWidget(self.combo,1);mv.addLayout(r1)
- r2=QHBoxLayout();r2.addWidget(QLabel("Formula:"));self.fe=QLineEdit();self.fe.setPlaceholderText("e.g. Fe5GeTe2")
- self.fe.setToolTip(
- "Type a chemical formula (e.g. 'Si', 'GaAs', 'Fe5GeTe2'),\n"
- "then click 'Match' to find the matching preset. Useful when\n"
- "you know the formula but not the exact preset name."
- )
- r2.addWidget(self.fe,1)
- mb2=QPushButton("Match");mb2.setStyleSheet("background:#4f98a3;padding:5px 10px;min-height:20px");mb2.setMaximumWidth(55)
- mb2.setToolTip("Find the preset that matches the typed formula.")
- mb2.clicked.connect(self._mt);r2.addWidget(mb2)
- cif_btn=QPushButton("Load .cif");cif_btn.setStyleSheet("background:#1f6feb;padding:5px 10px;min-height:20px");cif_btn.setMaximumWidth(70)
- cif_btn.setToolTip(
- "Import a Crystallographic Information File (.cif) as a custom\n"
- "material. The file is parsed for cell parameters, space group,\n"
- "and atomic basis. The new preset becomes selectable in the dropdown."
- )
- cif_btn.clicked.connect(self._load_cif);r2.addWidget(cif_btn)
+ r2=QHBoxLayout();r2.addWidget(QLabel("Formula:"));self.fe=QLineEdit();self.fe.setPlaceholderText("e.g. Fe5GeTe2");r2.addWidget(self.fe,1)
+ mb2=QPushButton("Match");mb2.setStyleSheet("background:#4f98a3;padding:5px 10px;min-height:20px");mb2.setMaximumWidth(55);mb2.clicked.connect(self._mt);r2.addWidget(mb2)
+ cif_btn=QPushButton("Load .cif");cif_btn.setStyleSheet("background:#1f6feb;padding:5px 10px;min-height:20px");cif_btn.setMaximumWidth(70);cif_btn.clicked.connect(self._load_cif);r2.addWidget(cif_btn)
mv.addLayout(r2);self.il=QLabel();self.il.setWordWrap(True);self.il.setStyleSheet("color:#8b949e;font-size:9pt");mv.addWidget(self.il);ll.addWidget(mb)
# Supercell
sb=QGroupBox("Supercell");sf=QFormLayout(sb);sf.setLabelAlignment(Qt.AlignmentFlag.AlignRight);sf.setVerticalSpacing(4)
row=QHBoxLayout();self.nx=QSpinBox();self.ny=QSpinBox();self.nz=QSpinBox()
- for s in(self.nx,self.ny,self.nz): s.setRange(1,2500);s.setValue(20);s.setMinimumWidth(55);s.valueChanged.connect(self._us)
- self.nx.setToolTip(
- "Number of unit cells along X axis. Sets the X dimension of\n"
- "the particle: physical size = nx × a (lattice constant)."
- )
- self.ny.setToolTip(
- "Number of unit cells along Y axis. Sets the Y dimension of\n"
- "the particle: physical size = ny × b."
- )
- self.nz.setToolTip(
- "Number of unit cells along Z axis. Sets the Z dimension of\n"
- "the particle: physical size = nz × c."
- )
+ for s in(self.nx,self.ny,self.nz): s.setRange(1,100000);s.setValue(20);s.setMinimumWidth(55);s.valueChanged.connect(self._us)
row.addWidget(QLabel("nx"));row.addWidget(self.nx);row.addWidget(QLabel("ny"));row.addWidget(self.ny);row.addWidget(QLabel("nz"));row.addWidget(self.nz);row.addStretch()
- sf.addRow("Cells:",row);self.shp=QComboBox()
- self.shp.setToolTip(
- "Particle shape carved from the rectangular supercell.\n"
- " • cube: the whole box\n"
- " • sphere: inscribed sphere\n"
- " • cylinder: inscribed cylinder along Z\n"
- " • hexagonal_prism: hexagonal cross-section\n"
- " • octahedron / dodecahedron: faceted shapes\n"
- "Available shapes depend on the material's symmetry."
- )
- sf.addRow("Shape:",self.shp);self.sl=QLabel();self.sl.setStyleSheet("color:#8b949e;font-size:9pt");sf.addRow(self.sl);ll.addWidget(sb)
- vr=QHBoxLayout();self.bc=QCheckBox("Bonds");self.bc.setToolTip("Draw atomic bonds (slows rendering with many atoms)")
- vr.addWidget(self.bc);self.stc=QCheckBox("Strain");self.stc.setToolTip("Color atoms by local strain magnitude (if strain is enabled)")
- vr.addWidget(self.stc);self.dvc=QCheckBox("Disloc. viz");self.dvc.setToolTip("Highlight atoms near the dislocation core")
- vr.addWidget(self.dvc);vr.addStretch();ll.addLayout(vr)
+ sf.addRow("Cells:",row)
+ nmr=QHBoxLayout();self.size_nm=_dbl(0.5,20000,8,1," nm");nmr.addWidget(self.size_nm)
+ bn=QPushButton("set cells");bn.setMaximumWidth(80);bn.setStyleSheet("background:#4f98a3;padding:4px 8px;min-height:20px")
+ bn.clicked.connect(self._from_nm);nmr.addWidget(bn);nmr.addStretch();sf.addRow("Size:",nmr)
+ self.shp=QComboBox();self.shp.currentTextChanged.connect(self._shape_changed);sf.addRow("Shape:",self.shp)
+ fr=QHBoxLayout()
+ self.fac_on=QCheckBox();self.wulff=_dbl(0.5,2.5,1.15,2);self.wulff.setEnabled(False)
+ self.fac_lbl=QLabel("Facet multiplicities:")
+ for w in (self.fac_on,self.wulff,self.fac_lbl): w.setToolTip('Controls the balance between the flat faces (facets) of the crystal.\\n\\nReal crystals grow flat faces whose relative size depends on which face is\\nmost stable. This number is the distance to the square {100} faces divided\\nby the distance to the triangular {111} faces.\\n\\n below 0.58 pure cube, only square faces\\n around 1.15 truncated octahedron, the usual shape of FCC metals\\n (aluminium, copper, gold)\\n above 1.73 pure octahedron, only triangular faces\\n\\nIn between you get a mix: raising the number grows the triangular faces\\nand shrinks the square ones. Leave the box unticked to use the standard 1.15.')
+ self.fac_on.toggled.connect(self.wulff.setEnabled)
+ fr.addWidget(self.fac_on);fr.addWidget(self.wulff);fr.addStretch()
+ sf.addRow(self.fac_lbl,fr);self.sl=QLabel();self.sl.setStyleSheet("color:#8b949e;font-size:9pt");sf.addRow(self.sl);ll.addWidget(sb)
+ vb=QGroupBox("Visualization");vv=QHBoxLayout(vb);vv.setSpacing(10)
+ self.bc=QCheckBox("Bonds");self.bc.setToolTip("Draw a line between neighbouring atoms. Clear for small cells, slow above ~80k atoms.")
+ self.stc=QCheckBox("Strain");self.stc.setToolTip("Colour every atom by how far it sits from its ideal lattice site (inferno scale).")
+ self.dvc=QCheckBox("Dislocation");self.dvc.setToolTip("Dim the undisturbed lattice and colour only the displaced atoms, so the defect stands out.")
+ for w in (self.bc,self.stc,self.dvc): vv.addWidget(w)
+ vv.addStretch();ll.addWidget(vb)
# Dislocation: line + loop tabs
db=QGroupBox("Defects");dv=QVBoxLayout(db);dv.setSpacing(4)
# Line dislocation
- self.dck=QCheckBox("Line dislocation");self.dck.setToolTip(
- "Add a single straight-line dislocation. The displacement field\n"
- "follows Volterra elasticity (edge: Burgers vector ⊥ line, screw:\n"
- "Burgers vector ∥ line, mixed: arbitrary angle)."
- );dv.addWidget(self.dck)
+ self.dck=QCheckBox("Line dislocation");dv.addWidget(self.dck)
self.df=QFrame();dg=QGridLayout(self.df);dg.setContentsMargins(2,2,2,2);dg.setHorizontalSpacing(8);dg.setVerticalSpacing(5)
- self.dt=QComboBox();self.dt.addItems(["edge","screw","mixed"])
- self.dt.setToolTip(
- "Dislocation character:\n"
- " • edge: Burgers vector perpendicular to line\n"
- " • screw: Burgers vector parallel to line\n"
- " • mixed: combination of both"
- )
- self.dd=QComboBox();self.dd.addItems(["Z","Y","X"])
- self.dd.setToolTip("Direction of the dislocation line in crystal axes.")
+ self.dt=QComboBox();self.dt.addItems(["edge","screw","mixed"]);self.dd=QComboBox();self.dd.addItems(["Z","Y","X"])
dg.addWidget(QLabel("Type:"),0,0,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.dt,0,1);dg.addWidget(QLabel("Line:"),0,2,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.dd,0,3)
- self.dpx=_dbl(0,1,.5,3);self.dpx.setToolTip("X position of the dislocation core as a fraction (0-1) of the particle.")
- self.dpy=_dbl(0,1,.5,3);self.dpy.setToolTip("Y position of the dislocation core as a fraction (0-1) of the particle.")
+ self.dpx=_dbl(0,1,.5,3);self.dpy=_dbl(0,1,.5,3)
dg.addWidget(QLabel("X:"),1,0,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.dpx,1,1);dg.addWidget(QLabel("Y:"),1,2,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.dpy,1,3)
- self.db_=_dbl(0,50,0,3," A");self.db_.setSpecialValueText("auto")
- self.db_.setToolTip(
- "Magnitude of the Burgers vector in Ångström.\n"
- "Set to 0 (= 'auto') to use the lattice constant 'a'."
- )
- self.dnu=_dbl(0,.5,.3,3);self.dnu.setToolTip(
- "Poisson's ratio ν. Controls the radial vs. tangential strain ratio\n"
- "for edge/mixed dislocations. Typical metal: 0.3."
- )
+ self.db_=_dbl(0,50,0,3," A");self.db_.setSpecialValueText("auto");self.dnu=_dbl(0,.5,.3,3)
dg.addWidget(QLabel("|b|:"),2,0,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.db_,2,1);dg.addWidget(QLabel("nu:"),2,2,Qt.AlignmentFlag.AlignRight);dg.addWidget(self.dnu,2,3)
self.df.setVisible(False);dv.addWidget(self.df);self.dck.toggled.connect(self.df.setVisible)
# Loop dislocation
- self.lck=QCheckBox("Dislocation loop (prismatic)")
- self.lck.setToolTip(
- "Add a prismatic dislocation loop (closed circular line dislocation).\n"
- "Common defect in irradiated and quenched materials."
- )
- dv.addWidget(self.lck)
+ self.lck=QCheckBox("Dislocation loop (prismatic)");dv.addWidget(self.lck)
self.lf=QFrame();lg=QGridLayout(self.lf);lg.setContentsMargins(2,2,2,2);lg.setHorizontalSpacing(8);lg.setVerticalSpacing(5)
- self.lcx=_dbl(0,1,.5,3);self.lcx.setToolTip("X position of the loop center (fraction 0-1).")
- self.lcy=_dbl(0,1,.5,3);self.lcy.setToolTip("Y position of the loop center (fraction 0-1).")
- self.lcz=_dbl(0,1,.5,3);self.lcz.setToolTip("Z position of the loop center (fraction 0-1).")
+ self.lcx=_dbl(0,1,.5,3);self.lcy=_dbl(0,1,.5,3);self.lcz=_dbl(0,1,.5,3)
lg.addWidget(QLabel("Cx:"),0,0,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.lcx,0,1);lg.addWidget(QLabel("Cy:"),0,2,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.lcy,0,3);lg.addWidget(QLabel("Cz:"),0,4,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.lcz,0,5)
- self.lr=_dbl(1,500,20,1," A");self.lr.setToolTip("Loop radius in Ångström.")
- self.lb=_dbl(0,50,0,3," A");self.lb.setSpecialValueText("auto")
- self.lb.setToolTip("Burgers vector magnitude in Ångström (= 'auto' uses 'a').")
- self.ln=QComboBox();self.ln.addItems(["Z","Y","X"])
- self.ln.setToolTip("Loop normal direction (the loop plane is perpendicular to this).")
- self.lnu2=_dbl(0,.5,.3,3);self.lnu2.setToolTip("Poisson's ratio ν (typical metal: 0.3).")
+ self.lr=_dbl(1,500,20,1," A");self.lb=_dbl(0,50,0,3," A");self.lb.setSpecialValueText("auto");self.ln=QComboBox();self.ln.addItems(["Z","Y","X"]);self.lnu2=_dbl(0,.5,.3,3)
lg.addWidget(QLabel("R:"),1,0,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.lr,1,1);lg.addWidget(QLabel("|b|:"),1,2,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.lb,1,3);lg.addWidget(QLabel("N:"),1,4,Qt.AlignmentFlag.AlignRight);lg.addWidget(self.ln,1,5)
self.lf.setVisible(False);dv.addWidget(self.lf);self.lck.toggled.connect(self.lf.setVisible)
ll.addWidget(db)
stb=QGroupBox("Analytical strain");stf=QFormLayout(stb);stf.setLabelAlignment(Qt.AlignmentFlag.AlignRight);stf.setVerticalSpacing(4)
- self.strc=QComboBox();self.strc.addItems(["none","radial_gradient","edge_dislocation","random"])
- self.strc.setToolTip(
- "Add a parameterized strain field on top of the rigid lattice:\n"
- " • none: no strain (default)\n"
- " • radial_gradient: strain grows from center outward\n"
- " • edge_dislocation: Volterra edge-dislocation field\n"
- " • random: Gaussian random field (smoothed)"
- )
- stf.addRow("Type:",self.strc)
- self.strm=_dbl(0,.1,1e-4,6);self.strm.setToolTip(
- "Peak strain magnitude (dimensionless). Typical BCDI: 1e-4 to 1e-3.\n"
- "0 = no strain."
- )
- stf.addRow("eps:",self.strm);ll.addWidget(stb)
- self.bb=QPushButton("Build & Visualize");self.bb.setStyleSheet("background:#1f6feb;min-height:34px;font-size:11pt")
- self.bb.setToolTip("Build the supercell, apply strain/dislocations, and render the 3D lattice.")
- self.bb.clicked.connect(self._bd);ll.addWidget(self.bb)
- self.pg=QProgressBar();ll.addWidget(self.pg);self.lg=QPlainTextEdit();self.lg.setReadOnly(True);self.lg.setMaximumHeight(65);ll.addWidget(self.lg);ll.addStretch();sc.setWidget(inner);root.addWidget(sc)
+ self.strc=QComboBox();self.strc.addItems(["none","radial_gradient","edge_dislocation","random"]);stf.addRow("Type:",self.strc)
+ self.strm=_dbl(0,.1,1e-4,6);stf.addRow("eps:",self.strm);ll.addWidget(stb)
+ self.bb=QPushButton("Build & Visualize");self.bb.setStyleSheet("background:#1f6feb;min-height:34px;font-size:11pt");self.bb.clicked.connect(self._bd);ll.addWidget(self.bb)
+ self.pg=QProgressBar();ll.addWidget(self.pg);self.lg=QPlainTextEdit();self.lg.setReadOnly(True);self.lg.setMaximumHeight(65);ll.addWidget(self.lg);ll.addStretch();sc.setWidget(inner);split.addWidget(sc)
right=QWidget();rv=QVBoxLayout(right);rv.setContentsMargins(0,0,0,0);rv.addWidget(QLabel("3D Lattice"))
if _HAS_WEB: self.viz=QWebEngineView();self.viz.setHtml(_ph("Build lattice."))
else: self.viz=QLabel("Need PyQt6-WebEngine")
- self.viz.setMinimumHeight(500);rv.addWidget(self.viz,1);root.addWidget(right,1);self._pc(self.combo.currentText());self._us()
+ self.viz.setMinimumHeight(400);rv.addWidget(self.viz,1);split.addWidget(right)
+ split.setStretchFactor(0,0);split.setStretchFactor(1,1);split.setSizes([480,1000]);root.addWidget(split)
+ self._pc(self.combo.currentText());self._us()
def _load_cif(self):
path,_=QFileDialog.getOpenFileName(self,"Load CIF","","CIF files (*.cif);;All (*)")
if not path: return
@@ -233,12 +165,23 @@ def _mt(self):
r=find_preset(q)
if r: self.combo.setCurrentText(r)
else: QMessageBox.information(self,"No match",f"No preset for '{q}'. Try Load .cif")
+ def _from_nm(self):
+ try:
+ n=cells_for_size_nm(self.combo.currentText(),self.size_nm.value())
+ for w,v in zip((self.nx,self.ny,self.nz),n): w.blockSignals(True);w.setValue(v);w.blockSignals(False)
+ self._us()
+ except Exception as e: self.lg.appendPlainText(f"size: {e}")
+ def _shape_changed(self,t):
+ on=(t=='wulff')
+ for w in (self.fac_on,self.fac_lbl): w.setEnabled(on)
+ self.wulff.setEnabled(on and self.fac_on.isChecked())
def _us(self):
try:
cfg=BCDIConfig(self.combo.currentText());cfg.SUPERCELL_MULT=(self.nx.value(),self.ny.value(),self.nz.value())
sz=cfg.particle_size_nm;nc=self.nx.value()*self.ny.value()*self.nz.value();na=nc*len(cfg.BASIS_COORDS)
f=lambda v:f"{v:.1f}nm" if v<100 else f"{v/1000:.3f}um"
- self.sl.setText(f"-> {f(sz[0])} x {f(sz[1])} x {f(sz[2])} {nc:,} cells ~{na:,} atoms")
+ w="" if na<2_000_000 else (" [heavy: minutes + GBs on most machines]" if na<2e7 else " [very heavy: expect long build times]")
+ self.sl.setText(f"-> {f(sz[0])} x {f(sz[1])} x {f(sz[2])} {nc:,} cells ~{na:,} atoms{w}")
except: self.sl.setText("")
def _bd(self):
n=self.combo.currentText()
@@ -251,6 +194,11 @@ def _bd(self):
if self.lck.isChecked():
lbv=self.lb.value() if self.lb.value()>0 else None
cfg.DISLOCATION_LOOP=DislocationLoopConfig(center_frac=(self.lcx.value(),self.lcy.value(),self.lcz.value()),radius_angstrom=self.lr.value(),b_angstrom=lbv,normal=self.ln.currentText(),nu=self.lnu2.value())
+ cfg.WULFF_RATIO=self.wulff.value() if self.fac_on.isChecked() else 1.15
+ na=cfg.SUPERCELL_MULT[0]*cfg.SUPERCELL_MULT[1]*cfg.SUPERCELL_MULT[2]*len(cfg.BASIS_COORDS)
+ if na>10_000_000 and QMessageBox.question(self,"Large supercell",
+ f"~{na:,} atoms.\nBuild time and memory scale with this and depend on your machine.\nContinue?"
+ )!=QMessageBox.StandardButton.Yes: return
self.config=cfg;self.bb.setEnabled(False);self.nb.setEnabled(False);self.pg.setValue(0);self.lg.clear()
self._w=BW(cfg);self._w.progress.connect(self.pg.setValue);self._w.log.connect(self.lg.appendPlainText);self._w.done.connect(self._ok);self._w.failed.connect(self._fl);self._w.start()
def _ok(self,b):
@@ -276,108 +224,47 @@ def _ui(self):
mh=QHBoxLayout();mh.setSpacing(8);lc=QVBoxLayout();lc.setSpacing(4)
# Beam
bb=QGroupBox("X-ray beam");bf=QFormLayout(bb);bf.setLabelAlignment(Qt.AlignmentFlag.AlignRight);bf.setVerticalSpacing(3)
- self.keV=_dbl(.5,100,10,2," keV")
- self.keV.setToolTip(
- "X-ray photon energy in kilo-electron-volts.\n"
- "Common BCDI beamline energies: 7–12 keV.\n"
- "Higher energy → shorter wavelength → tighter 2θ angles."
- )
- self.keV.valueChanged.connect(self._ul);self.keV.valueChanged.connect(lambda:self._rt.start())
- self.ll_=QLabel();self.ll_.setStyleSheet("color:#8b949e;font-size:9pt");self._ul()
- self.pol=_dbl(0,1,1,2)
- self.pol.setToolTip(
- "Polarization factor (0 to 1).\n"
- "1 = full horizontal polarization (typical synchrotron sigma-pol)\n"
- "0 = no polarization correction."
- )
- self.bsz=_dbl(0.1,1000,1,1," um")
- self.bsz.setToolTip(
- "Incident beam size (FWHM) at the sample in micrometers.\n"
- "Typical focused beam: 0.5–3 μm. Affects illumination function\n"
- "in coherent diffraction simulation."
- )
- bf.addRow("Energy:",self.keV);bf.addRow("",self.ll_);bf.addRow("Pol:",self.pol);bf.addRow("Beam size:",self.bsz);lc.addWidget(bb)
+ self.keV=_dbl(.5,100,10,2," keV");self.keV.valueChanged.connect(self._ul);self.keV.valueChanged.connect(lambda:self._rt.start())
+ self.ll_=QLabel();self.ll_.setStyleSheet("color:#8b949e;font-size:9pt");self._ul();self.pol=_dbl(0,1,1,2);self.bsz=_dbl(0.1,1000,1,1," um")
+ self.nph=_Sci(1e9);self.nph.setToolTip("Photons in the simulated exposure. Type it as 1e9, 2.5e12 or a plain number.\n"
+ "Only applied when Poisson noise is ticked in the Results tab.")
+ self.phmode=QComboBox();self.phmode.addItems(["total","peak"])
+ bf.addRow("Energy:",self.keV);bf.addRow("",self.ll_);bf.addRow("Pol:",self.pol);bf.addRow("Beam size:",self.bsz)
+ bf.addRow("Photons:",self.nph);bf.addRow("counted as:",self.phmode);lc.addWidget(bb)
# Detector with preset selector and NX x NY
db=QGroupBox("Detector");dv=QVBoxLayout(db);dv.setSpacing(3)
pr=QHBoxLayout();pr.addWidget(QLabel("Preset:"));self.det_combo=QComboBox()
- self.det_combo.setToolTip(
- "Pre-configured detector. Auto-fills pixel count and pixel size.\n"
- " • Maxipix (ID01): 516×516, 55 μm pixels\n"
- " • Eiger 2M: 1062×1028, 75 μm pixels\n"
- " • Custom: set values manually"
- )
for k in DETECTOR_PRESETS: self.det_combo.addItem(k)
self.det_combo.currentTextChanged.connect(self._det_preset);pr.addWidget(self.det_combo,1);dv.addLayout(pr)
df=QFormLayout();df.setLabelAlignment(Qt.AlignmentFlag.AlignRight);df.setVerticalSpacing(3)
self.det_d=_dbl(.05,10,.5,3," m")
- self.det_d.setToolTip(
- "Sample-to-detector distance in meters.\n"
- "Determines q-space resolution: closer = smaller q range, farther =\n"
- "finer q sampling. Typical BCDI: 0.5–2 m."
- )
# NX x NY row
- pxr=QHBoxLayout();self.det_nx=QSpinBox();self.det_nx.setRange(32,4096);self.det_nx.setValue(128)
- self.det_nx.setToolTip("Number of detector pixels in the horizontal (X) direction.")
- pxr.addWidget(self.det_nx);pxr.addWidget(QLabel("x"));self.det_ny=QSpinBox();self.det_ny.setRange(32,4096);self.det_ny.setValue(128)
- self.det_ny.setToolTip("Number of detector pixels in the vertical (Y) direction.")
- pxr.addWidget(self.det_ny)
+ pxr=QHBoxLayout();self.det_nx=QSpinBox();self.det_nx.setRange(32,4096);self.det_nx.setValue(256)
+ pxr.addWidget(self.det_nx);pxr.addWidget(QLabel("x"));self.det_ny=QSpinBox();self.det_ny.setRange(32,4096);self.det_ny.setValue(256);pxr.addWidget(self.det_ny)
+ for _w in (self.det_nx,self.det_ny):
+ _w.setToolTip("Detector panel in pixels. Editing it by hand also sets the image size below.")
+ _w.valueChanged.connect(self._panel_edited)
self.det_sz=_dbl(1,500,55,1," um")
- self.det_sz.setToolTip(
- "Physical pixel pitch in micrometers.\n"
- "Maxipix: 55 μm. Eiger 2M: 75 μm.\n"
- "Smaller pixels → finer angular resolution per pixel."
- )
df.addRow("Distance:",self.det_d);df.addRow("Pixels:",pxr);df.addRow("Pixel size:",self.det_sz)
# Rocking: "Steps" label (= number of rocking curve angular positions)
self.rock=_dbl(.0001,1,.00315,5," deg");self.rock.setDecimals(5)
- self.rock.setToolTip(
- "Angular step size between rocking curve points, in degrees.\n"
- "Sets the q-resolution along the rocking direction.\n"
- "Typical: 0.001–0.01°. Smaller = finer rocking sampling but more frames."
- )
self.steps=QSpinBox();self.steps.setRange(8,2048);self.steps.setValue(128)
- self.steps.setToolTip(
- "Number of rocking curve points (i.e., number of detector frames\n"
- "in the scan). Total rocking range = step × steps.\n"
- "Typical: 50–200 for full Bragg peak coverage."
- )
df.addRow("Rock step:",self.rock);df.addRow("Steps:",self.steps)
# Oversampling with tooltip
- self.os=_dbl(1,20,5,1);self.os.setToolTip(
- "Oversampling ratio: simulation grid = OS × Nyquist.\n"
- "Higher = finer q-resolution but slower.\n"
- "Typical BCDI: 3–8. Reconstruction is only possible if OS > 2."
- )
+ self.os=_dbl(1,20,5,1);self.os.setToolTip("Oversampling ratio: simulation grid = OS x Nyquist.\nHigher = finer q-resolution but slower.\nTypical BCDI: 3-8")
os_row=QHBoxLayout();os_row.addWidget(self.os);os_lbl=QLabel("(oversampling)");os_lbl.setStyleSheet("color:#8b949e;font-size:8pt");os_row.addWidget(os_lbl)
df.addRow("OS ratio:",os_row)
+ self.grid=QSpinBox();self.grid.setRange(32,512);self.grid.setValue(256)
+ self.grid.setToolTip("Pixels across the simulated image. More pixels sample each fringe more\n"
+ "finely, so the 2D slices get smoother with real data, not interpolation.\n"
+ "Oversampling follows automatically; override it below if you want.\n"
+ "Cost grows as N^3: 128 ~0.5 s, 256 ~5 s and ~1 GB, 384 ~16 s and ~3.6 GB.")
+ self.grid.valueChanged.connect(self._grid_changed)
+ df.addRow("Image size (px):",self.grid)
dv.addLayout(df);lc.addWidget(db)
# Coherence
cb=QGroupBox("Coherence");cf=QFormLayout(cb);cf.setLabelAlignment(Qt.AlignmentFlag.AlignRight);cf.setVerticalSpacing(3)
- self.sh=_dbl(0,10000,300,1," um")
- self.sh.setToolTip(
- "Horizontal source size (FWHM) at the source point, in micrometers.\n"
- "Affects horizontal transverse coherence at the sample.\n"
- "Typical synchrotron: 100–500 μm. Set to 0 to disable coherence model."
- )
- self.sv=_dbl(0,10000,10,1," um")
- self.sv.setToolTip(
- "Vertical source size (FWHM) at the source point, in micrometers.\n"
- "Affects vertical transverse coherence at the sample.\n"
- "Typical synchrotron: 5–50 μm (smaller than H — synchrotrons are\n"
- "vertically coherent by design)."
- )
- self.sd=_dbl(.1,500,50,1," m")
- self.sd.setToolTip(
- "Source-to-sample distance in meters.\n"
- "Longer distance → better transverse coherence (larger coherence\n"
- "length at the sample). Typical: 30–80 m."
- )
- self.bw=_dbl(1e-6,.1,1e-4,6);self.bw.setDecimals(6)
- self.bw.setToolTip(
- "Relative energy bandwidth ΔE/E of the monochromator.\n"
- "Sets the longitudinal coherence length.\n"
- "Si(111) monochromator: ~1.3×10⁻⁴. Multilayer: 10⁻² to 10⁻³."
- )
+ self.sh=_dbl(0,10000,300,1," um");self.sv=_dbl(0,10000,10,1," um");self.sd=_dbl(.1,500,50,1," m");self.bw=_dbl(1e-6,.1,1e-4,6);self.bw.setDecimals(6)
cf.addRow("Src H:",self.sh);cf.addRow("Src V:",self.sv);cf.addRow("Src D:",self.sd);cf.addRow("dE/E:",self.bw);lc.addWidget(cb);lc.addStretch();mh.addLayout(lc)
# Center: reflection chart
cc=QVBoxLayout();cc.setSpacing(4);cc.addWidget(QLabel("Reflection intensities"))
@@ -390,13 +277,48 @@ def _ui(self):
self.rt.setMinimumWidth(320);rc.addWidget(self.rt,1)
hr=QHBoxLayout();hr.addWidget(QLabel("hkl:"));self.he=QLineEdit();self.he.setPlaceholderText("e.g. 1 1 1");hr.addWidget(self.he,1);rc.addLayout(hr)
# Custom angle input
- ar=QHBoxLayout();ar.addWidget(QLabel("Custom 2th:"));self.custom_angle=_dbl(0,180,0,3," deg");self.custom_angle.setSpecialValueText("auto");ar.addWidget(self.custom_angle,1);rc.addLayout(ar)
+ ar=QHBoxLayout();ar.addWidget(QLabel("Custom 2th:"))
+ self.custom_angle=_dbl(0,180,0,3," deg");self.custom_angle.setSpecialValueText("auto")
+ self.custom_angle.setToolTip("Park the detector away from the exact Bragg angle.\n"
+ "The peak moves off centre and the recorded intensity falls, exactly as it does\n"
+ "when you step off a rocking curve. Leave at 0 to sit on the Bragg condition.")
+ self.custom_angle.valueChanged.connect(self._ang_info);ar.addWidget(self.custom_angle,1);rc.addLayout(ar)
+ self.ang_lbl=QLabel("");self.ang_lbl.setWordWrap(True);self.ang_lbl.setStyleSheet("color:#8b949e;font-size:9pt");rc.addWidget(self.ang_lbl)
mh.addLayout(rc,1);root.addLayout(mh,1)
self.sb_=QPushButton("Simulate");self.sb_.setStyleSheet("background:#1f6feb;min-height:34px;font-size:11pt");self.sb_.setEnabled(False);self.sb_.clicked.connect(self._rn);root.addWidget(self.sb_)
self.pg=QProgressBar();root.addWidget(self.pg);self.lg=QPlainTextEdit();self.lg.setReadOnly(True);self.lg.setMaximumHeight(55);root.addWidget(self.lg)
+ self._grid_changed(self.grid.value())
def _det_preset(self,name):
+ """A preset describes the physical panel; it must not blow up the FFT grid."""
if name in DETECTOR_PRESETS and name!='Custom':
- d=DETECTOR_PRESETS[name];self.det_nx.setValue(d['nx']);self.det_ny.setValue(d['ny']);self.det_sz.setValue(d['pixel_um'])
+ d=DETECTOR_PRESETS[name];self._lock=True
+ self.det_nx.setValue(d['nx']);self.det_ny.setValue(d['ny']);self.det_sz.setValue(d['pixel_um'])
+ self._lock=False
+
+ def _panel_edited(self):
+ if getattr(self,'_lock',False): return
+ self.grid.setValue(min(max(self.det_nx.value(),self.det_ny.value()),self.grid.maximum()))
+
+ def _grid_changed(self,v):
+ # keep the displayed q-window fixed: finer pixels mean finer sampling
+ self.os.setValue(round(v/25.6,1))
+ if hasattr(self,'ang_lbl'): self._ang_info()
+ def _ang_info(self):
+ from bcdi_core import offset_report
+ t=self.custom_angle.value()
+ if self.config is None or t<=0.01 or self._rdf is None or not len(self._rdf):
+ self.ang_lbl.setText("");return
+ near=self._rdf.iloc[(self._rdf['2theta']-t).abs().argsort().iloc[0]]
+ off=t-near['2theta']
+ c=self.config;c.TARGET_OVERSAMPLING=self.os.value();c.DETECTOR_PIXEL_UM=self.det_sz.value()
+ c.SAMPLE_DETECTOR_DISTANCE_M=self.det_d.value();c.SIM_GRID_MAX=self.grid.value()
+ c.DETECTOR_NX=self.det_nx.value();c.DETECTOR_NY=self.det_ny.value()
+ dq,vox,pix,gone=offset_report(c,np.radians(near['theta_B']),off)
+ msg=(f"nearest {near['hkl_display']} at {near['2theta']:.3f} deg -> offset {off:+.3f} deg\n"
+ f"dq = {dq:+.4f} 1/A : peak moves {vox:+.1f} voxels ({pix:+.0f} real pixels)")
+ self.ang_lbl.setText(msg+("\n peak leaves the simulated frame" if gone else ""))
+ self.ang_lbl.setStyleSheet("color:#f0883e;font-size:9pt" if gone else "color:#8b949e;font-size:9pt")
+
def _ul(self): lam=BCDIConfig.HC_KEV_ANG/self.keV.value() if self.keV.value()>0 else 0;self.ll_.setText(f"lam = {lam:.4f} A")
def _uc(self):
if self.config is None: self.cl.setText("Build lattice first.");self.cl.setStyleSheet("color:#f0883e;font-weight:600");self.sb_.setEnabled(False)
@@ -418,7 +340,7 @@ def _cr(self):
bcdi=df[df['BCDI_flag']]
if len(bcdi): self.rt.selectRow(bcdi.index[0])
elif len(df): self.rt.selectRow(0)
- self._dc(df)
+ self._dc(df);self._ang_info()
except Exception as e: self.lg.appendPlainText(f"Refl: {e}")
def _dc(self,df):
with matplotlib.rc_context(MPL_DARK):
@@ -443,8 +365,9 @@ def _rn(self):
if self.config is None: return
hkl=self._gh()
if hkl=='BAD': return
- c=self.config;c.BEAM_ENERGY_KEV=self.keV.value();c.POLARIZATION_FACTOR=self.pol.value();c.BEAM_SIZE_UM=self.bsz.value()
- c.SAMPLE_DETECTOR_DISTANCE_M=self.det_d.value();c.DETECTOR_NX=self.det_nx.value();c.DETECTOR_NY=self.det_ny.value()
+ c=self.config;c.BEAM_ENERGY_KEV=self.keV.value();c.POLARIZATION_FACTOR=self.pol.value();c.BEAM_SIZE_UM=self.bsz.value();c.N_PHOTONS=self.nph.value();c.PHOTON_MODE=self.phmode.currentText()
+ c.DET_COUNTING=bool(DETECTOR_PRESETS.get(self.det_combo.currentText(),{}).get('counting',True))
+ c.SAMPLE_DETECTOR_DISTANCE_M=self.det_d.value();c.DETECTOR_NX=self.det_nx.value();c.DETECTOR_NY=self.det_ny.value();c.SIM_GRID_MAX=self.grid.value()
c.DETECTOR_PIXEL_UM=self.det_sz.value();c.ROCKING_STEP_DEG=self.rock.value();c.ROCKING_STEPS=self.steps.value()
c.SOURCE_SIZE_H_UM=self.sh.value();c.SOURCE_SIZE_V_UM=self.sv.value();c.SOURCE_DISTANCE_M=self.sd.value();c.MONOCHR_BANDWIDTH=self.bw.value();c.TARGET_OVERSAMPLING=self.os.value()
self.lg.clear()
@@ -461,6 +384,11 @@ def _rn(self):
c.ANGULAR_OFFSET_DEG=0.0
else:
c.ANGULAR_OFFSET_DEG=0.0
+ if c.SIM_GRID_MAX>320 and QMessageBox.question(self,"Large image",
+ f"{c.SIM_GRID_MAX} px needs roughly {(c.SIM_GRID_MAX/384)**3*16:.0f} s and "
+ f"{(c.SIM_GRID_MAX/384)**3*3.6:.1f} GB on a typical machine.\nContinue?"
+ )!=QMessageBox.StandardButton.Yes:
+ self.sb_.setEnabled(True);return
self.sb_.setEnabled(False);self.nb.setEnabled(False);self.pg.setValue(0);self.sim_req.emit({'config':c,'builder':self.builder,'hkl':hkl})
# ======================= TAB 3 =======================
@@ -470,8 +398,16 @@ def _ui(self):
root=QVBoxLayout(self);root.setContentsMargins(12,12,12,12);root.setSpacing(6)
h=QHBoxLayout();h.addWidget(QLabel("Diffraction Results"));h.itemAt(0).widget().setStyleSheet("color:#4f98a3;font-size:13pt;font-weight:700")
h.addSpacing(15);h.addWidget(QLabel("Effects:"));h.itemAt(2).widget().setStyleSheet("color:#8b949e;font-size:9pt")
- self.np_=QCheckBox("Poisson");self.nr=QCheckBox("Readout");self.na=QCheckBox("Air");self.nd=QCheckBox("Dead")
- for c in(self.np_,self.nr,self.na,self.nd): h.addWidget(c)
+ self.np_=QCheckBox("Poisson")
+ self.np_.setToolTip("Counting statistics for the photon budget set in the Beam tab.\n"
+ "Leave off to see the ideal pattern: weak fringes survive only at high photon counts.")
+ self.nr=QCheckBox("Readout");self.nr.setToolTip("Integrating (CCD/CMOS) detectors only. Photon-counting "
+ "detectors (EIGER, PILATUS, MAXIPIX, MERLIN, TIMEPIX) have no read noise.")
+ self.na=QCheckBox("Air");self.nd=QCheckBox("Dead px");self.nh=QCheckBox("Hot px")
+ for c in(self.np_,self.nr,self.na,self.nd,self.nh): h.addWidget(c)
+ h.addSpacing(12);h.addWidget(QLabel("Frame:"));h.itemAt(h.count()-1).widget().setStyleSheet("color:#8b949e;font-size:9pt")
+ self.frame=QComboBox();self.frame.addItems(["sample (lab, orthogonal q)","detector (raw, skewed)"])
+ self.frame.setMaximumWidth(190);self.frame.currentTextChanged.connect(lambda _:self._rr());h.addWidget(self.frame)
self.rb=QPushButton("Re-render");self.rb.setEnabled(False);self.rb.setStyleSheet("background:#4f98a3;padding:4px 10px;min-height:20px;font-size:9pt");self.rb.clicked.connect(self._rr);h.addWidget(self.rb)
h.addStretch();root.addLayout(h)
self.il=QLabel("Run simulation.");self.il.setStyleSheet("color:#8b949e;font-size:9pt");root.addWidget(self.il)
@@ -480,7 +416,7 @@ def _ui(self):
else: self.bv=QLabel("No WebEngine")
self.bv.setMinimumWidth(520);ll.addWidget(self.bv,1);sp.addWidget(lp)
rp=QWidget();rl=QVBoxLayout(rp);rl.setContentsMargins(0,0,0,0);rl.setSpacing(4);rl.addWidget(QLabel("2D Slices"))
- with matplotlib.rc_context(MPL_DARK): self.f1=Figure(figsize=(5,3.4),dpi=150,tight_layout=True);self.f2=Figure(figsize=(5,3.4),dpi=150,tight_layout=True)
+ with matplotlib.rc_context(MPL_DARK): self.f1=Figure(figsize=(5,3.4),dpi=200,tight_layout={'pad':0.4});self.f2=Figure(figsize=(5,3.4),dpi=200,tight_layout={'pad':0.4})
self.c1=FigureCanvas(self.f1);self.c2=FigureCanvas(self.f2)
for c in(self.c1,self.c2):
c.setMinimumHeight(230);fr=QFrame();fr.setFrameShape(QFrame.Shape.StyledPanel);fl=QVBoxLayout(fr);fl.setContentsMargins(0,0,0,0)
@@ -492,12 +428,19 @@ def _ui(self):
self.exp_btn.setStyleSheet("background:#1f6feb;padding:8px 20px;min-height:28px");self.exp_btn.clicked.connect(self._export);eb.addWidget(self.exp_btn)
eb.addStretch();root.addLayout(eb)
def _gn(self):
- o={}
- if self.np_.isChecked(): o['poisson']=True
+ """None when nothing is ticked, so the ideal pattern is shown untouched."""
+ c=self._c;o={}
+ if self.np_.isChecked():
+ o['poisson']=True
+ o['n_photons']=getattr(c,'N_PHOTONS',0) if c is not None else 0
+ o['photon_mode']=getattr(c,'PHOTON_MODE','total') if c is not None else 'total'
if self.nr.isChecked(): o['readout_noise']=5.0
if self.na.isChecked(): o['air_scatter']=100.0
if self.nd.isChecked(): o['dead_pixels_frac']=0.002
- return o or None
+ if self.nh.isChecked(): o['hot_pixels_frac']=0.0005
+ if not o: return None
+ o.setdefault('poisson',False);o['counting']=bool(getattr(c,'DET_COUNTING',True))
+ return o
def _rr(self):
if self._s: self._di(self._s,self._r,self._c,self._gn())
def _export(self):
@@ -505,433 +448,56 @@ def _export(self):
path,_=QFileDialog.getSaveFileName(self,"Export measurement","bcdi_measurement","NumPy (*.npz);;HDF5 (*.h5)")
if not path: return
try:
+ self._c.Q_FRAME='det' if self.frame.currentIndex()==1 else 'lab'
export_measurement(self._s,self._r,self._c,path,self._gn());QMessageBox.information(self,"Exported",f"Saved to {path}")
except Exception as e: QMessageBox.critical(self,"Export Error",str(e))
def display(self,sim,refl,cfg): self._s=sim;self._r=refl;self._c=cfg;self.rb.setEnabled(True);self.exp_btn.setEnabled(True);self._di(sim,refl,cfg,self._gn())
def _di(self,sim,refl,cfg,no):
- self.il.setText(f"{cfg.MATERIAL_NAME} {refl['hkl_str']} lam={cfg.wavelength_angstrom:.3f}A 2th={np.degrees(refl['two_theta_rad']):.3f} beam={cfg.BEAM_SIZE_UM:.1f}um det={cfg.DETECTOR_NX}x{cfg.DETECTOR_NY}")
+ fl='detector frame (raw, skewed)' if self.frame.currentIndex()==1 else 'sample / lab frame (orthogonal q)'
+ self.nr.setEnabled(not bool(getattr(cfg,'DET_COUNTING',True)))
+ if not self.nr.isEnabled(): self.nr.setChecked(False)
+ self.il.setText(f"{cfg.MATERIAL_NAME} {refl['hkl_str']} lam={cfg.wavelength_angstrom:.3f}A "
+ f"2th={np.degrees(refl['two_theta_rad']):.3f} det={cfg.DETECTOR_NX}x{cfg.DETECTOR_NY} grid={cfg.DETECTOR_N_PIXELS}^3 "
+ f"photons={getattr(cfg,'N_PHOTONS',0):.3g} | {fl}")
+ vol=sim.diff_volume.copy()
+ if no: vol=add_experimental_noise(vol,**no)
+ vol,_=to_frame(sim,'det' if self.frame.currentIndex()==1 else 'lab',vol)
if _HAS_WEB:
try:
- html=bragg_html_with_controls(sim,2,1e-4,no)
+ html=bragg_html_with_controls(sim,2,1e-4,None,vol=vol,frame_label=fl)
if html: f=tempfile.NamedTemporaryFile('w',suffix='.html',delete=False,encoding='utf-8');f.write(html);f.close();self.bv.load(QUrl.fromLocalFile(f.name))
except Exception as e: print(f"3D: {e}")
- vol=sim.diff_volume.copy()
- if no: vol=add_experimental_noise(vol,**no)
qx=sim.q_grids['qx'];qy=sim.q_grids['qy'];qz=sim.q_grids['qz'];N=vol.shape[0];cn=N//2;lmx=np.log10(vol.max()+1.);lmn=max(0,lmx-6)
- # Up-sample slice data with cubic spline interpolation BEFORE imshow.
- # The simulation cube has unequal effective sampling per axis (the
- # rocking direction may have fewer real samples than the detector
- # axes), so a raw imshow shows blocky stripes. Zooming the 2D slice
- # 4x with cubic interpolation gives a visually clean rendering
- # without changing the underlying physics.
- from scipy.ndimage import zoom as _zoom
- zoom_factor = 4
for fig,title,data,qh,qv,xl,yl in [(self.f1,"qx-qy",vol[:,:,cn],qx,qy,"qx","qy"),(self.f2,"qx-qz",vol[:,cn,:],qx,qz,"qx","qz")]:
with matplotlib.rc_context(MPL_DARK):
- fig.clear();ax=fig.add_subplot(111);ax.set_facecolor('#000005')
- # Cubic interpolation in linear space first, then take log
- data_smooth = _zoom(data, zoom_factor, order=3, mode='nearest')
- # Clip to non-negative (cubic can over/undershoot)
- data_smooth = np.maximum(data_smooth, 0)
- ld=np.log10(data_smooth.T+1.)
- im=ax.imshow(ld,origin='lower',cmap='jet',aspect='auto',extent=[qh[0],qh[-1],qv[0],qv[-1]],vmin=lmn,vmax=lmx,interpolation='bilinear')
+ fig.clear();ax=fig.add_subplot(111);ax.set_facecolor('#000005');ld=np.log10(data.T+1.)
+ im=ax.imshow(ld,origin='lower',cmap='jet',aspect='auto',extent=[qh[0],qh[-1],qv[0],qv[-1]],vmin=lmn,vmax=lmx,interpolation='nearest')
tA=cfg.particle_size_angstrom;dq=2*np.pi/np.mean(tA[:2]);qz_=8*dq;ax.set_xlim(-qz_,qz_);ax.set_ylim(-qz_,qz_)
- div=make_axes_locatable(ax);cax=div.append_axes('right',size='4%',pad=.05);fig.colorbar(im,cax=cax)
- ax.set_title(title,fontsize=9);ax.set_xlabel(xl,fontsize=8);ax.set_ylabel(yl,fontsize=8);ax.tick_params(labelsize=7)
+ div=make_axes_locatable(ax);cax=div.append_axes('right',size='2.5%',pad=.035)
+ cb=fig.colorbar(im,cax=cax);cb.set_label('intensity arb.',fontsize=6);cb.ax.tick_params(labelsize=5)
+ ax.set_title(f'{title}',fontsize=7)
+ ax.set_xlabel(xl+' (1/A)',fontsize=6);ax.set_ylabel(yl+' (1/A)',fontsize=6);ax.tick_params(labelsize=5)
self.c1.draw();self.c2.draw()
-# ======================= REPORTS & SUGGESTIONS =======================
-class ReportsDialog(QWidget):
- """Small dialog: user enters email and message, sent to saidisoufiane@hotmail.com"""
- RECIPIENT = "saidisoufiane@hotmail.com"
- def __init__(self,parent=None):
- super().__init__(parent,Qt.WindowType.Dialog)
- self.setWindowTitle("Reports & Suggestions");self.setFixedSize(480,360);self.setStyleSheet(QSS)
- v=QVBoxLayout(self);v.setContentsMargins(16,16,16,16);v.setSpacing(10)
- title=QLabel("Reports & Suggestions");title.setStyleSheet("color:#4f98a3;font-size:14pt;font-weight:700");v.addWidget(title)
- info=QLabel(""
- "Both fields are required. "
- "Your default mail client will open with the message ready to send."
- "")
- info.setWordWrap(True);v.addWidget(info)
- v.addWidget(QLabel("Your email:"));self.email_le=QLineEdit();self.email_le.setPlaceholderText("you@example.com");v.addWidget(self.email_le)
- v.addWidget(QLabel("Message:"));self.msg_te=QPlainTextEdit();self.msg_te.setPlaceholderText("Describe the issue or suggestion...");self.msg_te.setMinimumHeight(140);v.addWidget(self.msg_te,1)
- self.status=QLabel("");self.status.setWordWrap(True);self.status.setStyleSheet("font-size:9pt");v.addWidget(self.status)
- br=QHBoxLayout();br.addStretch()
- cancel=QPushButton("Cancel");cancel.setStyleSheet("background:#30363d;padding:7px 14px");cancel.clicked.connect(self.close);br.addWidget(cancel)
- send=QPushButton("Send");send.setStyleSheet("background:#238636;padding:7px 18px;font-weight:600");send.clicked.connect(self._send);br.addWidget(send)
- v.addLayout(br)
- def _validate_email(self,e):
- import re
- return bool(re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$",e))
- def _send(self):
- email=self.email_le.text().strip();msg=self.msg_te.toPlainText().strip()
- if not email:
- self.status.setText("Email is required.");return
- if not self._validate_email(email):
- self.status.setText("Please enter a valid email address.");return
- if not msg:
- self.status.setText("Message is required.");return
- # Open the user's default mail client via mailto:
- import urllib.parse,webbrowser
- subject=urllib.parse.quote(f"BCDI Software - Report from {email}")
- body=urllib.parse.quote(f"From: {email}\n\n{msg}")
- url=f"mailto:{self.RECIPIENT}?subject={subject}&body={body}"
- try:
- webbrowser.open(url,new=1)
- self.status.setText(
- f"\u2713 Opening your mail client. "
- f"Click 'Send' there to deliver the message to {self.RECIPIENT}."
- )
- QTimer.singleShot(2500,self.close)
- except Exception as e:
- # Fallback: copy mailto URL to clipboard
- try:
- cb=QApplication.clipboard();cb.setText(f"To: {self.RECIPIENT}\nFrom: {email}\n\n{msg}")
- self.status.setText(
- f"Could not open mail client automatically. "
- f"The message has been copied to clipboard \u2014 please paste it into "
- f"an email to {self.RECIPIENT}."
- )
- except Exception as e2:
- self.status.setText(f"Failed: {e2}")
-
-
-def _make_reports_button(parent):
- """Create a small 'Reports & Suggestions' button anchored bottom-right.
- Note: Qt treats `&` in button labels as a keyboard accelerator marker
- (the next character becomes Alt+letter). To display a literal `&`,
- we use `&&`."""
- btn=QPushButton("Reports && Suggestions",parent)
- # 60% smaller than the previous version: smaller padding, smaller font, smaller height
- btn.setStyleSheet(
- "background:#1f6feb;color:#fff;padding:1px 6px;border-radius:3px;"
- "font-size:7pt;font-weight:600;border:none"
- )
- btn.setToolTip("Send feedback or report a bug")
- btn.setFixedHeight(16)
- btn.adjustSize()
- btn._dialog=None
- def _open():
- if btn._dialog is None or not btn._dialog.isVisible():
- btn._dialog=ReportsDialog(parent)
- btn._dialog.show();btn._dialog.raise_();btn._dialog.activateWindow()
- btn.clicked.connect(_open)
- return btn
-
-
-# ======================= SIMULATION WINDOW =======================
-class MW_Sim(QMainWindow):
- """Simulation-only window: Material, Beam, Results tabs (no numbering)."""
- def __init__(self,parent=None):
- super().__init__(parent);self.setWindowTitle("BCDI Simulation");self.resize(1500,960);self.setMinimumSize(1200,800)
- central=QWidget();vl=QVBoxLayout(central);vl.setContentsMargins(0,0,0,0);vl.setSpacing(0)
- self.tabs=QTabWidget();self.tabs.setDocumentMode(True)
- self.t1=T1();self.t2=T2();self.t3=T3()
- self.tabs.addTab(self.t1,"Material");self.tabs.addTab(self.t2,"Beam");self.tabs.addTab(self.t3,"Results")
- self.tabs.setTabEnabled(1,False);self.tabs.setTabEnabled(2,False)
- vl.addWidget(self.tabs,1)
- self.setCentralWidget(central)
- sb=QStatusBar();self.setStatusBar(sb);sb.showMessage("Build a lattice to begin.")
+# ======================= MAIN =======================
+class MW(QMainWindow):
+ def __init__(self):
+ super().__init__();self.setWindowTitle("BCDI Simulator v6.0");self.resize(1500,960);self.setMinimumSize(1200,800)
+ self.tabs=QTabWidget();self.tabs.setDocumentMode(True);self.t1=T1();self.t2=T2();self.t3=T3()
+ self.tabs.addTab(self.t1," 1 Material ");self.tabs.addTab(self.t2," 2 Beam ");self.tabs.addTab(self.t3," 3 Results ")
+ self.tabs.setTabEnabled(1,False);self.tabs.setTabEnabled(2,False);self.setCentralWidget(self.tabs)
+ sb=QStatusBar();self.setStatusBar(sb);sb.showMessage("Build lattice first.")
self.t1.built.connect(self._ob);self.t1.nb.clicked.connect(lambda:self.tabs.setCurrentIndex(1))
- self.t2.sim_req.connect(self._os);self.t2.nb.clicked.connect(lambda:self.tabs.setCurrentIndex(2))
- self._sw=None
- # Reports button bottom-right
- self.report_btn=_make_reports_button(self)
- self._reposition_reports()
- def resizeEvent(self,e):
- super().resizeEvent(e);self._reposition_reports()
- def _reposition_reports(self):
- if hasattr(self,'report_btn'):
- margin=10;sb_h=self.statusBar().height() if self.statusBar() else 0
- self.report_btn.move(self.width()-self.report_btn.width()-margin,
- self.height()-self.report_btn.height()-sb_h-margin)
- self.report_btn.raise_()
+ self.t2.sim_req.connect(self._os);self.t2.nb.clicked.connect(lambda:self.tabs.setCurrentIndex(2));self._sw=None
def _ob(self,c,b): self.t2.sc(c,b);self.tabs.setTabEnabled(1,True)
def _os(self,p):
self._sw=SW(p['config'],p['builder'],p['hkl']);self._sw.progress.connect(self.t2.pg.setValue);self._sw.log.connect(self.t2.lg.appendPlainText)
self._sw.done.connect(self._sd);self._sw.failed.connect(self._sf);self._sw.start()
def _sd(self,s,r,d): self.t2.sb_.setEnabled(True);self.t2.nb.setEnabled(True);self.t3.display(s,r,s.config);self.tabs.setTabEnabled(2,True)
def _sf(self,m): self.t2.sb_.setEnabled(True);self.t2.lg.appendPlainText(f"FAIL: {m}")
-
-
-# ======================= DATA ANALYSIS WINDOW =======================
-class MW_Analysis(QMainWindow):
- """Data analysis window: Generate Data, AutoPhaseNN, CDI NN, Reconstruction, 3D Viewer."""
- def __init__(self,parent=None):
- super().__init__(parent);self.setWindowTitle("BCDI Data Analysis");self.resize(1500,960);self.setMinimumSize(1200,800)
- if not _HAS_NN:
- QMessageBox.critical(self,"Missing module",
- "The data analysis module (nn_gui_tabs) is not available. "
- "Make sure all NN-related files are present.")
- QTimer.singleShot(0,self.close);return
- central=QWidget();vl=QVBoxLayout(central);vl.setContentsMargins(0,0,0,0);vl.setSpacing(0)
- self.tabs=QTabWidget();self.tabs.setDocumentMode(True)
- self.tg=T_Gen();self.t4=T4();self.t4s=T4_Sup();self.t5=T5();self.t6=T6()
- self.tabs.addTab(self.tg,"Generate Data")
- self.tabs.addTab(self.t4,"AutoPhase_NN")
- self.tabs.addTab(self.t4s,"CDI_NN")
- self.tabs.addTab(self.t5,"Reconstruction")
- self.tabs.addTab(self.t6,"3D Viewer")
- self.t5.recon_done.connect(self.t6.set_result)
- vl.addWidget(self.tabs,1)
- self.setCentralWidget(central)
- sb=QStatusBar();self.setStatusBar(sb);sb.showMessage("Generate training data, train models, then run reconstruction.")
- # Reports button bottom-right
- self.report_btn=_make_reports_button(self)
- self._reposition_reports()
- def resizeEvent(self,e):
- super().resizeEvent(e);self._reposition_reports()
- def _reposition_reports(self):
- if hasattr(self,'report_btn'):
- margin=10;sb_h=self.statusBar().height() if self.statusBar() else 0
- self.report_btn.move(self.width()-self.report_btn.width()-margin,
- self.height()-self.report_btn.height()-sb_h-margin)
- self.report_btn.raise_()
-
-
-# ======================= LAUNCHER (small first window) =======================
-class Launcher(QMainWindow):
- """Launcher window with the CDI-ST logo + two mode buttons + Exit."""
- def __init__(self):
- super().__init__()
- self.setWindowTitle("CDI-ST")
- # Sized so the 678x340 logo fits at a reasonable scale
- self.setFixedSize(560, 540)
- self.sim_window = None
- self.analysis_window = None
- central = QWidget()
- central.setStyleSheet("background:#0d1117")
- v = QVBoxLayout(central)
- v.setContentsMargins(20, 16, 20, 14)
- v.setSpacing(12)
- # Logo image at the top (matches splash screen)
- logo = _logo_path()
- if logo is not None:
- pix = QPixmap(logo)
- if not pix.isNull():
- # Scale so it fits in the launcher width comfortably
- pix = pix.scaledToWidth(520, Qt.TransformationMode.SmoothTransformation)
- logo_label = QLabel()
- logo_label.setPixmap(pix)
- logo_label.setAlignment(Qt.AlignmentFlag.AlignCenter)
- logo_label.setStyleSheet("background:transparent")
- v.addWidget(logo_label)
- else:
- self._add_text_header(v)
- else:
- self._add_text_header(v)
- v.addSpacing(6)
- choose = QLabel("Choose a module:")
- choose.setStyleSheet("color:#e6edf3;font-size:10pt;background:transparent")
- v.addWidget(choose)
- # Mode buttons (no tooltips — labels are self-explanatory)
- sim_btn = QPushButton("BCDI Simulation")
- sim_btn.setStyleSheet(
- "background:#238636;color:#fff;padding:14px;border-radius:6px;"
- "font-size:11pt;font-weight:600;text-align:center"
- )
- sim_btn.setMinimumHeight(48)
- sim_btn.clicked.connect(self._open_sim)
- v.addWidget(sim_btn)
- ana_btn = QPushButton("BCDI Data Analysis")
- ana_btn.setStyleSheet(
- "background:#1f6feb;color:#fff;padding:14px;border-radius:6px;"
- "font-size:11pt;font-weight:600;text-align:center"
- )
- ana_btn.setMinimumHeight(48)
- ana_btn.clicked.connect(self._open_analysis)
- v.addWidget(ana_btn)
- v.addSpacing(4)
- # Exit button (smaller, red)
- exit_btn = QPushButton("Exit")
- exit_btn.setStyleSheet(
- "background:#da3633;color:#fff;padding:8px;border-radius:5px;"
- "font-size:10pt;font-weight:600"
- )
- exit_btn.clicked.connect(self._exit_app)
- v.addWidget(exit_btn)
- v.addStretch()
- self.setCentralWidget(central)
- # Software credit bottom-right (small grey)
- self.credit = QLabel("Software developed by Soufiane SAIDI", self)
- self.credit.setStyleSheet(
- "color:#6e7681;font-size:8pt;background:transparent"
- )
- self.credit.adjustSize()
- self._reposition_credit()
- def _add_text_header(self, layout):
- """Fallback header if the logo image is missing — text only."""
- title = QLabel("CDI-ST")
- title.setStyleSheet(
- "color:#e6edf3;font-size:32pt;font-weight:700;"
- "background:transparent;letter-spacing:2px"
- )
- title.setAlignment(Qt.AlignmentFlag.AlignCenter)
- layout.addWidget(title)
- sub = QLabel("Coherent Diffraction Imaging Simulation Tools")
- sub.setStyleSheet("color:#8b949e;font-size:10pt;background:transparent")
- sub.setAlignment(Qt.AlignmentFlag.AlignCenter)
- layout.addWidget(sub)
- def resizeEvent(self, e):
- super().resizeEvent(e)
- self._reposition_credit()
- def _reposition_credit(self):
- if hasattr(self, 'credit'):
- margin = 8
- self.credit.move(
- self.width() - self.credit.width() - margin,
- self.height() - self.credit.height() - margin
- )
- self.credit.raise_()
- def _open_sim(self):
- if self.sim_window is None or not self.sim_window.isVisible():
- self.sim_window = MW_Sim(); self.sim_window.show()
- else:
- self.sim_window.raise_(); self.sim_window.activateWindow()
- def _open_analysis(self):
- if self.analysis_window is None or not self.analysis_window.isVisible():
- self.analysis_window = MW_Analysis(); self.analysis_window.show()
- else:
- self.analysis_window.raise_(); self.analysis_window.activateWindow()
- def _exit_app(self):
- if self.sim_window is not None: self.sim_window.close()
- if self.analysis_window is not None: self.analysis_window.close()
- QApplication.quit()
-
-
-# ======================= SPLASH SCREEN =======================
-def _logo_path():
- """Return the path to the CDI-ST logo, looking in several plausible places."""
- here = os.path.dirname(os.path.abspath(__file__))
- candidates = [
- os.path.join(here, "CDI_ST_logo.png"),
- os.path.join(here, "CDI_NN_logo.png"),
- os.path.join(here, "logo.png"),
- ]
- for c in candidates:
- if os.path.exists(c):
- return c
- return None
-
-
-class CDIStSplash(QSplashScreen):
- """
- Splash screen showing the CDI-ST logo with a loading bar underneath.
- Matches the dark theme of the rest of the application.
- """
- def __init__(self):
- # Build the splash pixmap: logo image + space below for loading bar
- logo = _logo_path()
- if logo is not None:
- base_pix = QPixmap(logo)
- if base_pix.isNull():
- base_pix = self._fallback_pixmap()
- else:
- base_pix = self._fallback_pixmap()
- # Scale to a reasonable splash size if needed
- if base_pix.width() > 700:
- base_pix = base_pix.scaledToWidth(700, Qt.TransformationMode.SmoothTransformation)
- # Create a final pixmap with extra space for the loading bar + status text
- BAR_H = 64 # extra height below the logo for bar + text
- full = QPixmap(base_pix.width(), base_pix.height() + BAR_H)
- full.fill(QColor("#0d1117")) # dark background matching app theme
- p = QPainter(full)
- p.drawPixmap(0, 0, base_pix)
- p.end()
- super().__init__(full, Qt.WindowType.WindowStaysOnTopHint)
- self.setFixedSize(full.size())
- self._logo_h = base_pix.height()
- self._bar_h = BAR_H
- self._progress = 0
- self._status = "Loading..."
- # Loading bar overlay (drawn in drawContents)
- def _fallback_pixmap(self):
- """If the logo file is missing, build a simple text-based banner."""
- pix = QPixmap(640, 280)
- pix.fill(QColor("#0d1117"))
- p = QPainter(pix)
- # Title
- p.setPen(QColor("#e6edf3"))
- f = QFont(); f.setPointSize(48); f.setBold(True); f.setLetterSpacing(QFont.SpacingType.AbsoluteSpacing, 4)
- p.setFont(f)
- p.drawText(QRect(0, 50, 640, 80), Qt.AlignmentFlag.AlignCenter, "CDI-ST")
- # Subtitle
- p.setPen(QColor("#8b949e"))
- f2 = QFont(); f2.setPointSize(13)
- p.setFont(f2)
- p.drawText(QRect(0, 150, 640, 30), Qt.AlignmentFlag.AlignCenter,
- "Coherent Diffraction Imaging Simulation Tools")
- p.end()
- return pix
- def set_progress(self, value, status=""):
- self._progress = max(0, min(100, int(value)))
- if status:
- self._status = status
- self.repaint()
- def drawContents(self, painter):
- # Draw loading bar + status text in the strip below the logo
- w = self.width()
- bar_y_top = self._logo_h + 18
- bar_x = 40
- bar_w = w - 80
- bar_h = 8
- # Track
- painter.setBrush(QBrush(QColor("#161b22")))
- painter.setPen(QPen(QColor("#30363d"), 1))
- painter.drawRoundedRect(bar_x, bar_y_top, bar_w, bar_h, 3, 3)
- # Filled portion
- if self._progress > 0:
- painter.setBrush(QBrush(QColor("#4f98a3")))
- painter.setPen(Qt.PenStyle.NoPen)
- painter.drawRoundedRect(bar_x, bar_y_top, int(bar_w * self._progress / 100.0), bar_h, 3, 3)
- # Status text
- painter.setPen(QColor("#8b949e"))
- f = QFont(); f.setPointSize(9)
- painter.setFont(f)
- painter.drawText(
- QRect(bar_x, bar_y_top + bar_h + 6, bar_w, 18),
- Qt.AlignmentFlag.AlignCenter,
- self._status
- )
-
-
-# ======================= MAIN =======================
def main():
- app = QApplication(sys.argv); app.setStyle("Fusion"); pal = QPalette()
- for r, c in [(QPalette.ColorRole.Window, "#0d1117"),
- (QPalette.ColorRole.WindowText, "#e6edf3"),
- (QPalette.ColorRole.Base, "#0d1117"),
- (QPalette.ColorRole.AlternateBase, "#161b22"),
- (QPalette.ColorRole.Text, "#e6edf3"),
- (QPalette.ColorRole.Button, "#21262d"),
- (QPalette.ColorRole.ButtonText, "#e6edf3"),
- (QPalette.ColorRole.Highlight, "#1f6feb"),
- (QPalette.ColorRole.HighlightedText, "#ffffff")]:
- pal.setColor(r, QColor(c))
- app.setPalette(pal); app.setStyleSheet(QSS)
- # Show splash screen first
- splash = CDIStSplash()
- splash.show(); app.processEvents()
- # Simulate a few-stage loading sequence so the user sees the bar fill
- import time
- stages = [
- (10, "Initializing Qt..."),
- (30, "Loading core modules..."),
- (55, "Loading NN modules..."),
- (80, "Preparing UI..."),
- (100, "Ready"),
- ]
- for pct, msg in stages:
- splash.set_progress(pct, msg)
- app.processEvents()
- time.sleep(0.18)
- # Build and show launcher
- win = Launcher()
- win.show()
- splash.finish(win)
- sys.exit(app.exec())
-
-
-if __name__ == "__main__":
- main()
+ app=QApplication(sys.argv);app.setStyle("Fusion");pal=QPalette()
+ for r,c in[(QPalette.ColorRole.Window,"#0d1117"),(QPalette.ColorRole.WindowText,"#e6edf3"),(QPalette.ColorRole.Base,"#0d1117"),(QPalette.ColorRole.AlternateBase,"#161b22"),(QPalette.ColorRole.Text,"#e6edf3"),(QPalette.ColorRole.Button,"#21262d"),(QPalette.ColorRole.ButtonText,"#e6edf3"),(QPalette.ColorRole.Highlight,"#1f6feb"),(QPalette.ColorRole.HighlightedText,"#ffffff")]:
+ pal.setColor(r,QColor(c))
+ app.setPalette(pal);app.setStyleSheet(QSS);win=MW();win.show();sys.exit(app.exec())
+if __name__=="__main__": main()
diff --git a/src/cdi_st/facets.py b/src/cdi_st/facets.py
new file mode 100644
index 0000000..0d5c6af
--- /dev/null
+++ b/src/cdi_st/facets.py
@@ -0,0 +1,82 @@
+from __future__ import annotations
+import numpy as np
+from itertools import permutations
+from scipy.optimize import linprog
+
+_CUBIC = {'100': (1, 0, 0), '110': (1, 1, 0), '111': (1, 1, 1), '311': (3, 1, 1)}
+_PRISM = [(1, 0, 0), (0, 1, 0), (-1, 1, 0), (-1, 0, 0), (0, -1, 0), (1, -1, 0)]
+_BASAL = [(0, 0, 1), (0, 0, -1)]
+_PYR = [(h, k, l) for (h, k, _) in _PRISM for l in (1, -1)]
+
+# name -> {facet family: relative distance from centre}; larger d shrinks that facet
+SHAPES = {
+ 'cube': {'100': 1.0},
+ 'octahedron': {'111': 1.0},
+ 'cuboctahedron': {'111': 1.0, '100': 0.82},
+ 'trunc_octahedron': {'111': 1.0, '100': 1.10},
+ 'trunc_cube': {'100': 1.0, '111': 0.90},
+ 'rhombic_dodeca': {'110': 1.0},
+ 'wulff': {'111': 1.0, '100': 1.15},
+ 'hex_prism': {'prism': 1.0, 'basal': 1.0},
+ 'hex_bipyramid': {'prism': 1.0, 'pyr': 0.95, 'basal': 1.3},
+ 'hex_trunc': {'prism': 1.0, 'pyr': 1.15, 'basal': 0.95},
+}
+HEX_SHAPES = [k for k, v in SHAPES.items() if 'prism' in v]
+CUBIC_SHAPES = [k for k, v in SHAPES.items() if 'prism' not in v]
+
+
+def _family(fam):
+ if fam == 'prism': return list(_PRISM)
+ if fam == 'basal': return list(_BASAL)
+ if fam == 'pyr': return list(_PYR)
+ g = _CUBIC[fam]; out = set()
+ for p in permutations(g):
+ for s in np.ndindex(2, 2, 2):
+ out.add(tuple(((-1) ** s[i]) * p[i] for i in range(3)))
+ return sorted(out)
+
+
+def normals(shape, B, wulff_ratio=1.15):
+ """Unit facet normals (Cartesian) and relative distances. B rows = reciprocal vectors."""
+ spec = dict(SHAPES[shape])
+ if shape == 'wulff': spec = {'111': 1.0, '100': float(wulff_ratio)}
+ N, D = [], []
+ for fam, d in spec.items():
+ for hkl in _family(fam):
+ n = np.asarray(hkl, float) @ B
+ nn = np.linalg.norm(n)
+ if nn < 1e-9: continue
+ N.append(n / nn); D.append(d)
+ return np.asarray(N), np.asarray(D)
+
+
+def _support(N, D, u):
+ r = linprog(-u, A_ub=N, b_ub=D, bounds=[(None, None)] * 3)
+ return float(-r.fun) if r.success else np.inf
+
+
+def extents(N, D):
+ """Bounding-box extent of the unit polyhedron along x, y, z."""
+ e = np.empty(3)
+ for a in range(3):
+ u = np.zeros(3); u[a] = 1.0
+ e[a] = _support(N, D, u) + _support(N, D, -u)
+ return e
+
+
+def scale_to_size(N, D, size_ang):
+ """Scale factor R so the polyhedron's largest extent equals size_ang."""
+ e = extents(N, D); m = np.nanmax(e[np.isfinite(e)]) if np.isfinite(e).any() else 0.
+ return float(size_ang / m) if m > 0 else 1.0
+
+
+def mask(pos, centre, N, D, R):
+ return np.all((pos - centre) @ N.T <= R * D[None, :], axis=1)
+
+
+def grid_mask(X, Y, Z, N, D, R):
+ """Facet mask on a meshgrid (used for the simulator support function)."""
+ m = np.ones(X.shape, bool)
+ for n, d in zip(N, D):
+ m &= (n[0] * X + n[1] * Y + n[2] * Z) <= R * d
+ return m
diff --git a/src/cdi_st/nn_autophase_infer.py b/src/cdi_st/nn_autophase_infer.py
index 1e4541c..8d643b1 100644
--- a/src/cdi_st/nn_autophase_infer.py
+++ b/src/cdi_st/nn_autophase_infer.py
@@ -1,39 +1,3 @@
-"""
-nn_autophase_infer.py — Inference and hybrid refinement for AutoPhaseNet3D.
-
-Runs the trained unsupervised network on new data (simulated or experimental),
-optionally followed by iterative refinement (ER + RAAR) from your existing
-nn_phase_retrieval module.
-
-Three modes:
- 1. 'nn_only' — just the network forward pass (~100 ms, like AutoPhaseNN Fig 3b)
- 2. 'refined' — NN prediction + short refinement (AutoPhaseNN + your RAAR)
- 3. 'compare' — run all modes and compare quality metrics
-
-Works with:
- - .npz files from your data generator (any sample)
- - .npz files converted from experiment via nn_experimental_loader.py
- - .h5 files directly (ID01 / 34-ID-C format, auto-loaded)
-
-Usage:
- # On a simulated test sample:
- python nn_autophase_infer.py \\
- --input training_data/sample_00123.npz \\
- --model checkpoints_autophase/best_model.pt \\
- --output recon_autophase.npz \\
- --mode refined
-
- # On an experimental .h5 file:
- python nn_autophase_infer.py \\
- --input scan_042.h5 \\
- --model checkpoints_autophase/best_model.pt \\
- --output recon_exp.npz \\
- --mode refined
-
- # Compare NN-only vs refined:
- python nn_autophase_infer.py --input data.npz --model best.pt --mode compare
-"""
-
from __future__ import annotations
import argparse, time
import numpy as np
@@ -66,18 +30,7 @@ class AutoPhaseResult:
def load_input(path: str, target_size: int = 64,
apply_gap_mask: bool = True) -> tuple:
- """
- Load a diffraction volume from .npz or .h5.
-
- Returns (diffraction, truth_dict_or_none, voxel_size_nm_or_none).
- `truth_dict` contains phase_true/support if available (simulated data).
- `voxel_size_nm` is a 3-vector (per axis) if available, else None.
-
- If `apply_gap_mask` and the data has NO ground truth (i.e. it's
- experimental rather than simulated), runs the detector-gap heuristic
- on the loaded volume. This catches multi-chip detector gaps that
- survived an earlier .npz conversion.
- """
+
p = Path(path)
if p.suffix == '.npz':
@@ -141,22 +94,7 @@ def nn_only_infer(
support_threshold: float = 0.05,
support_method: str = 'percentile',
) -> AutoPhaseResult:
- """
- Run the trained AutoPhaseNet3D in pure prediction mode (no refinement).
-
- Improvements vs naive amplitude-threshold support:
- - 'percentile' support method: keeps the top X% of voxels by amplitude,
- which preserves SHAPE (cube stays cube, hexagon stays hexagon) instead
- of forcing roundness through threshold-based truncation.
- - Crystal-aware target support fraction estimated from autocorrelation
- of the measured diffraction (gives realistic crystal volume).
-
- support_method:
- 'percentile' — keep top N voxels (N estimated from autocorrelation)
- Best for general use, preserves shape best.
- 'threshold' — keep voxels above support_threshold * amp.max()
- Original behavior, can clip cubes/hexagons.
- """
+
t0 = time.time()
device = torch.device(device or ('cuda' if torch.cuda.is_available() else 'cpu'))
@@ -339,21 +277,8 @@ def nn_only_infer(
)
-# ═══════════════════════════════════════════════════════════════════════════════
-# Hybrid refinement (NN + RAAR + ER from your existing module)
-# ═══════════════════════════════════════════════════════════════════════════════
-
def _center_object(obj_3d, amp, phase, support, voxel_threshold=0.10):
- """
- Shift the reconstruction so its center-of-mass lands on the grid center.
-
- Phase retrieval algorithms have a translation ambiguity: any spatial
- shift of the object produces the same diffraction magnitude. Without
- explicit centering, the object can end up anywhere in the volume —
- making the central XY/XZ/YZ slices show empty regions.
- Returns the shifted (object_3d, amplitude, phase, support).
- """
from scipy.ndimage import center_of_mass, shift as nd_shift
import numpy as _np
@@ -405,29 +330,7 @@ def refined_infer(
max_support_fraction: float = 0.20,
algorithm: str = 'hybrid',
) -> AutoPhaseResult:
- """
- AutoPhaseNet prediction + iterative refinement (HIO + RAAR + ER).
- Why this works when "RAAR + ER alone" fails on a partial NN solution:
-
- 1. The NN gives a partial reconstruction (R~0.6) that's stuck in a local
- minimum. RAAR alone can't always escape.
-
- 2. HIO (Fienup 1982) is more aggressive: it uses negative feedback outside
- the support, which is much better at escaping local minima than RAAR.
- Standard practice in BCDI is HIO → RAAR → ER, not RAAR alone.
-
- 3. We preserve the NN's amplitude prediction (not just phase) as the
- starting amplitude. This was being thrown away in the previous version.
-
- 4. We track the best iterate by R-factor and return that one, since
- HIO/RAAR oscillate.
-
- Schedule (matches Argonne 34-ID-C standard pipeline):
- n_hio iterations of HIO (default 50) — escape local minima
- n_raar iterations of RAAR (default 50) — smooth refinement
- n_er iterations of ER (default 20) — final polish
- """
t0 = time.time()
# Step 1: NN prediction
@@ -572,15 +475,7 @@ def supervised_nn_infer(
base_channels: int = 32,
device: str = None,
) -> AutoPhaseResult:
- """
- Inference using the SUPERVISED PhaseUNet3D model (predicts phase only,
- using ground-truth phase labels during training).
-
- This is complementary to AutoPhaseNet3D (unsupervised, dual-decoder).
- The supervised model often has better phase prediction inside the
- support but doesn't predict amplitude — we derive amplitude from
- autocorrelation seed.
- """
+
t0 = time.time()
device = torch.device(device or ('cuda' if torch.cuda.is_available() else 'cpu'))
@@ -664,31 +559,7 @@ def ensemble_infer(
n_er: int = 20,
weight_strategy: str = 'auto',
) -> AutoPhaseResult:
- """
- Ensemble two complementary networks:
- - AutoPhaseNet3D (unsupervised, predicts amplitude AND phase)
- - PhaseUNet3D (supervised, predicts phase only)
-
- Then optionally refine with HIO/RAAR/ER.
-
- Strategy:
- 1. Run both networks independently
- 2. Combine: average phases (weighted), use AutoPhaseNet's amplitude
- (since supervised model doesn't predict amplitude well)
- 3. Build initial support from union of both models' supports
- 4. Optionally feed combined object to RAAR refinement
-
- Why this helps: the two models have different biases. AutoPhaseNet sees
- only the diffraction so it learns physics; supervised model sees ground
- truth so it learns shape priors. Their errors are largely independent,
- so averaging reduces variance.
-
- weight_strategy:
- 'auto' — weight by inverse R-factor of each model
- 'equal' — 50/50
- 'autophase' — only use AutoPhaseNet (fallback if supervised fails)
- 'supervised' — only use supervised
- """
+
t0 = time.time()
# Run both models
diff --git a/src/cdi_st/nn_autophase_model.py b/src/cdi_st/nn_autophase_model.py
index 90f3108..20abb4d 100644
--- a/src/cdi_st/nn_autophase_model.py
+++ b/src/cdi_st/nn_autophase_model.py
@@ -1,47 +1,11 @@
-"""
-nn_autophase_model.py — Physics-aware dual-decoder network for BCDI.
-
-Merges AutoPhaseNN (Yao et al., npj Comp Mat 2022) architecture with your
-existing U-Net backbone. Key differences from nn_phase_model.py:
-
- 1. TWO decoder heads — one for amplitude, one for phase
- (previously: one head for phase only)
-
- 2. Physics-enforcing activations:
- - amplitude: sigmoid → [0, 1]
- - phase: tanh × π → [-π, π]
-
- 3. Zero-padding support layer:
- The amplitude is padded to N/2 in each dimension, enforcing the
- oversampling requirement for phase retrieval (Miao 2000).
- This means the reconstructed object is automatically confined
- to half the grid in each direction.
-
- 4. Shape support constraint:
- During the forward model, amplitude is thresholded at 10% max
- to create a dynamic support mask.
-
-Trained with the physics-aware unsupervised loss in nn_autophase_train.py:
-no ground-truth phase or amplitude needed, only diffraction intensity.
-
-Reference:
- Yao et al., "AutoPhaseNN: unsupervised physics-aware deep learning of 3D
- nanoscale Bragg coherent diffraction imaging", npj Comp. Mat. 8, 124 (2022).
-"""
-
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
from typing import Tuple
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# Building blocks (compatible with your nn_phase_model.py conventions)
-# ═══════════════════════════════════════════════════════════════════════════════
-
class ConvBlock3D(nn.Module):
- """Two 3×3×3 convolutions with InstanceNorm + LeakyReLU."""
+
def __init__(self, in_ch: int, out_ch: int):
super().__init__()
self.conv = nn.Sequential(
@@ -58,7 +22,7 @@ def forward(self, x):
class DownBlock(nn.Module):
- """Downsample 2× via strided conv, then ConvBlock3D."""
+
def __init__(self, in_ch: int, out_ch: int):
super().__init__()
self.down = nn.Conv3d(in_ch, in_ch, 2, stride=2, bias=False)
@@ -79,15 +43,8 @@ def forward(self, x, skip):
return self.conv(torch.cat([x, skip], dim=1))
-# ═══════════════════════════════════════════════════════════════════════════════
-# Decoder heads (one per output: amplitude and phase)
-# ═══════════════════════════════════════════════════════════════════════════════
-
class DecoderBranch(nn.Module):
- """
- One decoder branch: takes the bottleneck + skip connections,
- produces a single-channel output with custom final activation.
- """
+
def __init__(self, base_channels: int, activation: str):
super().__init__()
C = base_channels
@@ -121,33 +78,7 @@ def forward(self, bottleneck, s1, s2, s3):
# ═══════════════════════════════════════════════════════════════════════════════
class AutoPhaseNet3D(nn.Module):
- """
- Dual-decoder 3D U-Net predicting both amplitude and phase.
- Input:
- diffraction magnitude (log-normalized), shape [B, 1, N, N, N]
-
- Outputs:
- amplitude, shape [B, 1, N, N, N] ∈ [0, 1]
- phase, shape [B, 1, N, N, N] ∈ [-1, 1] (multiply by π for radians)
-
- Architecture (base_channels=32, N=64):
-
- Input → ConvBlock(32) → DownBlock(64) → DownBlock(128) → Bottleneck(256)
- ↓ ↓ ↓
- s1 s2 s3
- ↓ ↓ ↓
- ┌──── Amp decoder: UpBlocks(128,64,32) → 1×1 conv → sigmoid ──→ amplitude
- │
- └──── Phase decoder: UpBlocks(128,64,32) → 1×1 conv → tanh ───→ phase
-
- Parameters: ~2.3M at base_channels=32
-
- Zero-padding support:
- If enforce_oversampling=True, the amplitude is zeroed outside a central
- N/2 × N/2 × N/2 region. This implements the Miao oversampling condition
- as a hard constraint, matching AutoPhaseNN's zero-padding layers.
- """
def __init__(
self,
@@ -170,12 +101,7 @@ def __init__(
self.phase_decoder = DecoderBranch(C, activation='tanh')
def _zero_pad_mask(self, shape, device):
- """
- Create a central-N/2 support mask (zero-padding layer, AutoPhaseNN §3.1).
- This enforces the oversampling constraint: the real-space object must
- fit inside half the grid in each dimension.
- """
B, _, N1, N2, N3 = shape
mask = torch.zeros((1, 1, N1, N2, N3), device=device)
q1, q2, q3 = N1 // 4, N2 // 4, N3 // 4
@@ -183,17 +109,7 @@ def _zero_pad_mask(self, shape, device):
return mask
def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
- """
- Parameters
- ----------
- x : tensor [B, 1, N, N, N]
- Log-normalized diffraction magnitude.
-
- Returns
- -------
- amplitude : tensor [B, 1, N, N, N] ∈ [0, 1]
- phase : tensor [B, 1, N, N, N] ∈ [-1, 1] (multiply by π for radians)
- """
+
# Shared encoder
s1 = self.enc1(x) # [B, C, N, N, N]
s2 = self.enc2(s1) # [B, 2C, N/2, N/2, N/2]
@@ -212,25 +128,8 @@ def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
return amplitude, phase
-# ═══════════════════════════════════════════════════════════════════════════════
-# Physics forward model (used during training)
-# ═══════════════════════════════════════════════════════════════════════════════
-
class PhysicsForwardModel(nn.Module):
- """
- X-ray scattering forward model (AutoPhaseNN §Methods).
- Given predicted (amplitude, phase), forms the complex object, applies a
- dynamic shape support derived from the amplitude itself, and returns the
- predicted diffraction magnitude.
-
- This module has NO trainable parameters — it purely implements physics:
- ρ(r) = A(r) · exp(i · φ(r)) · S(r)
- F(q) = FFT[ ρ(r) ]
- |F(q)| = |FFT[ ρ(r) ]|
-
- where S(r) is a shape support computed by thresholding A(r) at `threshold`.
- """
def __init__(self, threshold: float = 0.1):
super().__init__()
@@ -241,17 +140,7 @@ def forward(
amplitude: torch.Tensor,
phase: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
- """
- Parameters
- ----------
- amplitude : [B, 1, N, N, N] ∈ [0, 1]
- phase : [B, 1, N, N, N] ∈ [-1, 1]
-
- Returns
- -------
- pred_diff_mag : [B, 1, N, N, N] predicted diffraction magnitude
- shape_support : [B, 1, N, N, N] dynamic support (for regularization)
- """
+
# 1. Dynamic shape support by thresholding amplitude
# Use a soft threshold (sigmoid) so it stays differentiable
amp_max = amplitude.amax(dim=(2, 3, 4), keepdim=True).clamp(min=1e-6)
@@ -276,27 +165,8 @@ def forward(
return pred_diff_mag, shape_support
-# ═══════════════════════════════════════════════════════════════════════════════
-# Unsupervised loss (AutoPhaseNN's MAE on sqrt(intensity))
-# ═══════════════════════════════════════════════════════════════════════════════
-
class UnsupervisedBCDILoss(nn.Module):
- """
- Physics-aware unsupervised loss (Eq. 1 in AutoPhaseNN paper):
-
- Loss = Σ |√I_estimated − √I_measured| / N³
- That is: mean absolute error on the diffraction AMPLITUDE (sqrt of
- intensity). Using sqrt compresses the high dynamic range (the center of
- the Bragg peak can be 10⁶× brighter than the fringes) and gives the
- fringes comparable weight to the peak.
-
- Optional regularizers:
- - support_smoothness : penalty on gradients of the shape support
- (discourages fragmented reconstructions)
- - tv_phase : total-variation on phase inside the support
- (encourages smooth strain fields)
- """
def __init__(
self,
@@ -315,17 +185,7 @@ def forward(
phase: torch.Tensor = None,
shape_support: torch.Tensor = None,
) -> dict:
- """
- Parameters
- ----------
- pred_diff_mag : [B, 1, N, N, N] predicted |F(q)|
- measured_diff_mag : [B, 1, N, N, N] measured |F(q)|
- amplitude, phase, shape_support : optional, for regularizers
-
- Returns
- -------
- dict with 'total', 'mae_diff', and optional regularizer values.
- """
+
# Normalize to match scales (AutoPhaseNN normalizes to max=1)
pred_norm = pred_diff_mag / (pred_diff_mag.amax(dim=(2,3,4), keepdim=True) + 1e-8)
meas_norm = measured_diff_mag / (measured_diff_mag.amax(dim=(2,3,4), keepdim=True) + 1e-8)
diff --git a/src/cdi_st/nn_autophase_train.py b/src/cdi_st/nn_autophase_train.py
index 3d68e7f..e1723db 100644
--- a/src/cdi_st/nn_autophase_train.py
+++ b/src/cdi_st/nn_autophase_train.py
@@ -1,31 +1,3 @@
-"""
-nn_autophase_train.py — Unsupervised training for AutoPhaseNet3D.
-
-Trains the dual-decoder network WITHOUT requiring ground-truth phase or
-support. Only the measured diffraction magnitude is used.
-
-Training loop:
- 1. Input diffraction magnitude → encoder
- 2. Predict amplitude and phase
- 3. Physics forward model: ρ(r) = A·S·e^{iφ} → FFT → |F|_pred
- 4. Loss = MAE(|F|_pred, |F|_measured)
- 5. Backprop through the physics model AND the network
-
-This is a direct implementation of the AutoPhaseNN training procedure
-(Yao et al., npj Comp Mat 2022), adapted to work with your existing
-dataset structure.
-
-Usage:
- # From scratch on simulated data (no phase_true needed):
- python nn_autophase_train.py --data_dir ./training_data --epochs 60
-
- # Fine-tune on experimental data (loaded from .npz or .h5):
- python nn_autophase_train.py \\
- --data_dir ./experimental_data \\
- --resume checkpoints_autophase/best_model.pt \\
- --lr 1e-5 --epochs 30
-"""
-
from __future__ import annotations
import os, sys, time, json, argparse, csv
import numpy as np
@@ -47,19 +19,7 @@
# ═══════════════════════════════════════════════════════════════════════════════
class UnsupervisedBCDIDataset(Dataset):
- """
- Dataset that ONLY loads the measured diffraction magnitude.
-
- Works with:
- - .npz files from nn_data_generator.py (simulated)
- - .npz files exported from .h5 via nn_experimental_loader.py
- - raw .h5 files from ID01 beamline (auto-detected)
-
- Returns a dict with:
- 'input' : log-normalized diffraction magnitude [1, N, N, N]
- 'measured': linear-scale diffraction magnitude [1, N, N, N]
- 'amp_scale': normalization factor for reconstruction
- """
+
def __init__(
self,
diff --git a/src/cdi_st/nn_data_generator.py b/src/cdi_st/nn_data_generator.py
index f57e4d2..fd1f5ec 100644
--- a/src/cdi_st/nn_data_generator.py
+++ b/src/cdi_st/nn_data_generator.py
@@ -1,26 +1,3 @@
-"""
-nn_data_generator.py — Generate training data for NN-accelerated BCDI phase retrieval.
-
-Uses bcdi_core.py to create thousands of (support, phase, diffraction_pattern) triplets
-by randomizing:
- - Material (from MATERIAL_PRESETS)
- - Crystal shape (cube, sphere, cylinder, hexagonal)
- - Supercell size (controls particle size → fringe spacing)
- - Strain type and magnitude
- - Dislocation parameters
- - Reflection (hkl)
-
-Each sample is saved as a compressed .npz file containing:
- - 'amplitude': sqrt(diffraction_intensity), shape [N, N, N]
- - 'phase_true': the ground-truth phase field, shape [N, N, N]
- - 'support': binary support mask, shape [N, N, N]
- - 'diffraction': the full diffraction volume, shape [N, N, N]
- - 'metadata': dict with generation parameters
-
-Usage:
- python nn_data_generator.py --output_dir ./training_data --num_samples 2000 --grid_size 64
-"""
-
from __future__ import annotations
import os, sys, json, argparse, time
import numpy as np
@@ -35,24 +12,17 @@
)
-# ═══════════════════════════════════════════════════════════════════════════════
-# Random parameter samplers
-# ═══════════════════════════════════════════════════════════════════════════════
-
def random_material(rng: np.random.Generator) -> str:
- """Pick a random material from presets."""
names = list(MATERIAL_PRESETS.keys())
return rng.choice(names)
def random_shape(material_name: str, rng: np.random.Generator) -> str:
- """Pick a compatible random shape for the material."""
shapes = compatible_shapes(material_name)
return rng.choice(shapes)
def random_supercell(rng: np.random.Generator, min_n: int = 10, max_n: int = 30) -> tuple:
- """Random supercell multipliers. Keeps aspect ratio reasonable."""
nx = rng.integers(min_n, max_n + 1)
ny = rng.integers(min_n, max_n + 1)
nz = rng.integers(min_n, max_n + 1)
@@ -60,7 +30,6 @@ def random_supercell(rng: np.random.Generator, min_n: int = 10, max_n: int = 30)
def random_strain(rng: np.random.Generator) -> tuple:
- """Random strain type and magnitude."""
strain_types = ['none', 'none', 'radial_gradient', 'edge_dislocation', 'random']
# Weight 'none' more heavily for balanced training
stype = rng.choice(strain_types)
@@ -71,7 +40,6 @@ def random_strain(rng: np.random.Generator) -> tuple:
def random_dislocation(rng: np.random.Generator, lattice_a: float) -> Optional[DislocationConfig]:
- """Randomly decide whether to add a dislocation, and configure it."""
if rng.random() < 0.6: # 60% no dislocation
return None
dtype = rng.choice(['edge', 'screw', 'mixed'])
@@ -87,7 +55,6 @@ def random_dislocation(rng: np.random.Generator, lattice_a: float) -> Optional[D
def random_reflection(reflections_df, rng: np.random.Generator):
- """Pick a reflection, biased toward high-intensity ones."""
if len(reflections_df) == 0:
return None
# Prefer BCDI-flagged reflections
@@ -99,10 +66,6 @@ def random_reflection(reflections_df, rng: np.random.Generator):
return row['hkl']
-# ═══════════════════════════════════════════════════════════════════════════════
-# Core generation function
-# ═══════════════════════════════════════════════════════════════════════════════
-
def generate_single_sample(
sample_id: int,
grid_size: int = 64,
@@ -112,32 +75,6 @@ def generate_single_sample(
randomize_dislocation: bool = True,
randomize_strain: bool = True,
) -> dict:
- """
- Generate one (support, phase, diffraction) triplet.
-
- Parameters
- ----------
- sample_id : int
- Unique identifier for this sample.
- grid_size : int
- Detector grid size N (diffraction volume is NxNxN).
- rng : numpy random generator
- For reproducibility.
- add_noise : bool
- Whether to add experimental noise to some samples.
- fixed_material : str or None
- If set, only use this material (useful for material-specific training).
- randomize_dislocation : bool
- If True, occasionally add a random line dislocation to the sample.
- If False, all samples are dislocation-free (clean crystals).
- randomize_strain : bool
- If True, occasionally apply random strain (radial, edge, random).
- If False, all samples are strain-free (uniform unit cells).
-
- Returns
- -------
- dict with keys: amplitude, phase_true, support, diffraction, metadata
- """
if rng is None:
rng = np.random.default_rng(sample_id)
@@ -251,10 +188,6 @@ def generate_single_sample(
}
-# ═══════════════════════════════════════════════════════════════════════════════
-# Batch generation
-# ═══════════════════════════════════════════════════════════════════════════════
-
def generate_dataset(
output_dir: str,
num_samples: int = 2000,
@@ -266,28 +199,7 @@ def generate_dataset(
randomize_dislocation: bool = True,
randomize_strain: bool = True,
):
- """
- Generate a full training dataset.
-
- Creates output_dir/sample_XXXXX.npz files and a manifest.json.
-
- Parameters
- ----------
- output_dir : str
- Directory to save .npz files.
- num_samples : int
- Total number of samples to generate.
- grid_size : int
- NxNxN grid for each diffraction volume.
- seed : int
- Base random seed for reproducibility.
- fixed_material : str or None
- Lock to one material for focused training.
- add_noise : bool
- Whether to add noise to some samples.
- resume : bool
- Skip already-generated samples.
- """
+
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
@@ -399,9 +311,6 @@ def generate_dataset(
print(f"{'='*60}")
-# ═══════════════════════════════════════════════════════════════════════════════
-# CLI
-# ═══════════════════════════════════════════════════════════════════════════════
if __name__ == '__main__':
parser = argparse.ArgumentParser(
diff --git a/src/cdi_st/nn_dataset.py b/src/cdi_st/nn_dataset.py
index 60b48df..bc91119 100644
--- a/src/cdi_st/nn_dataset.py
+++ b/src/cdi_st/nn_dataset.py
@@ -1,31 +1,3 @@
-"""
-nn_dataset.py — PyTorch Dataset for BCDI phase retrieval training.
-
-Loads the .npz files created by nn_data_generator.py and prepares them
-for training the U-Net phase predictor.
-
-Input to the network:
- - log10(amplitude + 1), normalized to [0, 1] → shape [1, N, N, N]
-
-Target:
- - phase field (ground truth) → shape [1, N, N, N]
- - support mask → shape [1, N, N, N]
-
-Augmentations (applied randomly during training):
- - Random 90° rotations along each axis
- - Random flips along each axis
- - Gaussian noise on the input amplitude
-
-These augmentations exploit the fact that BCDI diffraction patterns have
-symmetry under rotation/flip (Friedel's law in centrosymmetric cases),
-and the physics is invariant to these transformations.
-
-Usage:
- from cdi_st.nn_dataset import BCDIDataset
- ds = BCDIDataset('./training_data', augment=True)
- loader = torch.utils.data.DataLoader(ds, batch_size=8, shuffle=True)
-"""
-
from __future__ import annotations
import os, json
import numpy as np
@@ -36,14 +8,6 @@
class BCDIDataset(Dataset):
- """
- PyTorch Dataset for BCDI (amplitude → phase) training pairs.
-
- Each sample contains:
- input: log-scaled amplitude [1, N, N, N] float32
- target: ground-truth phase [1, N, N, N] float32
- support: binary mask [1, N, N, N] float32
- """
def __init__(
self,
@@ -53,26 +17,6 @@ def __init__(
max_samples: int = None,
grid_size: int = None,
):
- """
- Parameters
- ----------
- data_dir : str
- Directory containing sample_XXXXX.npz files.
- augment : bool
- Apply random augmentations (rotations, flips, noise).
- normalize_phase : bool
- Normalize phase to [-1, 1] range (divide by π).
- max_samples : int or None
- Limit number of samples loaded (for debugging).
- grid_size : int or None
- Expected NxNxN grid size of each sample. If set, samples whose
- shape doesn't match are skipped (with a warning) instead of
- crashing later in the data loader. If None (default), all
- samples are loaded regardless of size — relies on the dataset
- being internally consistent (the data generator enforces this).
- Accepted here primarily for API compatibility with
- UnsupervisedBCDIDataset and the GUI training worker.
- """
self.data_dir = Path(data_dir)
self.augment = augment
self.normalize_phase = normalize_phase
@@ -154,14 +98,6 @@ def _augment(
phase: np.ndarray,
support: np.ndarray,
) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
- """
- Apply physics-respecting augmentations.
-
- Rotations and flips are valid because:
- - The diffraction pattern of a rotated/flipped crystal
- is the rotated/flipped diffraction pattern
- - Phase changes sign under inversion (Friedel's law)
- """
rng = np.random.default_rng()
# Random 90° rotations (0, 1, 2, or 3 times) for each axis pair
@@ -202,30 +138,6 @@ def create_dataloaders(
max_samples: int = None,
seed: int = 42,
) -> Tuple[DataLoader, DataLoader, DataLoader]:
- """
- Create train/val/test dataloaders from a directory of .npz files.
-
- Parameters
- ----------
- data_dir : str
- Directory with sample_XXXXX.npz files.
- batch_size : int
- Batch size for training.
- val_fraction : float
- Fraction of data for validation.
- test_fraction : float
- Fraction of data for testing.
- num_workers : int
- Number of dataloader workers.
- max_samples : int or None
- Limit dataset size.
- seed : int
- Random seed for splitting.
-
- Returns
- -------
- (train_loader, val_loader, test_loader)
- """
# Full dataset (no augmentation for splitting)
full_ds = BCDIDataset(data_dir, augment=False, max_samples=max_samples)
diff --git a/src/cdi_st/nn_demo.py b/src/cdi_st/nn_demo.py
index f9c35e3..bdff313 100644
--- a/src/cdi_st/nn_demo.py
+++ b/src/cdi_st/nn_demo.py
@@ -1,35 +1,11 @@
-"""
-nn_demo.py — End-to-end demonstration of the NN-accelerated BCDI pipeline.
-
-This script ties everything together:
- 1. Generate a small training dataset (50 samples for demo)
- 2. Train the U-Net for a few epochs
- 3. Run phase retrieval in all 3 modes
- 4. Compare results and plot
-
-For a real workflow, you would:
- - Generate 2000+ samples: python nn_data_generator.py --num_samples 2000
- - Train for 50+ epochs: python nn_train.py --epochs 50
- - Reconstruct: python nn_phase_retrieval.py --input data.npz --mode hybrid
-
-Usage:
- python nn_demo.py # Full demo
- python nn_demo.py --skip_training # Skip to reconstruction (needs trained model)
- python nn_demo.py --num_samples 200 # More training data
- python nn_demo.py --material Au # Focus on gold nanocrystals
-"""
-
from __future__ import annotations
import os, sys, argparse, time
import numpy as np
from pathlib import Path
-# ═══════════════════════════════════════════════════════════════════════════════
-# Step 1: Generate training data
-# ═══════════════════════════════════════════════════════════════════════════════
def step_generate(num_samples=50, grid_size=64, material=None, output_dir='./demo_data'):
- """Generate training samples using bcdi_core."""
+
from cdi_st.nn_data_generator import generate_dataset
print("\n" + "█" * 60)
@@ -47,13 +23,10 @@ def step_generate(num_samples=50, grid_size=64, material=None, output_dir='./dem
)
-# ═══════════════════════════════════════════════════════════════════════════════
-# Step 2: Train the U-Net
-# ═══════════════════════════════════════════════════════════════════════════════
def step_train(data_dir='./demo_data', output_dir='./demo_checkpoints',
epochs=10, batch_size=4):
- """Train the phase prediction U-Net."""
+
from nn_train import train
print("\n" + "█" * 60)
@@ -76,12 +49,8 @@ def step_train(data_dir='./demo_data', output_dir='./demo_checkpoints',
)
-# ═══════════════════════════════════════════════════════════════════════════════
-# Step 3: Reconstruct and compare
-# ═══════════════════════════════════════════════════════════════════════════════
def step_reconstruct(test_sample_path, model_path='./demo_checkpoints/best_model.pt'):
- """Run phase retrieval in all 3 modes and compare."""
from cdi_st.nn_phase_retrieval import HybridPhaseRetriever
print("\n" + "█" * 60)
@@ -143,12 +112,7 @@ def step_reconstruct(test_sample_path, model_path='./demo_checkpoints/best_model
return results
-# ═══════════════════════════════════════════════════════════════════════════════
-# Step 4: Plot results
-# ═══════════════════════════════════════════════════════════════════════════════
-
def step_plot(results, test_sample_path, output_dir='./demo_results'):
- """Generate comparison plots."""
try:
import matplotlib
matplotlib.use('Agg')
@@ -253,10 +217,6 @@ def step_plot(results, test_sample_path, output_dir='./demo_results'):
plt.close('all')
-# ═══════════════════════════════════════════════════════════════════════════════
-# Main
-# ═══════════════════════════════════════════════════════════════════════════════
-
def main():
parser = argparse.ArgumentParser(description='BCDI NN Phase Retrieval Demo')
parser.add_argument('--num_samples', type=int, default=50,
diff --git a/src/cdi_st/nn_experimental_loader.py b/src/cdi_st/nn_experimental_loader.py
index e201d95..19608d5 100644
--- a/src/cdi_st/nn_experimental_loader.py
+++ b/src/cdi_st/nn_experimental_loader.py
@@ -1,65 +1,9 @@
-"""
-nn_experimental_loader.py — Load real BCDI experimental data from .h5 files.
-
-Supports:
- - ID01 / BLISS format (ESRF) — primary target
- - 34-ID-C format (APS) — AutoPhaseNN paper format
- - Generic .h5 with a 3D dataset — fallback
-
-Key operations:
- 1. Locate the 3D diffraction volume in the HDF5 tree
- 2. Threshold hot pixels and beamstop
- 3. Center the Bragg peak (center of mass)
- 4. Crop / pad to the target grid size (64 or 128)
- 5. Resample to the target oversampling ratio (~3.0, following AutoPhaseNN)
-
-The resulting preprocessed volume can be passed to either:
- - nn_phase_retrieval.py (supervised model)
- - nn_autophase_infer.py (unsupervised model)
- - The unsupervised training loop (for fine-tuning)
-
-Requires h5py. If you don't have it installed yet:
- pip install h5py
-
-Usage:
- # Inspect the structure of an unknown .h5 file:
- python nn_experimental_loader.py --input scan_123.h5 --inspect
-
- # Load and preprocess, save as .npz:
- python nn_experimental_loader.py --input scan_123.h5 --output preprocessed.npz
-
- # Specify dataset path explicitly if auto-detect fails:
- python nn_experimental_loader.py --input scan_123.h5 \\
- --dataset_path /entry_0000/measurement/merlin/data
-"""
-
from __future__ import annotations
import argparse, json
import numpy as np
from pathlib import Path
from typing import Optional, Tuple
-
-# ═══════════════════════════════════════════════════════════════════════════════
-# Auto-load HDF5 compression plugins for Eiger/Lambda/Pilatus data.
-#
-# Eiger detectors (used at PETRA III P10/P21 and many other beamlines) write
-# data with bitshuffle + LZ4 compression (filter ID 32008). This filter is
-# NOT bundled with h5py — it lives in the `hdf5plugin` package.
-#
-# Without hdf5plugin, attempting to read a compressed Eiger dataset gives
-# the cryptic error:
-# "Can't synchronously read data (can't open directory)"
-# This is HDF5 looking for the plugin DLL/SO in a directory that doesn't
-# exist on the user's machine.
-#
-# Importing hdf5plugin registers all common scientific compression filters
-# (bitshuffle, LZ4, zstd, blosc, fcidecomp) with the HDF5 library, after
-# which h5py can read the data transparently.
-#
-# We capture the availability status so we can give a USEFUL error message
-# if the user tries to read a compressed file without hdf5plugin installed.
-# ═══════════════════════════════════════════════════════════════════════════════
try:
import hdf5plugin # noqa: F401 — registers LZ4/bitshuffle/zstd filters
_HDF5_PLUGINS_AVAILABLE = True
@@ -82,15 +26,6 @@
def _dataset_requires_plugin(dataset) -> Optional[str]:
- """
- Check if a dataset uses a compression filter that requires hdf5plugin.
-
- Returns
- -------
- None if the dataset uses only built-in filters.
- A descriptive string (e.g. "Bitshuffle + LZ4") if the dataset needs
- hdf5plugin and the filter is one of the known plugin-requiring filters.
- """
try:
plist = dataset.id.get_create_plist()
for i in range(plist.get_nfilters()):
@@ -107,12 +42,6 @@ def _dataset_requires_plugin(dataset) -> Optional[str]:
def _is_compression_plugin_error(err: Exception) -> bool:
- """
- Heuristic: detect if an exception is the cryptic "missing HDF5 compression
- plugin" error. Symptom on Windows: 'Can't synchronously read data (can't
- open directory)'. On Linux: similar errors mentioning 'open directory' or
- 'plugin'.
- """
msg = str(err).lower()
return (
"open directory" in msg
@@ -153,18 +82,6 @@ def _is_compression_plugin_error(err: Exception) -> bool:
def _find_p10_master(data_path: str) -> Optional[str]:
- """
- For a P10 'data file' like 'align_03_01698_data_000001.h5', look for the
- corresponding 'master.h5' in the same directory.
-
- P10 (and most modern Eiger setups) writes scans as one 'master' file plus
- one or more numbered data chunks. The master file contains a Virtual
- Dataset (VDS) that links the chunks together. Reading a single chunk
- directly often fails or yields partial data — the master is the correct
- entry point.
-
- Returns the path to the master file if found, else None.
- """
import os, re
if not data_path:
return None
@@ -189,32 +106,6 @@ def _find_p10_master(data_path: str) -> Optional[str]:
def parse_p10_fio(fio_path: str) -> dict:
- """
- Parse a P10 (DESY) '.fio' metadata file.
-
- .fio is a custom ASCII format used at PETRA III. Structure:
-
- !
- ! Comments
- !
- %c
- scan_command_here
- %p
- param_name = value
- ...
- %d
- Col 1 motor_or_counter_name DOUBLE
- Col 2 mpx4inr INTEGER
- ...
- value1 value2 ...
- value1 value2 ...
-
- Returns a dict with:
- scan_command : str
- params : dict[str, str/float] # %p block
- columns : list[str] # column names from %d block
- data : dict[str, np.ndarray] # column data
- """
import re
result = {
"scan_command": "",
@@ -284,13 +175,6 @@ def _is_number(s: str) -> bool:
def diagnose_vds(dataset, dataset_path: str) -> str:
- """
- Diagnose why a Virtual Dataset (VDS) read might fail.
-
- Returns a human-readable string listing the VDS sources and whether each
- source file is reachable. Used to translate cryptic HDF5 errors into
- actionable messages.
- """
import os
lines = [f" Dataset '{dataset_path}' is a Virtual Dataset (VDS)."]
try:
@@ -344,19 +228,6 @@ def diagnose_vds(dataset, dataset_path: str) -> str:
def _read_dataset_safely(ds, dpath: str, max_frames: int = None,
verbose: bool = False):
- """
- Read a dataset, with graceful fallback if it's a VDS whose sources fail,
- or if it requires a compression plugin we don't have.
-
- Strategy:
- 0. If the dataset uses a compression filter that needs hdf5plugin
- and hdf5plugin is NOT installed, raise a clear error.
- 1. If the dataset is a VDS, check whether all its source files exist.
- If any are missing, REFUSE the bulk read and raise a clear error.
- 2. Otherwise, try bulk read.
- 3. If bulk read fails with "can't open directory" (= compression
- plugin missing, or VDS source missing), report a useful error.
- """
# Step 0: detect compression plugin requirements BEFORE attempting read
filter_desc = _dataset_requires_plugin(ds)
if filter_desc is not None and not _HDF5_PLUGINS_AVAILABLE:
@@ -471,11 +342,6 @@ def _read_dataset_safely(ds, dpath: str, max_frames: int = None,
def inspect_h5(path: str, max_depth: int = 6):
- """Print the full tree of an .h5 file to help locate the diffraction data.
-
- Unlike a plain visititems() walk, this also enumerates EXTERNAL LINKS
- (the most common P10/Eiger pattern), which are otherwise invisible.
- """
import os
try:
import h5py
@@ -539,10 +405,6 @@ def _show_links_in(grp, path_prefix, indent):
def _vds_is_readable(ds) -> bool:
- """
- Check if a VDS's source files are reachable WITHOUT actually reading.
- Returns True if not VDS, or if VDS and at least one source exists.
- """
import os
try:
if not (hasattr(ds, 'is_virtual') and ds.is_virtual):
@@ -568,28 +430,6 @@ def _vds_is_readable(ds) -> bool:
def enumerate_external_links(h5_file, group_path: str = "/entry/data") -> list:
- """
- Eiger / DESY HDF5 master files store frame data as a series of
- EXTERNAL LINKS inside `/entry/data/`. Each link has a name like
- `data_000001`, `data_000002`, ... and points to a separate chunk file.
-
- External links are NOT visited by `h5py.File.visititems()`, which is why
- `inspect_h5` shows `/entry/data/` as an empty group even though it
- contains hundreds of links.
-
- This function explicitly enumerates the group's keys (which DOES include
- external links) and returns information about each one.
-
- Returns
- -------
- list of dicts, one per entry:
- {
- 'name': 'data_000001',
- 'link_type': 'external' | 'soft' | 'dataset' | 'group',
- 'target_file': '_data_000001.h5' (only for external),
- 'target_path': '/entry/data/data' (only for external/soft),
- }
- """
import h5py
try:
grp = h5_file[group_path]
@@ -635,34 +475,6 @@ def p10_scan_info(
h5_path: str,
group_path: str = "/entry/data",
) -> dict:
- """
- Cheap preview of a P10 master file: enumerate external links, probe
- each chunk for frame count + frame shape, and return a summary suitable
- for showing in a GUI before the user clicks Convert.
-
- This does NOT decompress any data — it only reads HDF5 metadata, so it
- is essentially instant even for 381-chunk datasets.
-
- Parameters
- ----------
- h5_path : str
- Path to the master .h5 file (or any chunk file in the set).
- group_path : str
- HDF5 group containing the external links. Default ``/entry/data``.
-
- Returns
- -------
- info : dict with keys
- - ``master`` : the resolved master file path
- - ``n_chunks`` : number of external links found
- - ``n_missing`` : how many chunk files don't exist on disk
- - ``n_frames`` : total frames across all readable chunks
- - ``frame_h`` : detector height (pixels)
- - ``frame_w`` : detector width (pixels)
- - ``dtype`` : numpy dtype string (e.g. 'uint32')
- - ``size_full_gb``: GB needed to load all frames at full detector
- - ``frames_per_chunk``: list of (chunk_name, n_frames_in_chunk)
- """
import os, h5py
# Resolve to master if a chunk was passed
master = _find_p10_master(h5_path) or h5_path
@@ -728,48 +540,6 @@ def load_p10_from_external_links(
memory_budget_gb: float = 8.0,
verbose: bool = True,
) -> np.ndarray:
- """
- Read P10 / Eiger data from a master file by manually resolving its
- EXTERNAL LINKS to chunk files.
-
- This is the correct strategy when the master file contains entries like
- `/entry/data/data_000001` that are external links to separate chunk files
- (the standard Eiger/DESY layout). Each chunk file contains contiguous
- detector data at `/entry/data/data`.
-
- The reader applies optional ``frame_range`` and ``detector_roi`` *during*
- the read, so reading e.g. 100 frames out of 3000 only allocates the
- 100-frame volume — never the full 49-GB stack.
-
- Parameters
- ----------
- master_path : str
- Path to the master .h5 file (typically ``_master.h5``).
- group_path : str
- HDF5 group containing the external links. Default ``/entry/data``.
- max_frames : int or None
- Truncate the FINAL volume to this many central frames (legacy).
- Ignored if ``frame_range`` is also given.
- frame_range : tuple ``(start, stop)`` or None
- Read only frames ``[start, stop)`` (Python slice semantics) from the
- *global* frame index across all chunks. ``start=0, stop=None`` means
- "from start to end". This is applied chunk-by-chunk so memory usage
- scales with the requested range, not the total dataset size.
- detector_roi : tuple ``(y0, y1, x0, x1)`` or None
- Read only this detector region per frame. Indices are half-open
- like Python slices. Cuts memory by ``(2167*2070)/(roi area)`` for
- Eiger 4M data.
- memory_budget_gb : float
- Maximum acceptable memory footprint (in GB) for the concatenated
- volume. If the requested read would exceed this, the function
- raises ``MemoryError`` with a suggested ``frame_range`` and
- ``detector_roi`` instead of crashing the Python process.
- verbose : bool
-
- Returns
- -------
- volume : ndarray, shape (N_frames_selected, H, W), float32
- """
import os, h5py
master_dir = os.path.dirname(os.path.abspath(master_path))
@@ -987,17 +757,6 @@ def load_p10_from_external_links(
def find_diffraction_dataset(h5_file):
- """
- Try known paths first, then scan for any suitable 3D dataset.
-
- Skips Virtual Datasets whose source files are missing — picks readable
- contiguous datasets instead. This handles the P10 case where a chunk
- file's /entry/data/ group also contains VDS placeholders (`data_000001`,
- `data_000002`) pointing to *other* chunk files that the user doesn't
- have. The chunk's own contiguous data at `/entry/data/data` is preferred.
-
- Returns (path_str, dataset) or (None, None) if nothing found.
- """
import h5py
# Step 1: try the known paths directly
@@ -1062,19 +821,6 @@ def _walk(group, prefix=""):
def resolve_dataset_path(h5_file, user_path: str):
- """
- Resolve a user-provided HDF5 path to an actual Dataset.
-
- Handles the common confusion where the user types a Group path
- (e.g. '/entry/data') when they should have typed the Dataset path
- inside it (e.g. '/entry/data/data'). When the user's path lands on
- a group, this looks one level deeper for a child named 'data' or
- for any 3D+ dataset inside.
-
- Returns
- -------
- (resolved_path, dataset) or raises a helpful error.
- """
import h5py
if user_path not in h5_file:
raise KeyError(
@@ -1152,23 +898,6 @@ def resolve_dataset_path(h5_file, user_path: str):
def mask_detector_gaps(volume: np.ndarray,
detector: str = 'auto') -> np.ndarray:
- """
- Mask the inter-chip gaps of multi-chip BCDI detectors.
-
- Maxipix (ID01) is a 2×2 array of 256×256 chips with a gap of ~5 pixels
- between chips at rows/columns 255-260. These gaps appear as zeros and
- badly bias COM calculations and hot-pixel detection.
-
- The cdiutils / xrayutilities approach (which is what we follow here)
- is to FILL the gaps with the local median of surrounding voxels,
- not leave them as zeros. Zeros pull the COM toward the gap, while
- the median preserves the actual diffraction signal.
-
- Detector auto-detect:
- - 516×516 → Maxipix 2×2 (ID01)
- - 1062×1028 → Eiger 2M
- - other → use heuristic detection
- """
from scipy.ndimage import median_filter
out = volume.copy().astype(np.float32)
@@ -1251,18 +980,6 @@ def mask_detector_gaps(volume: np.ndarray,
def remove_hot_pixels(volume: np.ndarray, threshold_sigma: float = 50.0) -> np.ndarray:
- """
- Remove hot pixels (cosmic rays, defective detector pixels) by clipping
- values that are MUCH higher than their local neighborhood.
-
- Note: BCDI detector data has very high dynamic range. The Bragg peak
- itself has intensity 1e4-1e6× higher than the fringes around it. So a
- sigma-based threshold needs to be VERY high (50+) to avoid clipping
- the peak. Earlier value of 20 was too aggressive and ate the Bragg peak.
-
- Better approach: use a 3-pixel median filter and only clip pixels that
- are 100× the median (single bright pixels surrounded by dark).
- """
from scipy.ndimage import median_filter
# 3D median filter — more robust against the peak itself
local_median = median_filter(volume, size=3)
@@ -1279,18 +996,6 @@ def remove_hot_pixels(volume: np.ndarray, threshold_sigma: float = 50.0) -> np.n
def find_bragg_peak_box(volume: np.ndarray,
intensity_threshold_pct: float = 1.0,
margin_voxels: int = 8) -> tuple:
- """
- Locate the Bragg peak by finding the 3D bounding box of voxels above
- a threshold of the maximum intensity. Returns (z_slice, y_slice, x_slice).
-
- This is the standard approach in cdiutils, PyNX, BCDI-utilities:
- 1. Find the brightest voxel
- 2. Threshold around it (typically 1% of max — captures the peak + fringes)
- 3. Use the bounding box of the thresholded region
-
- This is much better than COM-based centering, which is biased by
- detector noise and beam stops.
- """
# Smooth lightly for robust peak finding (3-pixel Gaussian)
from scipy.ndimage import gaussian_filter
smoothed = gaussian_filter(volume.astype(np.float32), sigma=2.0)
@@ -1326,13 +1031,6 @@ def find_bragg_peak_box(volume: np.ndarray,
def crop_around_peak(volume: np.ndarray, center: tuple, N: int) -> np.ndarray:
- """
- Crop a (N, N, N) box around the given center in `volume`.
- If the box extends beyond the array, pad with zeros (NOT wrap around).
-
- This replaces both `center_on_bragg_peak` (which used np.roll = wrap)
- and `crop_or_pad_to` (which centered on grid center, not peak center).
- """
cz, cy, cx = center
half = N // 2
out = np.zeros((N, N, N), dtype=volume.dtype)
@@ -1354,21 +1052,6 @@ def _sliced(c, src_size):
def remove_beamstop_streaks(volume: np.ndarray) -> np.ndarray:
- """
- Remove beamstop / direct-beam streaks from BCDI data.
-
- These are bright lines along the rocking-axis direction (axis 0) that
- persist across all rocking frames — caused by the direct beam catching
- a beamstop edge or a saturated detector pixel.
-
- Critical: we ONLY check the detector plane (axes 1, 2). Streaks along
- the rocking axis are physical (Bragg fringes), so we never collapse
- along axis 0 to find "streaks" — that would destroy the actual signal.
-
- Detection: A streak is a (y, x) detector pixel where ALL rocking
- frames have intensity > 10× the global median. Real Bragg peaks
- have intensity at only some rocking angles.
- """
out = volume.copy().astype(np.float32)
nz = out[out > 0]
if len(nz) == 0:
@@ -1409,41 +1092,6 @@ def load_h5_diffraction(
max_frames: int = None,
verbose: bool = True,
) -> np.ndarray:
- """
- Load and preprocess a 3D diffraction volume from an .h5 file.
-
- Pipeline (matches cdiutils / BCDI-utilities standard):
- 1. Locate dataset in HDF5 tree
- 2. Convert to float32, clip negatives
- 3. Remove beamstop/direct-beam streaks
- 4. Remove hot pixels (conservative threshold to preserve Bragg peak)
- 5. Locate Bragg peak by intensity bounding box
- 6. Crop a (N, N, N) box AROUND the peak (not the array center)
-
- Parameters
- ----------
- path : str
- Path to .h5 file.
- dataset_path : str or None
- HDF5 path to the diffraction dataset. If None, auto-detect.
- target_size : int
- Output grid size N → returns (N, N, N) volume.
- clean_hot_pixels : bool
- Apply median filter to remove hot pixels.
- center_peak : bool
- Locate and crop around the Bragg peak.
- beamstop_threshold : float
- Values below this are set to 0 (removes beamstop artifacts).
- max_frames : int or None
- Limit the number of detector frames to use (helpful for memory).
- verbose : bool
- Print preprocessing diagnostics.
-
- Returns
- -------
- diffraction : ndarray [N, N, N]
- Preprocessed 3D diffraction volume (intensity, not amplitude).
- """
try:
import h5py
except ImportError:
@@ -1596,17 +1244,6 @@ def _walk(name, obj):
# ═══════════════════════════════════════════════════════════════════════════════
def list_spec_scans(spec_path: str) -> list:
- """
- Return a list of all scans in a SPEC file with summary info, so the GUI
- can show the user which scan number corresponds to which frame range.
-
- Returns a list of dicts with keys:
- scan_number — the #S N integer (1, 2, 3, ...)
- command — the scan command, e.g. "ascan eta 33.5 34.5 100 1"
- n_points — number of measurement points
- frame_min — first CCD frame number (from mpx4inr column)
- frame_max — last CCD frame number
- """
out = []
try:
from silx.io.specfile import SpecFile
@@ -1706,30 +1343,6 @@ def list_spec_scans(spec_path: str) -> list:
def read_spec_scan(spec_path: str, scan_number: int) -> dict:
- """
- Read motor positions for a scan from a SPEC file.
-
- SPEC files are the native ID01 (and many other beamlines') metadata
- format used before BLISS/HDF5 became standard. They contain:
- - Header with motor names + initial positions ('motors' dict)
- - Per-scan motor sweeps (eta or phi rocking)
- - CCD frame numbers (mpx4inr column)
-
- This function returns the motor positions needed for q-space
- orthogonalization: eta, phi, nu, delta arrays + the rocking type.
-
- Requires the `spec` Python package:
- pip install spec
- Or alternatively `silx.io.specfile` which is more modern.
-
- Returns
- -------
- dict with keys:
- 'eta', 'phi', 'nu', 'delta' — motor angle arrays (degrees)
- 'mpx4inr' — CCD frame numbers
- 'rocking_type' — 'eta' or 'phi'
- 'header_motors' — dict of all initial motor positions
- """
try:
# Try silx first (more modern, better maintained)
from silx.io.specfile import SpecFile
@@ -1802,35 +1415,6 @@ def read_spec_scan(spec_path: str, scan_number: int) -> dict:
def _filter_existing_edf_frames(edf_template: str, frame_numbers) -> tuple:
- """
- Probe the disk to find which EDF frames actually exist for a given
- template + frame number list.
-
- SPEC scans sometimes reference frame ranges that include frames the
- detector never wrote (e.g. acquisition was started a frame late or
- stopped a frame early). Loading those non-existent frames would either
- crash the reader or pad the volume with zeros, which contaminates the
- rocking curve. This helper returns just the frames that exist, plus
- the indices of the missing ones (so motor arrays can be trimmed
- consistently).
-
- Parameters
- ----------
- edf_template : str
- Full filename template with a single %d / %05d for the frame
- number, e.g. ``/data/scan1/data_mpx4_%05d.edf.gz``.
- frame_numbers : array-like of int
- Frame numbers to check.
-
- Returns
- -------
- keep_mask : ndarray of bool, shape (len(frame_numbers),)
- True where the EDF file exists on disk.
- existing_numbers : ndarray of int
- ``frame_numbers[keep_mask]`` — convenience for the caller.
- missing_numbers : ndarray of int
- Frame numbers that did NOT have a file on disk.
- """
import os
frame_numbers = np.asarray(frame_numbers, dtype=np.int64)
keep_mask = np.zeros(len(frame_numbers), dtype=bool)
@@ -1845,30 +1429,6 @@ def _filter_existing_edf_frames(edf_template: str, frame_numbers) -> tuple:
def read_edf_stack(edf_template: str, frame_numbers: list,
detector_shape: tuple = (516, 516)) -> np.ndarray:
- """
- Read a stack of EDF (European Data Format) detector frames.
-
- EDF is the legacy ID01 detector format. Each frame is a single .edf or
- .edf.gz file, named like 'data_mpx4_00123.edf.gz'. This function loads
- a list of frames into a 3D volume (n_frames, det_y, det_x).
-
- Requires `xrayutilities` (pip install xrayutilities) which has a
- convenient EDF reader.
-
- Parameters
- ----------
- edf_template : str
- Filename template with one %d for frame number, e.g.
- '/data/dir/data_mpx4_%05d.edf.gz'
- frame_numbers : list of int
- Frame numbers to load (typically the 'mpx4inr' column from SPEC).
- detector_shape : tuple
- Expected (rows, cols) of each detector frame.
-
- Returns
- -------
- volume : ndarray (n_frames, det_y, det_x)
- """
try:
import xrayutilities as xu
except ImportError:
@@ -1924,41 +1484,6 @@ def load_spec_edf_scan(
detector: str = 'maxipix',
verbose: bool = True,
) -> dict:
- """
- Full ID01 spec+EDF loader: read motor positions from a SPEC file,
- load corresponding EDF detector frames, preprocess (gap-mask, hot pixels,
- streaks, peak finding), and return everything needed for reconstruction
- PLUS q-space orthogonalization (if xrayutilities is available).
-
- This is the ID01-native loading path that mirrors what cdiutils does.
-
- Parameters
- ----------
- spec_path : str
- Path to .spec file containing the scan header.
- scan_number : int
- SPEC scan number to load.
- edf_dir : str
- Directory containing the .edf.gz detector frames.
- edf_template_name : str
- Filename template (default: 'data_mpx4_%05d.edf.gz' for ID01 maxipix)
- target_size : int
- Output grid size N → returns (N, N, N) volume.
- detector_shape : tuple
- Raw detector dimensions (default 516×516 for Maxipix).
- detector : str
- Detector name for gap masking ('maxipix', 'eiger2M', 'auto').
-
- Returns
- -------
- dict with keys:
- 'diffraction' — preprocessed (target_size,)*3 volume
- 'eta', 'phi', 'nu', 'delta' — motor arrays (degrees)
- 'rocking_type' — 'eta' or 'phi'
- 'frame_numbers' — original CCD frame numbers
- 'q_axes' — (qx, qy, qz) arrays in 1/Å, IF xrayutilities available
- 'voxel_size_nm' — real-space voxel pitch (3-vector) IF q_axes present
- """
import os
# 1. Read SPEC header
@@ -2140,24 +1665,6 @@ def _format_run(arr):
def _find_chunk_internal_data_path(h5_file, hint_path: str = None) -> Optional[str]:
- """
- For a P10-style chunk file, find the path to its own contiguous data.
-
- Different P10 configurations store data at different paths inside the
- chunk:
- - /entry/data/data (NeXus standard)
- - /entry/data/data_000001 (P10 with numbered datasets — common)
- - /entry/instrument/eiger_4m/data
- - /entry/instrument/eiger_500k/data
- - /entry/instrument/detector/data
- - /entry_1/data_1/data (CXI)
- - /entry/data_NNNN (some BLISS setups)
-
- This function scans the file and returns the path that:
- (1) is a real Dataset (not a Group, not a broken VDS)
- (2) is 3D with substantial size (> 10k voxels)
- (3) has reachable data (or is contiguous, not VDS)
- """
import h5py
# If hint_path is provided and valid, try it first
@@ -2256,31 +1763,6 @@ def load_p10_chunks_directly(
max_frames: int = None,
verbose: bool = True,
) -> np.ndarray:
- """
- BYPASS VDS — directly open each `*_data_NNNNNN.h5` chunk in the directory
- and concatenate their contiguous data. Auto-discovers the actual dataset
- path inside each chunk (P10 sometimes stores data at /entry/data/data,
- sometimes at /entry/data/data_NNNNNN, depending on the acquisition mode).
-
- Falls back to .cxi if no chunks are found.
-
- Parameters
- ----------
- one_h5_path : str
- Any file in the dataset's directory — chunk, master, or CXI.
- chunk_dataset_path : str
- HINT for the path inside each chunk. If None or the path doesn't
- exist in a particular chunk, auto-discovers an appropriate one.
- cxi_dataset_path : str
- Path inside .cxi files (for the CXI fallback).
- max_frames : int or None
- Truncate to at most this many frames (centered).
- verbose : bool
-
- Returns
- -------
- volume : ndarray shape (N_frames_total, H, W) float32
- """
import os, re, glob
try:
import h5py
@@ -2456,12 +1938,6 @@ def load_p10_chunks_directly(
def _preprocess_p10_volume(raw_volume: np.ndarray, target_size: int = 64,
verbose: bool = True) -> np.ndarray:
- """
- Apply the same preprocessing as load_h5_diffraction's tail, but starting
- from a raw frame stack (N_frames, H, W) — the output of
- load_p10_chunks_directly. Used as the second half of the direct-chunk
- fallback in p10_h5_to_npz.
- """
volume = np.maximum(raw_volume.astype(np.float32), 0)
if verbose:
print(f" Raw volume: shape={volume.shape} "
@@ -2515,47 +1991,6 @@ def p10_h5_to_npz(
detector_roi: tuple = None,
memory_budget_gb: float = 8.0,
):
- """
- Load a P10 (PETRA III / DESY) BCDI scan and save as a .npz that
- CDI-ST reconstruction can use directly.
-
- Robust to several P10 quirks:
- 1. Master file with external links → reads chunks directly (Eiger default)
- 2. Single chunk with broken VDS placeholders → reads its own data
- 3. Bitshuffle+LZ4 compressed data → requires ``hdf5plugin`` (auto-detected)
- 4. Large multi-thousand-frame scans → use ``frame_range`` / ``detector_roi``
- to read only a subset (avoids 49-GB memory blow-ups)
- 5. CXI (.cxi) files in the directory → used as fallback if no chunks
-
- Parameters
- ----------
- h5_path : str
- Any P10 file: master, data chunk, or .cxi.
- npz_path : str
- Output .npz filename.
- fio_path : str or None
- Optional .fio metadata path (motor positions). Auto-found if None.
- target_size : int
- Output cube size (target_size³). The cropped output is centered on
- the Bragg peak found in the loaded volume.
- dataset_path : str or None
- Explicit HDF5 path to detector data. Auto-detected if None.
- verbose : bool
- force_direct_chunks : bool
- If True, skip the master/VDS attempt and go straight to enumerating
- chunk files in the directory.
- frame_range : tuple ``(start, stop)`` or None
- Read only frames in ``[start, stop)`` (across all chunks). Useful for
- long rocking-curve scans where you only need the frames around the
- peak. Defaults to ``None`` (all frames).
- detector_roi : tuple ``(y0, y1, x0, x1)`` or None
- Read only this detector region per frame. Indices are Python slices.
- Defaults to ``None`` (full detector).
- memory_budget_gb : float
- Maximum estimated memory for the read. If exceeded, the function
- raises ``MemoryError`` with a suggested ``frame_range`` instead
- of crashing. Default 8 GB.
- """
import os
diffraction = None
@@ -2753,9 +2188,6 @@ def spec_edf_to_npz(
target_size: int = 64,
edf_template_name: str = "data_mpx4_%05d.edf.gz",
):
- """
- One-call helper: convert ID01 spec+EDF to a reconstruction-ready .npz.
- """
res = load_spec_edf_scan(
spec_path=spec_path,
scan_number=scan_number,
@@ -2789,10 +2221,6 @@ def h5_to_npz(
dataset_path: str = None,
target_size: int = 64,
):
- """
- Convert an experimental .h5 file to an .npz compatible with the dataset
- loaders (for either inference or fine-tuning the unsupervised model).
- """
diffraction = load_h5_diffraction(h5_path, dataset_path, target_size)
# Save as .npz with the same structure as simulated files
diff --git a/src/cdi_st/nn_gui_tabs.py b/src/cdi_st/nn_gui_tabs.py
index 789545f..1f72faf 100644
--- a/src/cdi_st/nn_gui_tabs.py
+++ b/src/cdi_st/nn_gui_tabs.py
@@ -1,10 +1,3 @@
-"""
-nn_gui_tabs.py — Three new tabs for the BCDI GUI:
- T4: NN Training — choose data, parameters, and train with live progress
- T5: BCDI Reconstruction — load .npz/.h5 and reconstruct with 4 figures
- T6: 3D Reconstruction Viewer — interactive 3D with phase/strain/density
-"""
-
import sys, os, tempfile, json, time
import numpy as np
from pathlib import Path
@@ -306,14 +299,6 @@ def run(self):
class SupervisedTrainingWorker(QThread):
- """
- Train the SUPERVISED PhaseUNet3D model.
-
- Unlike AutoPhaseNet (unsupervised, only diffraction), this needs
- ground-truth phase_true AND support in each .npz file. The data
- generator already produces these — they're ignored when training
- AutoPhaseNet.
- """
log = pyqtSignal(str)
progress = pyqtSignal(int)
epoch_done = pyqtSignal(int, float, float)
@@ -1547,18 +1532,6 @@ def _on_failed(self, msg):
# ═══════════════════════════════════════════════════════════════════════════════
class T4_Sup(QWidget):
- """
- Training tab for the SUPERVISED PhaseUNet3D model.
-
- Required dataset: .npz files with keys
- 'amplitude', 'phase_true', 'support', 'diffraction'
- Files generated by the 'Generate Data' tab include all of these.
- Files exported from the simulation tab include only 'diffraction_volume'
- and cannot be used for supervised training.
-
- Output: best_model.pt in the chosen output directory. Use this as
- 'Model 2' in the Reconstruct tab for ensemble mode.
- """
def __init__(self):
super().__init__()
diff --git a/src/cdi_st/nn_phase_model.py b/src/cdi_st/nn_phase_model.py
index 01f6b20..6edd0cb 100644
--- a/src/cdi_st/nn_phase_model.py
+++ b/src/cdi_st/nn_phase_model.py
@@ -1,40 +1,3 @@
-"""
-nn_phase_model.py — Lightweight 3D U-Net for BCDI phase prediction.
-
-Architecture:
- Input: log-scaled diffraction amplitude [B, 1, N, N, N]
- Output: predicted phase field [B, 1, N, N, N]
-
- Encoder path: 1 → 32 → 64 → 128 channels (3 levels)
- Bottleneck: 256 channels
- Decoder path: 128 → 64 → 32 → 1 channels (skip connections from encoder)
- Final: tanh activation (phase ∈ [-1, 1], multiply by π to get radians)
-
-Design choices:
- - Lightweight: ~1.9M parameters (fits on RTX 3070 with batch_size=8 at N=64)
- - Instance normalization instead of batch norm (more stable for small batches)
- - LeakyReLU activations (avoids dead neurons in phase regions)
- - Skip connections preserve high-frequency fringe information
- - tanh output naturally constrains phase to bounded range
-
-The model predicts phase INSIDE the support. Outside the support, phase is
-meaningless, so the loss function should mask with the support.
-
-Why U-Net for phase retrieval:
- The encoder captures the global structure of the diffraction pattern
- (overall shape, symmetry), while the decoder reconstructs local phase
- variations (strain fields, dislocations). Skip connections allow the
- decoder to reference fine fringes in the input when building the phase map.
-
-Usage:
- from cdi_st.nn_phase_model import PhaseUNet3D, count_parameters
- model = PhaseUNet3D(in_channels=1, base_channels=32)
- print(f"Parameters: {count_parameters(model):,}")
-
- x = torch.randn(4, 1, 64, 64, 64) # batch of 4, grid 64³
- phase_pred = model(x) # → [4, 1, 64, 64, 64]
-"""
-
from __future__ import annotations
import torch
import torch.nn as nn
@@ -42,14 +5,7 @@
class ConvBlock3D(nn.Module):
- """
- Two 3x3x3 convolutions with instance norm and LeakyReLU.
-
- InstanceNorm is preferred over BatchNorm for BCDI because:
- - Batch sizes are typically small (4-16) due to 3D volume memory
- - Each sample has very different intensity scales
- - InstanceNorm normalizes per-sample, per-channel
- """
+
def __init__(self, in_ch: int, out_ch: int):
super().__init__()
self.conv = nn.Sequential(
@@ -77,12 +33,7 @@ def forward(self, x):
class UpBlock(nn.Module):
- """
- Upsample by 2x, concatenate skip connection, then double conv block.
- Uses trilinear interpolation for upsampling (smoother than transposed conv
- for continuous phase fields — avoids checkerboard artifacts).
- """
def __init__(self, in_ch: int, skip_ch: int, out_ch: int):
super().__init__()
self.conv = ConvBlock3D(in_ch + skip_ch, out_ch)
@@ -96,36 +47,6 @@ def forward(self, x, skip):
class PhaseUNet3D(nn.Module):
- """
- 3D U-Net for predicting BCDI phase from diffraction amplitude.
-
- Architecture (for base_channels=32, N=64):
-
- Input [1, 64, 64, 64]
- ├─ ConvBlock → [32, 64, 64, 64] ← skip1
- ├─ DownBlock → [64, 32, 32, 32] ← skip2
- ├─ DownBlock → [128, 16, 16, 16] ← skip3
- ├─ DownBlock → [256, 8, 8, 8] ← bottleneck
- ├─ UpBlock → [128, 16, 16, 16] (+ skip3)
- ├─ UpBlock → [64, 32, 32, 32] (+ skip2)
- ├─ UpBlock → [32, 64, 64, 64] (+ skip1)
- └─ Conv 1x1 → [1, 64, 64, 64] + tanh
-
- Parameters: ~1.9M (at base_channels=32)
- Memory: ~1.2 GB per sample at N=64
-
- Parameters
- ----------
- in_channels : int
- Number of input channels (1 for amplitude-only).
- out_channels : int
- Number of output channels (1 for phase).
- base_channels : int
- Number of channels in first encoder level. Doubles each level.
- 32 → ~1.9M params (recommended for N=64, GPU ≥ 8GB)
- 48 → ~4.2M params (for N=64, GPU ≥ 12GB)
- 64 → ~7.5M params (for N=64, GPU ≥ 16GB)
- """
def __init__(
self,
@@ -158,19 +79,7 @@ def __init__(
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
- """
- Forward pass.
-
- Parameters
- ----------
- x : torch.Tensor
- Input amplitude, shape [B, 1, N, N, N].
-
- Returns
- -------
- torch.Tensor
- Predicted phase (normalized to [-1, 1]), shape [B, 1, N, N, N].
- """
+
# Encoder
s1 = self.enc1(x) # [B, C, N, N, N]
s2 = self.enc2(s1) # [B, 2C, N/2, N/2, N/2]
@@ -193,28 +102,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor:
# ═══════════════════════════════════════════════════════════════════════════════
class BCDIPhaseLoss(nn.Module):
- """
- Combined loss for BCDI phase prediction.
- Components:
- 1. Masked MSE loss on phase: |phase_pred - phase_true|² inside the
- support only (phase outside the crystal is physically meaningless).
- Controlled by ``alpha_amp`` (alias: ``alpha``).
-
- 2. FFT consistency loss: the predicted phase + measured amplitude should
- produce a real-space object that is consistent with the support.
- Acts as a physics-informed regularizer.
- Controlled by ``beta_phase`` (alias: ``beta``).
-
- 3. Gradient smoothness loss: penalizes sharp phase jumps inside the
- support (real strain fields are smooth, not noisy).
- Controlled by ``gamma_diff`` (alias: ``gamma``).
-
- Parameters can be passed using either the descriptive names
- (``alpha_amp``, ``beta_phase``, ``gamma_diff``) used in the GUI, or
- the Greek-letter aliases (``alpha``, ``beta``, ``gamma``) used in
- the original API. Descriptive names take precedence if both are passed.
- """
def __init__(self,
alpha_amp: float = None, beta_phase: float = None,
@@ -235,21 +123,7 @@ def forward(
amplitude: torch.Tensor = None,
diff_amp: torch.Tensor = None,
) -> dict:
- """
- Compute combined loss.
-
- Parameters
- ----------
- phase_pred : [B, 1, N, N, N] predicted phase (normalized [-1, 1])
- phase_true : [B, 1, N, N, N] ground truth phase (normalized [-1, 1])
- support : [B, 1, N, N, N] binary support mask
- amplitude : [B, 1, N, N, N] measured amplitude (for FFT loss).
- Accepted as either ``amplitude=`` or ``diff_amp=``.
-
- Returns
- -------
- dict with 'total', 'mse', 'fft_consistency', 'smoothness'
- """
+
# Accept either keyword for the amplitude tensor
if amplitude is None and diff_amp is not None:
amplitude = diff_amp
diff --git a/src/cdi_st/nn_phase_retrieval.py b/src/cdi_st/nn_phase_retrieval.py
index 6d14358..da7cb00 100644
--- a/src/cdi_st/nn_phase_retrieval.py
+++ b/src/cdi_st/nn_phase_retrieval.py
@@ -1,42 +1,3 @@
-"""
-nn_phase_retrieval.py — Hybrid NN-accelerated phase retrieval for BCDI.
-
-This is the core innovation: instead of starting RAAR/HIO from random phase,
-we use the trained U-Net to predict an initial phase estimate, then refine
-with a short run of iterative algorithms.
-
-Pipeline:
- 1. Load trained U-Net model
- 2. Predict initial phase from measured |F|²
- 3. Run 50-100 RAAR iterations (instead of 1000+)
- 4. Polish with 20 ER iterations
- 5. Return reconstructed complex object ρ(r)·e^{iφ(r)}
-
-Phase retrieval algorithms implemented:
- - ER (Error Reduction): simple projection, always reduces error
- - HIO (Hybrid Input-Output): Fienup's workhorse, escapes local minima
- - RAAR (Relaxed Averaged Alternating Reflections): state-of-the-art,
- smoother convergence than HIO
- - Shrink-wrap support refinement (optional)
-
-Comparison modes:
- - NN-only: just the U-Net prediction (fast, rough)
- - Classical: random start + 1000 RAAR + 100 ER (slow, standard)
- - Hybrid: NN start + 100 RAAR + 20 ER (fast, accurate)
-
-Usage:
- from cdi_st.nn_phase_retrieval import HybridPhaseRetriever
-
- retriever = HybridPhaseRetriever(model_path='checkpoints/best_model.pt')
- result = retriever.reconstruct(diffraction_volume)
-
- # result.object_3d → complex density [N, N, N]
- # result.phase → phase field [N, N, N]
- # result.amplitude → electron density [N, N, N]
- # result.support → refined support [N, N, N]
- # result.error_metric → list of R-factor at each iteration
-"""
-
from __future__ import annotations
import time
import numpy as np
@@ -47,13 +8,9 @@
from .nn_phase_model import PhaseUNet3D
-# ═══════════════════════════════════════════════════════════════════════════════
-# Data classes
-# ═══════════════════════════════════════════════════════════════════════════════
@dataclass
class ReconstructionResult:
- """Container for phase retrieval results."""
object_3d: np.ndarray # Complex object ρ·e^{iφ}, shape [N,N,N]
phase: np.ndarray # Phase field, shape [N,N,N]
amplitude: np.ndarray # |ρ|, shape [N,N,N]
@@ -64,41 +21,22 @@ class ReconstructionResult:
n_iterations: int = 0 # Total iterations
-# ═══════════════════════════════════════════════════════════════════════════════
-# Phase retrieval algorithms (pure NumPy, no ML dependency)
-# ═══════════════════════════════════════════════════════════════════════════════
def _fft3(x):
- """Centered 3D FFT."""
return np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(x)))
def _ifft3(x):
- """Centered 3D inverse FFT."""
return np.fft.fftshift(np.fft.ifftn(np.fft.ifftshift(x)))
def _r_factor(measured_amp, current_amp):
- """
- R-factor error metric (crystallographic convention).
-
- R = Σ|√I_meas - √I_calc| / Σ|√I_meas|
-
- Lower is better. Typical converged values: 0.01-0.10
- """
num = np.sum(np.abs(measured_amp - current_amp))
den = np.sum(np.abs(measured_amp))
return num / max(den, 1e-12)
def _modulus_projection(obj, measured_amplitude):
- """
- Fourier modulus constraint: replace amplitude with measured,
- keep the predicted phase.
-
- This is the fundamental constraint in phase retrieval:
- we know |F(q)| from the detector, so we enforce it.
- """
F = _fft3(obj)
phase = np.angle(F)
F_constrained = measured_amplitude * np.exp(1j * phase)
@@ -106,25 +44,6 @@ def _modulus_projection(obj, measured_amplitude):
def _support_projection(obj, support, mode='er', beta=0.9, obj_prev=None):
- """
- Real-space support constraint.
-
- Standard projections (Marchesini 2007 review):
- P_S(x) = x · 1_S (zero outside support)
-
- Algorithms used here:
- ER: x_{n+1} = P_S P_M x_n
- (simplest, monotonically decreases error but stagnates)
- HIO: x_{n+1}|_S = (P_M x_n)|_S
- x_{n+1}|_~S = (x_n - β · P_M x_n)|_~S
- (Fienup 1982 — escapes local minima)
- RAAR: x_{n+1} = β/2 (R_S R_M + I) x_n + (1-β) P_M x_n
- where R_S = 2 P_S - I, R_M = 2 P_M - I
- (Luke 2005 — smoother than HIO)
-
- For RAAR/HIO this function expects `obj` to be the *modulus-projected*
- iterate P_M(x_n), and `obj_prev` to be the *previous iterate* x_{n-1}.
- """
inside = support > 0.5
if mode == 'er':
@@ -134,7 +53,6 @@ def _support_projection(obj, support, mode='er', beta=0.9, obj_prev=None):
return result
elif mode == 'hio':
- # Hybrid Input-Output (Fienup 1982)
# Inside support: keep modulus-projected value
# Outside support: x_n - β · P_M x_n
if obj_prev is None:
@@ -148,7 +66,6 @@ def _support_projection(obj, support, mode='er', beta=0.9, obj_prev=None):
return result
elif mode == 'raar':
- # RAAR (Luke 2005)
# x_{n+1} = β/2 (R_S R_M + I) x_n + (1-β) P_M x_n
#
# Computing R_S R_M x_n:
@@ -183,22 +100,6 @@ def _support_projection(obj, support, mode='er', beta=0.9, obj_prev=None):
def _shrink_wrap(obj, support, threshold=0.10, sigma=2.0, max_support_fraction=0.30):
- """
- Shrink-wrap support update (Marchesini 2003).
-
- Periodically refines the support by:
- 1. Gaussian-blur the current reconstruction amplitude
- 2. Threshold at a fraction of the maximum
- 3. Use the thresholded region as new support
- 4. Hard cap: never let support exceed `max_support_fraction` of the grid
-
- The hard cap is essential — without it, shrink-wrap with a noisy
- reconstruction can grow the support to fill the entire grid, removing
- all constraints and giving a meaningless R=0 trivial solution.
-
- This allows the support to adapt to the actual crystal shape,
- which is important when the initial support guess is too large.
- """
from scipy.ndimage import gaussian_filter
amp = np.abs(obj)
blurred = gaussian_filter(amp, sigma=sigma)
@@ -236,45 +137,11 @@ def run_phase_retrieval(
algorithm: str = 'hybrid',
progress_cb=None,
) -> ReconstructionResult:
- """
- Iterative phase retrieval.
-
- Algorithm choices (literature-standard):
- 'er' — pure ER (rare, only stable case)
- 'hio' — Fienup HIO (escapes local minima)
- 'raar' — Luke RAAR (smoother than HIO)
- 'hybrid' — n_hio HIO → n_raar RAAR → n_er ER (recommended)
- (if n_hio=0 just runs RAAR → ER, classic)
-
- Tracks the best object by R-factor and returns that, not the last iterate
- (RAAR/HIO often oscillate; the absolute best may be mid-iteration).
-
- Parameters
- ----------
- measured_amplitude : ndarray [N, N, N]
- Square root of measured diffraction intensity.
- initial_phase : ndarray [N, N, N] or None
- Starting phase estimate. If None, random.
- initial_amplitude : ndarray [N, N, N] or None
- Starting amplitude inside support. If None, derived from
- |IFFT(measured_amplitude)|.real (autocorrelation seed). This
- preserves NN amplitude information when seeded by AutoPhaseNet.
- support : ndarray [N, N, N] or None
- Binary support mask.
- n_raar, n_er, n_hio : int
- Iteration counts per algorithm.
- beta : float
- Feedback parameter (0.7–0.9 typical for both HIO and RAAR).
- """
+
N = measured_amplitude.shape[0]
rng = np.random.default_rng(42)
- # ── Initialize support ────────────────────────────────────────────────
- # The autocorrelation A(r) = IFFT(|F|²) has support roughly twice the
- # diameter of the object (it's the convolution of object with its mirror).
- # In 3D this means autocorr volume is ~8× the object volume. A simple
- # threshold gives a too-loose support. We use a tighter threshold AND
- # cap the support fraction to a sensible upper bound.
+
if support is None:
auto_corr = np.abs(_ifft3(measured_amplitude ** 2))
ac_max = auto_corr.max()
@@ -339,13 +206,6 @@ def run_phase_retrieval(
total_iter = max(len(schedule), 1)
- # ── Main loop ────────────────────────────────────────────────────────
- # Note on Fienup HIO conventions:
- # g_{n+1}(x) = g'_n(x) if x in S
- # g_{n+1}(x) = g_n(x) - β g'_n(x) if x not in S
- # where g'_n = P_M(g_n) is the modulus projection of the CURRENT iterate.
- # So inside _support_projection, we pass obj=P_M(current) and obj_prev=current.
- # Variable `obj` here holds the current iterate g_n.
for i, mode in enumerate(schedule):
# Fourier modulus projection: g'_n = P_M(g_n)
obj_mod = _modulus_projection(obj, measured_amplitude)
@@ -409,38 +269,9 @@ def run_phase_retrieval(
)
-# ═══════════════════════════════════════════════════════════════════════════════
-# Hybrid NN + iterative retriever
-# ═══════════════════════════════════════════════════════════════════════════════
class HybridPhaseRetriever:
- """
- Main interface: uses trained U-Net for initialization,
- then refines with RAAR + ER.
-
- Three reconstruction modes:
-
- 1. 'hybrid' (recommended):
- NN predicts phase → 100 RAAR → 20 ER
- ~20× faster than classical, similar quality
-
- 2. 'nn_only':
- Just the NN prediction, no refinement
- ~1000× faster, rough quality (useful for screening)
-
- 3. 'classical':
- Random phase → 500 RAAR → 100 ER (no NN)
- Standard approach, slow but robust
-
- Parameters
- ----------
- model_path : str
- Path to trained checkpoint (best_model.pt).
- base_channels : int
- Must match the trained model architecture.
- device : str
- 'cuda' or 'cpu'.
- """
+
def __init__(
self,
@@ -468,19 +299,7 @@ def _load_model(self, path: str, base_channels: int):
@torch.no_grad()
def predict_phase_nn(self, diffraction: np.ndarray) -> np.ndarray:
- """
- Use U-Net to predict phase from diffraction intensity.
-
- Parameters
- ----------
- diffraction : ndarray [N, N, N]
- Measured diffraction intensity (not amplitude!).
-
- Returns
- -------
- ndarray [N, N, N]
- Predicted phase in radians.
- """
+
if self.model is None:
raise RuntimeError("No model loaded. Provide model_path.")
@@ -513,37 +332,12 @@ def reconstruct(
use_shrink_wrap: bool = False,
progress_cb=None,
) -> ReconstructionResult:
- """
- Full reconstruction pipeline.
-
- Parameters
- ----------
- diffraction : ndarray [N, N, N]
- Measured diffraction intensity.
- support : ndarray [N, N, N] or None
- Support mask (estimated if None).
- mode : str
- 'hybrid', 'nn_only', or 'classical'.
- n_raar : int
- RAAR iterations (100 for hybrid, 500 for classical).
- n_er : int
- ER iterations (20 for hybrid, 100 for classical).
- beta : float
- RAAR feedback parameter.
- use_shrink_wrap : bool
- Enable shrink-wrap support refinement.
- progress_cb : callable or None
- Progress callback.
-
- Returns
- -------
- ReconstructionResult
- """
+
t0 = time.time()
amplitude = np.sqrt(np.maximum(diffraction, 0)).astype(np.float32)
if mode == 'nn_only':
- # ── Fast NN-only prediction ───────────────────────────────────
+
phase = self.predict_phase_nn(diffraction)
if support is None:
@@ -566,7 +360,7 @@ def reconstruct(
)
elif mode == 'hybrid':
- # ── NN initialization + short iterative refinement ────────────
+
print(" [1/3] NN phase prediction...")
initial_phase = self.predict_phase_nn(diffraction)
@@ -587,7 +381,7 @@ def reconstruct(
result.elapsed_seconds = time.time() - t0
elif mode == 'classical':
- # ── Standard random-start approach ────────────────────────────
+
# Use more iterations to compensate for random start
if n_raar < 200:
n_raar = 500
@@ -622,12 +416,7 @@ def compare_methods(
diffraction: np.ndarray,
support: np.ndarray = None,
) -> dict:
- """
- Run all three methods on the same data for comparison.
- Returns dict with keys 'nn_only', 'hybrid', 'classical',
- each containing a ReconstructionResult.
- """
results = {}
print("\n── NN-only reconstruction ──")
@@ -661,10 +450,6 @@ def compare_methods(
return results
-# ═══════════════════════════════════════════════════════════════════════════════
-# Standalone usage
-# ═══════════════════════════════════════════════════════════════════════════════
-
if __name__ == '__main__':
import argparse
diff --git a/src/cdi_st/nn_train.py b/src/cdi_st/nn_train.py
index 030b688..bb6a4ce 100644
--- a/src/cdi_st/nn_train.py
+++ b/src/cdi_st/nn_train.py
@@ -1,24 +1,3 @@
-"""
-nn_train.py — Training loop for the BCDI phase prediction U-Net.
-
-Handles:
- - Training with mixed-precision (AMP) for faster GPU utilization
- - Cosine annealing learning rate schedule
- - Early stopping based on validation loss
- - Checkpoint saving (best model + periodic)
- - Logging training curves to CSV for plotting
- - Gradient clipping to prevent explosion
-
-Usage:
- python nn_train.py --data_dir ./training_data --epochs 50 --batch_size 8
-
- # Resume from checkpoint:
- python nn_train.py --data_dir ./training_data --resume checkpoints/best_model.pt
-
- # Smaller model for limited GPU:
- python nn_train.py --data_dir ./training_data --base_channels 16 --batch_size 4
-"""
-
from __future__ import annotations
import os, sys, time, json, argparse, csv
import numpy as np
@@ -35,7 +14,7 @@
def train_one_epoch(
model, loader, loss_fn, optimizer, scaler, device, epoch, max_grad_norm=1.0,
):
- """Train for one epoch. Returns dict of mean losses."""
+
model.train()
running = {'total': 0, 'mse': 0, 'fft_consistency': 0, 'smoothness': 0}
n_batches = 0
@@ -76,7 +55,7 @@ def train_one_epoch(
@torch.no_grad()
def validate(model, loader, loss_fn, device):
- """Validate. Returns dict of mean losses."""
+
model.eval()
running = {'total': 0, 'mse': 0, 'fft_consistency': 0, 'smoothness': 0}
n_batches = 0
@@ -99,7 +78,6 @@ def validate(model, loader, loss_fn, device):
def save_checkpoint(model, optimizer, scheduler, scaler, epoch, val_loss, path):
- """Save training state for resuming."""
torch.save({
'epoch': epoch,
'model_state_dict': model.state_dict(),
@@ -111,7 +89,7 @@ def save_checkpoint(model, optimizer, scheduler, scaler, epoch, val_loss, path):
def load_checkpoint(path, model, optimizer=None, scheduler=None, scaler=None):
- """Load training state."""
+
ckpt = torch.load(path, map_location='cpu', weights_only=False)
model.load_state_dict(ckpt['model_state_dict'])
if optimizer and 'optimizer_state_dict' in ckpt:
@@ -139,36 +117,7 @@ def train(
max_samples: int = None,
seed: int = 42,
):
- """
- Full training loop.
-
- Parameters
- ----------
- data_dir : str
- Directory with .npz training samples.
- output_dir : str
- Directory for checkpoints and logs.
- epochs : int
- Maximum training epochs.
- batch_size : int
- Batch size (reduce if OOM).
- lr : float
- Initial learning rate.
- base_channels : int
- U-Net base channels (32=~1.9M params, 16=~0.5M params).
- loss_alpha, loss_beta, loss_gamma : float
- Loss component weights.
- patience : int
- Early stopping patience (epochs without improvement).
- resume : str or None
- Path to checkpoint to resume from.
- num_workers : int
- DataLoader workers.
- max_samples : int or None
- Limit dataset size (for debugging).
- seed : int
- Random seed.
- """
+
# Setup
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
diff --git a/src/cdi_st/nn_visualize.py b/src/cdi_st/nn_visualize.py
index 4544f78..f780c7f 100644
--- a/src/cdi_st/nn_visualize.py
+++ b/src/cdi_st/nn_visualize.py
@@ -1,20 +1,3 @@
-"""
-nn_visualize.py — Visualize BCDI phase retrieval reconstruction results.
-
-Fixed version with:
- - Correct real-space vs reciprocal-space comparison
- - Phase unwrapping / global offset removal
- - Diffraction-space comparison (the scientifically meaningful one)
- - Better interpretation of convergence curves
- - Helper to find samples with actual strain for meaningful testing
-
-Usage:
- %run nn_visualize.py --input reconstruction.npz
- %run nn_visualize.py --input reconstruction.npz --ground_truth training_data/sample_00000.npz
- %run nn_visualize.py --find_strained # List samples WITH strain/dislocation
- %run nn_visualize.py --input reconstruction.npz --save_dir ./figures
-"""
-
from __future__ import annotations
import argparse, json
import numpy as np
@@ -28,7 +11,7 @@
# ═══════════════════════════════════════════════════════════════════════════════
def load_reconstruction(path: str) -> dict:
- """Load reconstruction.npz and reconstruct complex object."""
+
data = np.load(path)
out = dict(data)
if 'object_real' in data and 'object_imag' in data:
@@ -47,7 +30,7 @@ def load_reconstruction(path: str) -> dict:
def load_ground_truth(path: str) -> dict:
- """Load original sample .npz for comparison."""
+
data = np.load(path)
d = dict(data)
print(f"\nLoaded ground truth: {path}")
@@ -68,7 +51,7 @@ def load_ground_truth(path: str) -> dict:
def find_strained_samples(data_dir: str, limit: int = 20):
- """List samples that have actual strain or dislocations."""
+
data_dir = Path(data_dir)
metas = sorted(data_dir.glob("sample_*_meta.json"))
if not metas:
@@ -103,18 +86,8 @@ def find_strained_samples(data_dir: str, limit: int = 20):
print(f"{'='*76}\n")
-# ═══════════════════════════════════════════════════════════════════════════════
-# Preprocessing: remove trivial ambiguities
-# ═══════════════════════════════════════════════════════════════════════════════
-
def remove_global_phase(phase: np.ndarray, support: np.ndarray) -> np.ndarray:
- """
- Remove global phase offset (trivial phase-retrieval ambiguity).
- The diffraction pattern |F(q)|² is invariant under ρ(r) → ρ(r)·e^{iφ₀},
- so phase is only recoverable up to a constant. Subtract the circular mean
- over the support so comparisons are meaningful.
- """
mask = support > 0.5
if mask.sum() == 0:
return phase
@@ -123,9 +96,6 @@ def remove_global_phase(phase: np.ndarray, support: np.ndarray) -> np.ndarray:
return np.angle(np.exp(1j * unwrapped)) # Re-wrap to [-π, π]
-# ═══════════════════════════════════════════════════════════════════════════════
-# Figure 1: Real-space reconstruction overview
-# ═══════════════════════════════════════════════════════════════════════════════
def fig_overview(recon: dict, save_path: str = None):
support = recon.get('support', np.ones_like(recon['phase']))
@@ -165,16 +135,8 @@ def fig_overview(recon: dict, save_path: str = None):
plt.show()
-# ═══════════════════════════════════════════════════════════════════════════════
-# Figure 2: Diffraction-space comparison (the RIGHT one)
-# ═══════════════════════════════════════════════════════════════════════════════
-
def fig_diffraction_comparison(recon: dict, truth: dict, save_path: str = None):
- """
- FFT(reconstructed_object) vs measured amplitude.
- This is apples-to-apples: both are reciprocal-space amplitudes.
- """
obj = recon['object_3d']
F_recon = np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(obj)))
amp_recon_q = np.abs(F_recon).astype(np.float32)
@@ -223,10 +185,6 @@ def log_n(x):
plt.show()
-# ═══════════════════════════════════════════════════════════════════════════════
-# Figure 3: Phase comparison with offset removed
-# ═══════════════════════════════════════════════════════════════════════════════
-
def fig_phase_comparison(recon: dict, truth: dict, save_path: str = None):
support = recon.get('support', truth['support']).astype(np.float32)
@@ -285,10 +243,6 @@ def fig_phase_comparison(recon: dict, truth: dict, save_path: str = None):
plt.show()
-# ═══════════════════════════════════════════════════════════════════════════════
-# Figure 4: Convergence with annotations
-# ═══════════════════════════════════════════════════════════════════════════════
-
def fig_convergence(recon: dict, save_path: str = None):
if 'error_metric' not in recon or len(recon['error_metric']) == 0:
return
@@ -325,9 +279,6 @@ def fig_convergence(recon: dict, save_path: str = None):
plt.show()
-# ═══════════════════════════════════════════════════════════════════════════════
-# Main
-# ═══════════════════════════════════════════════════════════════════════════════
def main():
parser = argparse.ArgumentParser(description='Visualize BCDI reconstruction')
diff --git a/src/cdi_st/ortho.py b/src/cdi_st/ortho.py
new file mode 100644
index 0000000..4209f60
--- /dev/null
+++ b/src/cdi_st/ortho.py
@@ -0,0 +1,104 @@
+from __future__ import annotations
+import numpy as np
+from scipy.ndimage import map_coordinates
+
+LAB_AXES = {'x': np.array([1., 0, 0]), 'y': np.array([0, 1., 0]), 'z': np.array([0, 0, 1.])}
+
+
+def det_matrix(lam_ang, two_theta_deg, pix_um, det_dist_m, rock_step_deg,
+ gamma_deg=0.0, rock_axis='z'):
+ """Columns of M are dq/d(px), dq/d(py), dq/d(rocking step), in 1/A.
+
+ Beam along +x, vertical +z. two_theta is the in-plane scattering angle,
+ gamma the out-of-plane one. A non-orthogonal M is why raw BCDI data must be
+ interpolated onto a regular grid before the object is Fourier-inverted.
+ """
+ k = 2 * np.pi / lam_ang
+ d, g = np.radians(two_theta_deg), np.radians(gamma_deg)
+ nf = np.array([np.cos(g) * np.cos(d), np.cos(g) * np.sin(d), np.sin(g)])
+ eh = np.array([-np.sin(d), np.cos(d), 0.])
+ ev = np.array([-np.sin(g) * np.cos(d), -np.sin(g) * np.sin(d), np.cos(g)])
+ qB = k * (nf - np.array([1., 0, 0]))
+ s = k * (pix_um * 1e-6) / det_dist_m
+ dqr = -np.cross(LAB_AXES[rock_axis], qB) * np.radians(rock_step_deg)
+ return np.column_stack([s * eh, s * ev, dqr]), qB
+
+
+def skew(M):
+ """0 for an orthogonal sampling grid; rises as the grid shears."""
+ C = M / np.linalg.norm(M, axis=0, keepdims=True)
+ G = np.abs(C.T @ C); np.fill_diagonal(G, 0.0)
+ return float(G.max())
+
+
+def _match_scale(M, dq):
+ """Rescale M to the lab voxel size so only the shear, not the zoom, is shown."""
+ n = np.linalg.norm(M, axis=0)
+ if not np.all(n > 0): return M
+ return M * (float(np.mean(dq)) / float(np.mean(n)))
+
+
+def lab_to_det(vol, q, M, order=1, match_scale=True):
+ """Resample an orthogonal q-space volume onto the skewed detector grid."""
+ qx, qy, qz = q['qx'], q['qy'], q['qz']
+ dq = np.array([qx[1] - qx[0], qy[1] - qy[0], qz[1] - qz[0]])
+ if match_scale: M = _match_scale(M, dq)
+ n = vol.shape
+ q0 = np.array([qx[0], qy[0], qz[0]])
+ c0 = -q0 / dq # fractional index where q = 0
+ I, J, K = np.meshgrid(*[np.arange(s) - c0[a] for a, s in enumerate(n)], indexing='ij')
+ P = np.stack([I.ravel(), J.ravel(), K.ravel()])
+ Q = M @ P
+ c = np.stack([Q[a] / dq[a] + c0[a] for a in range(3)])
+ out = map_coordinates(vol, c, order=order, mode='constant', cval=0.0)
+ return out.reshape(n).astype(np.float32)
+
+
+def det_to_lab(vol, q, M, order=1, match_scale=True):
+ """Orthogonalise detector-frame data onto the regular lab q grid."""
+ qx, qy, qz = q['qx'], q['qy'], q['qz']
+ dq = np.array([qx[1] - qx[0], qy[1] - qy[0], qz[1] - qz[0]])
+ if match_scale: M = _match_scale(M, dq)
+ q0 = np.array([qx[0], qy[0], qz[0]])
+ c0 = -q0 / dq
+ Minv = np.linalg.inv(M)
+ QX, QY, QZ = np.meshgrid(qx, qy, qz, indexing='ij')
+ Q = np.stack([QX.ravel(), QY.ravel(), QZ.ravel()])
+ P = Minv @ Q
+ c = np.stack([P[a] + c0[a] for a in range(3)])
+ out = map_coordinates(vol, c, order=order, mode='constant', cval=0.0)
+ return out.reshape(vol.shape).astype(np.float32)
+
+
+def apply_photons(vol, n_photons, mode='total', poisson=True, seed=None):
+ """Scale to a photon budget, then draw counting statistics.
+
+ mode 'total' sets the integrated counts in the volume, 'peak' the brightest voxel.
+ """
+ v = np.asarray(vol, np.float64)
+ s = v.sum() if mode == 'total' else v.max()
+ if s <= 0: return np.zeros(v.shape, np.float32)
+ v = v * (float(n_photons) / s)
+ if poisson:
+ v = np.random.default_rng(seed).poisson(np.clip(v, 0, 1e15)).astype(np.float64)
+ return v.astype(np.float32)
+
+
+def detector_noise(vol, dark_cps=0.0, exposure_s=1.0, readout_e=0.0,
+ dead_frac=0.0, hot_frac=0.0, seed=None):
+ """Instrument effects applied after the photon budget.
+
+ readout_e is meaningful only for integrating (CCD/CMOS) detectors. Hybrid
+ photon-counting detectors (EIGER, PILATUS, MAXIPIX, MERLIN, TIMEPIX) have no
+ read noise -- leave it at zero and use dark_cps for their small dark rate.
+ """
+ rng = np.random.default_rng(seed)
+ v = np.asarray(vol, np.float64).copy()
+ if dark_cps > 0: v += rng.poisson(dark_cps * exposure_s, v.shape)
+ if readout_e > 0: v = np.maximum(v + rng.normal(0, readout_e, v.shape), 0)
+ if dead_frac > 0:
+ i = rng.choice(v.size, int(dead_frac * v.size), replace=False); v.flat[i] = 0
+ if hot_frac > 0:
+ i = rng.choice(v.size, int(hot_frac * v.size), replace=False)
+ v.flat[i] = max(v.max(), 1.0) * rng.uniform(1, 5, i.size)
+ return v.astype(np.float32)
diff --git a/src/cdi_st/recon.py b/src/cdi_st/recon.py
new file mode 100644
index 0000000..fd0eaa4
--- /dev/null
+++ b/src/cdi_st/recon.py
@@ -0,0 +1,198 @@
+from __future__ import annotations
+import os
+import numpy as np
+from scipy.ndimage import gaussian_filter, center_of_mass, shift as ndshift
+
+_F = lambda a: np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(a)))
+_Fi = lambda a: np.fft.fftshift(np.fft.ifftn(np.fft.ifftshift(a)))
+
+
+def parse_recipe(s):
+ """'HIO:200,ER:50' -> [('HIO',200),('ER',50)]"""
+ out = []
+ for part in str(s).replace(';', ',').split(','):
+ part = part.strip()
+ if not part: continue
+ a, _, n = part.partition(':')
+ a = a.strip().upper()
+ if a not in ('ER', 'HIO', 'RAAR'): raise ValueError(f'unknown algorithm {a}')
+ out.append((a, int(n) if n else 50))
+ return out
+
+
+def auto_support(data, thr=0.08):
+ ac = np.abs(_Fi(np.sqrt(np.maximum(data, 0))))
+ return ac > thr * ac.max()
+
+
+def shrinkwrap(rho, sigma, thr):
+ a = gaussian_filter(np.abs(rho).astype(np.float64), sigma)
+ return a > thr * a.max()
+
+
+def chi2(F, amp, w=None):
+ d = np.abs(F) - amp
+ if w is not None: d = d * w
+ n = (amp ** 2).sum()
+ return float((d ** 2).sum() / n) if n > 0 else np.inf
+
+
+def retrieve(data, support=None, recipe='HIO:200,ER:50', beta=0.9, bad_mask=None,
+ sw_every=20, sw_sigma=(3.0, 1.0), sw_thr=0.2, sw_max_frac=0.35, seed=0, cb=None):
+ """Iterative phase retrieval on a centred 3-D intensity volume.
+
+ data intensity (not amplitude), Bragg peak at the array centre
+ bad_mask True where the detector value is invalid -- those pixels float free
+ recipe e.g. 'RAAR:300,HIO:100,ER:50'
+ sw_thr shrinkwrap threshold on the smoothed amplitude (0.2 is a safe default)
+ cb(i, algo, err, support_frac) is called every iteration if given
+ Returns (rho, support, errors).
+ """
+ data = np.asarray(data, np.float64)
+ amp = np.sqrt(np.maximum(data, 0))
+ steps = parse_recipe(recipe)
+ total = sum(n for _, n in steps)
+ rng = np.random.default_rng(seed)
+ sup = auto_support(data) if support is None else np.asarray(support, bool)
+ if not sup.any(): sup = auto_support(data)
+
+ rho = _Fi(amp * np.exp(2j * np.pi * rng.random(amp.shape))) * sup
+ errs, it = [], 0
+ s0, s1 = sw_sigma
+
+ for algo, n in steps:
+ for _ in range(n):
+ F = _F(rho)
+ errs.append(chi2(F, amp, None if bad_mask is None else ~bad_mask))
+ ph = np.exp(1j * np.angle(F))
+ Fm = amp * ph if bad_mask is None else np.where(bad_mask, F, amp * ph)
+ rp = _Fi(Fm)
+ if algo == 'ER':
+ rho = rp * sup
+ elif algo == 'HIO':
+ rho = np.where(sup, rp, rho - beta * rp)
+ else:
+ rho = np.where(sup, rp, beta * rho + (1 - 2 * beta) * rp)
+ it += 1
+ if sw_every and it % sw_every == 0:
+ sig = s0 + (s1 - s0) * (it / max(total, 1))
+ new = shrinkwrap(rho, sig, sw_thr)
+ # reject a degenerate or runaway support: unguarded, noisy data
+ # bloats the support until the algorithm just fits the noise
+ if 8 < new.sum() < sw_max_frac * new.size: sup = new
+ if cb is not None: cb(it, algo, errs[-1], float(sup.mean()))
+
+ rho = centre(rho * sup)
+ return rho, sup, np.asarray(errs)
+
+
+def centre(rho):
+ a = np.abs(rho)
+ if a.max() <= 0: return rho
+ c = np.asarray(center_of_mass(a)) - np.asarray(rho.shape) / 2.0
+ return ndshift(rho.real, -c, order=1) + 1j * ndshift(rho.imag, -c, order=1)
+
+
+def remove_ramp(rho, sup=None):
+ """Strip the linear phase ramp (an arbitrary real-space origin shift)."""
+ m = (np.abs(rho) > 0.1 * np.abs(rho).max()) if sup is None else np.asarray(sup, bool)
+ if m.sum() < 16: return rho
+ idx = np.argwhere(m).astype(float)
+ ph = np.unwrap(np.angle(rho[m]))
+ A = np.column_stack([idx, np.ones(len(idx))])
+ c, *_ = np.linalg.lstsq(A, ph, rcond=None)
+ g = np.meshgrid(*[np.arange(s, dtype=float) for s in rho.shape], indexing='ij')
+ ramp = c[0] * g[0] + c[1] * g[1] + c[2] * g[2] + c[3]
+ return rho * np.exp(-1j * ramp)
+
+
+def prtf(rho, data, nbins=40):
+ """Single-run PRTF vs |q| (shell-averaged). Call on the raw retrieve() output:
+ remove_ramp shifts the pattern in q and would drag the PRTF down."""
+ amp = np.sqrt(np.maximum(np.asarray(data, np.float64), 0))
+ F = np.abs(_F(rho))
+ g = np.meshgrid(*[np.arange(s) - s // 2 for s in data.shape], indexing='ij')
+ r = np.sqrt(sum(x.astype(float) ** 2 for x in g))
+ b = np.clip((r / (r.max() + 1e-12) * nbins).astype(int), 0, nbins - 1)
+ num = np.bincount(b.ravel(), F.ravel(), nbins)
+ den = np.bincount(b.ravel(), amp.ravel(), nbins)
+ with np.errstate(invalid='ignore', divide='ignore'):
+ p = np.where(den > 0, num / den, np.nan)
+ return r.max() * (np.arange(nbins) + 0.5) / nbins, p
+
+
+# ---------------------------------------------------------------- external I/O
+
+_OBJ_KEYS = ['obj', 'object', 'rho', 'reconstruction', 'recon', 'complex_object',
+ 'image', 'data', 'result']
+_H5_PATHS = ['/entry_1/image_1/data', '/entry_1/data_1/data', '/entry/data/data',
+ '/object', '/obj', '/reconstruction', '/rho', '/data']
+
+
+def load_recon(path):
+ """Load a reconstruction produced elsewhere (PyNX, bcdi, cdiutils, MATLAB...).
+
+ Returns (complex array, metadata dict). Amplitude/phase pairs are recombined.
+ """
+ ext = os.path.splitext(path)[1].lower()
+ if ext == '.npy':
+ return np.asarray(np.load(path)), {'source': path}
+ if ext == '.npz':
+ z = np.load(path, allow_pickle=True)
+ meta = {'source': path, 'keys': list(z.keys())}
+ for k in _OBJ_KEYS:
+ if k in z and np.iscomplexobj(z[k]): return np.asarray(z[k]), meta
+ amp = next((z[k] for k in ('amp', 'amplitude', 'modulus') if k in z), None)
+ ph = next((z[k] for k in ('phase', 'phi', 'angle') if k in z), None)
+ if amp is not None:
+ return np.asarray(amp) * np.exp(1j * np.asarray(ph if ph is not None else 0)), meta
+ for k in _OBJ_KEYS:
+ if k in z and np.asarray(z[k]).ndim == 3:
+ return np.asarray(z[k]).astype(complex), meta
+ raise KeyError(f'no object array found; keys={list(z.keys())}')
+ if ext in ('.h5', '.hdf5', '.cxi', '.nxs'):
+ import h5py
+ with h5py.File(path, 'r') as f:
+ meta = {'source': path, 'attrs': dict(f.attrs)}
+ for p in _H5_PATHS:
+ if p in f:
+ a = np.asarray(f[p])
+ if a.ndim == 3: return a.astype(complex), meta
+ found = []
+ f.visit(lambda n: found.append(n))
+ for n in found:
+ try: d = f[n]
+ except Exception: continue
+ if hasattr(d, 'shape') and len(getattr(d, 'shape', ())) == 3:
+ return np.asarray(d).astype(complex), {**meta, 'path': n}
+ raise KeyError('no 3-D dataset found in file')
+ if ext == '.mat':
+ try:
+ from scipy.io import loadmat
+ m = loadmat(path)
+ for k in _OBJ_KEYS + list(m.keys()):
+ if k.startswith('__'): continue
+ a = np.asarray(m.get(k, []))
+ if a.ndim == 3: return a.astype(complex), {'source': path, 'key': k}
+ except NotImplementedError:
+ import h5py
+ with h5py.File(path, 'r') as f:
+ for n in f:
+ a = np.asarray(f[n])
+ if a.ndim == 3: return a.astype(complex), {'source': path, 'key': n}
+ raise KeyError('no 3-D array in .mat file')
+ raise ValueError(f'unsupported format: {ext}')
+
+
+def save_recon(path, rho, support=None, errs=None, **meta):
+ d = {'obj': np.asarray(rho)}
+ if support is not None: d['support'] = np.asarray(support)
+ if errs is not None: d['errors'] = np.asarray(errs)
+ for k, v in meta.items(): d[k] = np.asarray(v)
+ if path.endswith(('.h5', '.cxi')):
+ import h5py
+ with h5py.File(path, 'w') as f:
+ for k, v in d.items(): f.create_dataset(k, data=v, compression='gzip')
+ else:
+ np.savez_compressed(path, **d)
+ return path
diff --git a/src/cdi_st/trainlog.py b/src/cdi_st/trainlog.py
new file mode 100644
index 0000000..ba89742
--- /dev/null
+++ b/src/cdi_st/trainlog.py
@@ -0,0 +1,146 @@
+from __future__ import annotations
+import json, os, time
+from datetime import datetime
+import numpy as np
+
+
+def _finite_stats(a):
+ a = np.asarray(a, np.float64).ravel()
+ n = a.size
+ bad_n = int(np.isnan(a).sum()); bad_i = int(np.isinf(a).sum())
+ f = a[np.isfinite(a)]
+ return {'n': n, 'nan': bad_n, 'inf': bad_i,
+ 'min': float(f.min()) if f.size else None,
+ 'max': float(f.max()) if f.size else None,
+ 'absmax': float(np.abs(f).max()) if f.size else None,
+ 'mean': float(f.mean()) if f.size else None,
+ 'std': float(f.std()) if f.size else None}
+
+
+class TrainLog:
+ """Writes .log (human) and .jsonl (per-step metrics) next to each other.
+
+ tl = TrainLog('runs', 'autophase_v3')
+ ...
+ loss.backward()
+ if tl.step(ep, it, loss, model=net, lr=lr, inputs=x) != 'ok':
+ opt.zero_grad(); continue # skip the poisoned update
+ opt.step()
+ """
+
+ def __init__(self, outdir, run=None, grad_clip_warn=1e3, loss_jump=50.0, echo=True):
+ os.makedirs(outdir, exist_ok=True)
+ run = run or datetime.now().strftime('run_%Y%m%d_%H%M%S')
+ self.run = run
+ self.log_path = os.path.join(outdir, run + '.log')
+ self.jsonl_path = os.path.join(outdir, run + '.jsonl')
+ self.grad_clip_warn = grad_clip_warn
+ self.loss_jump = loss_jump
+ self.echo = echo
+ self.t0 = time.time()
+ self.n_bad = 0
+ self.last_loss = None
+ self._lf = open(self.log_path, 'a', buffering=1)
+ self._jf = open(self.jsonl_path, 'a', buffering=1)
+ self.write(f'=== {run} started {datetime.now().isoformat(timespec="seconds")} ===')
+
+ def write(self, msg):
+ line = f'[{time.time() - self.t0:8.1f}s] {msg}'
+ self._lf.write(line + '\n')
+ if self.echo: print(line, flush=True)
+
+ def config(self, **kw):
+ self.write('config ' + json.dumps(kw, default=str))
+
+ def _grad_norm(self, model):
+ tot, bad = 0.0, []
+ for n, p in model.named_parameters():
+ g = getattr(p, 'grad', None)
+ if g is None: continue
+ g = g.detach()
+ if not bool(g.isfinite().all()): bad.append(n)
+ tot += float(g.float().pow(2).sum())
+ return tot ** 0.5, bad
+
+ def _param_check(self, model):
+ return [n for n, p in model.named_parameters()
+ if not bool(p.detach().isfinite().all())]
+
+ def step(self, epoch, it, loss, model=None, lr=None, inputs=None, extra=None):
+ """Returns 'ok', 'nan', 'inf' or 'spike'. Dumps a diagnosis on anything but 'ok'."""
+ lv = float(loss.detach()) if hasattr(loss, 'detach') else float(loss)
+ rec = {'t': round(time.time() - self.t0, 2), 'epoch': epoch, 'it': it, 'loss': lv}
+ if lr is not None: rec['lr'] = float(lr)
+
+ gn, gbad = (None, [])
+ if model is not None:
+ try: gn, gbad = self._grad_norm(model)
+ except Exception as e: self.write(f'grad inspect failed: {e!r}')
+ if gn is not None: rec['grad_norm'] = gn
+
+ status = 'ok'
+ if lv != lv: status = 'nan'
+ elif lv in (float('inf'), float('-inf')): status = 'inf'
+ elif gbad: status = 'nan'
+ elif self.last_loss is not None and abs(lv) > self.loss_jump * max(abs(self.last_loss), 1e-12):
+ status = 'spike'
+ elif gn is not None and gn > self.grad_clip_warn:
+ self.write(f'ep{epoch} it{it} large grad norm {gn:.3e} (loss {lv:.4e})')
+
+ rec['status'] = status
+ if extra: rec.update(extra)
+ self._jf.write(json.dumps(rec) + '\n')
+
+ if status != 'ok':
+ self.n_bad += 1
+ self.write(f'--- {status.upper()} at epoch {epoch} it {it}: loss={lv!r} '
+ f'grad_norm={gn} lr={lr}')
+ if gbad: self.write(f' non-finite grads in: {gbad[:12]}')
+ if model is not None:
+ pbad = self._param_check(model)
+ if pbad: self.write(f' non-finite weights in: {pbad[:12]}')
+ if inputs is not None:
+ for name, arr in self._as_arrays(inputs):
+ s = _finite_stats(arr)
+ if s['nan'] or s['inf'] or (s['absmax'] or 0) > 1e8:
+ self.write(f' input {name}: {s}')
+ self.write(' likely causes: log/sqrt/div of 0 in the loss, un-normalised '
+ 'intensities, lr too high, fp16 overflow, or an empty support mask')
+ self.last_loss = lv if status == 'ok' else self.last_loss
+ return status
+
+ @staticmethod
+ def _as_arrays(x, prefix='x'):
+ out = []
+ if isinstance(x, dict): items = list(x.items())
+ elif isinstance(x, (list, tuple)): items = [(f'{prefix}{i}', v) for i, v in enumerate(x)]
+ else: items = [(prefix, x)]
+ for n, v in items:
+ try:
+ out.append((n, v.detach().cpu().numpy() if hasattr(v, 'detach') else np.asarray(v)))
+ except Exception:
+ pass
+ return out
+
+ def epoch_end(self, epoch, **metrics):
+ self.write(f'epoch {epoch} ' + ' '.join(f'{k}={v:.5g}' for k, v in metrics.items()))
+ self._jf.write(json.dumps({'epoch': epoch, 'summary': metrics,
+ 't': round(time.time() - self.t0, 2)}) + '\n')
+
+ def close(self):
+ self.write(f'=== finished, {self.n_bad} bad step(s), log: {self.log_path} ===')
+ self._lf.close(); self._jf.close()
+
+ def __enter__(self): return self
+ def __exit__(self, *a): self.close()
+
+
+def scan_dataset(arrays, names=None):
+ """Pre-flight check: report non-finite or absurdly scaled training data."""
+ names = names or [f'array{i}' for i in range(len(arrays))]
+ bad = []
+ for n, a in zip(names, arrays):
+ s = _finite_stats(a)
+ if s['nan'] or s['inf'] or s['absmax'] is None or s['absmax'] > 1e8:
+ bad.append((n, s))
+ return bad
diff --git a/src/cdi_st/viz3d.py b/src/cdi_st/viz3d.py
new file mode 100644
index 0000000..0c56378
--- /dev/null
+++ b/src/cdi_st/viz3d.py
@@ -0,0 +1,102 @@
+from __future__ import annotations
+import numpy as np
+
+BCDI = [[0, '#030d1e'], [.12, '#0a2a6e'], [.30, '#1060c8'], [.50, '#30b0e8'],
+ [.65, '#e8d830'], [.80, '#e87010'], [.92, '#d01808'], [1., '#ffffff']]
+
+CMAPS = {'BCDI': BCDI, 'Turbo': 'Turbo', 'Inferno': 'Inferno', 'Viridis': 'Viridis',
+ 'Plasma': 'Plasma', 'Magma': 'Magma', 'Cividis': 'Cividis', 'Jet': 'Jet',
+ 'Hot': 'Hot', 'Rainbow': 'Rainbow', 'Greys': 'Greys', 'Blackbody': 'Blackbody',
+ 'Electric': 'Electric', 'Twilight': 'Twilight'}
+CMAP_NAMES = list(CMAPS)
+# cyclic maps belong on phase, not intensity
+PHASE_CMAPS = ['Twilight', 'HSV', 'Phase']
+
+SCOPES = {'volume': 'whole volume (global min/max)',
+ 'shell': 'rendered shell only (between iso-min and iso-max)',
+ 'slice': 'current clip slice only'}
+
+
+def limits(vol, scope='volume', iso_lo=None, iso_hi=None, clip_index=None, clip_axis=0):
+ v = np.asarray(vol)
+ if scope == 'shell' and iso_lo is not None:
+ m = v >= iso_lo
+ if iso_hi is not None: m &= v <= iso_hi
+ s = v[m]
+ elif scope == 'slice' and clip_index is not None:
+ s = np.take(v, int(clip_index), axis=clip_axis).ravel()
+ else:
+ scope, s = 'volume', v.ravel()
+ s = s[np.isfinite(s)]
+ if s.size == 0: s = v.ravel()[np.isfinite(v.ravel())]
+ lo, hi = (float(s.min()), float(s.max())) if s.size else (0.0, 1.0)
+ return lo, hi, f'auto scale from {SCOPES.get(scope, scope)}: [{lo:.3g}, {hi:.3g}]'
+
+
+def clip_volume(vol, axis=0, index=None, side='below'):
+ v = np.asarray(vol, np.float32).copy()
+ n = v.shape[axis]
+ i = n // 2 if index is None else int(np.clip(index, 0, n - 1))
+ keep = [slice(None)] * 3
+ keep[axis] = slice(i, None) if side == 'below' else slice(None, i + 1)
+ out = np.full_like(v, np.nan)
+ out[tuple(keep)] = v[tuple(keep)]
+ return out, i
+
+
+def cut_face(fig, vol, q, axis, index, cmap, lo, hi, opacity=1.0):
+ import plotly.graph_objects as go
+ qx, qy, qz = q['qx'], q['qy'], q['qz']
+ sl = np.take(np.asarray(vol), int(index), axis=axis)
+ if axis == 0:
+ A, B = np.meshgrid(qy, qz, indexing='ij')
+ X = np.full(A.shape, qx[int(index)]); Y, Z = A, B
+ elif axis == 1:
+ A, B = np.meshgrid(qx, qz, indexing='ij')
+ Y = np.full(A.shape, qy[int(index)]); X, Z = A, B
+ else:
+ A, B = np.meshgrid(qx, qy, indexing='ij')
+ Z = np.full(A.shape, qz[int(index)]); X, Y = A, B
+ fig.add_trace(go.Surface(x=X, y=Y, z=Z, surfacecolor=sl, colorscale=CMAPS.get(cmap, cmap),
+ cmin=lo, cmax=hi, showscale=False, opacity=opacity,
+ name='cut', hovertemplate='%{surfacecolor:.3g}cut'))
+ return fig
+
+
+def volume_fig(vol, q, cmap='BCDI', iso_lo_pct=75, iso_hi_pct=100, opacity=1.0,
+ surfaces=21, scope='volume', clip=None, clip_fill=True, title=''):
+ """
+ clip = (axis, index, side) or None.
+ """
+ import plotly.graph_objects as go
+ v = np.asarray(vol, np.float32)
+ vmax = float(np.nanmax(v)) if np.isfinite(v).any() else 1.0
+ lo_i, hi_i = vmax * iso_lo_pct / 100.0, vmax * iso_hi_pct / 100.0
+ ci = clip[1] if clip else None
+ ca = clip[0] if clip else 0
+ lo, hi, label = limits(v, scope, lo_i, hi_i, ci, ca)
+ shown = v
+ if clip:
+ shown, ci = clip_volume(v, clip[0], clip[1], clip[2])
+ qx, qy, qz = q['qx'], q['qy'], q['qz']
+ QX, QY, QZ = np.meshgrid(qx, qy, qz, indexing='ij')
+ fig = go.Figure()
+ fig.add_trace(go.Volume(x=QX.ravel(), y=QY.ravel(), z=QZ.ravel(),
+ value=np.nan_to_num(shown, nan=-1e30).ravel(),
+ isomin=lo_i, isomax=hi_i, opacity=opacity,
+ surface_count=int(surfaces), colorscale=CMAPS.get(cmap, cmap),
+ cmin=lo, cmax=hi, showscale=True,
+ colorbar=dict(thickness=14, len=.9, tickfont=dict(size=10)),
+ caps=dict(x_show=False, y_show=False, z_show=False)))
+ if clip and clip_fill:
+ cut_face(fig, v, q, ca, ci, cmap, lo, hi)
+ ax = dict(backgroundcolor='#080a0e', gridcolor='rgba(255,255,255,.09)',
+ showbackground=True, color='#c0c4cc', zeroline=False)
+ fig.update_layout(template='plotly_dark', paper_bgcolor='#0d0e12',
+ scene=dict(xaxis=dict(**ax, title='qx(1/A)'), yaxis=dict(**ax, title='qy(1/A)'),
+ zaxis=dict(**ax, title='qz(1/A)'), aspectmode='data',
+ camera=dict(eye=dict(x=1.4, y=1.4, z=.7))),
+ font=dict(family='Consolas,monospace', size=11, color='#c0c4cc'),
+ title=dict(text=title, font=dict(size=12), x=.5, xanchor='center'),
+ margin=dict(l=0, r=0, t=36, b=0))
+ return fig, label