From 103778239d20f319fac1a234d7a6f06c7e989346 Mon Sep 17 00:00:00 2001 From: Refze Date: Sun, 8 Nov 2026 14:41:36 +0100 Subject: [PATCH] v0.3.0: faceted particles, photon counts, detector/sample frames, phase retrieval solo, training diagnostics --- .gitignore | 45 -- CHANGELOG.md | 189 +++---- src/cdi_st/__main__.py | 1 - src/cdi_st/bcdi_core.py | 164 +++--- src/cdi_st/bcdi_gui.py | 814 +++++++-------------------- src/cdi_st/facets.py | 82 +++ src/cdi_st/nn_autophase_infer.py | 137 +---- src/cdi_st/nn_autophase_model.py | 152 +---- src/cdi_st/nn_autophase_train.py | 42 +- src/cdi_st/nn_data_generator.py | 93 +-- src/cdi_st/nn_dataset.py | 88 --- src/cdi_st/nn_demo.py | 44 +- src/cdi_st/nn_experimental_loader.py | 572 ------------------- src/cdi_st/nn_gui_tabs.py | 27 - src/cdi_st/nn_phase_model.py | 132 +---- src/cdi_st/nn_phase_retrieval.py | 231 +------- src/cdi_st/nn_train.py | 59 +- src/cdi_st/nn_visualize.py | 55 +- src/cdi_st/ortho.py | 104 ++++ src/cdi_st/recon.py | 198 +++++++ src/cdi_st/trainlog.py | 146 +++++ src/cdi_st/viz3d.py | 102 ++++ 22 files changed, 1034 insertions(+), 2443 deletions(-) delete mode 100644 .gitignore create mode 100644 src/cdi_st/facets.py create mode 100644 src/cdi_st/ortho.py create mode 100644 src/cdi_st/recon.py create mode 100644 src/cdi_st/trainlog.py create mode 100644 src/cdi_st/viz3d.py diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 668083d..0000000 --- a/.gitignore +++ /dev/null @@ -1,45 +0,0 @@ -# Build artifacts -build/ -dist/ -*.egg-info/ -*.egg -__pycache__/ -*.py[cod] -*$py.class -.Python - -# Virtual environments -venv/ -env/ -.venv/ -.env/ -ENV/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -.DS_Store - -# Test / coverage -.pytest_cache/ -.coverage -htmlcov/ -.tox/ -.cache - -# Tools -.mypy_cache/ -.ruff_cache/ - -# Project-specific -training_data/ -checkpoints_*/ -*.pt -*.npz -*.h5 -*.vti - -# Logs -*.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 60be931..73e93e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,122 +1,91 @@ # Changelog -All notable changes to CDI-ST are documented here. -The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.3.0] - 2026-11-08 -## [Unreleased] +Faceted particles, photon-budget noise, detector/sample space, standalone phase +retrieval and training diagnostics. -## [0.2.2] — 2026-06-18 - -### Fixed -- **ID01 SPEC+EDF conversion now tolerates missing edge frames.** - Acquisitions are sometimes started a frame late or stopped a frame - early, so a SPEC scan can list frames 262–662 (401 points) while the - EDF directory only has 263–661 (399 files). The converter now probes - the disk first, loads only the EDF files that actually exist, and - trims the per-frame motor arrays (`eta`, `phi`, `nu`, `delta`, - `mpx4inr`) to stay in lock-step with the loaded data. A clear note - identifies which leading / trailing / interior frames were skipped. - New helper `_filter_existing_edf_frames()` exposes this filtering for - programmatic use. - -## [0.2.1] — 2026-06-15 +With thanks to **Dr. Gerards Hinsley**, whose review of v0.2.2 prompted most of +the changes in this release: the facet options, the photon-count control, the +detector-space toggle, the standalone reconstruction path, the training log, and +the corrections to the noise model and colour-scale controls. ### Added -- **PETRA III P10 / DESY Eiger compressed-data support.** Auto-imports - `hdf5plugin` at module load so bitshuffle + LZ4 compressed datasets - (the Eiger default at DESY P10 and many other beamlines) read - transparently. `hdf5plugin>=4.0` is now a runtime dependency. -- **HDF5 external-link resolution.** The P10 loader now enumerates and - resolves external links in master files instead of relying on h5py's - default link-following, which was unreliable on Windows. Each chunk is - opened directly to bypass any path-mangling. Master files with hundreds - of external links to chunk files (the standard Eiger layout) work out - of the box. -- **Scan range selection for P10** (new in the P10 converter dialog) — - similar to the ID01 SPEC+EDF scan browser. Avoids loading 49 GB of - detector frames into RAM when you only need the rocking-curve maximum: - - **Probe scan** button reads chunk metadata only (no decompression), - instantly reporting total frame count, detector shape, and estimated - full-load memory footprint. - - **Frame range** (`from`, `to`) reads only the requested global frame - range across all chunks. Loaded chunk-by-chunk so memory scales with - the chosen range, not the total dataset size. - - **Detector ROI** reads a centered square ROI from each frame instead - of the full detector. For BCDI you typically only need ~256×256 - around the Bragg peak — cuts memory by `(2167*2070)/(roi²)` for - Eiger 4M data. - - **Memory budget** (default 8 GB) — the loader refuses reads that - would exceed this and tells you which `frame_range` would fit, - instead of crashing the Python process with an OOM error. -- **Programmatic API additions** to `nn_experimental_loader`: - `p10_scan_info()` for cheap metadata previews, plus new - `frame_range`, `detector_roi`, `memory_budget_gb` parameters on - `load_p10_from_external_links()` and `p10_h5_to_npz()`. -### Fixed -- Cryptic "Can't synchronously read data (can't open directory)" error - on P10 / Eiger files. The message was HDF5 looking for the compression - plugin DLL in a non-existent directory — now caught and converted to - a clear "install hdf5plugin" instruction. -- AutoPhase_NN tab missing `_on_epoch` slot (introduced in 0.2.0 during - the live-curve refactor; AutoPhase_NN's `epoch_done` signal had no - receiver and emitted unhandled errors). Restored. -- CDI_NN `KeyError: 'phase_true'` — `BCDIDataset.__getitem__` now exposes - the GUI's expected keys (`phase_true`, `amplitude`) alongside the legacy - ones (`target_phase`, `input`). -- `BCDIPhaseLoss` accepts both the GUI's descriptive kwargs (`alpha_amp`, - `beta_phase`, `gamma_diff`, `diff_amp`) and the original short ones - (`alpha`, `beta`, `gamma`, `amplitude`). Either spelling now works. -- Training Stop button no longer crashes the program. `QThread.terminate()` - was killing PyTorch mid-`backward()` and corrupting CUDA / autograd - state — replaced with a cooperative `request_stop()` flag that the - training loop checks at batch / epoch boundaries, then exits cleanly - with a saved checkpoint. +**Simulation** +- Crystallographic facet shapes built from a Wulff half-space construction + (`facets.py`): octahedron, cuboctahedron, truncated octahedron, truncated cube, + rhombic dodecahedron, tunable Wulff, and hexagonal prism / bipyramid / + truncated bipyramid. Facet normals come from the reciprocal lattice, so the + construction is correct for every crystal system. +- Facet multiplicities control: a ratio of facet distances, off by default, + enabled by a tick box, with a plain-language tooltip giving the shape each + value produces. +- Particle size entered directly in nm; supercell multiples follow from the + lattice vectors. Cell limit raised from 2500 to 100000 per axis, with a size + notice above 2M atoms and a confirmation above 10M. +- Photon budget: the pattern is scaled to a chosen number of photons before + Poisson sampling, so the counting statistics mean something. The field accepts + scientific notation (1e9, 2.5e12). +- Detector / sample space toggle. `ortho.det_matrix` builds the real + (dq/dpx, dq/dpy, dq/drocking) transform; the Results tab switches between the + orthogonal lab q grid and the raw skewed detector grid and reports the skew. +- Custom 2theta with a live readout: the offset from the nearest reflection, the + resulting dq, and the peak shift in both simulation voxels and real detector + pixels, with a warning when the peak leaves the frame. +- Prismatic dislocation loop alongside the existing line dislocations. +- Separate simulation grid control, independent of the detector panel size. +- Visualization controls (bonds, strain, dislocation) grouped together. -### Changed -- Live training curve in AutoPhase_NN and CDI_NN — the dashed "running" - line updates every ~2 s during training so users can see progress - immediately on slow CPU runs (rather than staring at a blank plot - for an entire epoch). -- `inspect_h5()` now properly shows external links with their target - files and `[OK]` / `[MISSING]` markers. The empty-looking `/entry/data/` - group on Eiger masters is no longer misleading. +**Data processing** +- `recon.py` — standalone ER / HIO / RAAR phase retrieval with shrinkwrap, + bad-pixel masking, chi-squared history, centring, phase-ramp removal and PRTF. + No network needed. +- `recon.load_recon` — reads reconstructions from `.npz`, `.npy`, `.h5`, `.cxi`, + `.nxs` and `.mat`, including PyNX `/entry_1/image_1/data` and amplitude/phase + pairs, so work reconstructed outside CDI-St can be loaded. +- `trainlog.py` — per-step training log (`.log` + `.jsonl`) that flags NaN/Inf + losses, non-finite gradients and weights, and loss spikes, then dumps input + statistics and the likely cause. `scan_dataset()` pre-flights a training set. +- `viz3d.py` — 14 colormaps including Turbo; clip plane with a filled cut face + showing the data on the cut layer; explicit colour-scale scope (whole volume / + rendered shell / current slice) with a label stating which was used. -## [0.1.0] — Alpha — 2026-05-08 +### Changed -### Added -- **Initial public release** of CDI-ST as a packaged Python application. -- BCDI Simulation module (Material → Beam → Results) - - Material presets: Si, Ge, Pt, Au, SiC, GaAs, and more - - CIF import for arbitrary crystals - - Particle shapes: cube, sphere, cylinder, hexagonal prism, octahedron, dodecahedron - - Strain fields: radial gradient, edge dislocation, random - - Configurable line dislocations: edge / screw / mixed with arbitrary Burgers vector - - 3D atomic lattice viewer (Plotly) - - Detector presets: Maxipix 2×2, Eiger 2M, custom rectangular - - Analytical and FFT simulation paths -- BCDI Data Analysis module (Generate Data → AutoPhase_NN → CDI_NN → Reconstruction → 3D Viewer) - - Training-data generator with random material/shape/size/strain/dislocation - - Toggles for randomized strain and dislocations - - AutoPhaseNet3D (unsupervised, dual-decoder, physics-driven loss) - - PhaseUNet3D (supervised, BCDI loss with α/β/γ weights) - - Inference: NN-only, refined (HIO+RAAR+ER), ensemble, compare - - Iterative phase retrieval: HIO (β=0.9), RAAR (β=0.9), ER, with shrink-wrap - - Experimental data loader: HDF5, ID01 SPEC+EDF, q-space orthogonalization - - Detector chip-gap masking, hot pixel removal, beamstop streak removal - - 3D viewer with point cloud / isosurface / surface-only modes - - Per-voxel opacity weighted by intensity - - VTI export for ParaView -- Splash screen and theme-matched launcher -- In-app Reports & Suggestions feature -- Cross-platform support (Linux, macOS, Windows) via PyQt6 -- MIT license, full source on GitHub +- Read noise is applied only to integrating detectors. Hybrid photon-counting + detectors (EIGER, PILATUS, MAXIPIX, MERLIN, TIMEPIX) have Poisson statistics + only, and the Readout option disables itself when one is selected. A + `CCD (integrating)` preset was added for the case where read noise applies. +- Noise controls are named per effect (Poisson / Readout / Air / Dead px / + Hot px) instead of a single "Add noise", and all are off by default, so the + ideal pattern is what you see until you ask for noise. +- Default simulation grid raised from 128 to 192, giving 2.25x the pixels in the + 2D slices for about 2 s per simulation. +- The simulation panel is resizable; the fixed 520 px left column is a splitter. +- Frame transform uses cubic interpolation (0.7% round-trip error, against 3.6% + for linear). -### Known limitations (alpha) -- Q-space orthogonalization is ID01-specific (can be extended) -- Some checkpoints from earlier prototypes lack `grid_size` metadata -- Mask-aware modulus constraint not yet implemented (gaps are filled instead) +### Fixed -[Unreleased]: https://github.com/SAIDIsoufiane/CDI-St/compare/v0.1.0...HEAD -[0.1.0]: https://github.com/SAIDIsoufiane/CDI-St/releases/tag/v0.1.0 +- Poisson noise was on by default and applied the photon budget unconditionally, + which quantised 83% of the volume to zero counts and erased the weak fringes + in the 3D peak and the 2D slices. Noise is now opt-in. +- The simulation grid was capped by the detector panel size, so a 128 x 128 + panel forced 128^3 slices regardless of the grid setting. Panel geometry + (exported as metadata) and simulation grid are now independent. +- Reflections are labelled by their canonical Friedel mate, so selecting (1 1 1) + no longer displays (-1 -1 -1). Both label the same physical peak; the hkl + search simply reached the negative representative first. The raw index is kept + as `hkl_raw`. +- `select_reflection` returns full-precision d, 2theta, q, |F|^2 and LP. It + previously used the four-decimal values rounded for the table. +- Half-voxel registration error in the detector/lab transform. An orthogonal + transform is now an exact no-op for any q-grid convention. +- Shrinkwrap could inflate the support until the algorithm fit the noise (89% of + the array at 1e5 counts). An absolute cap keeps reconstructions stable from + 1e3 to 1e8 photons. +- Selecting a large detector preset set the transform size to the full panel: + Eiger2 4M attempted a 2068^3 complex FFT, about 140 GB. +- The facet ratio control was not read when building, and the large-supercell + notice was missing. diff --git a/src/cdi_st/__main__.py b/src/cdi_st/__main__.py index 509ccd8..7de9c83 100644 --- a/src/cdi_st/__main__.py +++ b/src/cdi_st/__main__.py @@ -1,4 +1,3 @@ -"""Entry point for `python -m cdi_st`.""" from cdi_st.bcdi_gui import main if __name__ == "__main__": diff --git a/src/cdi_st/bcdi_core.py b/src/cdi_st/bcdi_core.py index eb6ed34..5212718 100644 --- a/src/cdi_st/bcdi_core.py +++ b/src/cdi_st/bcdi_core.py @@ -1,4 +1,3 @@ -"""bcdi_core.py v6.0 — dislocation loops, CIF import, rect detector, detector presets, export.""" from __future__ import annotations import os,warnings,re,tempfile from pathlib import Path @@ -14,6 +13,7 @@ import xrayutilities as xu; _HAS_XU=True except: xu=None; _HAS_XU=False warnings.filterwarnings('ignore') +import facets, ortho def _6c_R3m(z): return [[0,0,z%1],[2/3,1/3,(z+1/3)%1],[1/3,2/3,(z+2/3)%1],[0,0,(1-z)%1],[2/3,1/3,(1-z+1/3)%1],[1/3,2/3,(1-z+2/3)%1]] @@ -22,17 +22,19 @@ def _3a_R3m(): def _3_from_6c_R3m(z): return [[0,0,z%1],[2/3,1/3,(z+1/3)%1],[1/3,2/3,(z+2/3)%1]] +# counting=True -> hybrid photon-counting pixel detector: Poisson only, no read noise DETECTOR_PRESETS={ - 'Custom':{'nx':128,'ny':128,'pixel_um':55.0}, - 'Eiger2 500K':{'nx':1028,'ny':512,'pixel_um':75.0}, - 'Eiger2 1M':{'nx':1028,'ny':1062,'pixel_um':75.0}, - 'Eiger2 4M':{'nx':2068,'ny':2162,'pixel_um':75.0}, - 'Maxipix 1x1':{'nx':256,'ny':256,'pixel_um':55.0}, - 'Maxipix 2x2':{'nx':516,'ny':516,'pixel_um':55.0}, - 'Merlin Quad':{'nx':512,'ny':512,'pixel_um':55.0}, - 'Pilatus 100K':{'nx':487,'ny':195,'pixel_um':172.0}, - 'Pilatus 300K':{'nx':487,'ny':619,'pixel_um':172.0}, - 'Timepix3':{'nx':256,'ny':256,'pixel_um':55.0}, + 'Custom':{'nx':128,'ny':128,'pixel_um':55.0,'counting':True,'dark_cps':0.0}, + 'Eiger2 500K':{'nx':1028,'ny':512,'pixel_um':75.0,'counting':True,'dark_cps':0.0}, + 'Eiger2 1M':{'nx':1028,'ny':1062,'pixel_um':75.0,'counting':True,'dark_cps':0.0}, + 'Eiger2 4M':{'nx':2068,'ny':2162,'pixel_um':75.0,'counting':True,'dark_cps':0.0}, + 'Maxipix 1x1':{'nx':256,'ny':256,'pixel_um':55.0,'counting':True,'dark_cps':0.0}, + 'Maxipix 2x2':{'nx':516,'ny':516,'pixel_um':55.0,'counting':True,'dark_cps':0.0}, + 'Merlin Quad':{'nx':512,'ny':512,'pixel_um':55.0,'counting':True,'dark_cps':0.0}, + 'Pilatus 100K':{'nx':487,'ny':195,'pixel_um':172.0,'counting':True,'dark_cps':0.0}, + 'Pilatus 300K':{'nx':487,'ny':619,'pixel_um':172.0,'counting':True,'dark_cps':0.0}, + 'Timepix3':{'nx':256,'ny':256,'pixel_um':55.0,'counting':True,'dark_cps':0.0}, + 'CCD (integrating)':{'nx':1024,'ny':1024,'pixel_um':13.5,'counting':False,'dark_cps':0.05}, } MATERIAL_PRESETS={ @@ -113,11 +115,18 @@ def default_shape_for_material(n): if ct in('hcp','wurtzite') or m['gamma']==120: return 'hexagonal' return 'cube' def compatible_shapes(n): - if n not in MATERIAL_PRESETS: return ['cube','rectangle','cylinder','hexagonal','sphere'] + base=['cube','rectangle','cylinder','sphere'] + ded=lambda L:list(dict.fromkeys(L)) + if n not in MATERIAL_PRESETS: return ded(base+['hexagonal']+facets.CUBIC_SHAPES+facets.HEX_SHAPES) m=MATERIAL_PRESETS[n];ct=m['crystal_type'] - if ct in('fcc','bcc','diamond'): return ['cube','rectangle','cylinder','sphere'] - if ct in('hcp','wurtzite') or m['gamma']==120: return ['hexagonal','cylinder','sphere'] - return ['cube','rectangle','cylinder','hexagonal','sphere'] + if ct in('fcc','bcc','diamond'): return ded(base+facets.CUBIC_SHAPES) + if ct in('hcp','wurtzite') or m['gamma']==120: return ['hexagonal','cylinder','sphere']+facets.HEX_SHAPES + return ded(base+['hexagonal']+facets.CUBIC_SHAPES+facets.HEX_SHAPES) + +def cells_for_size_nm(name,size_nm): + """Supercell multiples giving a particle of the requested size in nm.""" + c=BCDIConfig(name);M=c.lattice_vectors_cartesian;t=float(size_nm)*10. + return tuple(max(1,int(round(t/np.linalg.norm(M[i])))) for i in range(3)) def find_preset(q): q=q.strip() if q in MATERIAL_PRESETS: return q @@ -156,10 +165,16 @@ def __init__(self,material_name='Al'): self.ROCKING_STEPS=128;self.ROCKING_STEP_DEG=.00315 self.SOURCE_SIZE_H_UM=300.;self.SOURCE_SIZE_V_UM=10.;self.SOURCE_DISTANCE_M=50.;self.MONOCHR_BANDWIDTH=1e-4 self.STRAIN_TYPE='none';self.STRAIN_MAGNITUDE=1e-4;self.TARGET_OVERSAMPLING=5.;self.ANGULAR_OFFSET_DEG=0.0 + self.N_PHOTONS=1e9;self.PHOTON_MODE='total';self.SIM_GRID_MAX=192;self.WULFF_RATIO=1.15;self.Q_FRAME='lab';self.DET_COUNTING=True self.HKL_MAX_SEARCH=5;self.Q_MAX_INV_ANG=8.;self.TWO_THETA_MIN=10.;self.TWO_THETA_MAX=90. self.OUTPUT_DIR=Path('bcdi_output');self.OUTPUT_DIR.mkdir(exist_ok=True) @property - def DETECTOR_N_PIXELS(self): return max(self.DETECTOR_NX, self.DETECTOR_NY) + def DETECTOR_N_PIXELS(self): + """Side of the cubic FFT grid: the Bragg ROI cropped from the panel, as in a + real analysis. DETECTOR_NX/NY describe the physical panel and are exported + as metadata; they do not set the transform size (a 4 Mpixel panel would be + a 2068^3 complex FFT).""" + return int(self.SIM_GRID_MAX) @property def wavelength_angstrom(self): return self.HC_KEV_ANG/self.BEAM_ENERGY_KEV @property @@ -216,6 +231,9 @@ def apply_shape_filter(self): elif s=='sphere': R=0.47*min(Lx,Ly,Lz);mask=(r**2).sum(1)<=R**2 elif s=='hexagonal': s3=np.sqrt(3.)/2.;mask=np.maximum(np.abs(r[:,0]),np.maximum(np.abs(.5*r[:,0]+s3*r[:,1]),np.abs(.5*r[:,0]-s3*r[:,1])))<=R_xy + elif s in facets.SHAPES: + Nn,Dd=facets.normals(s,cfg.reciprocal_lattice_vectors,cfg.WULFF_RATIO) + R=facets.scale_to_size(Nn,Dd,0.94*min(Lx,Ly,Lz));mask=facets.mask(p,center,Nn,Dd,R) else: return self.supercell_positions_ang=p[mask];self.supercell_species=[sv for sv,m in zip(self.supercell_species,mask) if m] if self._cell_indices: self._cell_indices=[c for c,m in zip(self._cell_indices,mask) if m] @@ -351,7 +369,8 @@ def calculate(self): fk=self._fk(np.array([h,k,l]),ops);iu=fk not in seen if iu: seen.add(fk) else: iu=(h>=0)and(h>=k)and(k>=l)and(l>=0) - rows.append({'hkl':(h,k,l),'hkl_str':f'({h} {k} {l})','d_Ang':round(d,4),'2theta':round(tt,4),'theta_B':round(tt/2.,4),'q_Ang':round(q,4),'|F|':round(Fa,3),'F_sq':round(F2,2),'LP':round(LP,4),'unique':iu}) + rows.append({'hkl':(h,k,l),'hkl_str':f'({h} {k} {l})','d_Ang':round(d,4),'2theta':round(tt,4),'theta_B':round(tt/2.,4),'q_Ang':round(q,4),'|F|':round(Fa,3),'F_sq':round(F2,2),'LP':round(LP,4),'unique':iu, + '_d':d,'_tt':tt,'_q':q,'_F2':F2,'_LP':LP}) df=pd.DataFrame(rows).sort_values('q_Ang').reset_index(drop=True) df=df[df['unique']].copy().reset_index(drop=True) df['I_rel']=df['F_sq']*df['LP'];mx=df['I_rel'].max() if len(df) else 1. @@ -381,7 +400,11 @@ def select_reflection(self,hkl=None): if len(m): break if len(m)==0: raise ValueError(f'Reflection ({h} {k} {l}) not found') sel=m.iloc[0] - return{'hkl':sel['hkl'],'hkl_str':sel['hkl_str'],'d_hkl':sel['d_Ang'],'theta_B_rad':np.radians(sel['theta_B']),'two_theta_rad':np.radians(sel['2theta']),'q_magnitude':sel['q_Ang'],'F_squared':sel['F_sq'],'LP_factor':sel['LP'],'intensity_norm':sel['I_norm']} + # full-precision values; the rounded columns are for display only + return{'hkl':sel['hkl'],'hkl_str':sel.get('hkl_display',sel['hkl_str']),'hkl_raw':sel['hkl_str'], + 'd_hkl':float(sel['_d']),'theta_B_rad':np.radians(sel['_tt']/2.),'two_theta_rad':np.radians(sel['_tt']), + 'q_magnitude':float(sel['_q']),'F_squared':float(sel['_F2']),'LP_factor':float(sel['_LP']), + 'intensity_norm':float(sel['I_norm'])} class BCDISimulator: def __init__(self,config,reflection,crystal_builder=None): @@ -437,6 +460,9 @@ def _sup(self,N,OS): if s=='sphere': return np.sqrt(X**2+Y**2+Z**2)<=h if s=='hexagonal': s3=np.sqrt(3.)/2.;return(np.maximum(np.abs(X),np.maximum(np.abs(.5*X+s3*Y),np.abs(.5*X-s3*Y)))<=h)&(np.abs(Z)<=h) + if s in facets.SHAPES: + Nn,Dd=facets.normals(s,self.config.reciprocal_lattice_vectors,self.config.WULFF_RATIO) + return facets.grid_mask(X,Y,Z,Nn,Dd,facets.scale_to_size(Nn,Dd,2*h)) return(np.abs(X)<=h)&(np.abs(Y)<=h)&(np.abs(Z)<=h) def simulate(self,progress_cb=None): c=self.config;N=c.DETECTOR_N_PIXELS;tA=c.particle_size_angstrom;F2=self.reflection['F_squared'];Vc=c.unit_cell_volume_ang3;LP=self.reflection['LP_factor'];OS=c.TARGET_OVERSAMPLING @@ -448,13 +474,11 @@ def simulate(self,progress_cb=None): sup=self._sup(N,OS).astype(np.float32);ph=self._displacement(N,tA,OS);psi=sup*np.exp(1j*ph) if progress_cb: progress_cb(40) # Apply angular offset as phase ramp (shifts Bragg peak off-center) - # AND attenuate intensity by the kinematic rocking curve envelope if abs(c.ANGULAR_OFFSET_DEG)>1e-6: delta_th=np.radians(c.ANGULAR_OFFSET_DEG/2.) delta_q=(4*np.pi/c.wavelength_angstrom)*np.cos(self.reflection['theta_B_rad'])*delta_th - # Phase ramp along first spatial axis (real-space extent = OS*tA) - ext0=OS*tA[0] - rx=np.linspace(-ext0/2,ext0/2,N,dtype=np.float32) + # Phase ramp along first spatial axis + rx=np.linspace(-ext[0]/2,ext[0]/2,N,dtype=np.float32) phase_ramp=np.exp(1j*delta_q*rx)[:,None,None] psi=psi*phase_ramp pq=np.fft.fftshift(np.fft.fftn(np.fft.ifftshift(psi))) @@ -472,54 +496,64 @@ def simulate(self,progress_cb=None): diff=pf*(sx**2)[:,None,None]*(sy**2)[None,:,None]*(sz**2)[None,None,:] if progress_cb: progress_cb(70) diff=self._pc(diff.astype(np.float32),dqx,dqy,dqz) - # Angular offset: attenuate by kinematic rocking curve envelope - # Away from exact Bragg condition, intensity drops as sinc^2 - if abs(c.ANGULAR_OFFSET_DEG)>1e-6: - # Shape-aware physical particle size (not just supercell box) - # For sphere/cylinder/hexagonal, the inscribed shape is smaller than the box - shape=c.PARTICLE_SHAPE - if shape=='sphere': - D_phys=float(min(tA)) # diameter = smallest box dim - elif shape=='cylinder': - D_phys=float(min(tA[0],tA[1])) # diameter in basal plane - elif shape=='hexagonal': - D_phys=float(min(tA[0],tA[1])) # in-plane width - else: - D_phys=float(np.mean(tA)) # cube/rectangle: full extent - th_B=self.reflection['theta_B_rad'] - # Kinematic rocking curve FWHM: dw ~ 2*lambda/(D*sin(2*theta_B)) - dw_fwhm=2.*c.wavelength_angstrom/(D_phys*np.sin(2.*th_B)+1e-12) - delta_deg=abs(c.ANGULAR_OFFSET_DEG) - # Number of FWHM widths away - n_fwhm=delta_deg/max(np.degrees(dw_fwhm),1e-10) - # Gaussian attenuation (matches kinematic rocking curve envelope) - attenuation=np.exp(-2.*np.log(2)*(n_fwhm**2)) - diff*=attenuation if progress_cb: progress_cb(95) self.diff_volume=diff.astype(np.float32);self.q_grids={'qx':qx,'qy':qy,'qz':qz} xh,xv,xl=self._coh();self.coherence={'xi_h_A':xh,'xi_v_A':xv,'xi_l_A':xl} if progress_cb: progress_cb(100);return diff,self.q_grids -def add_experimental_noise(vol,poisson=True,readout_noise=0,air_scatter=0,dead_pixels_frac=0,seed=42): - rng=np.random.default_rng(seed);out=vol.copy().astype(np.float64) +def add_experimental_noise(vol,n_photons=None,photon_mode='total',poisson=True,air_scatter=0, + readout_noise=0,dark_cps=0,exposure_s=1.,dead_pixels_frac=0, + hot_pixels_frac=0,counting=True,seed=42): + """Photon budget first, then instrument effects. + + n_photons sets the counting statistics: everything else is relative to it. + readout_noise applies only to integrating detectors (counting=False); EIGER, + PILATUS, MAXIPIX, MERLIN and TIMEPIX are photon counting and have none. + """ + rng=np.random.default_rng(seed);out=np.asarray(vol,np.float64) if air_scatter>0: - N=vol.shape[0];c=N//2;r0=max(1,N//4);x=np.arange(N)-c;QX,QY,QZ=np.meshgrid(x,x,x,indexing='ij') - r_norm=(QX**2+QY**2+QZ**2)/(r0**2+1e-10);peak=max(1.0,vol.max());out+=peak*air_scatter*1e-6/(1.+r_norm**2) - if poisson: - scale=1e6/max(1,out.max());counts=np.clip(out*scale,0,1e12);out=rng.poisson(counts.astype(np.float64)).astype(np.float64)/scale - if readout_noise>0: out+=rng.normal(0,readout_noise*out.max()*1e-6,out.shape);out=np.maximum(out,0) - if dead_pixels_frac>0: - n_dead=int(dead_pixels_frac*out.size);idx=rng.choice(out.size,n_dead,replace=False);out.flat[idx]=0 - return out.astype(np.float32) + N=out.shape[0];c=N//2;r0=max(1,N//4);x=np.arange(N)-c;QX,QY,QZ=np.meshgrid(x,x,x,indexing='ij') + rn=(QX**2+QY**2+QZ**2)/(r0**2+1e-10);out=out+max(1.,out.max())*air_scatter*1e-6/(1.+rn**2) + if n_photons and n_photons>0: + out=ortho.apply_photons(out,n_photons,photon_mode,poisson=poisson,seed=seed).astype(np.float64) + elif poisson: + sc=1e6/max(1,out.max());out=rng.poisson(np.clip(out*sc,0,1e12)).astype(np.float64)/sc + return ortho.detector_noise(out,dark_cps=dark_cps,exposure_s=exposure_s, + readout_e=0. if counting else readout_noise, + dead_frac=dead_pixels_frac,hot_frac=hot_pixels_frac,seed=seed) + +def q_sampling(cfg,theta_B_rad=None): + """Compare the simulation q voxel with a real detector pixel. + + The simulated grid is set by the particle size and oversampling; the panel + grid by pixel size and distance. Matching them 1:1 needs N >> OS, so the + simulation is deliberately coarser. Returns (dq_sim, dq_pixel, implied_OS). + """ + k=2*np.pi/cfg.wavelength_angstrom + dq_sim=(2*np.pi/cfg.particle_size_angstrom[0])/cfg.TARGET_OVERSAMPLING + dq_pix=k*(cfg.DETECTOR_PIXEL_UM*1e-6)/cfg.SAMPLE_DETECTOR_DISTANCE_M + return dq_sim,dq_pix,(2*np.pi/cfg.particle_size_angstrom[0])/dq_pix + +def offset_report(cfg,theta_B_rad,d2th_deg): + """What a 2theta offset does: dq, shift in sim voxels and in real pixels.""" + dq=(4*np.pi/cfg.wavelength_angstrom)*np.cos(theta_B_rad)*np.radians(d2th_deg/2.) + ds,dp,_=q_sampling(cfg) + N=cfg.DETECTOR_N_PIXELS + return dq,dq/ds,dq/dp,abs(dq/ds)>N/2 + +def to_frame(sim,frame='lab',vol=None): + """Return (volume, label). 'lab' = orthogonal q; 'det' = raw skewed detector grid.""" + v=sim.diff_volume if vol is None else vol + if frame!='det': return v,'sample / lab frame (orthogonal q)' + c=sim.config;tt=np.degrees(sim.reflection['two_theta_rad']) + M,_=ortho.det_matrix(c.wavelength_angstrom,tt,c.DETECTOR_PIXEL_UM, + c.SAMPLE_DETECTOR_DISTANCE_M,c.ROCKING_STEP_DEG) + return ortho.lab_to_det(v,sim.q_grids,M,order=3),f'detector frame (raw grid, skew={ortho.skew(M):.3f})' def export_measurement(sim,refl,cfg,filepath,noise_opts=None): """Export simulation as .npz (like real BCDI measurement data).""" vol=sim.diff_volume.copy() if noise_opts: vol=add_experimental_noise(vol,**noise_opts) - # Real-space voxel pitch in nm = (object size * oversampling) / grid_size - N=cfg.DETECTOR_N_PIXELS - psize_arr=np.asarray(cfg.particle_size_nm,dtype=np.float32) - voxel_nm=(psize_arr*float(cfg.TARGET_OVERSAMPLING)/float(N)) data={'diffraction_volume':vol,'qx':sim.q_grids['qx'],'qy':sim.q_grids['qy'],'qz':sim.q_grids['qz'], 'hkl':np.array(refl['hkl']),'d_hkl':refl['d_hkl'],'two_theta':np.degrees(refl['two_theta_rad']), 'energy_keV':cfg.BEAM_ENERGY_KEV,'wavelength_A':cfg.wavelength_angstrom, @@ -529,7 +563,9 @@ def export_measurement(sim,refl,cfg,filepath,noise_opts=None): 'detector_nx':cfg.DETECTOR_NX,'detector_ny':cfg.DETECTOR_NY, 'rocking_step_deg':cfg.ROCKING_STEP_DEG,'rocking_steps':cfg.ROCKING_STEPS, 'beam_size_um':cfg.BEAM_SIZE_UM,'oversampling':cfg.TARGET_OVERSAMPLING, - 'particle_size_nm':psize_arr,'voxel_size_nm':voxel_nm} + 'n_photons':getattr(cfg,'N_PHOTONS',0),'photon_mode':getattr(cfg,'PHOTON_MODE','total'), + 'q_frame':getattr(cfg,'Q_FRAME','lab'),'detector_counting':getattr(cfg,'DET_COUNTING',True), + 'particle_size_nm':cfg.particle_size_nm} if filepath.endswith('.h5') or filepath.endswith('.hdf5'): import h5py with h5py.File(filepath,'w') as f: @@ -621,11 +657,12 @@ def lattice_html_with_controls(builder,max_atoms=50000,atom_size=3,show_bonds=Fa html=html.replace('',''+_loading_html('Rendering 3D Lattice...'),1) return html.replace('',PANEL+'',1) -def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=None): +def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=None,vol=None,frame_label=''): import plotly.graph_objects as go if sim.diff_volume is None: return None - vol=sim.diff_volume.copy() - if noise_opts: vol=add_experimental_noise(vol,**noise_opts) + if vol is None: + vol=sim.diff_volume.copy() + if noise_opts: vol=add_experimental_noise(vol,**noise_opts) d3=vol[::downsample,::downsample,::downsample] qx=sim.q_grids['qx'][::downsample];qy=sim.q_grids['qy'][::downsample];qz=sim.q_grids['qz'][::downsample] vl=np.log10(d3+1.).astype(np.float32);vm=float(vl.max()) @@ -641,6 +678,7 @@ def bragg_html_with_controls(sim,downsample=2,threshold_frac=1e-4,noise_opts=Non if noise_opts.get('air_scatter',0)>0: parts.append("air") if noise_opts.get('dead_pixels_frac',0)>0: parts.append("dead") if parts: noise_str=" | "+", ".join(parts) + if frame_label: noise_str+=" | "+frame_label fig.update_layout(template='plotly_dark',paper_bgcolor='#0d0e12',scene=dict(xaxis=dict(**ax,title='qx(1/A)'),yaxis=dict(**ax,title='qy(1/A)'),zaxis=dict(**ax,title='qz(1/A)'),camera=dict(eye=dict(x=1.4,y=1.4,z=.7)),aspectmode='data'),font=dict(family='Consolas,monospace',size=11,color='#c0c4cc'),title=dict(text=f'{cfg.MATERIAL_NAME} {sim.reflection["hkl_str"]} | {sz[0]:.0f}x{sz[1]:.0f}x{sz[2]:.0f}nm{noise_str}',font=dict(size=12),x=.5,xanchor='center'),margin=dict(l=0,r=0,t=40,b=0)) # Fringe measurement JS measure_js="""