diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ab4cd76 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +# Copy this file to .env (gitignored) and fill in your real key: +# cp .env.example .env +# +# Used by harness/agent.py, harness/mcp_agent.py and +# timelapse/model_trigger.py (loaded via python-dotenv). The MCP server +# itself and the mock-only tests never need it. +ANTHROPIC_API_KEY=sk-ant-... + +# Optional: where captures and logs are written (default: current directory). +# CONFOCAL_MCP_DATA_DIR=/path/to/data + +# Optional: PNG the mock camera serves for backend="mock" captures. +# CONFOCAL_MOCK_FRAME_PATH=/path/to/frame.png diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3a26a48 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: CI + +on: + push: # every branch, so a push runs even before a pull request exists + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # No hardware groups: the mock stage and mock camera need none, so the + # whole suite runs on a plain runner. 'harness' is pure Python and is + # needed to import the Claude loops for the approval-gate tests (no + # API key required - no model is ever called). + - run: pip install -e ".[harness,test]" + - run: python -m pytest -v tests # -v names each test in the log + # The MCP server must import on a core-only install (the camera and + # SDK stacks are lazy imports) - fail the build if that regresses. + # (tests/test_mcp_server.py goes further and actually runs it.) + - run: python -c "import mcp_server.server_loop, timelapse.scheduler" diff --git a/.gitignore b/.gitignore index 9384881..6d24c61 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ eaa_integration/memory/ eaa_integration/*.sqlite eaa_integration/*.sqlite-shm eaa_integration/*.sqlite-wal + +# Nikon Ti2 SDK documentation, originals and samples extracted from the +# Nikon installers. Licensed with SDK access - kept local, not published. +docs/nikon-sdk/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a54e519..2ac3cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,78 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow ## [Unreleased] +### Added +- `analysis/make_soundtrack.py` - original ambient soundtrack synthesised + with numpy (no samples, so no licensing questions), stretched to any + length. `--movie run.mp4` sizes it to the movie and muxes it on as AAC + with the video stream copied, writing `run_music.mp4`. At 39 s it + reproduces the Physarum Short's track exactly. +- `acquisition/calibration/ti2_inventory.py` - read-only inventory of every + device the Ti2 reports, with each one's valid range, unit, and `Control` + value. Field names come from the COM type library at runtime and rows are + grouped by the microscope's own `Control` value, so there is no device list + to keep in sync. `Control` is the practical guide to what can be driven: + `-1` devices (the D-LEDI channels, the DIA/EPI/AUX shutters, Intensilight, + TIRF, LAPP) reliably ignore writes; `>= 0` usually accepts, with measured + exceptions (`iDIC_PRISM`, `iTURRET2SHUTTER`). Note `Enabled` is *not* a + fitted-hardware flag - it reads True for all 88 devices. +- `acquisition/calibration/ti2_config.py` - `save` / `show` / `diff` / `apply` + for the full device configuration from the command line. `apply` requires + `--confirm`, skips stage/objective/TIRF unless `--include-motion`, and polls + the read-back rather than reading once (several devices report the old value + for up to a second after a successful write). +- `get_image(backend=...)`: `backend="mock"` returns a simulated frame via + `MockNIS.capture()` with no camera attached and no `confirm`, so capture + logic can be developed off the microscope PC. `backend="sdk"` (the + default) is unchanged and still requires `confirm=True`. The metadata dict + now carries `backend`. Both harnesses skip the hardware gate for mock + captures. +- `CONFOCAL_MOCK_FRAME_PATH`: PNG the mock capture serves (defaults to the + old `data/analysis/nd2_sample/frame_0.png` location). +- `timelapse/` package (not MCP tools): `change_detector` (model-free + per-frame change score), `frame_audit` (CLI: gaps / intensity jumps / + stage shifts vs. specimen change in an existing sequence), `scheduler` + (adaptive slow/burst acquisition loop with hard caps and a one-time + real-hardware approval), `model_trigger` (Claude behind the scheduler's + trigger hook: extend or end a burst from the before/after frames, with + call caps and fail-safe "no opinion"; runs on a background thread so + burst timing never waits on the model). See `docs/adaptive_timelapse.md`. +- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. Includes a + protocol-level test that spawns `mcp_server.server_loop` as a subprocess, + asserts the exact tool set, and drives every tool over MCP stdio. Also + tests of both harnesses' real-hardware approval gate: no `confirm` in any + model-facing schema, sdk calls stop at the gate and a decline executes + nothing, mock calls never prompt. And tests of `harness/context.py`'s + image pruning. +- `numpy` (2.4.x, the last line that supports Python 3.11) is now a core dependency; new `test` optional group (pytest). + +### Fixed +- README and code comments said the MCP surface was 4 tools; it has been 5 + since `estop` was added. The README's tool list now includes `estop`. +- `get_optical_configuration()` recorded no illumination state. It walked + `OPTICAL_CONFIG_PROPERTIES`, a hardcoded list that omitted + `iDIA_LAMP_Switch`/`iDIA_LAMP_Pos` entirely - so a saved configuration never + captured whether the transmitted lamp was on, the one setting that decides + whether a camera on the camera port sees anything. The `iDLED*` names it did + list are ignored by this microscope, whose D-LEDI is not driven through the + Ti2 body. Both it and `apply_optical_configuration()` now use the SDK's own + `DataGet`, which returns all ~90 properties in one call, and the hardcoded + list is gone. `apply_optical_configuration()` gains `include_motion=False`: + the snapshot is now the full device set, so without that guard restoring a + lamp setting would also drive the stage. + +- Runtime data no longer falls back to an unwritable working directory. MCP + clients choose the working directory their servers are launched with, and + Claude Desktop on Windows uses `C:\WINDOWS\system32` - so with + `CONFOCAL_MCP_DATA_DIR` unset, the first `get_image()` failed with an + "access is denied" `OSError` that read as a camera fault, and under an + elevated process would instead have written captures into a system + directory. `acquisition/paths.py` now rejects a working directory that is + inside the Windows directory or that it cannot create a file in, falling + back to `~/.confocal-mcp` with a warning on stderr. Setting + `CONFOCAL_MCP_DATA_DIR` is still honoured as-is, and a normal source + checkout still resolves to the checkout directory. + ## [0.1.0] - 2026-08-31 First packaged release. diff --git a/README.md b/README.md index ec86f72..9e70adb 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ MCP client). ## Tools -Exactly 4, by design - kept deliberately minimal: +Exactly 5, by design - kept deliberately minimal: - **`get_pos(backend)`** - current stage (x, y, z) position, in microns. A cheap, on-demand sync primitive, not something to call before every @@ -14,9 +14,11 @@ Exactly 4, by design - kept deliberately minimal: - **`move(x, y, backend, confirm)`** - move the XY stage to an absolute position, returns the *actual* resulting position. XY-only, on purpose (no absolute Z move is reachable from chat - crash risk into the sample). -- **`get_image(confirm, exposure_time_us, gain, crop, max_dimension)`** - - capture a real frame from the Baumer camera, paired with the exact - stage position it was taken at. Returns both the metadata and an +- **`get_image(confirm, exposure_time_us, gain, crop, max_dimension, backend)`** - + capture a frame from the Baumer camera (`backend="sdk"`, the default) + or a simulated one from the mock (`backend="mock"`, no hardware, no + `confirm` needed - set `CONFOCAL_MOCK_FRAME_PATH` to choose the PNG it + serves), paired with the exact stage position it was taken at. Returns both the metadata and an embedded image preview, so the model can actually see the picture, not just a file path. `crop` (optional `{"x","y","width","height"}` fractions of the full frame) restricts the embedded preview to a @@ -35,6 +37,12 @@ identifying that specific capture. the stage to this session, so "where have we already been" doesn't need to be re-derived from conversation history. +- **`estop(action)`** - emergency stop. `"engage"` forbids all stage motion + for every process on the machine via a flag file; `"status"` reports it. + Needs no `confirm`: a stop is always safe. Release is deliberately not + reachable from chat - a human clears it with + `python -m acquisition.estop release`. + `backend` is `"mock"` (default, safe, simulated) or `"sdk"` (real hardware - `move()`/`get_image()` require `confirm=True` for anything that touches real hardware). See `mcp_server/loop_tools.py`'s header @@ -45,6 +53,31 @@ No NIS-Elements involved anywhere - the camera is reached directly via GenICam/GenTL, and the stage via the Ti2 ActiveX SDK, both independent of whether NIS-Elements software is even running. +## Adaptive time-lapse (`timelapse/`) + +Not an MCP tool - a loop that sits beside the tools and calls `get_image()` +itself. It captures on a slow interval, scores each frame for change with +plain numpy (no model call), and switches to a fast burst when something +happens. `timelapse/frame_audit.py` applies the same scores to an existing +frame sequence to tell an acquisition gap, an illumination change or a +stage bump from real specimen change. Design, safety model and status: +[docs/adaptive_timelapse.md](docs/adaptive_timelapse.md). + +``` +python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-duration 5 --max-runtime 30 +python -m timelapse.scheduler --backend mock --model-trigger ... # Claude judges each trigger (needs API credentials) +python -m timelapse.frame_audit FRAME_DIR --timestamps times.csv --around 25h --window 1h +``` + +## Tests + +``` +pip install -e ".[test]" +python -m pytest -q tests +``` + +Mock backend only - nothing touches hardware. CI runs the same on every push. + ## Install This is a normal installable package (`confocal-mcp`) with a console entry @@ -99,9 +132,22 @@ project-level `.mcp.json` for Claude Code: { "mcpServers": { "confocal": { "command": "confocal-mcp" } } } ``` -Data (captures, move/frame history logs) is written under the current working -directory when the server runs from an installed package - launch it from a -stable location. +Data (captures, move/frame history logs, saved positions) is written under the +current working directory - so launch the server from a stable location, or set +`CONFOCAL_MCP_DATA_DIR` to pin it explicitly: + +```json +{ "mcpServers": { "confocal": { + "command": "confocal-mcp", + "env": { "CONFOCAL_MCP_DATA_DIR": "D:\\path\\to\\your\\data" } +} } } +``` + +Setting it is worth doing for any MCP client, because the client picks the +working directory and the caller has no say in it - Claude Desktop on Windows +launches servers in `C:\WINDOWS\system32`. When the working directory is +unusable like that, the server falls back to `~/.confocal-mcp` and says so on +stderr rather than failing at the first capture. ## Harness loop (`harness/agent.py`) @@ -113,10 +159,10 @@ unaffected either way - use whichever fits: MCP for Claude Desktop/Code, this loop for a standalone script. Requires `ANTHROPIC_API_KEY` set - either in a `.env` file at the repo -root (copy the commented-out line in `.env`, fill in your real key; this -file is gitignored and loaded automatically by `harness/agent.py`), as a +root (`cp .env.example .env`, fill in your real key; `.env` is gitignored +and loaded automatically by the harnesses and the model trigger), as a regular environment variable, or via `ant auth login`. Real hardware -(`backend="sdk"`, or any `get_image` call) always pauses for a live +(`backend="sdk"` on `move` or `get_image`) always pauses for a live "y/N" approval at the terminal before executing, regardless of what the model requests - see `harness/agent.py`'s header comment for why. Old captured images are pruned from the model's context after a couple of diff --git a/acquisition/backends/baumer_genicam.py b/acquisition/backends/baumer_genicam.py index 763465f..46182bb 100644 --- a/acquisition/backends/baumer_genicam.py +++ b/acquisition/backends/baumer_genicam.py @@ -119,13 +119,25 @@ def get_settings(self) -> dict: node_map = self._acquirer.remote_device.node_map exposure = node_map.ExposureTime gain = node_map.Gain - return { + settings = { "exposure_time_us": exposure.value, "exposure_time_range_us": [exposure.min, exposure.max], "gain": gain.value, "gain_range": [gain.min, gain.max], "pixel_format": node_map.PixelFormat.value, } + # REPORTED BECAUSE IT IS OTHERWISE INVISIBLE STATE. White balance + # persists on the camera across processes, is settable from any + # GenICam consumer, and silently rewrites every captured colour - + # a whole time-lapse was acquired on 2026-09-21 without anyone + # being able to tell from the saved metadata what the colour + # pipeline had done to it. BalanceRatio is not exposed on the + # VCXU-23C, so mode is all there is to record. + try: + settings["balance_white_auto"] = node_map.BalanceWhiteAuto.value + except Exception: + settings["balance_white_auto"] = None + return settings def set_settings(self, exposure_time_us: float | None = None, gain: float | None = None) -> dict: """Set exposure time (microseconds) and/or gain via the GenICam @@ -217,10 +229,25 @@ def capture(self) -> Path: elif pixel_format == "Mono8": rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_GRAY2RGB) elif pixel_format.startswith("Bayer"): - # Literal-name match to cv2's code; unverified against a - # known-colour target. If colours look swapped under real - # light, try COLOR_BayerGR/BG/GB2RGB instead. - rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_BayerRG2RGB) + # VERIFIED 2026-09-21 against the camera's OWN internal + # debayer, which is the ground truth available without a + # colour target: capturing the same scene in BGR8 and in + # RGB8 agreed exactly (organism hue 54 deg, saturation + # ~105), while decoding the BayerRG8 stream with + # COLOR_BayerRG2RGB - what this line used to do - gave hue + # 188 deg, very nearly the complement. That is R/B + # inversion, so the correct cv2 code for this body's + # "BayerRG8" is BayerBG2RGB, not BayerRG2RGB. + # + # PREFER BGR8 OVER ANY BAYER FORMAT ON THIS CAMERA. The + # Bayer stream also carries markedly less colour than the + # camera's own debayer of the same scene (channel spread + # 4.99 vs 11.93, saturation 56 vs 105), so it loses real + # information rather than just mislabelling it. The + # PixelFormat is whatever the camera was last left in by + # any GenICam consumer - if someone sets it back to Bayer, + # this branch at least renders the right hue. + rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_BayerBG2RGB) else: raise RuntimeError( f"Unsupported camera PixelFormat {pixel_format!r} - " diff --git a/acquisition/backends/nis_mock.py b/acquisition/backends/nis_mock.py index 3ee2d89..bd4ab50 100644 --- a/acquisition/backends/nis_mock.py +++ b/acquisition/backends/nis_mock.py @@ -15,6 +15,7 @@ # nis = MockNIS() # ------------------------------------------------------------ +import os import shutil import sys import time @@ -48,8 +49,15 @@ # Sample frame copied by capture() to simulate a real image capture. Falls # back to a generated placeholder if this isn't present (it usually isn't - -# it's a dev-only asset under the gitignored data/ tree). -SAMPLE_FRAME_PATH = _data_root() / "data" / "analysis" / "nd2_sample" / "frame_0.png" +# it's a dev-only asset under the gitignored data/ tree). Set +# CONFOCAL_MOCK_FRAME_PATH to point at any PNG instead - e.g. a real +# time-lapse frame - so mock captures return something worth analyzing. +# Read at call time (not cached) so a test or a dev script can swap the +# file, or reassign this module attribute, between captures. +SAMPLE_FRAME_PATH = Path( + os.environ.get("CONFOCAL_MOCK_FRAME_PATH") + or _data_root() / "data" / "analysis" / "nd2_sample" / "frame_0.png" +) CAPTURE_DIR = _captures_dir() diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index 9cc17fc..9866be6 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -51,6 +51,11 @@ import win32com.client import NkTi2Ax +# Imported at module level so the guard cannot be skipped by an import +# failing lazily inside a move. estop does not import this module, so +# there is no import cycle. +from acquisition import estop + # Raw-count-per-micron scale factors confirmed above. XY_COUNTS_PER_UM = 10.0 Z_COUNTS_PER_UM = 100.0 @@ -67,28 +72,15 @@ MAX_XY_STEP_UM = 5000.0 MAX_Z_STEP_UM = 50.0 -# Direct scalar properties making up an "optical configuration" (objective, -# filters, light path, illumination) - confirmed present in NkTi2Ax.py's -# INikonTi2AxMicroscope interface, same source as the stage properties -# above. Used by get_optical_configuration()/apply_optical_configuration() -# below - see acquisition/orchestration/imaging_profile.py for the -# save/list/apply-by-experiment-name layer on top of these two methods. -OPTICAL_CONFIG_PROPERTIES = [ - "iNOSEPIECE", # objective (turret slot 1-6) - "iDIC_PRISM", - "iDIC_POLARIZER", - "iANALYZER_POS", - "iANALYZER_SLOT", - "iLIGHTPATH", - "iCONDENSER", - "iOPTZOOM", - "iTURRET1POS", "iTURRET1SHUTTER", - "iTURRET2POS", "iTURRET2SHUTTER", - "iDLED1_POS", "iDLED1_SWITCH", - "iDLED2_POS", "iDLED2_SWITCH", - "iDLED3_POS", "iDLED3_SWITCH", - "iDLED4_POS", "iDLED4_SWITCH", -] +# There is deliberately no hardcoded list of "optical configuration" +# properties here any more. One used to live at this spot +# (OPTICAL_CONFIG_PROPERTIES) and it silently omitted iDIA_LAMP_Switch and +# iDIA_LAMP_Pos, so get_optical_configuration() never recorded whether the +# transmitted lamp was on - and every write to the iDLED* names it did list +# was ignored, because this microscope's D-LEDI is not driven through the +# Ti2 body. Both methods now enumerate whatever DataGet returns. See +# acquisition/calibration/ti2_inventory.py for what the interface actually +# reports per device, including which ones accept writes. # PFS offset units aren't calibrated to microns (unlike XY/Z - see the # module docstring), so nudge_pfs_offset's cap is expressed as a fraction @@ -164,6 +156,9 @@ def _get_com_thread() -> _ComThread: with _com_thread_lock: if _com_thread is None: _com_thread = _ComThread() + # First stage connection in this process: make sure the STOP + # button is on screen before anything can move. + estop.launch_panel() return _com_thread @@ -196,6 +191,12 @@ def XY_Move(self, x: float, y: float) -> None: than MAX_XY_STEP_UM, or if the target is outside the stage's own reported travel range (XPosition/YPosition .Lower/.Higher). """ + # E-STOP FIRST, before range checks, before anything. This is the + # lowest point every XY move passes through, which is the only + # place a stop can be effective against a caller that is already + # misbehaving - see acquisition/estop.py's header for the incident + # this exists because of. + estop.check() before = self.XY_GetPosition() x, y = to_plain_float(x), to_plain_float(y) step_um = ((x - before[0]) ** 2 + (y - before[1]) ** 2) ** 0.5 @@ -248,6 +249,10 @@ def Z_Move(self, z: float) -> None: nudge_pfs_offset() for routine focus adjustment when PFS is engaged, and reserve this for deliberate, small, checked steps. """ + # E-stop before anything else - Z is the axis that can drive the + # objective into the sample, so this is the most important guard + # in the file. + estop.check() before = self.Z_GetPosition() z = to_plain_float(z) step_um = abs(z - before) @@ -332,6 +337,10 @@ def nudge_pfs_offset(self, delta_counts: float) -> dict: capped to PFS_MAX_OFFSET_STEP_FRACTION of the SDK's own reported valid offset range. + Guarded by the e-stop like XY_Move/Z_Move: it is small and + relative, but it still moves focus, and "only a little" is not a + category the stop should recognise. + TODO(unconfirmed, 2026-08-10): writing iPFS_OFFSET while PFS is actively enabled/locked has been observed to silently no-op on real hardware - offset_before == offset_after, no exception - on @@ -347,6 +356,7 @@ def nudge_pfs_offset(self, delta_counts: float) -> dict: hardware. Failure mode observed so far is safe (no motion, no error) - not a functional feature yet, but not a hazard either. """ + estop.check() delta_counts = to_plain_float(delta_counts) def do_nudge(m): @@ -387,50 +397,76 @@ def do_nudge(m): # way an uncapped Z move is. def get_optical_configuration(self) -> dict: - """Snapshot the current optical/device configuration - objective - (nosepiece), DIC prism/polarizer, analyzer, light path, condenser, - zoom, both filter turrets, and the 4 D-LEDI illumination channels - - as a dict of raw SDK property values. Save this (e.g. to a JSON - file) and pass it to apply_optical_configuration() later to - reproduce the same setup. - - Property semantics (e.g. whether iDLED1_POS is an intensity - percent or something else) are NOT independently calibrated the - way XY/Z/PFS units were (see this module's docstring for that - methodology) - this assumes reading then writing back the same - raw value reproduces the same NIS-Elements UI state, which is - reasonable since these are exactly the properties NIS's own - "Optical Configuration" presets are built from, but it hasn't - been verified against a known physical reference the way stage - position was. + """Snapshot every device property the microscope reports, as a dict + of raw SDK values keyed by property name (iLIGHTPATH, + iDIA_LAMP_Switch, iNOSEPIECE, ...). Save this (e.g. to a JSON file) + and pass it to apply_optical_configuration() to reproduce the setup. + + Taken with one DataGet call, which fills an INikonTi2AxData object + with all ~90 properties at once. Pass None as its first argument - + it is declared [in,out] and win32com returns the filled object. + Constructing the object yourself does not work: the NikonTi2AxData + coclass is not registered (CoCreateInstance gives "Class not + registered"). + + This used to walk OPTICAL_CONFIG_PROPERTIES, a hardcoded list that + omitted iDIA_LAMP_Switch and iDIA_LAMP_Pos - so a saved config did + not record whether the transmitted lamp was on, which is the single + setting deciding whether a camera on the camera port sees anything + at all. Enumerating whatever DataGet returns removes the list, and + with it the chance of it drifting out of sync again. + + Property semantics are NOT independently calibrated the way XY/Z + units were (see this module's docstring) - this assumes reading then + writing back the same raw value reproduces the same state. """ def read(m): - return {name: getattr(m, name) for name in OPTICAL_CONFIG_PROPERTIES} + data = m.DataGet(None, 0) + values = {} + for name in sorted(getattr(type(data), "_prop_map_get_", {}) or {}): + try: + values[name] = getattr(data, name) + except Exception: + continue + return values return self._thread.call(read) - def apply_optical_configuration(self, config: dict) -> dict: + def apply_optical_configuration(self, config: dict, + include_motion: bool = False) -> dict: """Write back a config dict from get_optical_configuration(). - Only keys in OPTICAL_CONFIG_PROPERTIES are applied - unknown keys - (e.g. from a config saved by a newer version of this code) are - silently ignored rather than erroring. Returns the read-back - value of every property actually applied. - - Switching the objective (iNOSEPIECE) changes working distance, - which may invalidate prior Z-safety assumptions for whatever - position you're at - this method never touches Z itself, that's - on the caller to handle deliberately afterward if needed. + Returns the read-back value of every property actually applied. + Properties that raise are recorded as None rather than aborting the + rest, and a write the microscope declines to act on shows up as a + read-back that differs from the requested value - the Ti2 ignores + such writes silently rather than raising (the D-LEDI channels and + the DIA/EPI shutters do this consistently on this workstation). + + Motion devices - stage XY/Z, the nosepiece, TIRF - are skipped + unless include_motion=True. get_optical_configuration() now returns + the full device set rather than a curated subset, so without this + guard restoring a lamp setting from a file would also drive the + stage across the slide as a side effect. Switching the objective + also changes working distance, which can invalidate Z-safety + assumptions for the current position; this method never moves Z on + its own unless you opt in. """ def write(m): applied = {} for name, value in config.items(): - if name not in OPTICAL_CONFIG_PROPERTIES: + if not include_motion and any( + key in name.upper() + for key in ("XPOSITION", "YPOSITION", "ZPOSITION", + "NOSEPIECE", "TIRF", "ZESCAPE", "RESET")): continue - setattr(m, name, value) - applied[name] = getattr(m, name) + try: + setattr(m, name, value) + applied[name] = getattr(m, name) + except Exception: + applied[name] = None return applied return self._thread.call(write) diff --git a/acquisition/calibration/__init__.py b/acquisition/calibration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/acquisition/calibration/nis_port_probe.py b/acquisition/calibration/nis_port_probe.py new file mode 100644 index 0000000..26a0b4b --- /dev/null +++ b/acquisition/calibration/nis_port_probe.py @@ -0,0 +1,117 @@ +# IMPORTANT: 'limjob' must be imported like this (not from nor as) +import limjob + +# A JOBS Python task that serves a tiny HTTP control API from inside NIS for +# 10 minutes (or until Abort). From a terminal on this PC: +# curl http://127.0.0.1:8765/status +# curl "http://127.0.0.1:8765/move?dx=200&dy=0" relative, um +# curl "http://127.0.0.1:8765/objective?name=" +# +# NIS macro functions are called through ctypes on g5_regprocs.dll, the way +# Nikon's own limpy.macro calls WaitText. Signatures are from the NIS 6.20 +# macro reference (Docs/nis/eng_ar); macro char* is wchar_t* in this build. +# +# NIS calls are made only on the job's own thread (the run() loop); the HTTP +# thread just queues requests, since NIS is not known to be thread-safe. +# +# SAFETY: moves are relative and capped at MAX_STEP_UM per call; nothing +# moves while the e-stop file exists (same file acquisition/estop.py uses); +# Z is never moved; objective changes are limited to 4x and 10x, whose long +# working distances cannot reach the dish. +import ctypes as ct, http.server, json, os, queue, threading, time, urllib.parse + +PORT, RUN_S, MAX_STEP_UM = 8765, 600, 1000.0 +ESTOP = r"C:\Users\AX Confocal\.confocal-mcp\ESTOP" +ALLOWED_OBJECTIVES = ("4x", "10x") + +nis = ct.cdll.g5_regprocs +for fn, args in {"StgGetPosXY": [ct.POINTER(ct.c_double)] * 2, + "StgGetPosZ": [ct.POINTER(ct.c_double), ct.c_int], + "StgMoveXY": [ct.c_double, ct.c_double, ct.c_int], + "Stg_GetNosepiecePosition": [], + "Stg_GetNosepiecePositions": [], + "Stg_GetNosepieceObjectiveName": [ct.c_int, ct.c_wchar_p, ct.c_int], + "GetCurrentObjName": [ct.c_wchar_p], + "ChangeObjective": [ct.c_wchar_p]}.items(): + getattr(nis, fn).argtypes, getattr(nis, fn).restype = args, ct.c_int + + +def status(): + x, y, z = ct.c_double(), ct.c_double(), ct.c_double() + rc_xy = nis.StgGetPosXY(ct.byref(x), ct.byref(y)) + rc_z = nis.StgGetPosZ(ct.byref(z), 0) + cur = ct.create_unicode_buffer(255); nis.GetCurrentObjName(cur) + names = {} + for i in range(0, nis.Stg_GetNosepiecePositions() + 1): # index base unknown: try 0..n + b = ct.create_unicode_buffer(255) + if nis.Stg_GetNosepieceObjectiveName(i, b, 255) == 1 and b.value: + names[i] = b.value + return {"xy_um": [x.value, y.value], "xy_rc": rc_xy, "z_um": z.value, "z_rc": rc_z, + "nosepiece_position": nis.Stg_GetNosepiecePosition(), + "current_objective": cur.value, "nosepiece_objectives": names} + + +def move(dx, dy): + if os.path.exists(ESTOP): + return {"error": "e-stop engaged - not moving"} + if max(abs(dx), abs(dy)) > MAX_STEP_UM: + return {"error": f"step over {MAX_STEP_UM} um refused"} + before = status()["xy_um"] + rc = nis.StgMoveXY(dx, dy, 1) # 1 = MOVE_RELATIVE + return {"rc": rc, "before_um": before, "after_um": status()["xy_um"]} + + +def objective(name): + if os.path.exists(ESTOP): + return {"error": "e-stop engaged - not changing objective"} + if not any(k in name for k in ALLOWED_OBJECTIVES): + return {"error": f"only {ALLOWED_OBJECTIVES} objectives allowed here"} + before = status() + rc = nis.ChangeObjective(name) + after = status() + return {"rc": rc, "before": before["current_objective"], "after": after["current_objective"], + "nosepiece_before": before["nosepiece_position"], + "nosepiece_after": after["nosepiece_position"]} + + +jobs = queue.Queue() + + +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + u = urllib.parse.urlparse(self.path); q = dict(urllib.parse.parse_qsl(u.query)) + call = {"/status": lambda: status(), + "/move": lambda: move(float(q.get("dx", 0)), float(q.get("dy", 0))), + "/objective": lambda: objective(q.get("name", ""))}.get(u.path) + if call is None: + out = {"error": "use /status, /move?dx=&dy=, /objective?name="} + else: + box = queue.Queue(); jobs.put((call, box)) + try: + out = box.get(timeout=60) + except queue.Empty: + out = {"error": "timed out waiting for NIS"} + body = (json.dumps(out, indent=1) + "\n").encode() + self.send_response(200); self.send_header("Content-Type", "application/json") + self.end_headers(); self.wfile.write(body) + + +def run(imgs: tuple[limjob.Image], Job: limjob.JobParam, macro: limjob.MacroParam, ctx: limjob.RunContext): + s = http.server.ThreadingHTTPServer(("127.0.0.1", PORT), H) + threading.Thread(target=s.serve_forever, daemon=True).start() + print(f"listening on 127.0.0.1:{PORT}, pid {os.getpid()}") + abort = getattr(ctx, "shouldAbort", lambda: False) + t0 = time.time() + try: + while time.time() - t0 < RUN_S and not abort(): + try: + call, box = jobs.get(timeout=0.2) + except queue.Empty: + continue + try: + box.put(call()) + except Exception as e: + box.put({"error": f"{type(e).__name__}: {e}"}) + finally: + s.shutdown(); s.server_close() + print("server stopped") diff --git a/acquisition/calibration/ti2_config.py b/acquisition/calibration/ti2_config.py new file mode 100644 index 0000000..41889f9 --- /dev/null +++ b/acquisition/calibration/ti2_config.py @@ -0,0 +1,317 @@ +# ti2_config.py +# ------------------------------------------------------------ +# Save and restore the Ti2's device configuration from the command line. +# +# python -m acquisition.calibration.ti2_config save +# python -m acquisition.calibration.ti2_config save --out protocols/brightfield.json +# python -m acquisition.calibration.ti2_config show protocols/brightfield.json +# python -m acquisition.calibration.ti2_config diff protocols/brightfield.json +# python -m acquisition.calibration.ti2_config apply protocols/brightfield.json --confirm +# +# HOW THE SNAPSHOT IS TAKEN: INikonTi2AxMicroscope.DataGet fills an +# INikonTi2AxData object with all 90 device properties in one COM call, and +# it uses the writable i* names (iLIGHTPATH, iDIA_LAMP_Switch, ...) - so a +# saved file can be written straight back with setattr, no name mapping. +# +# data = microscope.DataGet(None, 0) +# +# Pass None for the first argument: it is declared [in,out] (VT_BYREF| +# VT_DISPATCH) and win32com returns the filled object. Do not try to +# construct the object yourself - the NikonTi2AxData coclass is not +# registered on this workstation (CoCreateInstance gives "Class not +# registered"), which is a dead end DataGet sidesteps entirely. +# +# WHY NOT NISSdk.get_optical_configuration(): it loops over +# OPTICAL_CONFIG_PROPERTIES, a hand-maintained list in nis_sdk.py that +# omits iDIA_LAMP_Switch and iDIA_LAMP_Pos. A configuration saved through +# it does not record whether the transmitted lamp was on - the one setting +# that decides whether the camera sees anything at all. +# +# WHY NOT DataSet FOR RESTORE: DataSet(newVal, ...) writes the whole device +# set in one call, so restoring a lamp setting would also drive the stage, +# Z and the nosepiece as an unavoidable side effect. apply() writes +# per-property instead, which allows skipping motion devices and reporting +# each write individually. --bulk opts into DataSet when that is what you +# actually want. +# +# Control: each Setting reports a Control value. It is a useful filter but +# NOT a guarantee, and the asymmetry matters: +# +# Control = -1 reliably refuses. The D-LEDI channels, the DIA/EPI/AUX +# shutters, Intensilight, TIRF and LAPP all sit here, and +# every write to them was silently ignored. Worth skipping +# rather than issuing writes that vanish without an error. +# Control >= 0 usually accepts, but not always. Measured exceptions on +# this Ti2-E: iDIC_PRISM (Control=0) and iTURRET2SHUTTER +# (Control=1) both refuse writes that never take, even +# given seconds to settle. Something - an interlock, or +# NIS-Elements holding those devices - overrides them. +# +# So treat a Control >= 0 "not-applied" as real, and check the device +# rather than assuming the report is a timing artifact. +# +# SAFETY: save/show/diff are read-only. apply moves real hardware, needs +# --confirm, and skips the motion devices unless --include-motion. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import datetime +import json +import re +import time +from pathlib import Path +from typing import Any + +from acquisition.calibration.ti2_inventory import inventory + +#: Some devices do not report a new value the instant the write returns - +#: the DIC prism and turret shutters are mechanical, the DIA lamp settles. +#: Reading back immediately made apply report DiaLampPos, DicPrism and +#: Turret2Shutter as failures when all three had in fact applied. Poll +#: instead of trusting one immediate read. +SETTLE_SECONDS = 1.5 +SETTLE_POLL_SECONDS = 0.25 + +#: Devices that physically move something substantial; skipped by apply +#: unless --include-motion. Matched as substrings of the i* property name. +_MOTION_PROPERTIES = ("XPOSITION", "YPOSITION", "ZPOSITION", "ZEscape", + "ZReset", "XReset", "YReset", "NOSEPIECE", "Tirf") + + +def _is_motion(prop: str) -> bool: + return any(key.lower() in prop.lower() for key in _MOTION_PROPERTIES) + + +def _normalise(name: str) -> str: + """Fold a property name to a comparable key. + + Only needed to join DataGet's i* names to the friendly names Settings + reports (iDIA_LAMP_Switch <-> DiaLampSwitch), so that a snapshot can + carry each property's Control value and valid range alongside it. + """ + stripped = name[1:] if name.startswith("i") else name + return re.sub(r"[^a-z0-9]", "", stripped.lower()) + + +def _data_properties(data: Any) -> list[str]: + """Property names on the INikonTi2AxData object, from the type library.""" + return sorted(getattr(type(data), "_prop_map_get_", {}) or {}) + + +def save_config(sdk: Any = None) -> dict: + """Snapshot every device property via DataGet, plus Settings metadata. + + Read-only. Values come from one DataGet call; Control/range/unit are + joined on from INikonTi2AxMicroscope.Settings so the saved file records + what can be written back and within what limits. + """ + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + + def snapshot(microscope: Any) -> dict: + data = microscope.DataGet(None, 0) + values = {} + for prop in _data_properties(data): + try: + values[prop] = getattr(data, prop) + except Exception: + continue + return values + + values = sdk._thread.call(snapshot) + meta = {_normalise(str(row.get("Name"))): row for row in inventory(sdk) if row.get("Name")} + + properties = {} + for prop, value in sorted(values.items()): + row = meta.get(_normalise(prop), {}) + properties[prop] = { + "value": value, + "control": row.get("Control"), + "lower": row.get("Lower"), + "higher": row.get("Higher"), + "unit": row.get("Unit"), + "friendly_name": row.get("Name"), + } + return { + "saved_at": datetime.datetime.now().isoformat(timespec="seconds"), + "property_count": len(properties), + "properties": properties, + } + + +def _writable(entry: dict) -> bool: + control = entry.get("control") + return control is not None and control >= 0 + + +def format_config(config: dict, only_writable: bool = False) -> str: + props = config.get("properties", {}) + lines = ["saved_at: %s (%d properties)" % (config.get("saved_at", "?"), len(props)), ""] + lines.append(" %-24s %10s %8s %s" % ("PROPERTY", "VALUE", "CONTROL", "RANGE")) + for prop, entry in sorted(props.items()): + if only_writable and not _writable(entry): + continue + rng = "" if entry.get("lower") is None else "%s .. %s" % (entry["lower"], entry["higher"]) + lines.append(" %-24s %10s %8s %s" + % (prop, entry.get("value"), entry.get("control"), rng)) + return "\n".join(lines) + + +def diff_config(config: dict, sdk: Any = None) -> list[dict]: + """Properties whose live value differs from the saved one. Read-only.""" + live = save_config(sdk)["properties"] + rows = [] + for prop, entry in sorted(config.get("properties", {}).items()): + now = live.get(prop, {}).get("value") + if now != entry.get("value"): + rows.append({ + "property": prop, "saved": entry.get("value"), "now": now, + "control": entry.get("control"), "motion": _is_motion(prop), + }) + return rows + + +def _write_and_settle(microscope: Any, prop: str, value: Any) -> tuple[Any, Any]: + """setattr, then poll the read-back until it matches or SETTLE_SECONDS. + + Returns (before, readback). Polling rather than one immediate read is + the whole point - see SETTLE_SECONDS. + """ + before = getattr(microscope, prop) + setattr(microscope, prop, value) + deadline = time.monotonic() + SETTLE_SECONDS + readback = getattr(microscope, prop) + while readback != value and time.monotonic() < deadline: + time.sleep(SETTLE_POLL_SECONDS) + readback = getattr(microscope, prop) + return before, readback + + +def apply_config(config: dict, sdk: Any = None, include_motion: bool = False, + bulk: bool = False) -> dict: + """Write a saved config back. Real hardware - see this module's header.""" + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + + props = config.get("properties", {}) + + def run(microscope: Any) -> dict: + if bulk: + data = microscope.DataGet(None, 0) + for prop, entry in props.items(): + try: + setattr(data, prop, entry.get("value")) + except Exception: + continue + microscope.DataSet(data, 0, 1) + return {"_bulk": {"status": "DataSet issued", "properties": len(props)}} + + report: dict[str, dict] = {} + for prop, entry in sorted(props.items()): + value = entry.get("value") + if not _writable(entry): + report[prop] = {"status": "skipped", "why": "control=%s" % entry.get("control")} + continue + if not include_motion and _is_motion(prop): + report[prop] = {"status": "skipped", "why": "motion device"} + continue + try: + if getattr(microscope, prop) == value: + report[prop] = {"status": "unchanged", "value": value} + continue + before, readback = _write_and_settle(microscope, prop, value) + report[prop] = { + "status": "written" if readback == value else "not-applied", + "from": before, "to": value, "readback": readback, + } + except Exception as exc: + report[prop] = {"status": "error", + "error": "%s: %s" % (type(exc).__name__, exc)} + return report + + return sdk._thread.call(run) + + +def _default_out() -> Path: + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + return Path("protocols") / ("optical_config_%s.json" % stamp) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Save and restore the Ti2 device configuration.") + sub = parser.add_subparsers(dest="command", required=True) + + p_save = sub.add_parser("save", help="snapshot the configuration (read-only)") + p_save.add_argument("--out", type=Path, default=None) + + p_show = sub.add_parser("show", help="print a saved configuration file") + p_show.add_argument("path", type=Path) + p_show.add_argument("--writable", action="store_true", + help="only properties this SDK can drive (control >= 0)") + + p_diff = sub.add_parser("diff", help="compare a saved file against the live scope") + p_diff.add_argument("path", type=Path) + + p_apply = sub.add_parser("apply", help="write a saved configuration back") + p_apply.add_argument("path", type=Path) + p_apply.add_argument("--confirm", action="store_true", + help="required: this moves real hardware") + p_apply.add_argument("--include-motion", action="store_true", + help="also restore stage/objective/TIRF positions") + p_apply.add_argument("--bulk", action="store_true", + help="use DataSet to write everything in one call " + "(ignores --include-motion; moves the stage)") + + args = parser.parse_args() + + if args.command == "save": + config = save_config() + out = args.out or _default_out() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(config, indent=2, default=str), encoding="utf-8") + print(format_config(config)) + print("\nwrote %s" % out) + + elif args.command == "show": + config = json.loads(args.path.read_text(encoding="utf-8")) + print(format_config(config, only_writable=args.writable)) + + elif args.command == "diff": + config = json.loads(args.path.read_text(encoding="utf-8")) + rows = diff_config(config) + if not rows: + print("no differences - live scope matches %s" % args.path) + else: + print(" %-24s %10s %10s %8s %s" + % ("PROPERTY", "SAVED", "NOW", "CONTROL", "RESTORABLE?")) + for row in rows: + if row["control"] is None or row["control"] < 0: + verdict = "no (control=%s)" % row["control"] + elif row["motion"]: + verdict = "only with --include-motion" + else: + verdict = "yes" + print(" %-24s %10s %10s %8s %s" + % (row["property"], row["saved"], row["now"], row["control"], verdict)) + print("\n%d propert%s differ" % (len(rows), "y" if len(rows) == 1 else "ies")) + + elif args.command == "apply": + if not args.confirm: + raise SystemExit("apply moves real hardware - re-run with --confirm") + config = json.loads(args.path.read_text(encoding="utf-8")) + report = apply_config(config, include_motion=args.include_motion, bulk=args.bulk) + counts: dict[str, int] = {} + for prop, result in sorted(report.items()): + counts[result["status"]] = counts.get(result["status"], 0) + 1 + if result["status"] in ("written", "not-applied", "error", "DataSet issued"): + print(" %-24s %s" % (prop, result)) + print("\n" + ", ".join("%s: %d" % kv for kv in sorted(counts.items()))) + + +if __name__ == "__main__": + main() diff --git a/acquisition/calibration/ti2_inventory.py b/acquisition/calibration/ti2_inventory.py new file mode 100644 index 0000000..a099799 --- /dev/null +++ b/acquisition/calibration/ti2_inventory.py @@ -0,0 +1,186 @@ +# ti2_inventory.py +# ------------------------------------------------------------ +# What this Ti2 reports about its own devices, straight from the SDK. +# +# python -m acquisition.calibration.ti2_inventory +# python -m acquisition.calibration.ti2_inventory --json out.json +# python -m acquisition.calibration.ti2_inventory --columns Name,Value,Control +# +# WHY THIS EXISTS: the Ti2 COM interface exposes ~88 writable i* properties +# covering every device a Ti2 *could* have - TIRF stages, LAPP branches, +# filter wheels, four D-LEDI channels, Intensilight, and so on. A write the +# microscope will not act on is silently ignored: no exception, no error, +# the read-back simply keeps the old value. So write-then-read-back cannot +# tell "device not under Ti2 control" from "value refused" from "write +# worked and something else reset it" - which is the hole a long dark-image +# hunt falls into. +# +# INikonTi2AxMicroscope.Settings is the metadata side of those properties: +# a tuple of INikonTi2AxSetting objects, one per device. +# +# Control The field that predicts whether a write will take. Measured +# against this Ti2-E: Control >= 0 accepted, Control = -1 was +# refused, for 7 of the 8 devices actually written to during +# the session that produced this module. The exception was +# Turret2Shutter (Control=1, refused at the time), so treat +# this as a strong signal rather than a guarantee. +# Enabled NOT a fitted-hardware flag, despite the name: it reads True +# for all 88 devices, including every one that refuses writes. +# Read it as "the SDK models this device". +# Lower/Higher valid range, so "which iLIGHTPATH values exist?" is a +# lookup (1-4) rather than a sweep, and "iDIA_LAMP_Pos = +# 2100 out of what?" is answerable (0-2100, i.e. maximum). +# Unit/Scale units where the SDK declares them (um for the stages). +# +# Field names are discovered from the COM type library at runtime, and rows +# are grouped by the Control value the microscope itself reports - nothing +# about the device set is written down here. That is deliberate: a +# hardcoded property list (OPTICAL_CONFIG_PROPERTIES in nis_sdk.py) is what +# silently dropped every illumination setting from +# get_optical_configuration() in the first place, and a hardcoded list here +# would rot the same way against a different Ti2 or a newer SDK. +# +# Read-only and side-effect free - it moves nothing and changes no state, +# so it is safe to run against live hardware mid-experiment. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import json +from typing import Any + +#: Zero-argument accessor methods on INikonTi2AxSetting that are safe to call. +#: An allowlist rather than "call everything callable", because the same +#: interface exposes mutators (SetLongName/SetShortName) and methods needing +#: arguments (ConvertDev2Phys, ConvertPhys2Dev, GetConvertParams). Plain +#: properties are not listed anywhere - those come from the type library. +_SAFE_ACCESSOR_METHODS = ("LongName", "ShortName") + +#: Shown by default. Any field present on the Setting can be asked for with +#: --columns; this is a display choice, not a claim about what exists. +_DEFAULT_COLUMNS = ("Name", "Value", "Lower", "Higher", "Unit", "Enabled") + + +def _setting_fields(setting: Any) -> list[str]: + """Field names for one Setting, read from the COM type library. + + win32com's generated wrapper records every readable property of the + interface in ``_prop_map_get_``, so reading that gives whatever this + SDK version actually exposes - including fields added after this module + was written. + """ + cls = type(setting) + names = sorted(getattr(cls, "_prop_map_get_", {}) or {}) + names += [m for m in _SAFE_ACCESSOR_METHODS if hasattr(cls, m)] + return names + + +def _read_field(setting: Any, name: str) -> Any: + """Read one field, calling it if it is one of the accessor methods. + + Returns None on failure - one field that raises should not cost us the + other eleven. + """ + try: + value = getattr(setting, name) + except Exception: + return None + if callable(value): + try: + value = value() + except Exception: + return None + return value + + +def _read_settings(microscope: Any) -> list[dict]: + """Snapshot every Setting. Runs on the COM thread - see inventory().""" + try: + settings = microscope.Settings + except Exception as exc: + return [{"_error": "%s: %s" % (type(exc).__name__, exc)}] + rows = [] + for index, setting in enumerate(settings): + row = {"index": index} + for name in _setting_fields(setting): + row[name] = _read_field(setting, name) + rows.append(row) + return rows + + +def inventory(sdk: Any = None) -> list[dict]: + """Every device Setting the microscope reports, as a list of dicts. + + Pass an existing NISSdk to reuse its COM thread; omit it to build one. + NISSdk is imported lazily so this module stays importable without + pywin32 or the Ti2 SDK installed (same pattern as the lazy backend + imports in mcp_server/loop_tools.py). + """ + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + return sdk._thread.call(_read_settings) + + +def _control_label(control: Any) -> str: + """Section heading for one Control value - see this module's header.""" + if control is None: + return "Control unknown" + if control < 0: + return "Control = %s (writes refused on every device tested)" % control + return "Control = %s (writes took effect)" % control + + +def format_inventory(rows: list[dict], columns: list[str] | None = None) -> str: + """Render inventory() grouped by the Control value the microscope reports.""" + if rows and "_error" in rows[0]: + return "could not read Settings: " + rows[0]["_error"] + + columns = list(columns or _DEFAULT_COLUMNS) + widths = { + col: max([len(col)] + [len(str(r.get(col, "") or "")) for r in rows]) + for col in columns + } + + def row_text(values: dict) -> str: + return " " + " ".join( + str(values.get(col, "") if values.get(col) is not None else "").ljust(widths[col]) + for col in columns + ) + + lines = [] + for control in sorted({r.get("Control") for r in rows}, key=lambda c: (c is None, c)): + group = [r for r in rows if r.get("Control") == control] + lines.append("[%s] %d devices" % (_control_label(control), len(group))) + lines.append(row_text({col: col for col in columns})) + for row in sorted(group, key=lambda r: str(r.get("Name") or "")): + lines.append(row_text(row)) + lines.append("") + + discovered = sorted(set(rows[0]) - {"index"}) if rows else [] + lines.append("%d devices. Fields discovered from the type library: %s" + % (len(rows), ", ".join(discovered))) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Inventory the Ti2's device settings (read-only).") + parser.add_argument("--json", metavar="PATH", + help="also write the full raw inventory to this JSON file") + parser.add_argument("--columns", metavar="A,B,C", + help="comma-separated fields to show instead of the default") + args = parser.parse_args() + + rows = inventory() + columns = args.columns.split(",") if args.columns else None + print(format_inventory(rows, columns=columns)) + if args.json: + with open(args.json, "w", encoding="utf-8") as handle: + json.dump(rows, handle, indent=2, default=str) + print("\nwrote %s" % args.json) + + +if __name__ == "__main__": + main() diff --git a/acquisition/estop.py b/acquisition/estop.py new file mode 100644 index 0000000..45ef17b --- /dev/null +++ b/acquisition/estop.py @@ -0,0 +1,210 @@ +# estop.py +# ------------------------------------------------------------ +# Emergency stop for every process that can move this microscope. +# +# python -m acquisition.estop engage # STOP EVERYTHING, NOW +# python -m acquisition.estop status +# python -m acquisition.estop release # deliberate, separate action +# +# WHY THIS EXISTS. On 2026-09-21 a mosaic raster with a bad focus plane +# began driving Z toward the sample. Stopping it took several minutes and +# three separate interventions, because every mechanism available was the +# wrong shape: +# +# * stopping the background SHELL left its python CHILD running, still +# issuing moves - the process list, not the task list, was the truth +# * a tool call the operator REJECTED had already spawned its process, +# which kept looping Z moves regardless of the rejection +# * a "STOP file" checked once per mosaic tile is not a stop; between +# checks it still issued dozens of moves +# * the Ti2 SDK has NO abort/halt command (searched: only ZESCAPE, which +# is a retract, not a stop), so there is nothing to "cancel" with +# +# The lesson is that an e-stop must sit BELOW whatever is misbehaving. So +# this is enforced inside nis_sdk's move primitives themselves: no caller, +# no matter how confused, can move an axis while the stop is engaged, +# because the check happens after the caller has already decided to move. +# +# WHY A FILE. It must work across processes that do not know about each +# other - that was the whole failure. A file is visible to every process, +# survives the death of whoever set it, needs no server or port, and can +# be set by a human from any shell (or by creating the file by hand) when +# the agent itself is the thing malfunctioning. +# +# WHY A FIXED PATH, NOT data_root(). data_root() resolves relative to the +# working directory, and an MCP server launched by a desktop client gets a +# different cwd than a terminal. Two processes would then disagree about +# where the e-stop lives, which is precisely the failure this must not +# have. ESTOP_PATH is absolute and identical for every process on the +# machine. +# +# FAIL SAFE: if the state cannot be determined (unreadable directory, odd +# filesystem error), check() treats the stop as ENGAGED. An e-stop that +# fails open is not a safety device. The cost of the opposite error is a +# refused move and a clear message, which is recoverable; the cost of +# failing open is a driven objective. +# +# WHAT IT DOES NOT DO: it cannot stop a move the controller has already +# accepted. A move in flight runs to its end; the stop takes effect at the +# next move. move_xy splits long moves into HOP_UM hops, so an XY move +# stops within one hop (measured: 0.54 s, <= 4 mm). +# +# WHY NO HALT. engage() used to also write the current position back as +# the target, to freeze an axis mid-travel. estop_inflight_test showed +# that is worse than nothing: position reads are cached for the whole move +# (XY and Z), so the "current" position is the START, and the write queues +# behind the move. A 1 mm Z move ran to its end, then the halt drove Z all +# the way back - a second move, issued by the stop. Never add it back. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import subprocess +import sys +from pathlib import Path + +#: Absolute and machine-wide. Never derive this from the working +#: directory - see the module header. +ESTOP_PATH = Path(os.environ.get("CONFOCAL_ESTOP_FILE", + Path.home() / ".confocal-mcp" / "ESTOP")) + + +#: Held by the running STOP panel for its whole life, so every process can +#: tell whether one is already on screen. Windows frees it if the panel dies. +PANEL_MUTEX = r"Local\confocal-mcp-estop-panel" + + +class EStopEngaged(RuntimeError): + """Raised by any motion primitive while the e-stop is engaged.""" + + +def is_engaged() -> bool: + """True if motion is currently forbidden. + + Any error resolving the state counts as engaged - see FAIL SAFE. + """ + try: + return ESTOP_PATH.exists() + except OSError: + return True + + +def details() -> dict | None: + """Who engaged the stop, when, and why - or None if it is clear.""" + try: + if not ESTOP_PATH.exists(): + return None + return json.loads(ESTOP_PATH.read_text(encoding="utf-8")) + except (OSError, ValueError): + # Present but unreadable still means engaged; say so without a reason. + return {"reason": "(e-stop file present but unreadable)"} + + +def check() -> None: + """Raise EStopEngaged if motion is forbidden. Called by every move.""" + if is_engaged(): + info = details() or {} + raise EStopEngaged( + "E-STOP ENGAGED - motion refused. " + f"engaged_at={info.get('engaged_at', '?')} " + f"by={info.get('by', '?')} reason={info.get('reason', '?')}. " + f"Release with: python -m acquisition.estop release " + f"(or delete {ESTOP_PATH})" + ) + + +def engage(reason: str = "manual") -> dict: + """Forbid all motion immediately. Sets the flag and nothing else. + + It never commands the stage - see WHY NO HALT in the module header. + """ + info = { + "engaged_at": datetime.datetime.now().isoformat(timespec="seconds"), + "by": f"pid {os.getpid()}", + "reason": reason, + } + ESTOP_PATH.parent.mkdir(parents=True, exist_ok=True) + ESTOP_PATH.write_text(json.dumps(info, indent=2), encoding="utf-8") + return info + + +def release() -> None: + """Allow motion again. Deliberately a separate, explicit action.""" + try: + ESTOP_PATH.unlink() + except FileNotFoundError: + pass + + + +def panel_running() -> bool: + """True if a STOP panel is already on screen (Windows only).""" + if sys.platform != "win32": + return False + import ctypes + k32 = ctypes.windll.kernel32 + h = k32.OpenMutexW(0x00100000, False, PANEL_MUTEX) # SYNCHRONIZE + if h: + k32.CloseHandle(h) + return True + return False + + +def launch_panel() -> None: + """Put the STOP panel on screen unless one is already there. + + Called when a process first connects to the real stage, so the button + appears however a run was started - MCP server, mosaic, timelapse, a + one-liner. The 2026-09-21 run was started from a terminal, where no + panel existed; a safety control you have to remember to start is one + you will not have. + + Detached, not a child: the panel must outlive this process. Failures + go to stderr - never stdout, which is the MCP JSON-RPC channel - and + never stop the caller. CONFOCAL_NO_ESTOP_PANEL=1 turns it off. + """ + if os.environ.get("CONFOCAL_NO_ESTOP_PANEL"): + return + try: + if panel_running(): + return + flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) | getattr(subprocess, "DETACHED_PROCESS", 0) + subprocess.Popen([sys.executable, "-m", "acquisition.estop_panel"], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, creationflags=flags) + except Exception as exc: + print(f"[estop] could not start the STOP panel ({exc}). " + f"Stop manually with: python -m acquisition.estop engage", file=sys.stderr) + + +def main() -> None: + ap = argparse.ArgumentParser(description="Emergency stop for microscope motion.") + ap.add_argument("action", choices=("engage", "release", "status")) + ap.add_argument("--reason", default="manual") + args = ap.parse_args() + + if args.action == "engage": + info = engage(args.reason) + print("E-STOP ENGAGED") + for k, v in info.items(): + print(f" {k}: {v}") + print(f"\nflag file: {ESTOP_PATH}") + print("All motion is now refused. Release with: python -m acquisition.estop release") + elif args.action == "release": + release() + print(f"e-stop released - motion permitted again ({ESTOP_PATH} removed)") + else: + if is_engaged(): + print("ENGAGED - motion is refused") + for k, v in (details() or {}).items(): + print(f" {k}: {v}") + sys.exit(2) + print(f"clear - motion permitted (no {ESTOP_PATH})") + + +if __name__ == "__main__": + main() diff --git a/acquisition/estop_inflight_test.py b/acquisition/estop_inflight_test.py new file mode 100644 index 0000000..75caa2a --- /dev/null +++ b/acquisition/estop_inflight_test.py @@ -0,0 +1,162 @@ +# estop_inflight_test.py +# ------------------------------------------------------------ +# Can the e-stop stop a move that is ALREADY running? +# +# .venv/Scripts/python -m acquisition.estop_inflight_test z # Test A +# .venv/Scripts/python -m acquisition.estop_inflight_test xy # Test B +# +# The e-stop flag reliably refuses NEW moves. What is unknown is what +# happens to a move the controller has already accepted. XY reads are +# cached for the whole move (read_during_move_test.py), so writing the +# "current" XY back reverses the stage instead of freezing it. +# +# TEST A (Z). The Ti2 pushed Z-position events every ~500 ms during a +# move, which XY never did, so Z reads may be live. One raw Z move of +# Z_TEST_DROP_UM DOWNWARD - away from the sample, the only direction this +# test ever moves on its own - with a reader thread logging Z every 50 ms +# and a halt that writes the current Z back after HALT_AFTER_S. If Z reads +# are live, Z should stop partway. If they are cached, the write-back +# commands the start position, which is where we came from: the worst +# case of this test is going back up to where it began, never beyond. +# Afterwards Z is left where it stopped; restore focus by hand. +# +# TEST B (XY). No halt is possible, so the stop is the flag checked +# between move_xy's hops. A multi-hop move of XY_TEST_BY_UM in +X runs on +# a worker thread; the flag is engaged after ENGAGE_AFTER_S. +# Reported: how long the stage kept moving after the engage, and how far +# it travelled in total. The stop is left ENGAGED - a human releases it. +# +# Raw COM, not NISSdk, for Test A: NISSdk caps a Z step at 50 um, which +# is over before any halt could land. Test B goes through NISSdk and +# move_xy on purpose, since that is the path real moves take. +# ------------------------------------------------------------ + +import os +import sys +import threading +import time + +# Multi-threaded COM, so the reader and halt threads share the microscope +# object with the thread that is blocked in the move. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + +from acquisition import estop + +Z_COUNTS_PER_UM = 100.0 +Z_TEST_DROP_UM = 1000.0 +HALT_AFTER_S = 0.2 + +XY_TEST_BY_UM = 20000.0 +ENGAGE_AFTER_S = 1.5 + + +def log(msg): + now = time.time() + stamp = f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d}" + print(f"{stamp} [{threading.current_thread().name:>6}] {msg}", flush=True) + + +def test_z(): + m = win32com.client.Dispatch(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID) + z0 = m.iZPOSITION + lo = m.ZPosition.Lower + target = z0 - round(Z_TEST_DROP_UM * Z_COUNTS_PER_UM) + log(f"start Z {z0 / Z_COUNTS_PER_UM:.2f} um, target {target / Z_COUNTS_PER_UM:.2f} um (down)") + if target < lo: + log(f"target below travel range ({lo / Z_COUNTS_PER_UM:.1f} um) - not moving") + return + estop.check() + + stop = threading.Event() + + def reader(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + while not stop.is_set(): + log(f"Z read {m.iZPOSITION / Z_COUNTS_PER_UM:.2f}") + time.sleep(0.05) + + def halter(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + z = m.iZPOSITION + log(f"HALT: read Z {z / Z_COUNTS_PER_UM:.2f}, writing it back") + try: + m.iZPOSITION = z + log("HALT: write-back accepted") + except Exception as e: + log(f"HALT: write-back failed: {e}") + + threading.Thread(target=reader, name="reader", daemon=True).start() + time.sleep(0.2) # a few idle reads first, as a baseline + threading.Timer(HALT_AFTER_S, halter).start() + + t0 = time.perf_counter() + log("set iZPOSITION start") + try: + m.iZPOSITION = target + log(f"set iZPOSITION done after {time.perf_counter() - t0:.2f} s") + except Exception as e: + log(f"set iZPOSITION failed after {time.perf_counter() - t0:.2f} s: {e}") + + time.sleep(1.5) # let whatever is still moving settle + stop.set() + z1 = m.iZPOSITION + travelled = (z0 - z1) / Z_COUNTS_PER_UM + log(f"final Z {z1 / Z_COUNTS_PER_UM:.2f} um - moved {travelled:.2f} of {Z_TEST_DROP_UM:.0f} um down") + if abs(travelled) < 1: + log("RESULT: Z back at start - halt read a stale position (or the move never ran)") + elif travelled < Z_TEST_DROP_UM - 1: + log("RESULT: Z stopped partway - halt works on Z") + else: + log("RESULT: Z reached target - halt did not stop it") + + +def test_xy(): + from acquisition.backends.nis_sdk import NISSdk + from acquisition.move_xy import move_to + + sdk = NISSdk() + x0, y0 = sdk.XY_GetPosition() + log(f"start ({x0:.1f}, {y0:.1f}), moving +{XY_TEST_BY_UM:.0f} um in X") + estop.check() + + at_engage = {} + + def engager(): + # Flag first: NISSdk has one COM thread, busy for the whole hop in + # flight, so reading the position before engaging would delay the + # engage by up to a hop - the very latency being measured. + estop.engage("estop_inflight_test") + at_engage["t"] = time.perf_counter() + log("ENGAGED") + + threading.Timer(ENGAGE_AFTER_S, engager).start() + t0 = time.perf_counter() + try: + move_to(sdk, x0 + XY_TEST_BY_UM, y0) + log("move finished WITHOUT being stopped") + except estop.EStopEngaged: + log("move refused its next hop - stopped by e-stop") + t_end = time.perf_counter() + + x1, y1 = sdk.XY_GetPosition() + log(f"final ({x1:.1f}, {y1:.1f}) after {t_end - t0:.2f} s") + if at_engage: + log(f"RESULT: kept moving {t_end - at_engage['t']:.2f} s after engage " + f"(travelled {x1 - x0:.0f} of {XY_TEST_BY_UM:.0f} um in total)") + log("e-stop left ENGAGED - release by hand: python -m acquisition.estop release") + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ("z", "xy"): + print(__doc__ or "usage: python -m acquisition.estop_inflight_test z|xy") + raise SystemExit(2) + threading.current_thread().name = "main" + # Hard stop in case anything hangs. + threading.Timer(60.0, lambda: (log("60s safety timeout"), os._exit(1))).start() + (test_z if sys.argv[1] == "z" else test_xy)() + log("exit") + os._exit(0) diff --git a/acquisition/estop_panel.py b/acquisition/estop_panel.py new file mode 100644 index 0000000..0718b49 --- /dev/null +++ b/acquisition/estop_panel.py @@ -0,0 +1,174 @@ +# estop_panel.py +# ------------------------------------------------------------ +# An always-on-top STOP button for the microscope. +# +# python -m acquisition.estop_panel +# +# Small, borderless, stays above other windows, drag it anywhere. One big +# button. Hitting it engages acquisition.estop, which every motion +# primitive checks before it moves - see that module's header. +# +# WHY A WINDOW AND NOT JUST THE CLI: during the 2026-09-21 incident the +# stage was moving while the operator hunted for a terminal, typed a +# command, and waited for python to start. Seconds matter and typing does +# not scale under stress. A button that is already on screen costs one +# click, with no target to find and nothing to remember. +# +# WHY IT POLLS THE FLAG FILE rather than tracking its own state: the stop +# can also be engaged from a terminal, from the MCP tool, by another +# process, or by a human creating the file. The panel must show what is +# actually true, not what it last did - a panel reading "CLEAR" while the +# stop is engaged elsewhere would be worse than no panel. +# +# WHY RELEASE IS DELIBERATELY AWKWARD: the release control is small, +# visually quiet, and asks for confirmation, while STOP is huge and +# instant. The asymmetry is the point - the cost of an accidental stop is +# a few seconds, the cost of an accidental release is an unguarded +# microscope. Engaging is also idempotent, so panic-clicking is harmless. +# +# ONE PANEL. It opens by itself whenever a process connects to the stage +# (estop.launch_panel), so many processes ask for it. The first takes the +# PANEL_MUTEX; any later copy sees it and exits without drawing. +# +# TKINTER because it ships with CPython: a safety control that depends on +# `pip install` is a safety control that is missing on the day it matters. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import sys +import tkinter as tk +from tkinter import messagebox + +from acquisition import estop + +POLL_MS = 300 + +_RED, _RED_DARK = "#c62828", "#8e0000" +_GREEN, _GREY = "#2e7d32", "#9e9e9e" +_BG = "#1b1b1b" + + +class Panel: + def __init__(self, root: tk.Tk, topmost: bool = True, borderless: bool = True): + self.root = root + root.title("E-STOP") + root.configure(bg=_BG) + root.resizable(False, False) + if topmost: + root.attributes("-topmost", True) + if borderless: + # No title bar - it would be most of a window this small, and + # the drag handler below replaces the only thing it was for. + root.overrideredirect(True) + root.geometry("+40+40") + + self.button = tk.Button( + root, text="STOP", font=("Segoe UI", 26, "bold"), + bg=_RED, fg="white", activebackground=_RED_DARK, activeforeground="white", + relief="raised", bd=3, width=8, height=1, cursor="hand2", + command=self.on_stop) + self.button.pack(padx=8, pady=(8, 4)) + + self.status = tk.Label(root, text="checking...", font=("Segoe UI", 9, "bold"), + bg=_BG, fg=_GREY) + self.status.pack() + + bar = tk.Frame(root, bg=_BG) + bar.pack(fill="x", padx=8, pady=(2, 6)) + # Quiet, small, and confirmed - see the header on asymmetry. + self.release_btn = tk.Button(bar, text="release", font=("Segoe UI", 8), + bg="#2a2a2a", fg="#bbb", relief="flat", + cursor="hand2", command=self.on_release) + self.release_btn.pack(side="left") + tk.Button(bar, text="✕", font=("Segoe UI", 8), bg="#2a2a2a", fg="#bbb", + relief="flat", cursor="hand2", command=root.destroy).pack(side="right") + + # Drag from anywhere on the background, since there is no title bar. + for w in (root, self.status, bar): + w.bind("", self._press) + w.bind("", self._drag) + self._dx = self._dy = 0 + + self._last = None + self.tick() + + def _press(self, e) -> None: + self._dx, self._dy = e.x_root - self.root.winfo_x(), e.y_root - self.root.winfo_y() + + def _drag(self, e) -> None: + self.root.geometry(f"+{e.x_root - self._dx}+{e.y_root - self._dy}") + + def on_stop(self) -> None: + # Engage first and report afterwards: the flag write is what makes + # the microscope safe, and it must not wait behind a dialog. + try: + estop.engage("STOP button") + except Exception as exc: + messagebox.showerror("E-STOP", f"Could not engage:\n{exc}") + self.refresh(force=True) + + def on_release(self) -> None: + if not estop.is_engaged(): + return + if messagebox.askyesno("Release e-stop", + "Allow the microscope to move again?\n\n" + "Only do this once you know why it stopped."): + estop.release() + self.refresh(force=True) + + def refresh(self, force: bool = False) -> None: + engaged = estop.is_engaged() + if engaged == self._last and not force: + return + self._last = engaged + if engaged: + self.status.configure(text="ENGAGED - motion refused", fg=_RED) + self.button.configure(text="STOPPED", bg=_RED_DARK) + self.release_btn.configure(state="normal") + else: + self.status.configure(text="clear - motion permitted", fg=_GREEN) + self.button.configure(text="STOP", bg=_RED) + self.release_btn.configure(state="disabled") + + def tick(self) -> None: + self.refresh() + self.root.after(POLL_MS, self.tick) + + +_mutex = None # kept for the life of the process; Windows frees it on exit + + +def _claim_single_instance() -> bool: + """Take the panel mutex; False if another panel already holds it.""" + global _mutex + if sys.platform != "win32": + return True + import ctypes + k32 = ctypes.windll.kernel32 + k32.CreateMutexW.restype = ctypes.c_void_p + _mutex = k32.CreateMutexW(None, False, estop.PANEL_MUTEX) + return k32.GetLastError() != 183 # ERROR_ALREADY_EXISTS + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--no-topmost", action="store_true") + ap.add_argument("--titlebar", action="store_true", help="keep a normal window frame") + args = ap.parse_args() + if not _claim_single_instance(): + print("a STOP panel is already open", file=sys.stderr) + return + try: + root = tk.Tk() + except tk.TclError as exc: + print(f"no display available for the e-stop panel ({exc}). " + f"Use: python -m acquisition.estop engage", file=sys.stderr) + raise SystemExit(1) + Panel(root, topmost=not args.no_topmost, borderless=not args.titlebar) + root.mainloop() + + +if __name__ == "__main__": + main() diff --git a/acquisition/move_xy.py b/acquisition/move_xy.py new file mode 100644 index 0000000..35715f1 --- /dev/null +++ b/acquisition/move_xy.py @@ -0,0 +1,115 @@ +# move_xy.py +# ------------------------------------------------------------ +# Drive the XY stage by hand, from a terminal. +# +# python -m acquisition.move_xy # just show where we are +# python -m acquisition.move_xy --by 500 0 # relative, microns +# python -m acquisition.move_xy --to -1848 -10021 # absolute, microns +# python -m acquisition.move_xy --by 20000 0 --yes # skip the confirmation +# +# XY ONLY, ON PURPOSE. Z is what drives the objective into the sample, and +# it is not reachable from here - use the focus knob. This mirrors the +# same choice in the MCP tools. +# +# LONG MOVES ARE SPLIT AUTOMATICALLY. nis_sdk refuses any single step over +# MAX_XY_STEP_UM (5 mm) so that a fat-fingered coordinate cannot become a +# stage-length dash. That guard is worth keeping, but it makes a legitimate +# 15 mm move fail halfway with the stage somewhere unintended - which is +# exactly what happened on 2026-09-21, leaving the stage 8 mm from where +# anyone thought it was. So this walks the distance in legal hops and +# reports where it actually ended up. +# +# ANYTHING OVER --confirm-above ASKS FIRST, because the difference between +# a 200 um nudge and a 20000 um traverse is one keystroke, and only one of +# them can put the objective under the dish holder. +# +# The e-stop is checked by nis_sdk on every hop, so engaging it mid-move +# stops this at the next hop rather than at the end of the journey. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import sys +import time + +import numpy as np + +from acquisition import estop + +#: Kept under nis_sdk's 5000 um hard limit so a rounding error at the +#: boundary cannot trip the very guard we are working within. +HOP_UM = 4000.0 + +#: Moves longer than this ask for confirmation first. +CONFIRM_ABOVE_UM = 2000.0 + + +def move_to(sdk, x: float, y: float, quiet: bool = False) -> tuple[float, float]: + """Absolute move in microns, split into legal hops.""" + while True: + cx, cy = sdk.XY_GetPosition() + dx, dy = x - cx, y - cy + dist = float(np.hypot(dx, dy)) + if dist < 1.0: + return cx, cy + f = min(1.0, HOP_UM / dist) + sdk.XY_Move(cx + dx * f, cy + dy * f) + time.sleep(0.4) + if not quiet and dist > HOP_UM: + nx, ny = sdk.XY_GetPosition() + print(f" ... at ({nx:.1f}, {ny:.1f}), {dist - HOP_UM:.0f} um to go", flush=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description="Move the XY stage manually (microns).") + g = ap.add_mutually_exclusive_group() + g.add_argument("--to", nargs=2, type=float, metavar=("X", "Y"), help="absolute target") + g.add_argument("--by", nargs=2, type=float, metavar=("DX", "DY"), help="relative offset") + ap.add_argument("--yes", action="store_true", help="do not ask about long moves") + ap.add_argument("--confirm-above", type=float, default=CONFIRM_ABOVE_UM) + args = ap.parse_args() + + if estop.is_engaged(): + info = estop.details() or {} + print("E-STOP IS ENGAGED - the stage will not move.") + print(f" reason: {info.get('reason', '?')} at {info.get('engaged_at', '?')}") + print(" release with: python -m acquisition.estop release") + raise SystemExit(2) + + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + x0, y0 = sdk.XY_GetPosition() + z0 = sdk.Z_GetPosition() + print(f"current: X={x0:.2f} Y={y0:.2f} (Z={z0:.2f}, not touched)") + + if not args.to and not args.by: + return + + tx, ty = (args.to[0], args.to[1]) if args.to else (x0 + args.by[0], y0 + args.by[1]) + dist = float(np.hypot(tx - x0, ty - y0)) + print(f"target : X={tx:.2f} Y={ty:.2f} (moving {dist:.1f} um)") + + if dist > args.confirm_above and not args.yes: + # Interactive only: piped/automated use must pass --yes explicitly + # rather than have a prompt silently read EOF and proceed. + if not sys.stdin.isatty(): + print("long move needs --yes when not run interactively."); raise SystemExit(3) + if input(f" move {dist:.0f} um? [y/N] ").strip().lower() not in ("y", "yes"): + print(" cancelled - nothing moved."); return + + try: + x1, y1 = move_to(sdk, tx, ty) + except estop.EStopEngaged as exc: + cx, cy = sdk.XY_GetPosition() + print(f"\nSTOPPED BY E-STOP at ({cx:.2f}, {cy:.2f}) - target not reached.") + print(f" {exc}") + raise SystemExit(2) + + err = float(np.hypot(x1 - tx, y1 - ty)) + print(f"arrived: X={x1:.2f} Y={y1:.2f} (moved {np.hypot(x1-x0, y1-y0):.1f} um, " + f"{err:.2f} um from target)") + + +if __name__ == "__main__": + main() diff --git a/acquisition/orchestration/mosaic.py b/acquisition/orchestration/mosaic.py new file mode 100644 index 0000000..643b768 --- /dev/null +++ b/acquisition/orchestration/mosaic.py @@ -0,0 +1,459 @@ +# mosaic.py +# ------------------------------------------------------------ +# Whole-dish time-lapse by tiling: raster a grid of fields with the stage, +# stitch them into one large image, and repeat the grid on an interval. +# +# # plan only - prints the grid, moves nothing +# python -m acquisition.orchestration.mosaic --grid 6x6 --dry-run +# +# # one mosaic now, to check focus and framing before committing +# python -m acquisition.orchestration.mosaic --grid 6x6 --rounds 1 +# +# # overnight: a mosaic every 20 min for 12 h +# python -m acquisition.orchestration.mosaic --grid 6x6 --interval-min 20 --hours 12 +# +# WHY TILING RATHER THAN A SECOND CAMERA: the Baumer has no lens of its +# own - it is a bare sensor fed by the microscope optics, so it cannot +# image a whole dish directly. The stage, however, travels +/-57 mm in X +# and +/-37.5 mm in Y, far more than a 35 mm dish. So the wide view is +# assembled from many narrow ones. +# +# STITCHING IS BY STAGE COORDINATES, NOT FEATURE MATCHING. The stage is +# accurate (commanded 150.00 um -> reported 150.00) and the image scale +# was measured (1.4901 um/px), so every tile's position on the canvas is +# computed, not searched for. That matters for a living sample: feature +# matching on an organism that moves between tiles can align to the +# organism instead of the substrate and warp the mosaic. Geometry cannot. +# +# THE STAGE->IMAGE MAPPING IS MEASURED, NOT ASSUMED. calibrate_axes() +# moves in X and then in Y and phase-correlates each, recovering a full +# 2x2 matrix. This gets the SIGNS right - image Y usually points down +# while stage Y points up, so assuming a sign is a coin flip that +# silently mirrors the mosaic - and absorbs any small camera rotation. +# Pass --scale to skip it only if you already know the mapping. +# +# LIGHT: Physarum is negatively phototactic, and a tiled run illuminates +# any given spot only for the moment its tile is taken rather than +# continuously. That is gentler than parking on one field for hours - a +# real advantage of this approach for an overnight run, not just a +# side effect. +# +# FOCUS IS THE KNOWN WEAKNESS: Z is held fixed across the whole grid, so +# a non-level agar surface will drift out of focus toward the edges. +# Run --rounds 1 first and look at the corner tiles before trusting an +# overnight run. Each tile's focus score is written to tiles.csv so the +# drift is measurable rather than a surprise in the morning. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +import sys +import time +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from acquisition.paths import data_root + +#: Per-call stage limit in nis_sdk. Longer hops are broken into chunks. +_MAX_STEP_UM = 4500.0 + +#: Seconds to let the stage settle before grabbing. The stage reports the +#: move complete before vibration has died away; a frame taken too early +#: is motion-blurred in a way no later processing can undo. +SETTLE_S = 0.8 + + +def _move_safe(sdk, x: float, y: float) -> tuple[float, float]: + """Absolute move, split into legal-sized hops. + + nis_sdk refuses a single step over MAX_XY_STEP_UM, which the jump from + the end of one grid back to the start of the next will exceed. + """ + while True: + cx, cy = sdk.XY_GetPosition() + dx, dy = x - cx, y - cy + dist = float(np.hypot(dx, dy)) + if dist < 1.0: + return cx, cy + if dist <= _MAX_STEP_UM: + sdk.XY_Move(x, y) + return sdk.XY_GetPosition() + f = _MAX_STEP_UM / dist + sdk.XY_Move(cx + dx * f, cy + dy * f) + + +#: A computed focus Z may never depart from the reference focus by more +#: than this. The sample tilt actually measured across a 15.6 mm span was +#: 306 um, so 400 um is generous for any real surface while still being +#: far short of the objective's working distance. See FocusPlane. +MAX_FOCUS_DEVIATION_UM = 400.0 + + +class FocusPlane: + """z = z_ref + b*(x-x0) + c*(y-y0), fitted to measured focus points. + + THIS CLASS EXISTS BECAUSE A NAIVE FIT DROVE THE OBJECTIVE AT THE DISH. + On 2026-09-21 a plane was fitted to TWO points with + `lstsq` on [1, x, y]. Two points cannot determine three coefficients, + so lstsq returned its minimum-norm solution - and because it minimises + a^2+b^2+c^2, and x,y are ~10^4 while z is ~4.6x10^3, the cheapest way + to satisfy both equations was to collapse the intercept to ZERO and + use enormous gradients: z = 0 + 0.209*x + 1.470*y. At the far corner + of the grid that evaluates to 10734 um against a true focus of 4573 - + about 6 mm of objective travel toward the sample. The raster was + stopped at tile 15 of 48 before reaching it. + So this class refuses the conditions that produced that: + * at least THREE points, and not collinear - anything less cannot + define a plane, and pretending otherwise is what caused the fault + * coordinates are centred before fitting, so the free parameter is + the mean measured focus rather than an intercept at x=y=0 tens of + millimetres outside the sample + * every evaluated Z is hard-clamped to MAX_FOCUS_DEVIATION_UM around + the reference, so no arithmetic error downstream can command a + large move even if the fit is somehow still wrong + """ + + def __init__(self, points: list[tuple[float, float, float]], + max_dev: float = MAX_FOCUS_DEVIATION_UM): + if len(points) < 3: + raise ValueError( + f"a focus plane needs at least 3 measured points, got {len(points)}. " + "Two points cannot define a plane - fitting them is what drove the " + "objective 6 mm off focus on 2026-09-21. Use a fixed Z instead.") + pts = np.asarray(points, dtype=float) + self.x0, self.y0 = pts[:, 0].mean(), pts[:, 1].mean() + dx, dy = pts[:, 0] - self.x0, pts[:, 1] - self.y0 + # Collinear points leave the perpendicular gradient undetermined, + # which is the same degenerate case by another route. + spread = np.linalg.svd(np.column_stack([dx, dy]), compute_uv=False) + if spread.min() < 1e-6 * max(spread.max(), 1.0): + raise ValueError( + "focus points are collinear - they cannot determine tilt " + "perpendicular to the line joining them. Add a point off that line.") + A = np.column_stack([np.ones_like(dx), dx, dy]) + coef, *_ = np.linalg.lstsq(A, pts[:, 2], rcond=None) + self.z_ref, self.b, self.c = (float(v) for v in coef) + self.max_dev = float(max_dev) + self.residuals = [abs(self.z(px, py) - pz) for px, py, pz in points] + + def z(self, x: float, y: float) -> float: + raw = self.z_ref + self.b * (x - self.x0) + self.c * (y - self.y0) + return float(np.clip(raw, self.z_ref - self.max_dev, self.z_ref + self.max_dev)) + + def would_clamp(self, x: float, y: float) -> bool: + raw = self.z_ref + self.b * (x - self.x0) + self.c * (y - self.y0) + return abs(raw - self.z_ref) > self.max_dev + + def __str__(self) -> str: + return (f"z = {self.z_ref:.2f} {self.b:+.5f}*(x-{self.x0:.0f}) " + f"{self.c:+.5f}*(y-{self.y0:.0f}) clamped to +/-{self.max_dev:.0f} um") + + +def _move_z(sdk, z: float, limit: float = 45.0) -> float: + """Z in steps under nis_sdk's 50 um per-call cap.""" + for _ in range(200): + cz = sdk.Z_GetPosition() + d = z - cz + if abs(d) < 0.3: + return cz + sdk.Z_Move(cz + max(-limit, min(limit, d))) + time.sleep(0.2) + return sdk.Z_GetPosition() + + +def _grab_gray(cam) -> np.ndarray: + p = cam.capture() + a = np.asarray(Image.open(p)).mean(axis=2).astype(np.float32) + p.unlink(missing_ok=True) + return a + + +def calibrate_axes(cam, sdk, step_um: float = 150.0) -> np.ndarray: + """Measure M where (image shift in px) = M @ (stage delta in um). + + Returns a 2x2 matrix. Inverting the sign of M gives where a tile + belongs on the canvas: if moving the stage +X slides features left in + the image, then the tile taken at larger X shows sample further right. + """ + x0, y0 = sdk.XY_GetPosition() + base = _grab_gray(cam) + win = cv2.createHanningWindow((base.shape[1], base.shape[0]), cv2.CV_32F) + + cols = [] + for axis in (0, 1): + tx = x0 + (step_um if axis == 0 else 0.0) + ty = y0 + (step_um if axis == 1 else 0.0) + _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S + 0.7) + xa, ya = sdk.XY_GetPosition() + actual = (xa - x0) if axis == 0 else (ya - y0) + moved = _grab_gray(cam) + (dx, dy), resp = cv2.phaseCorrelate(base, moved, win) + if abs(actual) < 1.0 or resp < 0.05: + raise SystemExit( + f"axis {'XY'[axis]} calibration failed (moved {actual:.2f} um, " + f"confidence {resp:.3f}) - too little texture in view, or the stage did not move" + ) + cols.append([dx / actual, dy / actual]) + print(f" axis {'XY'[axis]}: moved {actual:+.2f} um -> image ({dx:+.2f}, {dy:+.2f}) px, " + f"confidence {resp:.3f}") + _move_safe(sdk, x0, y0) + time.sleep(SETTLE_S) + + M = np.array(cols).T # columns are the X and Y responses + scale = float(np.hypot(M[0, 0], M[1, 0])) + print(f" => {1.0 / scale:.4f} um/px" if scale else " => degenerate mapping") + return M + + +def grid_positions(cx: float, cy: float, nx: int, ny: int, + step_x: float, step_y: float) -> list[tuple[float, float]]: + """Serpentine raster - each row reverses, so the stage never makes a + long dash back to the start of the next row. Less travel, less + vibration, less time per mosaic.""" + xs = (np.arange(nx) - (nx - 1) / 2.0) * step_x + cx + ys = (np.arange(ny) - (ny - 1) / 2.0) * step_y + cy + out = [] + for j, y in enumerate(ys): + row = xs if j % 2 == 0 else xs[::-1] + out.extend((float(x), float(y)) for x in row) + return out + + +def stitch(tiles: list[tuple[np.ndarray, float, float]], M: np.ndarray, + ref: tuple[float, float]) -> np.ndarray: + """Paste tiles onto one canvas using their stage coordinates.""" + Minv = -M # canvas offset is the negated image shift + offs = [Minv @ np.array([sx - ref[0], sy - ref[1]]) for _, sx, sy in tiles] + h, w = tiles[0][0].shape[:2] + ox = np.array([o[0] for o in offs]); oy = np.array([o[1] for o in offs]) + ox -= ox.min(); oy -= oy.min() + # Size from the ROUNDED placements, not the raw maxima: a tile whose + # offset is x.6 is written at x+1, which overruns a canvas sized on + # int(x). Rounding first makes the bound exact. + xi_all = np.round(ox).astype(int); yi_all = np.round(oy).astype(int) + canvas = np.zeros((int(yi_all.max()) + h, int(xi_all.max()) + w, 3), np.uint8) + for (img, _, _), xi, yi in zip(tiles, xi_all, yi_all): + canvas[yi:yi + h, xi:xi + w] = img + return canvas + + +def run(grid: tuple[int, int], overlap: float, rounds: int, interval_s: float, + exposure_us: float, mosaic_scale: float, keep_tiles: bool, + scale_um_px: float | None, tag: str | None, + focus_points: list[tuple[float, float, float]] | None = None, + centre: tuple[float, float] | None = None) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / (f"mosaic_{stamp}" + (f"_{tag}" if tag else "")) + out.mkdir(parents=True, exist_ok=True) + + sdk = NISSdk() + cam = BaumerGenICam() + cx = cy = 0.0 + try: + cam.set_settings(exposure_time_us=exposure_us, gain=1.0) + # AUTO WHITE BALANCE MUST BE OFF FOR A MOSAIC. Left on Continuous, + # the camera re-neutralises every tile independently - a tile of + # bare agar and a tile of organism then get different corrections, + # and the stitched result has visible colour seams at every tile + # boundary that no amount of post-processing can unpick. It resets + # itself to Continuous across power cycles, so force it per run + # rather than trusting whatever it was left in. + try: + node = cam._acquirer.remote_device.node_map.BalanceWhiteAuto + if node.value != "Off": + print(f"BalanceWhiteAuto was {node.value!r} - forcing 'Off' for consistent tiles") + node.value = "Off" + except Exception as exc: + print(f"WARNING: could not force BalanceWhiteAuto off ({exc}); " + "tiles may not colour-match", file=sys.stderr) + print(f"camera: {cam.get_settings()}") + if centre is not None: + cx, cy = centre + _move_safe(sdk, cx, cy) + cx, cy = sdk.XY_GetPosition() + else: + cx, cy = sdk.XY_GetPosition() + print(f"grid centre: ({cx:.1f}, {cy:.1f}) um") + + plane = None + z_fixed = sdk.Z_GetPosition() + if focus_points: + plane = FocusPlane(focus_points) + print(f"focus plane {plane} (from {len(focus_points)} points, " + f"max residual {max(plane.residuals):.1f} um)") + else: + print(f"FIXED focus: Z held at {z_fixed:.2f} um for every tile " + f"(no --focus-plane given)") + + if scale_um_px: + M = np.array([[1.0 / scale_um_px, 0.0], [0.0, 1.0 / scale_um_px]]) + print(f"using supplied scale {scale_um_px} um/px (axes assumed square and unflipped)") + else: + print("calibrating stage->image mapping...") + M = calibrate_axes(cam, sdk) + + probe = _grab_gray(cam) + h, w = probe.shape + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + step_x = w * umpx * (1.0 - overlap) + step_y = h * umpx * (1.0 - overlap) + nx, ny = grid + pts = grid_positions(cx, cy, nx, ny, step_x, step_y) + print(f"{nx}x{ny} tiles, step {step_x:.0f} x {step_y:.0f} um " + f"-> covers {nx * step_x / 1000:.2f} x {ny * step_y / 1000:.2f} mm") + + (out / "mosaic.json").write_text(json.dumps({ + "grid": [nx, ny], "overlap": overlap, "um_per_px": round(umpx, 4), + "step_um": [round(step_x, 1), round(step_y, 1)], + "centre_um": [cx, cy], "exposure_us": exposure_us, + "stage_to_image_matrix": M.tolist(), + "covers_mm": [round(nx * step_x / 1000, 3), round(ny * step_y / 1000, 3)], + # The saved mosaic_NNN.png is resized by this factor, so its + # pixels are um_per_px / mosaic_scale - make_movie needs it + # to draw a scale bar that is true on the stitched image. + "mosaic_scale": mosaic_scale, + }, indent=2), encoding="utf-8") + + # An out-of-band kill switch. Stopping a background shell does NOT + # always take its python child with it - on 2026-09-21 a stopped + # raster kept driving the stage afterwards. Creating this file makes + # the run halt itself at the next tile boundary, without needing to + # find and kill a process. + stop_file = out / "STOP" + print(f"to halt this run at any point, create: {stop_file}") + + fields = ("round", "tile", "wall_clock", "x_um", "y_um", "z_um", + "mean", "focus", "error") + log = (out / "tiles.csv").open("w", newline="", encoding="utf-8") + writer = csv.DictWriter(log, fieldnames=fields) + writer.writeheader() + + t_start = time.monotonic() + for r in range(rounds): + target = t_start + r * interval_s + if time.monotonic() < target: + time.sleep(target - time.monotonic()) + r_dir = out / f"round_{r:03d}" + if keep_tiles: + r_dir.mkdir(exist_ok=True) + print(f"\n[round {r + 1}/{rounds}] {datetime.datetime.now():%H:%M:%S}", flush=True) + + tiles = [] + for i, (tx, ty) in enumerate(pts): + row = {k: "" for k in fields} + row.update(round=r, tile=i, + wall_clock=datetime.datetime.now().isoformat(timespec="seconds")) + try: + if stop_file.exists(): + print(f" STOP FILE {stop_file.name} present - halting", flush=True) + raise KeyboardInterrupt + ax, ay = _move_safe(sdk, tx, ty) + if plane is not None: + _move_z(sdk, plane.z(ax, ay)) + row["z_um"] = round(sdk.Z_GetPosition(), 2) + time.sleep(SETTLE_S) + p = cam.capture() + img = np.asarray(Image.open(p)) + if keep_tiles: + p.replace(r_dir / f"t{i:03d}.png") + else: + p.unlink(missing_ok=True) + g = img.mean(axis=2) + lap = (g[:-2, 1:-1] + g[2:, 1:-1] + g[1:-1, :-2] + g[1:-1, 2:] + - 4 * g[1:-1, 1:-1]) + row.update(x_um=round(ax, 1), y_um=round(ay, 1), + mean=round(float(g.mean()), 2), focus=round(float(lap.var()), 2)) + tiles.append((img, ax, ay)) + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + print(f" tile {i}: {row['error']}", flush=True) + writer.writerow(row); log.flush() + + if tiles: + canvas = stitch(tiles, M, (cx, cy)) + if mosaic_scale != 1.0: + canvas = cv2.resize( + canvas, (int(canvas.shape[1] * mosaic_scale), + int(canvas.shape[0] * mosaic_scale)), + interpolation=cv2.INTER_AREA) + dest = out / f"mosaic_{r:03d}.png" + Image.fromarray(canvas).save(dest) + print(f" {len(tiles)}/{len(pts)} tiles -> {dest.name} " + f"({canvas.shape[1]}x{canvas.shape[0]})", flush=True) + + log.close() + finally: + try: + _move_safe(sdk, cx, cy) + print(f"returned to grid centre ({cx:.1f}, {cy:.1f}) um") + except Exception as exc: + print(f"WARNING: could not return to centre: {exc}", file=sys.stderr) + cam.close() + print("released") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--grid", default="6x6", help="tiles as NXxNY (default 6x6)") + ap.add_argument("--overlap", type=float, default=0.10, help="tile overlap fraction") + ap.add_argument("--rounds", type=int, help="number of mosaics (default: from --hours)") + ap.add_argument("--hours", type=float, help="run length; with --interval-min sets rounds") + ap.add_argument("--interval-min", type=float, default=20.0) + ap.add_argument("--exposure-us", type=float, default=10000.0) + ap.add_argument("--mosaic-scale", type=float, default=0.5, + help="downsample the stitched mosaic (tiles are kept full-res)") + ap.add_argument("--no-tiles", action="store_true", help="do not keep individual tiles") + ap.add_argument("--scale", type=float, help="known um/px; skips axis calibration") + ap.add_argument("--tag") + ap.add_argument("--focus-plane", + help="measured focus references as 'x,y,z;x,y,z;...' (um). Z is " + "interpolated per tile from a least-squares plane through them") + ap.add_argument("--centre", help="grid centre as 'x,y' (um); default is the current position") + ap.add_argument("--dry-run", action="store_true", help="print the plan, move nothing") + args = ap.parse_args() + + fpts = None + if args.focus_plane: + fpts = [tuple(float(v) for v in grp.split(",")) + for grp in args.focus_plane.split(";") if grp.strip()] + if any(len(p) != 3 for p in fpts): + ap.error("--focus-plane entries must each be x,y,z") + ctr = tuple(float(v) for v in args.centre.split(",")) if args.centre else None + + nx, ny = (int(v) for v in args.grid.lower().split("x")) + if args.rounds: + rounds = args.rounds + elif args.hours: + rounds = max(1, int(round(args.hours * 60 / args.interval_min))) + else: + rounds = 1 + + if args.dry_run: + umpx = args.scale or 1.4901 + sx = 1920 * umpx * (1 - args.overlap) / 1000 + sy = 1200 * umpx * (1 - args.overlap) / 1000 + print(f"{nx}x{ny} = {nx * ny} tiles, step {sx:.3f} x {sy:.3f} mm") + print(f"covers {nx * sx:.2f} x {ny * sy:.2f} mm at {umpx} um/px") + print(f"{rounds} rounds every {args.interval_min} min " + f"= {rounds * args.interval_min / 60:.1f} h") + print(f"~{nx * ny * 3:.0f} MB of tiles per round, ~{rounds * nx * ny * 3 / 1000:.1f} GB total") + return + + run((nx, ny), args.overlap, rounds, args.interval_min * 60.0, args.exposure_us, + args.mosaic_scale, not args.no_tiles, args.scale, args.tag, fpts, ctr) + + +if __name__ == "__main__": + main() diff --git a/acquisition/orchestration/timelapse.py b/acquisition/orchestration/timelapse.py new file mode 100644 index 0000000..0af66ee --- /dev/null +++ b/acquisition/orchestration/timelapse.py @@ -0,0 +1,175 @@ +# timelapse.py +# ------------------------------------------------------------ +# Fixed-position brightfield time-lapse from the Baumer camera. +# +# python -m acquisition.orchestration.timelapse --minutes 60 --interval 10 +# python -m acquisition.orchestration.timelapse --frames 12 --interval 5 --tag test +# +# WHY 10 s BY DEFAULT: written for Physarum polycephalum, whose +# cytoplasmic shuttle streaming reverses on a ~100-130 s contraction +# rhythm. A 10 s interval puts ~10-13 samples in each cycle - enough to +# fit the oscillation - while an hour of it still spans ~30 cycles, so a +# periodogram has something to work with. Slower front advance falls out +# of the same series for free. +# +# THE CAMERA IS OPENED ONCE for the whole run, not per frame. Connecting +# costs seconds and a GenICam camera can only be held by one process at a +# time, so a per-frame open/close would both blow the interval budget and +# leave a window where anything else could grab the device mid-run. +# +# ILLUMINATION IS LEFT ALONE. The DIA lamp stays at whatever it was set +# to, steady, for the run's whole duration - shuttering or dimming +# between frames would save the sample some light but makes every frame's +# exposure history different, which is precisely what ruins a +# quantitative intensity series. Set the lamp before starting. +# +# DRIFT: frames are scheduled against a fixed start time, not by sleeping +# `interval` after each capture. Capture takes real time (exposure + +# fetch + PNG encode), so sleep-after-capture accumulates a lag that +# grows all run - fatal for a periodogram. A frame whose slot has already +# passed is taken immediately and flagged `late` in the log rather than +# skipped. +# +# ROBUSTNESS: one failed frame does not end the run - it is recorded with +# its error in the CSV and the series continues on schedule. A run that +# dies at frame 200 of 360 still leaves 200 usable frames plus their +# metadata. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import shutil +import sys +import time +from pathlib import Path + +import numpy as np +from PIL import Image + +from acquisition.paths import data_root + +#: Per-frame columns. `mean`/`p1`/`p50`/`p99`/`sat` are written at capture +#: time because they are what tells you mid-run whether the series is +#: still well exposed and in focus, without reopening 300 PNGs. +_FIELDS = ("frame", "t_seconds", "wall_clock", "filename", "exposure_us", + "gain", "mean", "p1", "p50", "p99", "sat_frac", "focus", "late_s", "error") + + +def _series_dir(tag: str | None) -> Path: + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + name = f"timelapse_{stamp}" + (f"_{tag}" if tag else "") + return data_root() / "data" / name + + +def _stats(path: Path) -> dict: + """Exposure and focus numbers for one frame. + + `focus` is the variance of the Laplacian - the standard cheap + focus proxy. Only ever compare it between frames of one series at one + illumination; it is not an absolute scale. + """ + a = np.asarray(Image.open(path)) + g = a.mean(axis=2) + lap = (g[:-2, 1:-1] + g[2:, 1:-1] + g[1:-1, :-2] + g[1:-1, 2:] - 4 * g[1:-1, 1:-1]) + return { + "mean": round(float(g.mean()), 3), + "p1": round(float(np.percentile(g, 1)), 2), + "p50": round(float(np.percentile(g, 50)), 2), + "p99": round(float(np.percentile(g, 99)), 2), + "sat_frac": round(float((a >= 254).mean()), 6), + "focus": round(float(lap.var()), 2), + } + + +def run(frames: int, interval: float, exposure_us: float, gain: float, + tag: str | None = None) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + + out = _series_dir(tag) + out.mkdir(parents=True, exist_ok=True) + log_path = out / "frames.csv" + + cam = BaumerGenICam() + try: + settings = cam.set_settings(exposure_time_us=exposure_us, gain=gain) + print(f"[timelapse] camera: {settings}", flush=True) + print(f"[timelapse] {frames} frames every {interval}s " + f"(~{frames * interval / 60:.1f} min) -> {out}", flush=True) + + # One throwaway grab so the first logged frame is taken under the + # exposure just set, not the one before it. + cam.capture().unlink(missing_ok=True) + + start = time.monotonic() + wall_start = datetime.datetime.now() + + with log_path.open("w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=_FIELDS) + writer.writeheader() + + for i in range(frames): + target = start + i * interval + late = time.monotonic() - target + if late < 0: + time.sleep(-late) + late = 0.0 + + row = {f: "" for f in _FIELDS} + row.update(frame=i, + t_seconds=round(time.monotonic() - start, 3), + wall_clock=datetime.datetime.now().isoformat(timespec="seconds"), + exposure_us=exposure_us, gain=gain, + late_s=round(late, 3)) + try: + src = cam.capture() + dest = out / f"f{i:05d}.png" + shutil.move(str(src), dest) + row["filename"] = dest.name + row.update(_stats(dest)) + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + + writer.writerow(row) + fh.flush() # so the CSV is readable mid-run, not just at the end + + if row["error"]: + print(f"[timelapse] frame {i}: {row['error']}", flush=True) + elif i % 10 == 0 or i == frames - 1: + print(f"[timelapse] {i + 1}/{frames} t={row['t_seconds']:.0f}s " + f"mean={row['mean']} focus={row['focus']} " + f"sat={row['sat_frac']}", flush=True) + + elapsed = (datetime.datetime.now() - wall_start).total_seconds() + print(f"[timelapse] done in {elapsed / 60:.1f} min -> {out}", flush=True) + finally: + cam.close() + + return out + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + g = p.add_mutually_exclusive_group() + g.add_argument("--minutes", type=float, help="run length; with --interval sets frame count") + g.add_argument("--frames", type=int, help="explicit frame count") + p.add_argument("--interval", type=float, default=10.0, help="seconds between frames (default 10)") + p.add_argument("--exposure-us", type=float, default=40_000.0) + p.add_argument("--gain", type=float, default=1.0) + p.add_argument("--tag", help="appended to the output directory name") + args = p.parse_args() + + if args.frames: + frames = args.frames + elif args.minutes: + frames = int(round(args.minutes * 60 / args.interval)) + 1 + else: + p.error("pass --minutes or --frames") + + run(frames, args.interval, args.exposure_us, args.gain, args.tag) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/acquisition/orchestration/worm_follow.py b/acquisition/orchestration/worm_follow.py new file mode 100644 index 0000000..72e8dca --- /dev/null +++ b/acquisition/orchestration/worm_follow.py @@ -0,0 +1,557 @@ +# worm_follow.py +# ------------------------------------------------------------ +# Follow one crawling worm across a dish: take a small mosaic (3x3 by +# default) around it, find the worm in the stitched image, re-centre the +# next mosaic on it, and repeat back-to-back. The result is the worm's +# path across the dish, one point every ~15 s, plus a mosaic per round. +# +# # detection check on a saved image, moves nothing +# python -m acquisition.orchestration.worm_follow --detect-only img.png --um-per-px 1.49 +# +# # one round, to check focus, exposure and that the worm is found +# python -m acquisition.orchestration.worm_follow --rounds 1 +# +# # follow for an hour +# python -m acquisition.orchestration.worm_follow --minutes 60 --tag celegans +# +# START WITH THE WORM IN THE MIDDLE OF THE FIELD. The first round takes +# the qualifying object nearest the grid centre; after that each round +# takes the one nearest the last position. Debris the size of an adult +# worm is rare, so size plus proximity is enough to stay locked on. +# +# WHY A SMALL MOSAIC AND NOT THE WHOLE DISH. A crawling adult covers +# 0.1-0.2 mm/s. A whole-dish round takes ~4.5 min, in which the worm can +# cross the dish, and tiles one row apart are ~25 s apart, so it is cut +# at seams or appears twice. A 3x3 round is ~13 s: the worm moves at most +# ~2 mm, so it stays inside the 7.7 x 4.8 mm block and is found again. +# +# WHY A DEADBAND ON RE-CENTRING. Worms react to vibration (tap response). +# The stage only jumps when the worm has moved more than --deadband-mm +# from the block centre, so a dwelling worm is not shaken every round. +# +# SAFETY: every re-centre is clamped to --limit-mm around the dish centre +# (default: where the run started), so a misdetection cannot walk the +# stage off the glass window. Z is fixed - the worm crawls on the agar +# surface, and a 3x3 block is small enough that tilt barely matters. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +import sys +import time +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from acquisition.orchestration.mosaic import (SETTLE_S, _move_safe, calibrate_axes, + grid_positions) +from acquisition.paths import data_root + +#: Stage->image mapping measured at 4x on 2026-09-24 (night-2 Physarum +#: run, 1.4877 um/px). Same camera, same objective, so reused rather than +#: re-measured - calibration needs texture, and clean agar has little. +M_4X = np.array([[-0.6721844559295096, 0.0007000999870820124], + [0.0007312652036724406, 0.6711263094676038]]) + +#: Detection runs on the stitched block shrunk by this factor (~6 um/px +#: at 4x). An adult is ~50 um wide, so still ~8 px across. +DETECT_SCALE = 0.25 + + +def force_camera_autos_off(cam) -> None: + """Any auto colour/exposure function drifts a series mid-run + (2026-09-23: ColorTransformationAuto turned mosaics green). Camera + Explorer can switch them back on, so force them per run.""" + nm = cam._acquirer.remote_device.node_map + for name in ("BalanceWhiteAuto", "ColorTransformationAuto", "ExposureAuto", "GainAuto"): + try: + node = getattr(nm, name) + if node.value != "Off": + print(f"{name} was {node.value!r} - forcing 'Off'") + node.value = "Off" + except Exception: + pass + # Explorer also leaves its own colour pipeline behind: on 2026-09-29 it + # had Gamma 1.45, an adapted colour matrix that zeroed green, and red/ + # blue gains of 5.9/4.4 - every 4x frame was solid 255. Restore the + # linear, neutral state measured on clear agar on 2026-09-23. + try: + nm.Gamma.value = 1.0 + for s in nm.ColorTransformationValueSelector.symbolics: + nm.ColorTransformationValueSelector.value = s + nm.ColorTransformationValue.value = 1.0 if s[-2] == s[-1] else 0.0 + for s, v in (("Red", 1.689), ("GreenRed", 1.0), ("GreenBlue", 1.0), ("Blue", 2.884)): + nm.GainSelector.value = s + nm.Gain.value = v + nm.GainSelector.value = "All" + print("camera colour: Gamma 1.0, identity matrix, gains R 1.689 / G 1.0 / B 2.884") + except Exception as exc: + print(f"WARNING: could not restore camera colour state ({exc})", file=sys.stderr) + + +def stitch_with_origin(tiles, M, ref): + """mosaic.stitch, but also returns the canvas origin offset so canvas + pixels can be mapped back to stage coordinates.""" + offs = np.array([-M @ np.array([sx - ref[0], sy - ref[1]]) for _, sx, sy in tiles]) + omin = offs.min(axis=0) + pos = np.round(offs - omin).astype(int) + h, w = tiles[0][0].shape[:2] + canvas = np.zeros((pos[:, 1].max() + h, pos[:, 0].max() + w, 3), np.uint8) + for (img, _, _), (xi, yi) in zip(tiles, pos): + canvas[yi:yi + h, xi:xi + w] = img + return canvas, omin, (w, h) + + +def canvas_to_stage(px, py, M, ref, omin, tile_wh) -> tuple[float, float]: + """Stage position that would put canvas pixel (px, py) at the image + centre. A feature at stage-centre S appears in a tile taken at stage s + at c + M(s - S); with the tile's canvas offset -M(s-ref) - omin this + reduces to S = ref - M^-1 (P + omin - c), independent of the tile.""" + c = np.array(tile_wh, float) / 2.0 + S = np.array(ref) - np.linalg.solve(M, np.array([px, py]) + omin - c) + return float(S[0]), float(S[1]) + + +def find_worms(rgb: np.ndarray, um_per_px: float, min_area_mm2: float, + max_area_mm2: float, dark_ratio: float) -> tuple[list[dict], np.ndarray]: + """Dark objects of worm size against a locally-estimated background. + + Returns candidates (full-resolution centroid, area, length) and the + shrunk mask for inspection. The background is a morphological CLOSE + with a kernel wider than a worm: that fills in anything dark and thin + while following slow illumination falloff and vignetting, so the + threshold is relative to the local agar rather than a global level. + """ + small = cv2.resize(rgb, None, fx=DETECT_SCALE, fy=DETECT_SCALE, interpolation=cv2.INTER_AREA) + g = small.mean(axis=2).astype(np.float32) + # Canvas outside every tile is zero. The camera is rotated ~0.06 deg + # to the stage, so the stitched block has zero slivers a few px wide + # along its edges; shrunk, they turn into thin grey lines that read as + # dark objects. Judge validity at full resolution (a 2 px sliver is + # only half a shrunk pixel, never dark enough to test as empty), keep + # only shrunk pixels that were entirely inside tiles, then erode. + full = rgb.any(axis=2).astype(np.float32) + valid = cv2.resize(full, (g.shape[1], g.shape[0]), interpolation=cv2.INTER_AREA) > 0.999 + valid = cv2.erode(valid.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 + upx = um_per_px / DETECT_SCALE + k = int(round(150 / upx)) | 1 # 150 um: 3x a worm's width + bg = cv2.morphologyEx(g, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))) + bg = cv2.GaussianBlur(bg, (0, 0), k / 2) + mask = ((g < dark_ratio * np.maximum(bg, 1)) & valid).astype(np.uint8) + mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, np.ones((2, 2), np.uint8)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8)) + n, lab, stats, cents = cv2.connectedComponentsWithStats(mask, connectivity=8) + px_mm2 = (upx / 1000.0) ** 2 + out = [] + for i in range(1, n): + area = stats[i, cv2.CC_STAT_AREA] * px_mm2 + if not (min_area_mm2 <= area <= max_area_mm2): + continue + comp = (lab == i).astype(np.uint8) + # Skeleton-free length estimate: area / mean width, with width from + # the distance transform. Curled worms still read as long. + dt = cv2.distanceTransform(comp, cv2.DIST_L2, 3) + width_um = max(2.0 * float(dt.max()) * upx, 1.0) + length_mm = area / (width_um / 1000.0) + out.append({"x": cents[i][0] / DETECT_SCALE, "y": cents[i][1] / DETECT_SCALE, + "area_mm2": area, "width_um": width_um, "length_mm": length_mm}) + return out, mask * 255 + + +def pick(cands: list[dict], near_px: tuple[float, float], max_px: float, + area_mm2: float | None = None) -> dict | None: + """Nearest candidate within max_px. With area_mm2 (the tracked worm's + last area), candidates more than 2x bigger or smaller are ignored, so + the track does not hop onto a larva or another adult passing close by + - plates are crowded (2026-09-29: a second adult 2.5 mm away).""" + best = None + for c in cands: + if area_mm2 and not (0.5 <= c["area_mm2"] / area_mm2 <= 2.0): + continue + d = float(np.hypot(c["x"] - near_px[0], c["y"] - near_px[1])) + if d <= max_px and (best is None or d < best["dist_px"]): + best = {**c, "dist_px": d} + return best + + +def detect_only(path: Path, umpx: float, args) -> None: + rgb = np.asarray(Image.open(path).convert("RGB")) + cands, mask = find_worms(rgb, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + h, w = rgb.shape[:2] + for c in cands: + print(f" ({c['x']:.0f}, {c['y']:.0f}) px area {c['area_mm2']:.4f} mm2 " + f"width {c['width_um']:.0f} um length ~{c['length_mm']:.2f} mm") + best = pick(cands, (w / 2, h / 2), float("inf")) + print(f"{len(cands)} candidate(s); picked: {best and (round(best['x']), round(best['y']))}") + Image.fromarray(mask).save(path.with_name(path.stem + "_mask.png")) + + +#: Setup focus sweep never leaves this band around the starting Z. The 4x +#: has a 20 mm working distance, so this is far from any contact; it only +#: has to cover the parfocal offset from the 10x (a few tens of um). +SETUP_Z_RANGE_UM = 150.0 + + +def _focus_score(rgb: np.ndarray) -> float: + g = cv2.resize(rgb, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA).mean(axis=2) + return float(cv2.Laplacian(g.astype(np.float32), cv2.CV_32F).var()) + + +def setup(args) -> Path: + """Put the microscope in the state a run starts from: 4x objective, + exposure set once from the agar, focus found by a capped Z sweep, and + the worm moved to the middle of the field. Saves every sweep frame and + a final frame so the result can be checked by eye before a run.""" + from acquisition import estop + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + from acquisition.orchestration.mosaic import _move_z + + estop.check() # the nosepiece write below has no guard of its own + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / f"wormsetup_{stamp}" + out.mkdir(parents=True, exist_ok=True) + sdk = NISSdk() + cam = BaumerGenICam() + + def grab() -> np.ndarray: + p = cam.capture() + a = np.asarray(Image.open(p)); p.unlink(missing_ok=True) + return a + + try: + z_start = sdk.Z_GetPosition() + nose = sdk._thread.call(lambda m: int(m.iNOSEPIECE)) + if nose != args.nosepiece: + print(f"nosepiece {nose} -> {args.nosepiece} (Z {z_start:.2f} um before)", flush=True) + estop.check() + sdk._thread.call(lambda m: setattr(m, "iNOSEPIECE", args.nosepiece)) + deadline = time.monotonic() + 20 + while sdk._thread.call(lambda m: int(m.iNOSEPIECE)) != args.nosepiece: + if time.monotonic() > deadline: + raise SystemExit("nosepiece did not reach the requested position in 20 s") + time.sleep(0.25) + time.sleep(1.0) + z0 = sdk.Z_GetPosition() + print(f"nosepiece {args.nosepiece}, Z {z0:.2f} um", flush=True) + + force_camera_autos_off(cam) + exp = args.exposure_us + for _ in range(10): + cam.set_settings(exposure_time_us=exp, gain=1.0) + grab() # first frame after a change can be stale + p90 = float(np.percentile(grab().mean(axis=2), 90)) + print(f"exposure {exp / 1000:.2f} ms -> agar 90th percentile {p90:.0f}/255", flush=True) + if 170 <= p90 <= 230: + break + # A clipped frame says nothing about how far over it is, so + # halve until it reads; the response is linear after that. + exp = exp / 2 if p90 >= 250 else exp * 200.0 / max(p90, 1.0) + exp = float(np.clip(exp, 100.0, 60000.0)) + + def sweep(centre: float, half: float, step: float, tag: str) -> float: + scores = [] + for z in np.arange(centre - half, centre + half + 0.1, step): + z = float(np.clip(z, z0 - SETUP_Z_RANGE_UM, z0 + SETUP_Z_RANGE_UM)) + _move_z(sdk, z) + time.sleep(0.4) + img = grab() + s = _focus_score(img) + print(f" {tag} Z {sdk.Z_GetPosition():.1f} um focus {s:.1f}", flush=True) + Image.fromarray(cv2.resize(img, None, fx=0.4, fy=0.4)).save( + out / f"{tag}_z{z:.0f}.jpg", quality=85) + scores.append((s, z)) + # A flat curve (e.g. every frame clipped - 2026-09-29 sent Z to + # the top of the range on all-zero scores) means no information: + # stay where the sweep was centred rather than at an end. + vals = [s for s, _ in scores] + if max(vals) < 1.0 or max(vals) < 1.2 * min(vals): + print(f" {tag} sweep is flat - keeping Z {centre:.1f} um", flush=True) + return centre + return max(scores)[1] + + z_best = sweep(z0, SETUP_Z_RANGE_UM, 25.0, "coarse") + _move_z(sdk, z_best) + + M = M_4X + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + img = grab() + h, w = img.shape[:2] + cands, _ = find_worms(img, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + best = pick(cands, (w / 2, h / 2), float("inf")) + if best: + # A feature at image pixel q is centred by moving the stage by + # -M^-1 (q - c) - same geometry as canvas_to_stage. + x, y = sdk.XY_GetPosition() + d = np.linalg.solve(M, np.array([best["x"] - w / 2, best["y"] - h / 2])) + tx, ty = x - d[0], y - d[1] + print(f"worm at ({best['x']:.0f}, {best['y']:.0f}) px, area {best['area_mm2']:.3f} mm2 " + f"-> centring: stage ({x:.1f}, {y:.1f}) -> ({tx:.1f}, {ty:.1f}) um", flush=True) + _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S) + else: + print(f"NO WORM FOUND in the 4x field ({len(cands)} candidates) - stage not moved", + flush=True) + + z_fine = sweep(z_best, 20.0, 5.0, "fine") + _move_z(sdk, z_fine) + time.sleep(0.4) + final = grab() + cands, _ = find_worms(final, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + b = pick(cands, (w / 2, h / 2), float("inf")) + mark = final.copy() + if b: + cv2.circle(mark, (int(b["x"]), int(b["y"])), int(700 / umpx), (255, 0, 0), 4) + Image.fromarray(final).save(out / "final.png") + Image.fromarray(cv2.resize(mark, None, fx=0.5, fy=0.5)).save(out / "final_marked.jpg", quality=90) + x, y = sdk.XY_GetPosition() + summary = {"nosepiece": args.nosepiece, "z_um": sdk.Z_GetPosition(), "z_before_um": z_start, + "xy_um": [x, y], "exposure_us": exp, "worm_found": bool(b), + "worm_px": [b["x"], b["y"]] if b else None} + (out / "setup.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(f"READY: {summary}\nimages in {out}", flush=True) + finally: + cam.close() + return out + + +def run(args) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / (f"wormfollow_{stamp}" + (f"_{args.tag}" if args.tag else "")) + (out / "crops").mkdir(parents=True, exist_ok=True) + stop_file = out / "STOP" + + sdk = NISSdk() + cam = BaumerGenICam() + cx = cy = 0.0 + try: + force_camera_autos_off(cam) + cam.set_settings(exposure_time_us=args.exposure_us, gain=1.0) + cx, cy = sdk.XY_GetPosition() + dish = tuple(float(v) for v in args.dish_centre.split(",")) if args.dish_centre else (cx, cy) + z = sdk.Z_GetPosition() + print(f"start ({cx:.1f}, {cy:.1f}) um, Z fixed at {z:.2f} um; " + f"stage confined to {args.limit_mm} mm around ({dish[0]:.0f}, {dish[1]:.0f})") + + if args.calibrate: + M = calibrate_axes(cam, sdk) + else: + M = M_4X + print("using the 4x stage->image mapping measured 2026-09-24 (pass --calibrate to re-measure)") + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + + probe = cam.capture() + img = np.asarray(Image.open(probe)); probe.unlink(missing_ok=True) + h, w = img.shape[:2] + bright = float(np.percentile(img.mean(axis=2), 90)) + print(f"camera {cam.get_settings()} - probe frame 90th percentile {bright:.0f}/255") + if bright > 245 or bright < 60: + print("WARNING: agar is " + ("clipped" if bright > 245 else "dark") + + " - adjust --exposure-us or the lamp", file=sys.stderr) + + step = (w * umpx * (1 - args.overlap), h * umpx * (1 - args.overlap)) + (out / "run.json").write_text(json.dumps({ + "grid": args.grid, "search_grid": args.search_grid, "overlap": args.overlap, + "um_per_px": round(umpx, 4), "stage_to_image_matrix": M.tolist(), + "step_um": [round(s, 1) for s in step], "z_um": z, "exposure_us": args.exposure_us, + "dish_centre_um": dish, "limit_mm": args.limit_mm, "deadband_mm": args.deadband_mm, + "mosaic_scale": args.mosaic_scale, "min_area_mm2": args.min_area_mm2, + "max_area_mm2": args.max_area_mm2, "dark_ratio": args.dark_ratio, + }, indent=2), encoding="utf-8") + print(f"to halt this run at any point, create: {stop_file}") + + fields = ("round", "wall_clock", "t_s", "grid", "centre_x_um", "centre_y_um", + "found", "worm_x_um", "worm_y_um", "area_mm2", "length_mm", + "n_candidates", "recentred", "error") + log = (out / "track.csv").open("w", newline="", encoding="utf-8") + writer = csv.DictWriter(log, fieldnames=fields) + writer.writeheader() + + last = None # last worm position, stage um + last_area = None + misses = 0 + t0 = time.monotonic() + r = 0 + while (args.rounds is None or r < args.rounds) and \ + (args.minutes is None or time.monotonic() - t0 < args.minutes * 60): + if args.interval_s: + target = t0 + r * args.interval_s + if time.monotonic() < target: + time.sleep(target - time.monotonic()) + gspec = args.search_grid if misses else args.grid + nx, ny = (int(v) for v in gspec.lower().split("x")) + row = {k: "" for k in fields} + row.update(round=r, wall_clock=datetime.datetime.now().isoformat(timespec="seconds"), + t_s=round(time.monotonic() - t0, 1), grid=gspec, + centre_x_um=round(cx, 1), centre_y_um=round(cy, 1), found=0) + try: + tiles = [] + for tx, ty in grid_positions(cx, cy, nx, ny, *step): + if stop_file.exists(): + print("STOP file present - halting", flush=True) + raise KeyboardInterrupt + ax, ay = _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S) + p = cam.capture() + tiles.append((np.asarray(Image.open(p)), ax, ay)) + p.unlink(missing_ok=True) + canvas, omin, twh = stitch_with_origin(tiles, M, (cx, cy)) + cands, _ = find_worms(canvas, umpx, args.min_area_mm2, args.max_area_mm2, + args.dark_ratio) + row["n_candidates"] = len(cands) + # Where the previous worm position (or, first round, the + # block centre) sits on this canvas. Gate generously: a + # worm reversing and sprinting can do ~0.3 mm/s. + anchor = last or (cx, cy) + ap = -M @ (np.array(anchor) - np.array((cx, cy))) - omin + np.array(twh) / 2 + gate_px = (args.gate_mm * 1000 if last else args.first_gate_mm * 1000) / umpx + if misses: + gate_px *= 2 + best = pick(cands, (ap[0], ap[1]), gate_px, last_area) + + if best: + wx, wy = canvas_to_stage(best["x"], best["y"], M, (cx, cy), omin, twh) + last, misses, last_area = (wx, wy), 0, best["area_mm2"] + row.update(found=1, worm_x_um=round(wx, 1), worm_y_um=round(wy, 1), + area_mm2=round(best["area_mm2"], 4), + length_mm=round(best["length_mm"], 2)) + half = int(args.crop_mm * 1000 / umpx / 2) + bx, by = int(best["x"]), int(best["y"]) + crop = canvas[max(0, by - half):by + half, max(0, bx - half):bx + half] + Image.fromarray(crop).save(out / "crops" / f"crop_{r:04d}.jpg", quality=92) + else: + misses += 1 + + small = cv2.resize(canvas, None, fx=args.mosaic_scale, fy=args.mosaic_scale, + interpolation=cv2.INTER_AREA) + if best: + cv2.circle(small, (int(best["x"] * args.mosaic_scale), + int(best["y"] * args.mosaic_scale)), + int(0.7 * 1000 / umpx * args.mosaic_scale), (255, 0, 0), 3) + Image.fromarray(small).save(out / f"round_{r:04d}.jpg", quality=90) + + # Re-centre, only past the deadband, clamped to the dish. + if last and np.hypot(last[0] - cx, last[1] - cy) > args.deadband_mm * 1000: + nxp, nyp = last + dx, dy = nxp - dish[0], nyp - dish[1] + rad = float(np.hypot(dx, dy)); lim = args.limit_mm * 1000 + if rad > lim: + nxp, nyp = dish[0] + dx * lim / rad, dish[1] + dy * lim / rad + print(f" re-centre clamped to the {args.limit_mm} mm limit", flush=True) + cx, cy = nxp, nyp + row["recentred"] = 1 + state = (f"worm at ({row['worm_x_um']}, {row['worm_y_um']}) um, " + f"{row['area_mm2']} mm2" if best else f"NOT FOUND (miss {misses}, " + f"{len(cands)} candidates) - next round searches {args.search_grid}") + print(f"[{r:04d}] {datetime.datetime.now():%H:%M:%S} {gspec} {state}" + + (" -> re-centred" if row["recentred"] else ""), flush=True) + except KeyboardInterrupt: + writer.writerow(row); log.flush() + break + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + print(f"[{r:04d}] ERROR {row['error']}", flush=True) + if "e-stop" in str(exc).lower() or "estop" in str(exc).lower(): + writer.writerow(row); log.flush() + break + writer.writerow(row); log.flush() + r += 1 + log.close() + plot_track(out) + finally: + try: + _move_safe(sdk, cx, cy) + except Exception as exc: + print(f"WARNING: could not park at the last centre: {exc}", file=sys.stderr) + cam.close() + print(f"released - output in {out}") + return out + + +def plot_track(out: Path) -> None: + """track.png: the path drawn the way the mosaics show it (stage +X + moves features left, +Y moves them down, so X runs right-to-left and + Y bottom-to-top), coloured blue -> yellow by time, with a 1 mm bar.""" + with (out / "track.csv").open(encoding="utf-8") as fh: + rows = [r for r in csv.DictReader(fh) if r["found"] == "1"] + if not rows: + return + x = np.array([float(r["worm_x_um"]) for r in rows]) / 1000 + y = np.array([float(r["worm_y_um"]) for r in rows]) / 1000 + t = np.array([float(r["t_s"]) for r in rows]) / 60 + size, pad = 900, 70 + span = max(np.ptp(x), np.ptp(y), 2.0) + s = (size - 2 * pad) / span + cxm, cym = (x.max() + x.min()) / 2, (y.max() + y.min()) / 2 + pts = np.column_stack([size / 2 - (x - cxm) * s, size / 2 - (y - cym) * s]).astype(np.int32) + img = np.full((size, size, 3), 255, np.uint8) + cols = cv2.applyColorMap(np.linspace(0, 255, len(pts)).astype(np.uint8)[:, None], + cv2.COLORMAP_VIRIDIS)[:, 0] + for i in range(1, len(pts)): + cv2.line(img, tuple(pts[i - 1]), tuple(pts[i]), [int(v) for v in cols[i]], 2, cv2.LINE_AA) + cv2.circle(img, tuple(pts[0]), 9, (0, 0, 0), 2, cv2.LINE_AA) + cv2.circle(img, tuple(pts[-1]), 7, [int(v) for v in cols[-1]], -1, cv2.LINE_AA) + cv2.line(img, (pad, size - 30), (pad + int(s), size - 30), (0, 0, 0), 3) + cv2.putText(img, "1 mm", (pad, size - 40), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 1, cv2.LINE_AA) + d = np.hypot(np.diff(x), np.diff(y)).sum() + cv2.putText(img, f"{len(rows)} points, {t[-1]:.1f} min, {d:.1f} mm travelled " + f"(o = start, colour = time)", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, + 0.6, (0, 0, 0), 1, cv2.LINE_AA) + cv2.imwrite(str(out / "track.png"), img) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--grid", default="3x3", help="block while locked on (default 3x3)") + ap.add_argument("--search-grid", default="5x5", help="block after a miss (default 5x5)") + ap.add_argument("--overlap", type=float, default=0.10) + ap.add_argument("--rounds", type=int) + ap.add_argument("--minutes", type=float) + ap.add_argument("--interval-s", type=float, default=0.0, + help="seconds between round starts; 0 = back to back (default)") + ap.add_argument("--exposure-us", type=float, default=1600.0, + help="4x at lamp 1072: 1.6 ms puts agar at ~200/255 (2026-09-29)") + ap.add_argument("--deadband-mm", type=float, default=0.5) + ap.add_argument("--gate-mm", type=float, default=3.0, + help="max distance from the last position to accept a detection") + ap.add_argument("--first-gate-mm", type=float, default=2.0, + help="first round: max distance from the block centre") + ap.add_argument("--limit-mm", type=float, default=10.0, + help="never centre farther than this from the dish centre") + ap.add_argument("--dish-centre", help="'x,y' um; default is the start position") + ap.add_argument("--min-area-mm2", type=float, default=0.012) + ap.add_argument("--max-area-mm2", type=float, default=0.25) + ap.add_argument("--dark-ratio", type=float, default=0.85, + help="a pixel is worm if darker than this fraction of local agar") + ap.add_argument("--crop-mm", type=float, default=1.8) + ap.add_argument("--mosaic-scale", type=float, default=0.5) + ap.add_argument("--calibrate", action="store_true", help="re-measure the stage->image mapping") + ap.add_argument("--tag") + ap.add_argument("--detect-only", type=Path, help="run detection on an image and exit") + ap.add_argument("--um-per-px", type=float, default=1.4877, help="for --detect-only") + ap.add_argument("--setup", action="store_true", + help="switch objective, set exposure, focus and centre the worm, then exit") + ap.add_argument("--nosepiece", type=int, default=1, help="for --setup (1 = 4x)") + args = ap.parse_args() + if args.setup: + setup(args) + return + if args.detect_only: + detect_only(args.detect_only, args.um_per_px, args) + return + if args.rounds is None and args.minutes is None: + args.rounds = 1 + run(args) + + +if __name__ == "__main__": + main() diff --git a/acquisition/paths.py b/acquisition/paths.py index b1001e1..1e673e6 100644 --- a/acquisition/paths.py +++ b/acquisition/paths.py @@ -1,25 +1,98 @@ # paths.py # ------------------------------------------------------------ # Where the server writes runtime data (captured frames, move/frame history -# logs). Anchored to the *working directory*, not to __file__ - once this is -# installed as a package, __file__ lives in site-packages and must never be -# written to. +# logs, saved stage positions). Anchored to the *working directory*, not to +# __file__ - once this is installed as a package, __file__ lives in +# site-packages and must never be written to. # -# CONFOCAL_MCP_DATA_DIR if set, the base directory to use -# (unset) the current working directory +# CONFOCAL_MCP_DATA_DIR if set, the base directory to use - always +# honoured as-is, no second-guessing +# (unset) the current working directory, if it is a usable +# place to write; otherwise FALLBACK_DATA_ROOT # -# Resolved once at import time, from wherever the server process was launched - -# so launch it from a stable location (or set the env var). +# WHY THE CWD IS NOT TRUSTED BLINDLY: an MCP client picks the working +# directory its servers are launched with, and the caller has no say in it. +# Claude Desktop on Windows launches them in C:\WINDOWS\system32, so a plain +# Path.cwd() sends captures to C:\WINDOWS\system32\data\captures - which +# fails with a bare "access is denied" OSError at the first capture, a long +# way from this module and looking for all the world like a camera fault. +# Running elevated is worse than the error: the write then *succeeds*, and +# scatters image files through a system directory. +# +# So an unset env var resolves the cwd through _usable_data_root(), which +# rejects anything under the Windows directory and anything it cannot +# actually create a file in, and falls back to a per-user directory with a +# warning on stderr (never stdout - that is the JSON-RPC channel). +# +# Resolved once per process and cached: every caller reads it into a +# module-level constant at import time anyway. Tests that manipulate the env +# var must call data_root.cache_clear() to see the change. # ------------------------------------------------------------ +import functools import os +import sys +import tempfile from pathlib import Path +#: Used when the working directory is not a usable data root. Under the +#: user's home, so it is writable without elevation and easy to find. +FALLBACK_DATA_ROOT = Path.home() / ".confocal-mcp" + + +def _is_system_dir(path: Path) -> bool: + """True if `path` is inside the Windows directory. + + Checked separately from writability because an elevated process *can* + write to the Windows system directory - that is precisely the case + worth refusing rather than permitting. + """ + system_root = os.environ.get("SystemRoot") + if not system_root: + return False + try: + return path == Path(system_root) or Path(system_root) in path.parents + except OSError: + return False + + +def _is_writable(path: Path) -> bool: + """True if a file can actually be created in `path`. + + Probes with a real file rather than os.access(), which on Windows + reports only the read-only attribute and happily returns True for + directories that ACLs deny. + """ + try: + path.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path, prefix=".confocal-write-test-"): + pass + return True + except OSError: + return False + + +def _usable_data_root(cwd: Path) -> Path: + """`cwd` if it is a sane place to write runtime data, else the fallback.""" + if not _is_system_dir(cwd) and _is_writable(cwd): + return cwd + + print( + f"[confocal-mcp] Working directory {cwd} is not usable for runtime data " + f"(system directory or not writable); using {FALLBACK_DATA_ROOT} instead. " + f"Set CONFOCAL_MCP_DATA_DIR to choose the location explicitly.", + file=sys.stderr, + ) + return FALLBACK_DATA_ROOT + +@functools.lru_cache(maxsize=1) def data_root() -> Path: """Base directory for all runtime data. See module docstring.""" env = os.environ.get("CONFOCAL_MCP_DATA_DIR") - return Path(env).expanduser().resolve() if env else Path.cwd() + if env: + return Path(env).expanduser().resolve() + return _usable_data_root(Path.cwd()) def captures_dir() -> Path: diff --git a/acquisition/raw_move_xy.py b/acquisition/raw_move_xy.py new file mode 100644 index 0000000..c95c770 --- /dev/null +++ b/acquisition/raw_move_xy.py @@ -0,0 +1,55 @@ +import os +import sys +import threading +import time + +# Multi-threaded COM, so the watchdog thread can use the same microscope +# object as the main thread. Must be set before pythoncom is imported. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + + +def log(msg): + now = time.time() + print(f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d} {msg}", flush=True) + + +def watchdog(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + log("watchdog: 500ms elapsed, halting") + x, y = m.iXPOSITION, m.iYPOSITION + m.iXPOSITION, m.iYPOSITION = x, y + log(f"watchdog: halted at ({x}, {y})") + + +log("dispatch start") +m = win32com.client.Dispatch(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID) +log("dispatch done") +log(f"start position ({m.iXPOSITION}, {m.iYPOSITION})") + +# Armed after the connect, so the halt always has a microscope to talk to. +timer = threading.Timer(0.5, watchdog) +timer.start() +log("watchdog armed (500ms)") + +log("set iXPOSITION start") +m.iXPOSITION = 0 +log("set iXPOSITION done") + +log("set iYPOSITION start") +m.iYPOSITION = 0 +log("set iYPOSITION done") + +# Track progress until the halt, then for 1s after to see where it settles. +halted_at = None +while halted_at is None or time.time() - halted_at < 1.0: + log(f"position ({m.iXPOSITION}, {m.iYPOSITION})") + if halted_at is None and not timer.is_alive(): + halted_at = time.time() + time.sleep(0.05) + +log("exit") +os._exit(0) diff --git a/acquisition/read_during_move_test.py b/acquisition/read_during_move_test.py new file mode 100644 index 0000000..e3e7693 --- /dev/null +++ b/acquisition/read_during_move_test.py @@ -0,0 +1,93 @@ +# Does reading iXPOSITION block while a move is in progress, or return a +# cached value? And does the SDK push position/moving events during a move? +# +# Moves XY to (0, 0) - X then Y, with blocking writes on the main thread - +# while a reader thread times every read and an event sink logs callbacks. +import os +import sys +import threading +import time + +# Multi-threaded COM: the reader thread shares the microscope object, and +# events fire on the SDK's own thread without needing a message loop. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + +TARGET = (0, 0) +XY_STAGE_MASK = 0x2 # MIC_ACCESSORY_MASK_XYSTAGE + +GENERAL_EVENTS = {0: "DataSet_PositionChanged", 1: "RemCtrl_PositionChanged", + 3: "XYStageLogicalLimitsReached", 20: "DataSetReady", + 21: "DataSetFinished", 22: "DataSetAborted"} +DEDICATED_EVENTS = {25: "Ti2_IsBusy", 38: "MovingStatus_Changed"} + + +def log(msg): + now = time.time() + stamp = f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d}" + print(f"{stamp} [{threading.current_thread().name:>6}] {msg}", flush=True) + + +class Events: + def OnGeneralCallback(self, event, data): + d = win32com.client.Dispatch(data, None, NkTi2Ax.INikonTi2AxData) + log(f"EVENT general {event} {GENERAL_EVENTS.get(event, '?')} " + f"mask=0x{d.uiDataUsageMask:x} data=({d.iXPOSITION}, {d.iYPOSITION})") + + def OnDedicatedCallback(self, event, data): + log(f"EVENT dedicated {event} {DEDICATED_EVENTS.get(event, '?')} data={data!r}") + + def OnMetaDataCallback(self, mask): + log(f"EVENT metadata 0x{mask:x}") + + def OnEnabledCallback(self, mask): + log(f"EVENT enabled 0x{mask:x}") + + +def reader(stop): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + while not stop.is_set(): + t0 = time.perf_counter() + log("read start") + x, y = m.iXPOSITION, m.iYPOSITION + log(f"read done ({x}, {y}) took {(time.perf_counter() - t0) * 1000:.1f} ms") + time.sleep(0.05) + + +# Hard stop in case anything hangs. +threading.Timer(20.0, lambda: (log("20s safety timeout"), os._exit(1))).start() + +threading.current_thread().name = "main" +log("dispatch start") +m = win32com.client.DispatchWithEvents(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID, Events) +log("dispatch done") + +for cmd, arg in (("SET_MOVING_STATE_NOTIFICATION", str(XY_STAGE_MASK)), + ("GET_MOVING_STATE_NOTIFICATION", "")): + try: + log(f"{cmd} -> {m.DedicatedCommand(cmd, arg)!r}") + except Exception as e: + log(f"{cmd} failed: {e}") + +log(f"start position ({m.iXPOSITION}, {m.iYPOSITION})") + +stop = threading.Event() +threading.Thread(target=reader, args=(stop,), name="reader").start() +time.sleep(0.2) # a few idle reads first, as a baseline + +log("set iXPOSITION start") +m.iXPOSITION = TARGET[0] +log("set iXPOSITION done") + +log("set iYPOSITION start") +m.iYPOSITION = TARGET[1] +log("set iYPOSITION done") + +time.sleep(0.5) # a few reads after the move +stop.set() +time.sleep(0.2) +log("exit") +os._exit(0) diff --git a/analysis/aml18_survey.py b/analysis/aml18_survey.py new file mode 100644 index 0000000..6d557f1 --- /dev/null +++ b/analysis/aml18_survey.py @@ -0,0 +1,318 @@ +# aml18_survey.py +# ------------------------------------------------------------ +# Population survey of AML18 worms (pan-neuronal nuclear GFP + tagRFP) +# from a single NIS-Elements .nd2 frame: find every worm in the +# transmitted-light (TD) channel, measure its length along the body, +# estimate its life stage, and use the neuronal fluorescence to tell the +# head (nerve ring = brightest neuron cluster) from the tail. +# +# python -m analysis.aml18_survey D:\...\CelegansAML-18\Test1.nd2 +# python -m analysis.aml18_survey file.nd2 --neuron-channel GFP +# +# Writes _survey/ (or --out): worms.csv, survey.png (annotated +# overlay); for a time-lapse, survey_tNNN.png per time point and counts.csv. +# +# WHICH FLUORESCENCE CHANNEL. The neuron channel must be captured at the +# same instant as TD, or it shows where a crawling worm used to be. On +# the AX, TD is read out with the 561 nm laser, so RFP lines up with TD +# and a sequentially-scanned GFP does not (2026-09-29, Test1.nd2: 11.7 s +# frames, GFP offset from every worm). Default is RFP for that reason; +# use --neuron-channel GFP only for simultaneous acquisitions. +# +# WORMS VS EGGS. Both are dark in TD. Eggs are compact (~50 x 30 um); +# worms are long and thin. Objects are kept only if their body length +# (longest path through the skeleton) is > --min-length-um and at least +# 4x their width. +# +# LIFE STAGE is a rough guide from body length alone (N2 at 20 C: +# L1 ~0.25, L2 ~0.36, L3 ~0.49, L4 ~0.62-0.9, adult ~1.0-1.3 mm). A +# curled or partly hidden worm reads short, so treat it as indicative. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import cv2 +import nd2 +import numpy as np +from scipy import ndimage as ndi +from scipy.sparse import coo_matrix +from scipy.sparse.csgraph import dijkstra +from skimage.morphology import remove_small_objects, skeletonize + +STAGES = [(0.30, "L1"), (0.42, "L2"), (0.56, "L3"), (0.90, "L4"), (99.0, "adult")] + + +def load(path: Path) -> tuple[list[tuple[float, dict[str, np.ndarray]]], float]: + """Every time point as (seconds, {channel: 2-D image}). + + A Z-stack is collapsed per time point: fluorescence by maximum + projection (a neuron is counted wherever it is in focus), TD by + taking the single sharpest plane (a projection of brightfield smears + every plane's blur together). XY multipoint files are not handled. + """ + with nd2.ND2File(path) as f: + extra = set(f.sizes) - {"T", "Z", "C", "Y", "X"} + if extra: + raise SystemExit(f"unsupported dimensions {extra} in {f.sizes}") + arr = f.asarray().astype(np.float32) + dims = list(f.sizes) + names = [c.channel.name for c in f.metadata.channels] + um_px = float(f.voxel_size().x) + try: + times = [e.get("Time [s]", i) for i, e in enumerate(f.events())][: f.sizes.get("T", 1)] + except Exception: + times = [] + # Put the array in a fixed T, Z, C, Y, X order, adding missing axes. + for d in ("T", "Z", "C"): + if d not in dims: + arr = arr[np.newaxis] + dims.insert(0, d) + arr = np.transpose(arr, [dims.index(d) for d in ("T", "Z", "C", "Y", "X")]) + frames = [] + for t in range(arr.shape[0]): + ch = {} + for i, name in enumerate(names): + stack = arr[t, :, i] # Z, Y, X + if name == "TD": + sharp = [float(cv2.Laplacian(z, cv2.CV_32F).var()) for z in stack] + ch[name] = np.ascontiguousarray(stack[int(np.argmax(sharp))]) + else: + ch[name] = np.ascontiguousarray(stack.max(axis=0)) + frames.append((float(times[t]) if t < len(times) else float(t), ch)) + return frames, um_px + + +def segment_worms(td: np.ndarray, um_px: float, dark_ratio: float) -> np.ndarray: + """Dark, thin objects relative to local background (closing removes + anything narrower than the kernel, leaving the agar).""" + k = int(round(160 / um_px)) | 1 # 160 um: 3x an adult's width + bg = cv2.morphologyEx(td, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))) + bg = cv2.GaussianBlur(bg, (0, 0), k / 3) + mask = td < dark_ratio * bg + mask = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_OPEN, np.ones((2, 2), np.uint8)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8)).astype(bool) + return remove_small_objects(mask, max_size=int(2000 / um_px ** 2)) # <= 2000 um2 is debris + + +def longest_path(skel: np.ndarray) -> tuple[np.ndarray, float]: + """Pixels of the longest geodesic path through a skeleton, and its + length in pixels (diagonal steps count sqrt 2).""" + ys, xs = np.nonzero(skel) + idx = -np.ones(skel.shape, int) + idx[ys, xs] = np.arange(len(ys)) + rows, cols, w = [], [], [] + for dy, dx in ((0, 1), (1, 0), (1, 1), (1, -1)): + y2, x2 = ys + dy, xs + dx + ok = (y2 >= 0) & (y2 < skel.shape[0]) & (x2 >= 0) & (x2 < skel.shape[1]) + ok[ok] = skel[y2[ok], x2[ok]] + rows += list(idx[ys[ok], xs[ok]]); cols += list(idx[y2[ok], x2[ok]]) + w += [np.hypot(dy, dx)] * int(ok.sum()) + n = len(ys) + g = coo_matrix((w, (rows, cols)), shape=(n, n)).tocsr() + d0 = dijkstra(g, directed=False, indices=0) + a = int(np.nanargmax(np.where(np.isinf(d0), -1, d0))) + da, pred = dijkstra(g, directed=False, indices=a, return_predecessors=True) + b = int(np.nanargmax(np.where(np.isinf(da), -1, da))) + path = [b] + while path[-1] != a: + path.append(pred[path[-1]]) + return np.column_stack([ys[path], xs[path]]), float(da[b]) + + +def neuron_mask(ch: np.ndarray, um_px: float) -> tuple[np.ndarray, float]: + """Fluorescent neurons: well above the (mostly zero) background.""" + bg = cv2.medianBlur(ch.astype(np.float32), 5) if ch.max() > 0 else ch + noise = float(np.percentile(ch, 99)) # >99% of the frame is empty agar + thr = max(3.0 * noise, noise + 5.0) + return (cv2.GaussianBlur(ch, (0, 0), 1.0) > thr), thr + + +def survey_frame(ch: dict[str, np.ndarray], um_px: float, neuron_channel: str, + min_length_um: float, dark_ratio: float, title: str): + """One frame: worm rows and the annotated overlay (BGR).""" + if "TD" not in ch or neuron_channel not in ch: + raise SystemExit(f"need TD and {neuron_channel}; file has {list(ch)}") + td, neu = ch["TD"], ch[neuron_channel] + + worms = segment_worms(td, um_px, dark_ratio) + nmask, thr = neuron_mask(neu, um_px) + lab, n = ndi.label(worms) + dist = ndi.distance_transform_edt(worms) + near = ndi.distance_transform_edt(~worms) <= 25 / um_px # neurons may sit just outside the TD edge + + rows = [] + shape = td.shape + + def add_worm(obj: np.ndarray, width_map: np.ndarray | None, source: str) -> bool: + skel = skeletonize(obj) + if skel.sum() < 3: + return False + path_px, length_px = longest_path(skel) + length_um = length_px * um_px + width_um = 2 * float(np.median(width_map[skel])) * um_px if width_map is not None else None + if length_um < min_length_um or (width_um and length_um < 4 * width_um): + return False # egg or debris + ys, xs = np.nonzero(obj) + edge = bool(ys.min() == 0 or xs.min() == 0 or ys.max() == shape[0] - 1 + or xs.max() == shape[1] - 1) + # Neuron signal belonging to this worm: inside the body or within + # 25 um of it. + zone = ndi.binary_dilation(obj, iterations=int(25 / um_px)) + if source == "TD": + zone &= near + sig = np.where(zone & nmask, neu, 0) + total = float(sig.sum()) + if total == 0: + # Every AML18 worm has glowing neurons; a dark TD object with + # none is agar texture or debris (Loop 16:13 worm 4). + return False + # Head = the end whose first 15% of body length holds more signal. + seg = max(3, int(0.15 * len(path_px))) + r = int(60 / um_px) + + def end_signal(pts): + m = np.zeros(shape, np.uint8) + for y, x in pts: + cv2.circle(m, (int(x), int(y)), r, 1, -1) + return float(sig[m.astype(bool)].sum()) + + s_a, s_b = end_signal(path_px[:seg]), end_signal(path_px[-seg:]) + if max(s_a, s_b) < 1.5 * min(s_a, s_b) + 1: + head = None # no clear winner + else: + head = tuple(path_px[0] if s_a > s_b else path_px[-1]) + # An end at the image border is where the frame cut the worm, + # not a real end - its neurons may be off-frame, so the + # comparison is meaningless (Test1 worm 6 was called at the cut). + hy, hx = head + if min(hy, hx) <= 3 or hy >= shape[0] - 4 or hx >= shape[1] - 4: + head = None + stage = next(name for lim, name in STAGES if length_um / 1000 < lim) + cy, cx = ndi.center_of_mass(obj) + rows.append({ + "worm": len(rows) + 1, "found_by": source, "x_px": round(cx), "y_px": round(cy), + "length_mm": round(length_um / 1000, 3), + "width_um": round(width_um, 1) if width_um else "", + "stage_estimate": stage + (" (cut by edge)" if edge else ""), + "neuron_signal": round(total), "neuron_px": int((sig > 0).sum()), + "head_x_px": head[1] if head else "", "head_y_px": head[0] if head else "", + "head_confidence": round(max(s_a, s_b) / (min(s_a, s_b) + 1), 1), + "_path": path_px, "_obj": obj, + }) + return True + + for i in range(1, n + 1): + add_worm(lab == i, dist, "TD") + + # Worms seen only by their neurons: small larvae are too faint and + # thin in 4x brightfield (Timelapse1: ~6 glowing larvae unoutlined), + # but their neuron chain is a clear line of signal. Join each chain + # into one object and keep it if it is worm-shaped and not already + # part of a brightfield worm. Their length follows the neurons, so it + # can read a little short of the body. + claimed = np.zeros(shape, bool) + for w in rows: + claimed |= w["_obj"] + claimed = ndi.binary_dilation(claimed, iterations=int(40 / um_px)) + glow = ndi.binary_dilation(nmask, iterations=max(1, int(8 / um_px))) + glow = cv2.morphologyEx(glow.astype(np.uint8), cv2.MORPH_CLOSE, + cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))).astype(bool) + glab, gn = ndi.label(glow) + for i in range(1, gn + 1): + obj = glab == i + if (obj & claimed).any() or (obj & nmask).sum() < 8: + continue + add_worm(obj, None, "neurons") + + # Annotated overlay: TD grey, neuron channel magenta, worm outlines cyan. + g = np.clip((td - np.percentile(td, 0.5)) / (np.percentile(td, 99.5) - np.percentile(td, 0.5)), 0, 1) + f = np.clip(neu / max(np.percentile(neu[nmask], 95) if nmask.any() else 1, 1), 0, 1) + img = np.dstack([g * 0.7 + f, g * 0.7, g * 0.7 + f]) + img = (np.clip(img, 0, 1) * 255).astype(np.uint8)[..., ::-1].copy() # BGR + img = cv2.resize(img, None, fx=2, fy=2, interpolation=cv2.INTER_NEAREST) + for w in rows: + cnts, _ = cv2.findContours(w["_obj"].astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + colour = (255, 255, 0) if w["found_by"] == "TD" else (0, 165, 255) + cv2.drawContours(img, [c * 2 for c in cnts], -1, colour, 2 if w["found_by"] != "TD" else 1) + if w["head_x_px"] != "": + cv2.circle(img, (2 * w["head_x_px"], 2 * w["head_y_px"]), 14, (0, 255, 255), 3) + lbl = f"{w['worm']}: {w['stage_estimate'].split()[0]} {w['length_mm']:.2f}mm" + p = (2 * w["x_px"] + 30, 2 * w["y_px"]) + cv2.putText(img, lbl, p, cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 0), 5, cv2.LINE_AA) + cv2.putText(img, lbl, p, cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA) + px = int(round(500 / um_px)) * 2 + h, wd = img.shape[:2] + cv2.rectangle(img, (wd - 40 - px, h - 50), (wd - 40, h - 38), (255, 255, 255), -1) + cv2.putText(img, "500 um", (wd - 40 - px, h - 60), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2) + cv2.putText(img, f"{title}: {len(rows)} worms | cyan = brightfield outline, orange = found by neurons, yellow ring = head " + f"({neuron_channel} neurons, magenta)", (20, 40), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2, cv2.LINE_AA) + return rows, img, thr + + +def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float, + out: Path | None = None) -> Path: + frames, um_px = load(path) + out = out or path.with_name(path.stem + "_survey") + out.mkdir(exist_ok=True) + multi = len(frames) > 1 + all_rows, counts = [], [] + print(f"{path.name}: {um_px:.3f} um/px, {len(frames)} time point(s), neuron channel {neuron_channel}") + for ti, (t_s, ch) in enumerate(frames): + title = f"{path.name}" + (f" t={t_s / 60:.1f} min ({ti + 1}/{len(frames)})" if multi else "") + rows, img, thr = survey_frame(ch, um_px, neuron_channel, min_length_um, dark_ratio, title) + dest = out / (f"survey_t{ti:03d}.png" if multi else "survey.png") + # cv2.imwrite returns False instead of raising - a file held open + # elsewhere silently kept the previous run's image on 2026-09-29. + if not cv2.imwrite(str(dest), img): + raise SystemExit(f"could not write {dest} - is it open in another program?") + for w in rows: + w.update(t_index=ti, t_s=round(t_s, 1)) + all_rows += rows + stages = [w["stage_estimate"].split()[0] for w in rows] + counts.append({"t_index": ti, "t_s": round(t_s, 1), "worms": len(rows), + **{st: stages.count(st) for _, st in STAGES}, + "heads_found": sum(1 for w in rows if w["head_x_px"] != "")}) + if not multi: + print(f"threshold {thr:.0f}; {len(rows)} worms") + for w in rows: + head = "head found" if w["head_x_px"] != "" else "head unclear" + print(f" #{w['worm']:>2} {w['stage_estimate']:<22} {w['length_mm']:.2f} mm, " + + (f"{w['width_um']:.0f} um wide, " if w["width_um"] != "" else "found by neurons, ") + + f"neuron px {w['neuron_px']:>4}, {head} " + f"(ratio {w['head_confidence']})") + else: + c = counts[-1] + print(f" t={t_s / 60:5.1f} min: {len(rows)} worms " + + " ".join(f"{st}:{c[st]}" for _, st in STAGES) + f" heads {c['heads_found']}") + + keys = ["t_index", "t_s"] + [k for k in (all_rows[0] if all_rows else {"worm": 0}) + if not k.startswith("_") and k not in ("t_index", "t_s")] + with (out / "worms.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=keys, extrasaction="ignore") + wr.writeheader(); wr.writerows(all_rows) + if multi: + with (out / "counts.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(counts[0])) + wr.writeheader(); wr.writerows(counts) + print(f"wrote {out}") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("nd2", type=Path) + ap.add_argument("--neuron-channel", default="RFP") + ap.add_argument("--min-length-um", type=float, default=150.0) + ap.add_argument("--dark-ratio", type=float, default=0.85) + ap.add_argument("--out", type=Path, help="output folder (default: _survey)") + a = ap.parse_args() + analyse(a.nd2, a.neuron_channel, a.min_length_um, a.dark_ratio, a.out) + + +if __name__ == "__main__": + main() diff --git a/analysis/aml18_track.py b/analysis/aml18_track.py new file mode 100644 index 0000000..f43055f --- /dev/null +++ b/analysis/aml18_track.py @@ -0,0 +1,177 @@ +# aml18_track.py +# ------------------------------------------------------------ +# Second step after aml18_survey on a time-lapse: link the worms found in +# each frame into tracks, measure how far and how fast each one moved, +# check whether the neuron signal fades over the run, and render the +# outlined frames as a movie. +# +# python -m analysis.aml18_track D:\...\CelegansAML-18\Timelapse_1h.nd2 +# +# Reads _survey/worms.csv and survey_tNNN.png, writes into the same +# folder: tracks.csv (one row per worm per frame, with a track id), +# track_summary.csv (one row per track), tracks.png (all paths on the +# first frame), signal.csv (neuron-channel brightness per frame) and +# survey_movie.mp4. +# +# LINKING. Frames are a minute apart, so a roaming worm can move several +# mm between them while a feeding one barely moves. Each worm is joined +# to the nearest unclaimed worm of the previous frame within --max-step-mm +# and of similar length (0.5-2x), closest pairs first. Anything further +# starts a new track. So a fast worm crossing the field may be split into +# pieces, and a worm leaving the 3.9 mm field simply ends - tracks are a +# lower bound on how long each worm was followed, not identities. +# +# FADING. The 99.9th percentile of the neuron channel per frame tracks the +# brightest neurons in view. Worms entering and leaving change it too, so +# only a steady decline over the whole run points to photobleaching. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import cv2 +import numpy as np + + +def link(rows: list[dict], um_px: float, max_step_mm: float) -> list[dict]: + frames = sorted({int(r["t_index"]) for r in rows}) + by_t = {t: [r for r in rows if int(r["t_index"]) == t] for t in frames} + next_id = 1 + prev: list[dict] = [] + for t in frames: + cur = by_t[t] + pairs = [] + for i, c in enumerate(cur): + for j, p in enumerate(prev): + d_mm = np.hypot(float(c["x_px"]) - float(p["x_px"]), + float(c["y_px"]) - float(p["y_px"])) * um_px / 1000 + ratio = float(c["length_mm"]) / max(float(p["length_mm"]), 1e-6) + if d_mm <= max_step_mm and 0.5 <= ratio <= 2.0: + pairs.append((d_mm, i, j)) + used_c, used_p = set(), set() + for d_mm, i, j in sorted(pairs): + if i in used_c or j in used_p: + continue + cur[i]["track"] = prev[j]["track"] + cur[i]["step_mm"] = round(d_mm, 3) + used_c.add(i); used_p.add(j) + for i, c in enumerate(cur): + if i not in used_c: + c["track"] = next_id + c["step_mm"] = "" + next_id += 1 + prev = cur + return rows + + +def summarise(rows: list[dict]) -> list[dict]: + out = [] + for tid in sorted({r["track"] for r in rows}): + tr = sorted((r for r in rows if r["track"] == tid), key=lambda r: int(r["t_index"])) + t0, t1 = float(tr[0]["t_s"]), float(tr[-1]["t_s"]) + dist = sum(float(r["step_mm"]) for r in tr if r["step_mm"] != "") + lengths = [float(r["length_mm"]) for r in tr] + stages = [r["stage_estimate"].split()[0] for r in tr] + out.append({ + "track": tid, "frames": len(tr), "start_min": round(t0 / 60, 1), + "end_min": round(t1 / 60, 1), "distance_mm": round(dist, 2), + "mean_speed_mm_per_min": round(dist / ((t1 - t0) / 60), 3) if t1 > t0 else "", + "median_length_mm": round(float(np.median(lengths)), 3), + "stage": max(set(stages), key=stages.count), + }) + return out + + +def draw_tracks(first_png: Path, rows: list[dict], dest: Path) -> None: + img = cv2.imread(str(first_png)) + if img is None: + return + rng = np.random.default_rng(1) + for tid in sorted({r["track"] for r in rows}): + tr = sorted((r for r in rows if r["track"] == tid), key=lambda r: int(r["t_index"])) + pts = np.array([[2 * int(r["x_px"]), 2 * int(r["y_px"])] for r in tr], np.int32) + col = tuple(int(v) for v in rng.integers(80, 256, 3)) + if len(pts) > 1: + cv2.polylines(img, [pts], False, col, 3, cv2.LINE_AA) + cv2.circle(img, tuple(pts[-1]), 7, col, -1) + cv2.putText(img, str(tid), tuple(pts[-1] + 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, col, 2) + cv2.imwrite(str(dest), img) + + +def movie(folder: Path, fps: int) -> Path | None: + import imageio_ffmpeg + frames = sorted(folder.glob("survey_t*.png")) + if len(frames) < 2: + return None + first = cv2.imread(str(frames[0])) + h, w = (first.shape[0] // 2) * 2, (first.shape[1] // 2) * 2 + dest = folder / "survey_movie.mp4" + wr = imageio_ffmpeg.write_frames( + str(dest), (w, h), fps=fps, codec="libx264", quality=None, macro_block_size=2, + output_params=["-crf", "22", "-movflags", "+faststart"]) # write_frames already outputs yuv420p + wr.send(None) + for f in frames: + img = cv2.imread(str(f))[:h, :w] + wr.send(np.ascontiguousarray(img[..., ::-1])) + wr.close() + return dest + + +def signal_per_frame(nd2_path: Path, channel: str) -> list[dict]: + from analysis.aml18_survey import load + frames, _ = load(nd2_path) + return [{"t_min": round(t / 60, 2), + "p99_9": round(float(np.percentile(ch[channel], 99.9)), 1), + "max": round(float(ch[channel].max()), 1)} for t, ch in frames] + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("nd2", type=Path) + ap.add_argument("--survey", type=Path, help="survey folder (default _survey)") + ap.add_argument("--max-step-mm", type=float, default=1.0) + ap.add_argument("--channel", default="RFP") + ap.add_argument("--fps", type=int, default=6) + a = ap.parse_args() + folder = a.survey or a.nd2.with_name(a.nd2.stem + "_survey") + + import nd2 + with nd2.ND2File(a.nd2) as f: + um_px = float(f.voxel_size().x) + with (folder / "worms.csv").open(encoding="utf-8") as fh: + rows = list(csv.DictReader(fh)) + rows = link(rows, um_px, a.max_step_mm) + with (folder / "tracks.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=["track", "step_mm"] + [k for k in rows[0] + if k not in ("track", "step_mm")]) + wr.writeheader(); wr.writerows(rows) + summ = summarise(rows) + with (folder / "track_summary.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(summ[0])) + wr.writeheader(); wr.writerows(summ) + draw_tracks(folder / "survey_t000.png", rows, folder / "tracks.png") + + sig = signal_per_frame(a.nd2, a.channel) + with (folder / "signal.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(sig[0])) + wr.writeheader(); wr.writerows(sig) + mv = movie(folder, a.fps) + + long_tracks = [s for s in summ if s["frames"] >= 3] + print(f"{len(summ)} tracks, {len(long_tracks)} followed for 3+ frames") + for s in sorted(summ, key=lambda s: -s["frames"])[:15]: + print(f" track {s['track']:>3}: {s['stage']:<6} {s['frames']:>3} frames " + f"({s['start_min']}-{s['end_min']} min), {s['distance_mm']} mm, " + f"{s['mean_speed_mm_per_min']} mm/min") + first, last = sig[0]["p99_9"], sig[-1]["p99_9"] + print(f"{a.channel} brightest-neuron level: {first} -> {last} " + f"({(last / first - 1) * 100 if first else 0:+.0f}%) over {sig[-1]['t_min']} min") + print(f"wrote tracks.csv, track_summary.csv, tracks.png, signal.csv" + + (f", {mv.name}" if mv else "") + f" in {folder}") + + +if __name__ == "__main__": + main() diff --git a/analysis/live_view.py b/analysis/live_view.py new file mode 100644 index 0000000..d5704c6 --- /dev/null +++ b/analysis/live_view.py @@ -0,0 +1,150 @@ +# live_view.py +# ------------------------------------------------------------ +# Watch an acquisition as it happens, without touching the camera. +# +# python -m analysis.live_view # newest run, auto-detected +# python -m analysis.live_view data/mosaic_20260921_194543_region +# python -m analysis.live_view --fps 2 --width 1100 +# +# Then open the printed file:// link in a browser and leave it there. +# +# WHY NOT AN ACTUAL LIVE FEED: a GenICam camera can be held by exactly one +# process. Anything that opened the camera to stream preview frames would +# take it away from the run that is acquiring - the failure this repo hits +# repeatedly (AccessDeniedException) whenever two things want the device. +# +# So this never opens the camera. It watches the FILES the running +# acquisition is already writing and republishes the newest one as a fixed +# filename that a browser can poll. The acquisition does not know this +# exists and cannot be affected by it: worst case, the viewer shows a +# slightly stale frame. +# +# WHY A COPY RATHER THAN POINTING THE PAGE AT THE TILE ITSELF: the newest +# file changes name constantly, and a browser cannot discover that from a +# file:// page. Republishing to one stable name (latest.jpg) makes the +# page a two-line poll. JPEG rather than PNG because it is a preview - +# it re-encodes in milliseconds where a 3 MB PNG does not. +# +# PARTIAL WRITES: a file picked up mid-write decodes as a truncated or +# corrupt image. Each candidate is therefore only published once its size +# has stopped changing between polls, and any decode error is swallowed +# and retried rather than killing the viewer. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from PIL import Image + +from acquisition.paths import data_root + +_PAGE = """ +microscope live view + +
+ watching __DIR__ + frame - + updated - + poll __MS__ ms +
+
waiting for the first frame...
+ +""" + + +def newest_image(root: Path) -> Path | None: + best, best_m = None, -1.0 + for p in root.rglob("*.png"): + if p.name == "latest.jpg": + continue + try: + m = p.stat().st_mtime + except OSError: + continue + if m > best_m: + best, best_m = p, m + return best + + +def newest_run(base: Path) -> Path | None: + runs = [p for p in base.glob("*") if p.is_dir() + and (p.name.startswith("mosaic_") or p.name.startswith("timelapse_"))] + return max(runs, key=lambda p: p.stat().st_mtime) if runs else None + + +def serve(watch: Path, out: Path, fps: float, width: int) -> None: + out.mkdir(parents=True, exist_ok=True) + period = max(0.2, 1.0 / max(fps, 0.1)) + page = out / "index.html" + page.write_text( + _PAGE.replace("__DIR__", watch.name).replace("__MS__", str(int(period * 1000))), + encoding="utf-8") + print(f"watching : {watch}") + print(f"OPEN THIS: {page.resolve().as_uri()}") + print("(Ctrl-C to stop - this never touches the camera)") + + published, last_size = None, -1 + while True: + try: + src = newest_image(watch) + if src is not None and src != published: + size = src.stat().st_size + if size == last_size and size > 0: + try: + im = Image.open(src) + im.thumbnail((width, width)) + im.convert("RGB").save(out / "latest.jpg", quality=85) + published = src + print(f" {time.strftime('%H:%M:%S')} {src.name}", flush=True) + except Exception: + last_size = -1 # truncated - wait and retry + else: + last_size = size + except Exception: + pass + time.sleep(period) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("watch", nargs="?", type=Path, + help="run folder to watch (default: newest under data/)") + ap.add_argument("--out", type=Path, help="where to write the viewer (default: /_live)") + ap.add_argument("--fps", type=float, default=1.0) + ap.add_argument("--width", type=int, default=1200) + args = ap.parse_args() + + watch = args.watch or newest_run(data_root() / "data") + if watch is None or not watch.exists(): + raise SystemExit("no run folder found to watch - pass one explicitly") + serve(watch, args.out or (watch / "_live"), args.fps, args.width) + + +if __name__ == "__main__": + main() diff --git a/analysis/make_movie.py b/analysis/make_movie.py new file mode 100644 index 0000000..04bf98f --- /dev/null +++ b/analysis/make_movie.py @@ -0,0 +1,227 @@ +# make_movie.py +# ------------------------------------------------------------ +# Turn a timelapse series into a watchable MP4. +# +# python -m analysis.make_movie data/timelapse_20260921_163741_oats +# python -m analysis.make_movie --fps 24 --out front.mp4 +# python -m analysis.make_movie --scale 0.5 --no-overlay +# python -m analysis.make_movie data/mosaic_ # one frame per round +# +# A folder of 500 PNGs is data; a 20-second movie is something a person +# can actually look at and see an organism move. This is the step that +# turns one into the other. +# +# SCALE BAR: the pixel size is not guessable from the image - it depends +# on the objective and the C-mount adapter. UM_PER_PX below was MEASURED +# on 2026-09-21 by moving the stage a known 150.00 um and phase- +# correlating the before/after frames (shift 100.66 px, confidence 0.95, +# dy 0.18 px so the camera is square to the stage axes). It is valid ONLY +# for nosepiece position 1 with that adapter - change the objective and +# it is wrong. Pass --um-per-px to override, or --no-overlay to omit the +# bar rather than draw a scale you do not trust. +# +# TIMESTAMPS come from frames.csv's t_seconds, not from frame index x +# interval: a run that dropped or lagged a frame would otherwise show a +# time that drifts from reality, and the whole point of the overlay is to +# let a viewer judge rate. +# +# WHY mp4v: cv2's ffmpeg build on Windows reliably has it. H.264 gives +# smaller files but is not always present in the wheel, and a movie that +# fails to write is worse than one that is a few MB larger. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +from pathlib import Path + +import cv2 +import numpy as np + +#: Measured, not assumed - see module header. Objective-specific. +UM_PER_PX = 1.4901 + + +def _mosaic_rows(series_dir: Path) -> list[dict]: + """One row per stitched round of a mosaic run, shaped like frames.csv. + + A round's time is the wall clock of its first tile, relative to round + 0's first tile - the same "time since start" meaning t_seconds has in + a fixed-position series. + """ + starts: dict[int, datetime.datetime] = {} + with (series_dir / "tiles.csv").open(encoding="utf-8") as fh: + for r in csv.DictReader(fh): + rnd = int(r["round"]) + if rnd not in starts: + starts[rnd] = datetime.datetime.fromisoformat(r["wall_clock"]) + if not starts: + return [] + t0 = starts[min(starts)] + rows = [] + for rnd in sorted(starts): + name = f"mosaic_{rnd:03d}.png" + if (series_dir / name).exists(): + rows.append({"filename": name, + "t_seconds": (starts[rnd] - t0).total_seconds()}) + return rows + + +def mosaic_um_per_px(series_dir: Path) -> float | None: + """Pixel size of a mosaic run's stitched images, or None if not a mosaic. + + Runs from before mosaic_scale was recorded used the 0.5 default. + """ + meta_path = series_dir / "mosaic.json" + if not meta_path.exists(): + return None + meta = json.loads(meta_path.read_text(encoding="utf-8")) + return meta["um_per_px"] / meta.get("mosaic_scale", 0.5) + + +def _rows(series_dir: Path) -> list[dict]: + if (series_dir / "mosaic.json").exists(): + return _mosaic_rows(series_dir) + with (series_dir / "frames.csv").open(encoding="utf-8") as fh: + rows = [r for r in csv.DictReader(fh) if r.get("filename") and not r.get("error")] + return [r for r in rows if (series_dir / r["filename"]).exists()] + + +def wb_gains(img: np.ndarray) -> np.ndarray: + """Per-channel gains that make the brightest UNCLIPPED pixels neutral. + + In transmitted brightfield the bright background is light that missed + the specimen, so it is a legitimate white reference - but only where + it is not clipped. Clipped pixels are (255,255,255) by definition, so + including them forces the gains to 1.0 and silently reports "already + balanced" no matter how strong the cast really is. That mistake was + made once on this data; hence the explicit mask. + + Measured on this rig 2026-09-21: the camera's native response with + BalanceWhiteAuto off is markedly blue (background R=145 G=168 B=237), + needing roughly 1.26/1.09/0.77. + """ + chans = [img[:, :, i].astype(np.float32) for i in range(3)] + g = img.mean(axis=2) + unclipped = (img < 250).all(axis=2) + if unclipped.sum() < 1000: + return np.ones(3, dtype=np.float32) + ref = unclipped & (g > np.percentile(g[unclipped], 92)) + if ref.sum() < 200: + return np.ones(3, dtype=np.float32) + means = np.array([c[ref].mean() for c in chans], dtype=np.float32) + if (means <= 1).any(): + return np.ones(3, dtype=np.float32) + return means.mean() / means + + +def _scale_bar(img: np.ndarray, um_per_px: float) -> None: + """Draw a scale bar whose length is a round number of microns. + + Picks the largest 'nice' length that stays under a quarter of the + frame width, so the bar is meaningful at any magnification instead of + a fixed pixel count that means something different on every objective. + """ + h, w = img.shape[:2] + max_um = (w * 0.25) * um_per_px + nice = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + target = max([n for n in nice if n <= max_um], default=nice[0]) + px = int(round(target / um_per_px)) + + x1, y2 = w - 40, h - 40 + x0 = x1 - px + cv2.rectangle(img, (x0, y2 - 9), (x1, y2), (255, 255, 255), -1) + cv2.rectangle(img, (x0, y2 - 9), (x1, y2), (0, 0, 0), 1) + label = f"{target} um" if target < 1000 else f"{target / 1000:g} mm" + (tw, _), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2) + cv2.putText(img, label, (x0 + (px - tw) // 2, y2 - 16), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 3) + cv2.putText(img, label, (x0 + (px - tw) // 2, y2 - 16), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1) + + +def _stamp(img: np.ndarray, t_s: float, i: int, n: int) -> None: + hms = f"{int(t_s) // 3600:d}:{(int(t_s) % 3600) // 60:02d}:{int(t_s) % 60:02d}" + txt = f"t + {hms} ({i + 1}/{n})" + cv2.putText(img, txt, (24, 44), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 0), 4) + cv2.putText(img, txt, (24, 44), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2) + + +def build(series_dir: Path, out: Path, fps: int, scale: float, + um_per_px: float, overlay: bool, white_balance: bool = True) -> Path: + rows = _rows(series_dir) + if not rows: + raise SystemExit(f"no usable frames in {series_dir}") + + first = cv2.imread(str(series_dir / rows[0]["filename"])) + if first is None: + raise SystemExit(f"could not read {rows[0]['filename']}") + + # ONE set of gains for the whole movie, from the first frame. Computing + # them per frame would re-neutralise every frame independently, so the + # colour would shift as the organism covers more or less of the field - + # visible as flicker, and fatal to any comparison across time. + gains = wb_gains(first) if white_balance else np.ones(3, dtype=np.float32) + if white_balance: + print(f"white balance gains (B,G,R) = " + f"{gains[0]:.3f}, {gains[1]:.3f}, {gains[2]:.3f}") + h, w = first.shape[:2] + if scale != 1.0: + w, h = int(w * scale), int(h * scale) + w -= w % 2; h -= h % 2 # some encoders reject odd dimensions + + writer = cv2.VideoWriter(str(out), cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + if not writer.isOpened(): + raise SystemExit("could not open the video writer - is the mp4v codec available?") + + n = len(rows) + try: + for i, r in enumerate(rows): + img = cv2.imread(str(series_dir / r["filename"])) + if img is None: + continue # skip, do not abort the whole movie + if white_balance: + img = np.clip(img.astype(np.float32) * gains[None, None, :], + 0, 255).astype(np.uint8) + if (img.shape[1], img.shape[0]) != (w, h): + img = cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA) + if overlay: + _stamp(img, float(r["t_seconds"]), i, n) + _scale_bar(img, um_per_px / scale) + writer.write(img) + if i % 50 == 0: + print(f" {i + 1}/{n}", flush=True) + finally: + writer.release() + + span = float(rows[-1]["t_seconds"]) - float(rows[0]["t_seconds"]) + print(f"\n{n} frames spanning {span / 3600:.2f} h -> {n / fps:.1f} s of video") + print(f"time compression: {span / (n / fps):.0f}x") + print(f"wrote {out} ({out.stat().st_size / 1e6:.1f} MB)") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--out", type=Path, help="default: /movie.mp4") + ap.add_argument("--fps", type=int, default=30) + ap.add_argument("--scale", type=float, default=1.0, help="resize factor (0.5 = half size)") + ap.add_argument("--um-per-px", type=float, + help=f"default: from mosaic.json for a mosaic run, else {UM_PER_PX}") + ap.add_argument("--no-overlay", action="store_true", help="omit timestamp and scale bar") + ap.add_argument("--no-white-balance", action="store_true", + help="keep the camera's raw (blue-biased) colour") + args = ap.parse_args() + + out = args.out or (args.series_dir / "movie.mp4") + um_per_px = args.um_per_px or mosaic_um_per_px(args.series_dir) or UM_PER_PX + build(args.series_dir, out, args.fps, args.scale, um_per_px, + not args.no_overlay, not args.no_white_balance) + + +if __name__ == "__main__": + main() diff --git a/analysis/make_soundtrack.py b/analysis/make_soundtrack.py new file mode 100644 index 0000000..c297615 --- /dev/null +++ b/analysis/make_soundtrack.py @@ -0,0 +1,165 @@ +# make_soundtrack.py +# ------------------------------------------------------------ +# Original ambient soundtrack for a time-lapse movie, synthesised with +# numpy - no samples and no third-party audio, so there are no licensing +# questions when a movie is posted or presented. +# +# python -m analysis.make_soundtrack --duration 39 --out track.wav +# python -m analysis.make_soundtrack --movie run.mp4 # -> run_music.mp4 +# +# With --movie the track is made exactly as long as the movie and muxed +# onto it (video stream copied untouched, audio AAC), using the ffmpeg +# that imageio-ffmpeg bundles. +# +# THE ARRANGEMENT is the one written for the 2026-09-25 Physarum Short: +# a quiet opening, a chord progression that builds with a rising +# arpeggio, a sparse held middle, a lighter second rise and a warm major +# resolve at the end. It was composed on a 39 s timeline; other lengths +# stretch that timeline, so sections keep their proportions. Notes keep +# their own length, so very long movies (minutes) get sparser, not +# slower-sounding - it suits roughly 20-120 s. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import subprocess +import wave +from pathlib import Path + +import numpy as np + +SR = 44100 +SCORE_S = 39.0 # the timeline the arrangement was written on + + +def midi(m: float) -> float: + return 440.0 * 2 ** ((m - 69) / 12) + + +def render(duration: float, seed: int = 7) -> np.ndarray: + """Stereo float mix in [-1, 1], `duration` seconds long.""" + n = int(SR * duration) + t = np.arange(n) / SR + k = duration / SCORE_S # score seconds -> real seconds + rng = np.random.default_rng(seed) + + def env_between(a, b, fade): + e = np.minimum(np.clip((t - a * k) / fade, 0, 1), np.clip((b * k - t) / fade, 0, 1)) + return 0.5 - 0.5 * np.cos(np.pi * e) + + def pad(notes, a, b, level, bright=0.3): + """Soft pad: each note = detuned sines + weak harmonics, slow swell.""" + out = np.zeros((n, 2)) + e = env_between(a, b, fade=2.0) + for note in notes: + f = midi(note) + for det, pan in ((-0.12, 0.3), (0.0, 0.5), (0.12, 0.7)): + ff = f * 2 ** (det / 12) + ph = rng.uniform(0, 2 * np.pi) + s = (np.sin(2 * np.pi * ff * t + ph) + + bright * 0.35 * np.sin(2 * np.pi * 2 * ff * t + ph) + + bright * 0.12 * np.sin(2 * np.pi * 3 * ff * t + ph)) + out[:, 0] += s * (1 - pan) + out[:, 1] += s * pan + return out * (e * level / (len(notes) * 3))[:, None] + + def pluck(note, at, level, pan=0.5, decay=1.2): + out = np.zeros((n, 2)) + i0 = int(at * k * SR) + if i0 >= n: + return out + L = min(n - i0, int(SR * decay * 4)) + tt = np.arange(L) / SR + f = midi(note) + s = (np.sin(2 * np.pi * f * tt) + 0.3 * np.sin(2 * np.pi * 2 * f * tt) + + 0.1 * np.sin(2 * np.pi * 3 * f * tt)) + s *= np.exp(-tt / decay) * np.clip(tt / 0.005, 0, 1) * level + out[i0:i0 + L, 0] += s * (1 - pan) + out[i0:i0 + L, 1] += s * pan + return out + + mix = np.zeros((n, 2)) + # A minor world resolving to C major: Am, F, C, G + AM, F, C, G = [57, 64, 69, 72], [53, 60, 65, 69], [48, 55, 64, 67], [55, 62, 67, 71] + CMAJ = [48, 55, 60, 64, 67, 72] + + mix += pad([45, 57, 64], 0.0, 7.0, 0.55, bright=0.15) # opening + for s0, ch in ((6.0, AM), (8.75, F), (11.5, C), (14.25, G)): # progression + mix += pad(ch, s0 - 0.5, s0 + 3.4, 0.7 if s0 < 11 else 0.85, + bright=0.3 if s0 < 11 else 0.55) + for i, at in enumerate(np.arange(6.0, 11.0, 0.6875)): # root pulse + mix += pluck(45 if i % 2 == 0 else 52, at, 0.10, pan=0.4, decay=0.6) + arp = [69, 72, 76, 79, 81, 84, 81, 79] + for i, at in enumerate(np.arange(11.0, 17.2, 0.34375)): # rising arpeggio + note = arp[i % len(arp)] + (12 if at > 14.5 else 0) + mix += pluck(note - 12, at, 0.10 + 0.05 * (at - 11) / 6, pan=0.3 + 0.4 * (i % 2), decay=0.9) + mix += pad([45, 52, 60, 64], 16.5, 27.5, 0.5, bright=0.12) # held middle + for at, note in ((18.5, 76), (21.0, 72), (23.5, 71), (25.5, 69)): + mix += pluck(note, at, 0.07, pan=0.6, decay=2.0) + mix += pad(F, 26.8, 31.0, 0.6, bright=0.35) # second rise + mix += pad(G, 30.5, 34.5, 0.6, bright=0.35) + for i, at in enumerate(np.arange(27.2, 34.0, 0.4125)): + mix += pluck([72, 76, 79, 76][i % 4], at, 0.08, pan=0.35 + 0.3 * (i % 2), decay=0.8) + mix += pad(CMAJ, 33.8, 40.5, 0.9, bright=0.4) # resolve + for dt, note, lv in ((0.0, 60, 0.12), (0.15, 64, 0.10), (0.3, 67, 0.10)): + mix += pluck(note, 34.0 + dt, lv, decay=3.0) + + # Stereo reverb: convolve with decaying noise (FFT). + ir_len = int(SR * 2.2) + ir = rng.standard_normal((ir_len, 2)) * np.exp(-np.arange(ir_len) / (SR * 0.55))[:, None] + ir[0] = 0 + nfft = 1 << int(np.ceil(np.log2(n + ir_len))) + wet = np.stack([np.fft.irfft(np.fft.rfft(mix[:, c], nfft) * np.fft.rfft(ir[:, c], nfft), nfft)[:n] + for c in range(2)], axis=1) + wet *= np.abs(mix).max() / (np.abs(wet).max() + 1e-9) + out = 0.72 * mix + 0.38 * wet + + # Master: short fade-in, 2.5 s fade-out, normalise to -1 dBFS. + out *= (np.clip(t / 0.3, 0, 1) * np.clip((duration - t) / 2.5, 0, 1))[:, None] + return out * 10 ** (-1 / 20) / np.abs(out).max() + + +def write_wav(mix: np.ndarray, path: Path) -> None: + with wave.open(str(path), "wb") as w: + w.setnchannels(2); w.setsampwidth(2); w.setframerate(SR) + w.writeframes((mix * 32767).astype(np.int16).tobytes()) + + +def movie_seconds(movie: Path) -> float: + import imageio_ffmpeg + frames, secs = imageio_ffmpeg.count_frames_and_secs(str(movie)) + return float(secs) + + +def mux(movie: Path, wav: Path, out: Path) -> None: + import imageio_ffmpeg + cmd = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-loglevel", "error", + "-i", str(movie), "-i", str(wav), "-map", "0:v:0", "-map", "1:a:0", + "-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-shortest", + "-movflags", "+faststart", str(out)] + subprocess.run(cmd, check=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--duration", type=float, help="seconds (default: the movie's length, else 39)") + ap.add_argument("--movie", type=Path, help="add the track to this movie -> _music.mp4") + ap.add_argument("--out", type=Path, help="output .wav (default beside the movie, or soundtrack.wav)") + ap.add_argument("--seed", type=int, default=7, help="changes pad phases and reverb, not the notes") + a = ap.parse_args() + + dur = a.duration or (movie_seconds(a.movie) if a.movie else SCORE_S) + wav = a.out or (a.movie.with_name(a.movie.stem + "_soundtrack.wav") if a.movie + else Path("soundtrack.wav")) + mix = render(dur, a.seed) + write_wav(mix, wav) + print(f"wrote {wav} ({dur:.1f} s)") + if a.movie: + dest = a.movie.with_name(a.movie.stem + "_music.mp4") + mux(a.movie, wav, dest) + print(f"wrote {dest}") + + +if __name__ == "__main__": + main() diff --git a/analysis/oat_approach.py b/analysis/oat_approach.py new file mode 100644 index 0000000..694631d --- /dev/null +++ b/analysis/oat_approach.py @@ -0,0 +1,171 @@ +# oat_approach.py +# ------------------------------------------------------------ +# Is the plasmodium moving toward the food? Measure it from a mosaic run. +# +# python -m analysis.oat_approach data/mosaic_ --oat-box 0,3350,1200,7300 +# +# For every stitched round (mosaic_NNN.png) this segments plasmodium and +# reports how much of it lies within each distance band of the oat, plus +# the area-weighted mean distance to the oat. A plasmodium heading for the +# food shows up as area moving into the near bands and the mean distance +# falling, which is a number rather than an impression from the movie. +# +# THE OAT IS LOCATED BY HAND, ONCE. It is a flake that does not move, and +# in brightfield it is the only thing that is truly black (grey ~0-2 on a +# 0-255 scale; plasmodium bottoms out around 10). But the dish rim is just +# as black, so --oat-box says where to look: pixels <= OAT_MAX_GREY inside +# that box and inside the dish, in round 0, are the oat for every round. +# +# THRESHOLDS WERE MEASURED on 2026-09-23 (16.7 ms, gamma 1.0, neutral +# gains; quarter-size mosaic): agar ~219, plasmodium fans and trunk +# 10-40, an out-of-focus smear 110-180, oat 0-1, bare glass 250-255. +# PLASMODIUM_MAX_GREY sits in the gap so the smear is not counted as +# organism. Re-check them if the exposure or lamp changes. +# +# ONLY THE AGAR BLOCK IS MEASURED - organism cannot be anywhere else, so +# glass, pen marks and the rim are excluded by construction. What is NOT +# excluded is the dark shadow along the block's top edge near the oat, +# which is as dark and as textured as real fans. It does not change +# between rounds, so it offsets the near bands but not their trend: read +# this output as change over time, not as absolute coverage. +# +# DISTANCES are straight-line over the agar, from the oat's edge. Most of +# this dish's oat lies beyond the glass window; only its visible strip is +# the reference, so distances run from where the oat enters the view. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + +import cv2 +import numpy as np + +from analysis.make_movie import _mosaic_rows + +#: Work at this fraction of the stitched size - ~12 um/px, plenty for mm-scale fronts. +WORK_SCALE = 0.25 +OAT_MAX_GREY = 4 +PLASMODIUM_MAX_GREY = 90 +#: Bare glass beside the agar block reads 250-255; agar ~220. +GLASS_MIN_GREY = 246 +#: Band edges in mm from the oat's edge. +BANDS_MM = (0, 3, 6, 9, 12, 15, 20, 30) +#: Ordinal blue ramp, near = dark (BGR for cv2). +BAND_COLOURS = [(0x6b, 0x36, 0x0d), (0x95, 0x4f, 0x18), (0xbf, 0x6a, 0x25), + (0xe5, 0x87, 0x39), (0xe7, 0x98, 0x55), (0xec, 0xa7, 0x6d), + (0xef, 0xb6, 0x86)] + + +def _grey(path: Path) -> np.ndarray: + g = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) + return cv2.resize(g, None, fx=WORK_SCALE, fy=WORK_SCALE, interpolation=cv2.INTER_AREA) + + +def _hull(mask: np.ndarray) -> np.ndarray: + pts = np.column_stack(np.nonzero(mask)[::-1]).astype(np.int32) + out = np.zeros(mask.shape, dtype=np.uint8) + cv2.fillConvexPoly(out, cv2.convexHull(pts), 1) + return out + + +def dish_mask(g: np.ndarray) -> np.ndarray: + """The glass window: convex hull of all bare-glass pixels, pulled in so + the dark gradient at the rim is not counted as plasmodium.""" + return cv2.erode(_hull(g >= GLASS_MIN_GREY), np.ones((41, 41), np.uint8)) + + +def agar_mask(g: np.ndarray, dish: np.ndarray) -> np.ndarray: + """The agar block. Plasmodium only grows on agar, so everything outside + it - bare glass, the block's shadowed edge, pen marks on the dish - is + excluded. Everything in the dish that is not bare glass is agar, + organism or oat; the largest such region, as a convex outline, is the + block (the network cuts the visible agar into pieces, hence the hull).""" + m = ((g < GLASS_MIN_GREY) & (dish > 0)).astype(np.uint8) + m = cv2.morphologyEx(m, cv2.MORPH_OPEN, np.ones((9, 9), np.uint8)) + n, lab, st, _ = cv2.connectedComponentsWithStats(m) + big = 1 + int(np.argmax(st[1:, cv2.CC_STAT_AREA])) + return cv2.erode(_hull(lab == big), np.ones((21, 21), np.uint8)) + + +def oat_mask(g: np.ndarray, dish: np.ndarray, box: tuple[int, int, int, int]) -> np.ndarray: + x0, y0, x1, y1 = (int(v * WORK_SCALE) for v in box) + m = np.zeros_like(g, dtype=np.uint8) + m[y0:y1, x0:x1] = (g[y0:y1, x0:x1] <= OAT_MAX_GREY) & (dish[y0:y1, x0:x1] > 0) + n, lab, st, _ = cv2.connectedComponentsWithStats(m) + if n < 2: + raise SystemExit("no black object found inside --oat-box") + big = 1 + int(np.argmax(st[1:, cv2.CC_STAT_AREA])) + return (lab == big).astype(np.uint8) + + +def run(series_dir: Path, box: tuple[int, int, int, int]) -> Path: + meta = json.loads((series_dir / "mosaic.json").read_text(encoding="utf-8")) + um_px = meta["um_per_px"] / meta.get("mosaic_scale", 0.5) / WORK_SCALE + rows = _mosaic_rows(series_dir) + if not rows: + raise SystemExit(f"no stitched rounds in {series_dir}") + + g0 = _grey(series_dir / rows[0]["filename"]) + dish = dish_mask(g0) + oat = oat_mask(g0, dish, box) + agar = agar_mask(g0, dish) + # distance (mm) of every pixel from the oat's edge + dist_mm = cv2.distanceTransform((1 - oat).astype(np.uint8), cv2.DIST_L2, 5) * um_px / 1000 + valid = (agar > 0) & (oat == 0) + px_mm2 = (um_px / 1000) ** 2 + print(f"oat area {oat.sum() * px_mm2:.2f} mm^2, agar block {agar.sum() * px_mm2:.1f} mm^2, " + f"{um_px:.1f} um/px") + + edges = list(zip(BANDS_MM[:-1], BANDS_MM[1:])) + fields = (["round", "t_min", "total_mm2"] + [f"band_{a}_{b}mm_mm2" for a, b in edges] + + ["mean_dist_mm", "nearest_mm"]) + out = series_dir / "oat_approach.csv" + with out.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + for i, r in enumerate(rows): + g = g0 if i == 0 else _grey(series_dir / r["filename"]) + if g.shape != g0.shape: + g = cv2.resize(g, g0.shape[::-1], interpolation=cv2.INTER_AREA) + plas = (g <= PLASMODIUM_MAX_GREY) & (g > OAT_MAX_GREY) & valid + d = dist_mm[plas] + row = {"round": i, "t_min": round(r["t_seconds"] / 60, 1), + "total_mm2": round(plas.sum() * px_mm2, 2), + "mean_dist_mm": round(float(d.mean()), 3) if d.size else "", + "nearest_mm": round(float(d.min()), 3) if d.size else ""} + for a, b in edges: + row[f"band_{a}_{b}mm_mm2"] = round(((d >= a) & (d < b)).sum() * px_mm2, 2) + w.writerow(row) + print(f"round {i:3d} t={row['t_min']:6.1f} min total {row['total_mm2']:7.2f} mm^2 " + f"mean dist {row['mean_dist_mm']} mm") + + # One check image, so the oat, dish and bands can be verified by eye. + check = cv2.cvtColor(g0, cv2.COLOR_GRAY2BGR) + for (a, b), col in zip(edges, BAND_COLOURS): + ring = (dist_mm >= a) & (dist_mm < b) & valid & (g0 <= PLASMODIUM_MAX_GREY) & (g0 > OAT_MAX_GREY) + check[ring] = col + check[oat > 0] = (0, 0, 220) + for m, col in ((dish, (0, 180, 0)), (agar, (0, 200, 255))): + cnt, _ = cv2.findContours(m, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + cv2.drawContours(check, cnt, -1, col, 3) + cv2.imwrite(str(series_dir / "oat_approach_check.png"), check) + print(f"wrote {out} and oat_approach_check.png") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--oat-box", required=True, + help="x0,y0,x1,y1 in round-0 stitched-mosaic pixels bounding the oat") + args = ap.parse_args() + box = tuple(int(v) for v in args.oat_box.split(",")) + run(args.series_dir, box) + + +if __name__ == "__main__": + main() diff --git a/analysis/physarum_rhythm.py b/analysis/physarum_rhythm.py new file mode 100644 index 0000000..3851c10 --- /dev/null +++ b/analysis/physarum_rhythm.py @@ -0,0 +1,308 @@ +# physarum_rhythm.py +# ------------------------------------------------------------ +# Quantify a fixed-position brightfield time-lapse of a Physarum +# plasmodium: the contraction rhythm, and how the organism's footprint +# changes over the run. +# +# python -m analysis.physarum_rhythm data/timelapse_20260921_150051_physarum +# python -m analysis.physarum_rhythm --json out.json +# +# THE MEASUREMENT: in transmitted light a plasmodium's local brightness +# tracks its local thickness - a thicker strand absorbs/scatters more, so +# it reads darker. Physarum drives cytoplasm through its network with a +# peristaltic contraction wave, so thickness at a fixed point rises and +# falls, and transmitted intensity oscillates with it. The period of that +# oscillation is the quantity of interest; the textbook figure for +# P. polycephalum is ~100-130 s at room temperature. +# +# WHY A PER-PIXEL PERIODOGRAM AND NOT JUST THE FRAME MEAN: neighbouring +# regions of one plasmodium oscillate at a common period but *out of +# phase* - that phase gradient is the peristaltic wave. Averaging the +# whole frame first lets antiphase regions cancel, which can bury a +# strong local rhythm in a flat global trace. So the period is taken from +# the median of per-pixel dominant frequencies inside the mask, and the +# global trace is reported alongside as a sanity check, not as the +# primary estimate. +# +# DETRENDING: the slow drift from the organism advancing across the field +# (and any lamp drift) is a large low-frequency component that would +# otherwise dominate every periodogram. Each pixel's series is linearly +# detrended and Hann-windowed before the FFT, and frequencies below +# MIN_PERIOD_S/MAX_PERIOD_S are excluded from the peak search. +# +# SCALE: areas are reported in mm^2 when the series folder carries a +# calibration.json with a measured `um_per_px`, and in PIXELS otherwise - +# never by assuming a magnification. acquisition/orchestration's runs +# since 2026-09-21 write that file; older series predate it and will +# report pixels, which is the honest answer for them. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +#: Period search window, seconds. Wide enough to contain the textbook +#: ~100-130 s without presupposing it - a result landing hard against +#: either edge should be read as "outside the searched band", not as a +#: measurement. +MIN_PERIOD_S = 30.0 +MAX_PERIOD_S = 600.0 + +#: Spatial downsample for the per-pixel analysis. The rhythm is a +#: large-scale thickness wave, not a fine texture, so full resolution buys +#: nothing here and costs ~25x the memory (a full-res float stack of a +#: 360-frame run is several GB). +DOWNSAMPLE = 5 + + +def _um_per_px(series_dir: Path) -> float | None: + """Measured pixel size for this series, or None if it was never measured. + + Returning None rather than a default is deliberate: a plausible-looking + wrong scale silently turns every area into a wrong physical number, + which is worse than an honest pixel count. + """ + path = series_dir / "calibration.json" + if not path.exists(): + return None + try: + value = json.loads(path.read_text(encoding="utf-8")).get("um_per_px") + return float(value) if value else None + except (ValueError, OSError): + return None + + +def _load_series(series_dir: Path) -> tuple[np.ndarray, np.ndarray, list[str]]: + """Return (times_s, small_stack, filenames) for the frames that exist. + + Frames whose row carries an `error`, or whose file is missing, are + skipped rather than faked - a gap in the series is better than an + interpolated frame in a periodogram. + """ + rows = list(csv.DictReader((series_dir / "frames.csv").open(encoding="utf-8"))) + times, frames, names = [], [], [] + for r in rows: + if r.get("error") or not r.get("filename"): + continue + path = series_dir / r["filename"] + if not path.exists(): + continue + g = np.asarray(Image.open(path)).mean(axis=2).astype(np.float32) + frames.append(g[::DOWNSAMPLE, ::DOWNSAMPLE]) + times.append(float(r["t_seconds"])) + names.append(r["filename"]) + if len(frames) < 8: + raise SystemExit(f"only {len(frames)} usable frames in {series_dir} - too few to analyse") + return np.asarray(times), np.stack(frames), names + + +#: Band a plasmodium contraction rhythm is expected to fall in. Used ONLY +#: to score which Otsu region is the organism (see _regions), never to +#: constrain the reported period - that is searched over the full +#: MIN/MAX_PERIOD_S window so a result outside this band can still come out. +_BIO_BAND_S = (60.0, 200.0) + + +def _regions(stack: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The two Otsu classes of the time-averaged frame, as (bright, dark). + + Otsu on the mean image rather than per frame: a per-frame threshold + would itself flicker with the contraction rhythm, writing the signal + being measured into the very region it is measured over. + + WHICH ONE IS THE ORGANISM IS NOT DECIDED HERE, deliberately. A thick + pigmented plasmodium usually reads darker than bare agar in + transmitted light, but that depends on the illumination, the + substrate and how thin the organism has spread - and getting it + backwards would measure the substrate with full confidence. analyse() + scores both regions for rhythmicity instead and lets the oscillation + identify the living one. + """ + mean_img = stack.mean(axis=0) + u8 = cv2.normalize(mean_img, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + _, th = cv2.threshold(u8, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + k = np.ones((3, 3), np.uint8) + th = cv2.morphologyEx(th, cv2.MORPH_OPEN, k, iterations=2) + th = cv2.morphologyEx(th, cv2.MORPH_CLOSE, k, iterations=2) + bright = th > 0 + return bright, ~bright + + +def _dominant_period(series: np.ndarray, dt: float) -> tuple[np.ndarray, np.ndarray]: + """Per-pixel dominant period (s) and its spectral power fraction. + + `series` is (T, N) - time along axis 0. Returns (period, strength), + each length N. `strength` is the peak's share of total in-band power: + a clean oscillation concentrates power in one bin, noise spreads it, + so it is the natural weight for "how periodic is this pixel". + """ + T = series.shape[0] + t = np.arange(T, dtype=np.float64) + + # Linear detrend per pixel (closed form - avoids scipy). + tm = t.mean() + denom = ((t - tm) ** 2).sum() + slope = ((t - tm)[:, None] * (series - series.mean(axis=0))).sum(axis=0) / denom + detr = series - (slope[None, :] * (t - tm)[:, None] + series.mean(axis=0)) + + win = np.hanning(T)[:, None] + spec = np.abs(np.fft.rfft(detr * win, axis=0)) ** 2 + freqs = np.fft.rfftfreq(T, d=dt) + + band = (freqs >= 1.0 / MAX_PERIOD_S) & (freqs <= 1.0 / MIN_PERIOD_S) + if not band.any(): + raise SystemExit("period search band contains no FFT bins - run longer or sample faster") + + spec_b, freqs_b = spec[band], freqs[band] + idx = spec_b.argmax(axis=0) + peak = spec_b[idx, np.arange(spec_b.shape[1])] + total = spec_b.sum(axis=0) + strength = np.divide(peak, total, out=np.zeros_like(peak), where=total > 0) + return 1.0 / freqs_b[idx], strength + + +def _score_region(flat: np.ndarray, mflat: np.ndarray, dt: float) -> dict: + """Periodogram summary for one region, plus a rhythmicity score. + + `rhythmic_fraction` - the share of the region's pixels that both + oscillate cleanly (peak holds >=15% of in-band power) and do so + within _BIO_BAND_S - is what distinguishes a contracting organism + from substrate that merely drifts or flickers. + """ + period, strength = _dominant_period(flat[:, mflat], dt) + lo, hi = _BIO_BAND_S + rhythmic = (strength >= 0.15) & (period >= lo) & (period <= hi) + good = strength >= np.percentile(strength, 75) + return { + "pixels": int(mflat.sum()), + "mean_intensity": round(float(flat[:, mflat].mean()), 2), + "period_s": round(float(np.median(period[good])), 1), + "period_iqr_s": [round(float(np.percentile(period[good], 25)), 1), + round(float(np.percentile(period[good], 75)), 1)], + "rhythmic_fraction": round(float(rhythmic.mean()), 3), + "_period": period, "_strength": strength, "_good": good, + } + + +def analyse(series_dir: Path) -> dict: + times, stack, names = _load_series(series_dir) + dt = float(np.median(np.diff(times))) + umpx = _um_per_px(series_dir) + bright, dark = _regions(stack) + + T = stack.shape[0] + flat = stack.reshape(T, -1) + + scored = {name: _score_region(flat, m.reshape(-1), dt) + for name, m in (("bright", bright), ("dark", dark))} + + # The organism is whichever region actually oscillates in the + # biological band - not whichever is brighter. If the two scores are + # close, that is reported rather than papered over: it means the + # segmentation did not separate organism from substrate, and the + # period below should not be trusted without looking at the frames. + order = sorted(scored, key=lambda k: scored[k]["rhythmic_fraction"], reverse=True) + organism, other = order[0], order[1] + margin = scored[organism]["rhythmic_fraction"] - scored[other]["rhythmic_fraction"] + + sel = scored[organism] + mflat = (bright if organism == "bright" else dark).reshape(-1) + period, strength, good = sel["_period"], sel["_strength"], sel["_good"] + period_est = sel["period_s"] + + # Global trace (sanity check - see module header on why it is not primary). + trace = flat[:, mflat].mean(axis=1) + + # Footprint over time, from a fixed threshold so area changes reflect + # the organism, not a moving threshold. + u8m = cv2.normalize(stack.mean(axis=0), None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + thr, _ = cv2.threshold(u8m, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + lo, hi = float(stack.min()), float(stack.max()) + level = lo + (thr / 255.0) * (hi - lo) + side = (stack > level) if organism == "bright" else (stack < level) + area = side.reshape(T, -1).sum(axis=1).astype(float) + area_px = area * (DOWNSAMPLE ** 2) + + return { + "series_dir": str(series_dir), + "frames_used": T, + "duration_s": float(times[-1] - times[0]), + "interval_s": dt, + "organism_region": organism, + "organism_identified_by": "rhythmicity", + "rhythmic_margin": round(float(margin), 3), + "region_scores": {k: {kk: vv for kk, vv in v.items() if not kk.startswith("_")} + for k, v in scored.items()}, + "mask_pixels_downsampled": int(mflat.sum()), + "period_s": round(period_est, 1), + "period_iqr_s": [round(float(np.percentile(period[good], 25)), 1), + round(float(np.percentile(period[good], 75)), 1)], + "cycles_observed": round(float(times[-1] - times[0]) / period_est, 1), + "oscillating_pixel_fraction": round(float((strength > 0.15).mean()), 3), + "area_start_px": round(area_px[0], 0), + "area_end_px": round(area_px[-1], 0), + "area_change_pct": round(100.0 * (area_px[-1] - area_px[0]) / area_px[0], 2), + "um_per_px": umpx, + "area_start_mm2": round(area_px[0] * umpx ** 2 / 1e6, 4) if umpx else None, + "area_end_mm2": round(area_px[-1] * umpx ** 2 / 1e6, 4) if umpx else None, + "area_change_mm2": (round((area_px[-1] - area_px[0]) * umpx ** 2 / 1e6, 4) + if umpx else None), + "growth_rate_mm2_per_h": ( + round((area_px[-1] - area_px[0]) * umpx ** 2 / 1e6 + / max((times[-1] - times[0]) / 3600.0, 1e-9), 4) if umpx else None), + "trace_t_s": [round(float(x), 1) for x in times], + "trace_intensity": [round(float(x), 4) for x in trace], + "trace_area_px": [round(float(x), 0) for x in area_px], + "period_map_percentiles": { + str(p): round(float(np.percentile(period[good], p)), 1) + for p in (5, 25, 50, 75, 95) + }, + } + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--json", type=Path, help="also write the full result here") + args = ap.parse_args() + + res = analyse(args.series_dir) + if args.json: + args.json.write_text(json.dumps(res, indent=2), encoding="utf-8") + + print(f"frames {res['frames_used']} over {res['duration_s'] / 60:.1f} min " + f"@ {res['interval_s']:.1f}s") + rs = res["region_scores"] + print(f"organism region {res['organism_region']} " + f"(rhythmic fraction {rs[res['organism_region']]['rhythmic_fraction']} vs " + f"{min(v['rhythmic_fraction'] for v in rs.values())}, " + f"margin {res['rhythmic_margin']})") + if res["rhythmic_margin"] < 0.05: + print(" WARNING: the two regions are near-equally rhythmic - segmentation") + print(" did not separate organism from substrate. Treat the period below") + print(" as unverified and look at the frames.") + print(f"contraction period {res['period_s']} s " + f"(IQR {res['period_iqr_s'][0]}-{res['period_iqr_s'][1]} s, " + f"{res['cycles_observed']} cycles observed)") + print(f"oscillating pixels {res['oscillating_pixel_fraction'] * 100:.1f}% of the plasmodium mask") + if res["um_per_px"]: + print(f"footprint {res['area_change_pct']:+.2f}% " + f"({res['area_start_mm2']:.4f} -> {res['area_end_mm2']:.4f} mm2, " + f"{res['area_change_mm2']:+.4f} mm2)") + print(f"growth rate {res['growth_rate_mm2_per_h']:+.4f} mm2/h " + f"(scale {res['um_per_px']} um/px, measured)") + else: + print(f"footprint {res['area_change_pct']:+.2f}% " + f"({res['area_start_px']:.0f} -> {res['area_end_px']:.0f} px) " + f"- no calibration.json, so pixels only") + + +if __name__ == "__main__": + main() diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md new file mode 100644 index 0000000..4f80a38 --- /dev/null +++ b/docs/adaptive_timelapse.md @@ -0,0 +1,142 @@ +# Adaptive time-lapse: proposal and status + +## The question behind this + +In the long time-lapse, the specimen pulled back from one region at about +the 25 hour mark, and it looked instantaneous. Two readings are possible: + +- **Biology.** One cell, no nutrients in that region, so it withdraws its + material and redirects it. A retraction moves mass toward the main body, + so the connecting tubes and body should thicken in the frames just after. +- **Filming artefact.** A stalled acquisition, a focus loss, an + illumination change, or a stage bump can all make a region "vanish" + between two frames. + +The rendered video cannot settle this. It may drop or duplicate frames, +and it carries no timestamps. The raw frames can. + +## How to answer it (when at the microscope) + +`timelapse/frame_audit.py` scores every consecutive pair of frames in a +sequence and flags, per interval: + +| Flag | Meaning | +|---|---| +| `GAP x3.2` | the interval was 3.2x the median: the acquisition stalled, the change is a missing stretch of time | +| `INTENSITY +25%` | the whole field got brighter or darker: illumination or exposure, not the specimen | +| `SHIFT 12px` | the whole field translated: stage or sample moved | +| `CHANGE` | the specimen's footprint or pixels changed with none of the above: biology | + +Run it on the frames around the event with the real timestamps: + +``` +python -m timelapse.frame_audit RAW_FRAME_DIR --timestamps frame_times.csv --around 25h --window 1h +``` + +`frame_times.csv` is one timestamp per frame, in frame order, exported from +the raw dataset's metadata. Without it the tool falls back to file +modification times, which are only trustworthy if the files were written +as they were captured. + +**Finer time slices exist only if the raw dataset has more frames than +the video.** Compare the two frame counts first. If the run was a fixed +interval and the video used every frame, there is nothing finer to +recover, and the honest answer is that the interval was too coarse. + +## The proposal: image slowly, look cheaply, image fast when it matters + +The fixed-interval recorder on this branch, +`acquisition/orchestration/timelapse.py`, stays as it is: a steady 10 s +series is the right tool for the shuttle-streaming rhythm, and its +drift-free scheduling is reused here. The adaptive loop is for the long, +slow runs where the interesting minutes are rare and unpredictable. + +A fixed interval spends light, disk and time evenly on the boring hours +and the interesting minutes alike. The adaptive loop puts the fast +frames where the change is: + +1. **Slow mode.** One capture per slow interval, previewed at low + resolution. +2. **Cheap change score, no model.** Each frame is compared with a + running median of the last few frames: pixel change relative to the + baseline's own noise, change in the specimen's footprint area, and a + whole-field shift check. This is numpy on a 256 pixel thumbnail and + costs nothing per frame, so it can run for days. +3. **Burst mode.** When the score crosses the threshold, capture at the + fast interval for a fixed window. Further change inside the window + extends it. When the window lapses, return to slow. +4. **Model only at the trigger.** A hook fires once per trigger, not + once per frame. That is where a vision model looks at the before and + after frames and says "extend", "ignore", or where a notification + goes out. This keeps model calls rare and gives a written reason per + event. +5. **Shifts never start a burst.** A whole-field translation is the + stage or the sample moving. It is logged as its own event. Burst + imaging a drifted field is wasted light. + +### Safety and budget + +- **Hard caps.** Maximum captures and maximum runtime end the run in any + mode. +- **One approval per run, not per frame.** The unattended loop cannot + stop for a human on every capture. On real hardware the whole plan + (intervals, burst length, caps, thresholds) is printed and approved + once at the terminal before the first frame. The scheduler then + supplies the per-call hardware confirmation itself. No model ever + supplies it. +- **Light budget.** For brightfield the slow loop is nearly free. For + fluorescence the slow loop adds phototoxicity, so the preview channel + should be the least damaging one, and the burst window should be + short. +- **Nothing else moves.** The loop only captures. It does not move the + stage, refocus, or change exposure. Those stay under the existing + human-gated tools. + +### What is built (branch `adaptive-timelapse`) + +- `get_image(backend="mock")`: a simulated capture with no camera, so all + of this runs off the microscope PC. The real path is unchanged and + still gated. +- `timelapse/change_detector.py`: the change score. +- `timelapse/frame_audit.py`: the audit tool above. +- `timelapse/scheduler.py`: the slow/burst loop, with the caps, the + one-time approval, and the trigger hook. +- `timelapse/model_trigger.py`: Claude behind that hook. At each trigger + it sees the frame before, the frame that fired, and the detector's + numbers, and answers "extend" or "ignore" with a one-sentence reason + that is logged. Capped calls per run, never more than one ask per 30 s, + and every failure (no key, no network, refusal, odd reply) is a + harmless "no opinion". Enabled with `--model-trigger`. Verified live on + the mock on 2026-09-28: at a synthetic shrink event the model answered + "extend" and gave the right reason (specimen contracted, no field shift + or illumination change). The call runs on a background thread: a + second live run showed burst spacing held exactly while the model took + 9 s to answer. A reply that lands after its burst has ended is logged + as late and changes nothing. Verified against the mock: + swapping the served frame mid-run started a burst within one slow + interval and returned to slow afterwards. +- A test suite (24 tests, synthetic frames with known answers) and a CI + workflow that runs it on every push. + +Try it on any machine: + +``` +python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-duration 5 --max-runtime 30 +``` + +### What is next + +1. **Tune thresholds on a real recording.** Run the audit on the existing + time-lapse. The scores at the known 25 hour event and during quiet + stretches give the starting thresholds for the scheduler. +2. **First real run, attended.** Short slow interval, low caps, someone + watching, brightfield only. +3. **Decide on focus.** Long runs drift. Either the Perfect Focus System + holds it, or the loop needs a bounded refocus step, which would be a + new, gated capability. + +### One request + +Export the per-frame timestamps from the raw 25 hour dataset, or just +report the raw frame count next to the video frame count. That settles +whether finer slices exist before anyone spends time looking for them. diff --git a/docs/mcp_harness.md b/docs/mcp_harness.md index bf40889..66b28ba 100644 --- a/docs/mcp_harness.md +++ b/docs/mcp_harness.md @@ -165,7 +165,7 @@ MCP tool results (`CallToolResult.content`) are typed content blocks - |---|---| | `TextContent(text=...)` | `{"type": "text", "text": ...}` | | `ImageContent(data=..., mime_type=...)` | `{"type": "image", "source": {"type": "base64", "media_type": mime_type, "data": data}}` | -| anything else (audio, resource links, ...) | a text block noting the unsupported type - `loop_tools.py`'s 4 tools never actually produce these, so this is a defensive fallback, not a real code path | +| anything else (audio, resource links, ...) | a text block noting the unsupported type - `loop_tools.py`'s 5 tools never actually produce these, so this is a defensive fallback, not a real code path | Conveniently, MCP's `Tool.input_schema` is already plain JSON Schema in the same shape Anthropic's tool `input_schema` expects, so @@ -209,7 +209,7 @@ approval, same as `agent.py`. Both with the mock backend (no real hardware needed) and real Claude API calls: -- `list_tools()` correctly discovers all 4 tools, with `confirm` absent +- `list_tools()` correctly discovers all 5 tools, with `confirm` absent from every schema Claude sees. - `call_tool()` round-trips correctly for `get_pos`, `move`, and `get_move_history` - including error cases (unknown tool name, a diff --git a/eaa_integration/run_session.py b/eaa_integration/run_session.py index 1672e61..c0c5228 100644 --- a/eaa_integration/run_session.py +++ b/eaa_integration/run_session.py @@ -291,7 +291,7 @@ def main() -> None: # also fixes a real crash: several of those built-ins use dotted # names (e.g. "simple_python_eval_tool.evaluate_python_expression"), # which OpenAI's function-calling API rejects outright (names must - # match ^[a-zA-Z0-9_-]+$, no dots) - our own 4 tools are named + # match ^[a-zA-Z0-9_-]+$, no dots) - our own 5 tools are named # cleanly and were never the problem. task_manager.tool_manager.disable_bash_coding_tool() task_manager.tool_manager.disable_python_coding_tool() diff --git a/harness/agent.py b/harness/agent.py index cba83c9..f38ed7d 100644 --- a/harness/agent.py +++ b/harness/agent.py @@ -79,7 +79,7 @@ - Always use backend="mock" (the default) unless the user has clearly \ asked you to control the real, physical microscope. Only pass \ backend="sdk" when real hardware action is actually intended. -- Every real-hardware action (get_image always; move when \ +- Every real-hardware action (get_image and move when \ backend="sdk") pauses for a live human approval at the terminal before \ it executes - expect that pause, and explain to the user what you're \ about to do and why before calling it, so the approval makes sense to \ @@ -100,14 +100,20 @@ { "name": "get_image", "description": ( - "Capture one frame from the real camera, paired with the exact " - "stage position it was taken at. Always touches real hardware - " - "pauses for human approval before executing. Optionally crop and/or " - "resize the embedded preview." + "Capture one frame from the camera, paired with the exact " + "stage position it was taken at. backend=\"sdk\" (default) is the " + "real camera and pauses for human approval before executing; " + "backend=\"mock\" returns a simulated frame with no hardware. " + "Optionally crop and/or resize the embedded preview." ), "input_schema": { "type": "object", "properties": { + "backend": { + "type": "string", + "enum": ["mock", "sdk"], + "description": "\"sdk\" for the real camera (default), \"mock\" for a simulated frame.", + }, "exposure_time_us": { "type": ["number", "null"], "description": "Camera exposure time in microseconds. Omit to keep the current setting.", @@ -213,9 +219,11 @@ def _execute_tool(name: str, tool_input: dict) -> tuple[list[dict], bool]: tool_input = dict(tool_input) try: if name == "get_image": - if not _confirm_real_hardware_action(name, tool_input): - return _text_content("User declined this real-hardware capture. Not executed."), True - metadata, image = loop_tools.get_image(confirm=True, **tool_input) + if tool_input.get("backend", "sdk") == "sdk": + if not _confirm_real_hardware_action(name, tool_input): + return _text_content("User declined this real-hardware capture. Not executed."), True + tool_input["confirm"] = True + metadata, image = loop_tools.get_image(**tool_input) image_content = image.to_image_content() return [ {"type": "text", "text": json.dumps(metadata)}, diff --git a/harness/mcp_agent.py b/harness/mcp_agent.py index feafa6b..2666326 100644 --- a/harness/mcp_agent.py +++ b/harness/mcp_agent.py @@ -92,7 +92,7 @@ - Always use backend="mock" (the default) unless the user has clearly \ asked you to control the real, physical microscope. Only pass \ backend="sdk" when real hardware action is actually intended. -- Every real-hardware action (get_image always; move when \ +- Every real-hardware action (get_image and move when \ backend="sdk") pauses for a live human approval at the terminal before \ it executes - expect that pause, and explain to the user what you're \ about to do and why before calling it, so the approval makes sense to \ @@ -155,7 +155,7 @@ async def _confirm_real_hardware_action(tool_name: str, tool_input: dict) -> boo def _mcp_content_to_anthropic_content(mcp_content: list) -> list[dict]: """Convert a CallToolResult's content blocks (TextContent/ImageContent/ ...) to Anthropic tool_result content blocks. Only text and image are - handled - loop_tools.py's 4 tools never return audio/resource blocks. + handled - loop_tools.py's 5 tools never return audio/resource blocks. """ blocks = [] for block in mcp_content: @@ -178,7 +178,7 @@ async def _execute_tool(mcp_client: Client, name: str, tool_input: dict) -> tupl """ tool_input = dict(tool_input) try: - if name == "get_image": + if name == "get_image" and tool_input.get("backend", "sdk") == "sdk": if not await _confirm_real_hardware_action(name, tool_input): return _text_content("User declined this real-hardware capture. Not executed."), True tool_input["confirm"] = True diff --git a/mcp_server/loop_tools.py b/mcp_server/loop_tools.py index 9a030a1..6b92ef0 100644 --- a/mcp_server/loop_tools.py +++ b/mcp_server/loop_tools.py @@ -5,8 +5,9 @@ # get_pos() - a lightweight, on-demand position sync primitive # move() - move XY, return the ACTUAL resulting position # get_move_history() - every point move() has visited this session +# estop() - emergency stop: forbid all motion, machine-wide # -# Exactly 4 tools, by explicit direction - do not add more without +# Exactly 5 tools, by explicit direction - do not add more without # checking first. Calibration, historical-frame lookup, etc. should be # done by the model reasoning over get_image()'s embedded picture, not # by adding a dedicated tool per capability. @@ -75,7 +76,7 @@ # _append_frame_history), and get_frame(frame_id) resolves a frame_id # back to that record - but get_frame() is a plain Python function, NOT # a 5th MCP tool: team direction is to keep the model-facing surface at -# exactly 4 tools, so history/lookup logic can grow underneath without +# exactly 5 tools, so history/lookup logic can grow underneath without # growing what the model itself can call. # # WHY move() IS XY-ONLY (not x/y/z): a blind absolute Z move must never @@ -84,8 +85,11 @@ # # WHY get_image() REQUIRES confirm=True: unlike get_pos()/move(), it # fires a real camera - same safety-gate pattern used for anything that -# touches real hardware, even when (as here) there's no backend="mock" -# equivalent to fall back to. +# touches real hardware. backend="sdk" (the default - a capture is real +# hardware unless the caller says otherwise) requires confirm=True; +# backend="mock" routes to nis_mock.MockNIS.capture() (copies a sample +# frame to data/captures/) so the capture-analyze-decide loop can be +# developed and tested off the microscope PC with no camera attached. # # WHY get_image() RETURNS [metadata, image] INSTEAD OF JUST A PATH: MCP # tool results can embed real image content (base64 + mime type), not @@ -312,7 +316,7 @@ def get_frame(frame_id: int) -> dict: returned for it. NOT registered as an MCP tool (see server_loop.py) - by explicit team - direction the chat-facing surface stays at 4 tools. This is a plain + direction the chat-facing surface stays at 5 tools. This is a plain importable function for harness/analysis code that needs to resolve "frame 42" to its full-res file and the position it was captured at, without re-reading FRAME_HISTORY_PATH by hand. @@ -449,6 +453,7 @@ def get_image( gain: float | None = None, crop: dict | None = None, max_dimension: int | None = None, + backend: str = "sdk", ) -> list: """Grab one frame from the Baumer GenICam camera (see acquisition.backends.baumer_genicam.BaumerGenICam) and return it @@ -473,11 +478,16 @@ def get_image( No NIS-Elements involved - connects directly to the camera via its GenTL producer (see BaumerGenICam/find_cti_files), independent of any - NIS-Elements process. Real hardware only (no mock equivalent - there's - nothing to simulate a camera trigger against) - requires confirm=True, - same safety-gate pattern as every other real-hardware-touching tool in - this repo. Position is read via backend="sdk" (Ti2 ActiveX SDK), the - same NIS-Elements-independent hardware path move()/get_pos() use. + NIS-Elements process. backend="sdk" (default) is real hardware and + requires confirm=True, same safety-gate pattern as every other + real-hardware-touching tool in this repo; position is then read via + the Ti2 ActiveX SDK, the same NIS-Elements-independent path + move()/get_pos() use. backend="mock" needs no confirm: it copies a + sample frame via nis_mock.MockNIS.capture() (override the source + file with CONFOCAL_MOCK_FRAME_PATH) and reads the mock stage's + position, so everything downstream of a capture - preview, crop, + frame_id, frame history - can be exercised with no camera attached. + exposure_time_us/gain are accepted and ignored on the mock. exposure_time_us, gain: optional - if given, applied via BaumerGenICam.set_settings() before capturing (raises ValueError if @@ -509,19 +519,18 @@ def get_image( already-small region can afford more pixels). Omit to use the default. """ - if not confirm: - raise PermissionError( - "get_image fires a real camera and requires confirm=True. " - "Refusing to proceed without explicit confirmation." - ) + _require_confirm_for_sdk(backend, confirm) if crop is not None: _validate_crop(crop) - camera = _get_camera() - if exposure_time_us is not None or gain is not None: - camera.set_settings(exposure_time_us=exposure_time_us, gain=gain) - image_path = camera.capture() - pos = get_pos(backend="sdk") + if backend == "mock": + image_path = _get_backend("mock").capture() + else: + camera = _get_camera() + if exposure_time_us is not None or gain is not None: + camera.set_settings(exposure_time_us=exposure_time_us, gain=gain) + image_path = camera.capture() + pos = get_pos(backend=backend) metadata = { "image": str(image_path), @@ -531,6 +540,7 @@ def get_image( "captured_at": pos["measured_at"], "monotonic_ms": pos["monotonic_ms"], "crop": crop, + "backend": backend, } _append_frame_history(metadata) preview = _make_preview_image( @@ -562,3 +572,53 @@ def get_move_history(limit: int = 50) -> dict: records = [json.loads(line) for line in lines[-limit:]] return {"history": records, "returned": len(records), "total_moves": len(lines)} + + +def estop(action: str = "status", reason: str = "requested via MCP") -> dict: + """EMERGENCY STOP: immediately forbid all microscope motion. + + Call this the moment anything looks wrong - an unexpected move, a + position that doesn't match what you asked for, a stage that seems to + be travelling when it shouldn't be, or an instruction from the + operator to stop. It is cheap, it is instant, and a needless stop + costs nothing but a release. Do not deliberate: stop first, diagnose + afterwards. + + Once engaged, EVERY motion primitive refuses - not just this session's, + but every process on the machine that drives this microscope, + including runs started by something else entirely. The flag lives in a + file, so it outlives whatever set it and cannot be lost by a process + dying. It cannot stop a move already in flight (the Ti2 has no abort): + that move runs to its end, and every move after it is refused. Long XY + moves are split into hops, so the stage stops within one hop. + + action: "engage" to stop everything, or "status" to report the current + state. RELEASE IS DELIBERATELY NOT AVAILABLE HERE - a stop that the + agent can lift by itself is not a safety device. A human clears it + from a terminal with `python -m acquisition.estop release`. + + Returns the resulting state. This tool never needs confirm=True: a + stop is always safe to perform, and requiring a confirmation step for + an emergency control would defeat its purpose. + """ + from acquisition import estop as _estop + + if action == "engage": + info = _estop.engage(reason) + return { + "engaged": True, + "detail": info, + "release_with": "python -m acquisition.estop release", + "note": "All motion is now refused for every process on this machine.", + } + if action == "status": + return { + "engaged": _estop.is_engaged(), + "detail": _estop.details(), + "flag_file": str(_estop.ESTOP_PATH), + } + raise ValueError( + f"Unknown estop action {action!r}. Use 'engage' to stop motion, or " + "'status' to check. Releasing is a human action performed at a " + "terminal: python -m acquisition.estop release" + ) diff --git a/mcp_server/server_loop.py b/mcp_server/server_loop.py index 4e8250d..4845bb9 100644 --- a/mcp_server/server_loop.py +++ b/mcp_server/server_loop.py @@ -1,8 +1,8 @@ # server_loop.py # ------------------------------------------------------------ # Minimal MCP server entry point: registers ONLY loop_tools.py's -# functions (get_image, get_pos, move, get_move_history) - kept -# deliberately minimal, 4 tools total, by explicit team direction. +# functions (get_image, get_pos, move, get_move_history, estop) - kept +# deliberately minimal, 5 tools total, by explicit team direction. # # Run, either way: # confocal-mcp (installed console script - see pyproject.toml) @@ -14,6 +14,8 @@ from mcp.server.mcpserver import MCPServer +from acquisition import estop + from mcp_server import loop_tools as tools mcp = MCPServer("ConfocalOrchestrator-Loop") @@ -22,10 +24,13 @@ mcp.add_tool(tools.get_pos) mcp.add_tool(tools.move) mcp.add_tool(tools.get_move_history) +mcp.add_tool(tools.estop) def main() -> None: - """Console entry point (``confocal-mcp``): serve the 4 tools over stdio.""" + """Console entry point (``confocal-mcp``): serve the 5 tools over stdio.""" + # The STOP panel, on screen for as long as this server runs. + estop.launch_panel() mcp.run(transport="stdio") diff --git a/pyproject.toml b/pyproject.toml index 5a7cf24..abb28dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "mcp==2.0.0", "Pillow==12.3.0", "PyYAML==6.0.3", + "numpy==2.4.6", ] [project.optional-dependencies] @@ -35,6 +36,10 @@ camera = [ sdk = [ "pywin32==312", ] +# Running the test suite (tests/ - mock backend only, no hardware). +test = [ + "pytest==9.1.1", +] # The standalone Claude harness loops in harness/ (not the MCP server). harness = [ "anthropic==1.0.0", @@ -42,7 +47,7 @@ harness = [ ] # Everything, for a full workstation install. all = [ - "confocal-mcp[camera,sdk,harness]", + "confocal-mcp[camera,sdk,harness,test]", ] [project.scripts] @@ -53,4 +58,4 @@ Homepage = "https://github.com/BioNanomics/microscope-agent" Repository = "https://github.com/BioNanomics/microscope-agent" [tool.hatch.build.targets.wheel] -packages = ["mcp_server", "acquisition", "harness"] +packages = ["mcp_server", "acquisition", "harness", "timelapse"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f9ed12b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +# Runtime data (captures, history logs) is anchored to CONFOCAL_MCP_DATA_DIR +# at *import* time (acquisition/paths.py), so it must be set before any +# test module imports mcp_server.loop_tools or acquisition.backends.nis_mock. +import os +import tempfile +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +_DATA_DIR = tempfile.mkdtemp(prefix="confocal-mcp-tests-") +os.environ["CONFOCAL_MCP_DATA_DIR"] = _DATA_DIR + + +def make_blob_frame( + size: int = 128, + center: tuple[float, float] = (64, 64), + radius: float = 20.0, + background: float = 200.0, + foreground: float = 60.0, + noise: float = 3.0, + seed: int = 0, +) -> np.ndarray: + """Brightfield-like frame: a dark disc on a bright background, plus + Gaussian noise. uint8 array of shape (size, size).""" + rng = np.random.default_rng(seed) + yy, xx = np.mgrid[0:size, 0:size] + disc = (xx - center[0]) ** 2 + (yy - center[1]) ** 2 <= radius ** 2 + frame = np.full((size, size), background, dtype=np.float32) + frame[disc] = foreground + frame += rng.normal(0.0, noise, frame.shape).astype(np.float32) + return np.clip(frame, 0, 255).astype(np.uint8) + + +def save_frame(array: np.ndarray, path: Path) -> Path: + Image.fromarray(array, mode="L").convert("RGB").save(path) + return path + + +@pytest.fixture +def blob_frame(): + return make_blob_frame + + +@pytest.fixture +def write_frame(tmp_path): + def _write(name: str, **kwargs) -> Path: + return save_frame(make_blob_frame(**kwargs), tmp_path / name) + return _write diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py new file mode 100644 index 0000000..c2a9bfa --- /dev/null +++ b/tests/test_change_detector.py @@ -0,0 +1,71 @@ +import numpy as np + +from timelapse.change_detector import ChangeDetector, otsu_threshold, phase_correlation_shift + + +def _feed_quiet(det, blob_frame, n=6): + for i in range(n): + det.score_array(blob_frame(seed=i).astype(np.float32)) + + +def test_first_frame_is_never_interesting(blob_frame): + det = ChangeDetector() + sc = det.score_array(blob_frame().astype(np.float32)) + assert sc.interesting is False + assert sc.baseline_frames == 0 + + +def test_quiet_sequence_stays_quiet(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(seed=99).astype(np.float32)) + assert sc.interesting is False, sc + assert sc.diff_score < 2.0 + + +def test_shrinking_blob_triggers_area_delta(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(radius=10.0, seed=7).astype(np.float32)) + assert sc.interesting is True + assert sc.area_delta < -0.04 + assert "shrank" in sc.reason + + +def test_stage_shift_is_reported_as_shift(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(center=(76, 64), seed=7).astype(np.float32)) + assert sc.interesting is True + assert sc.shift_px > 8 + assert "shift" in sc.reason + + +def test_frame_size_change_resets_baseline(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(size=64).astype(np.float32)) + assert "reset" in sc.reason + assert sc.baseline_frames == 0 + + +def test_otsu_splits_bimodal(): + arr = np.concatenate([np.full(500, 50.0), np.full(500, 200.0)]) + thr = otsu_threshold(arr) + assert 50 < thr < 200 + + +def test_phase_correlation_recovers_integer_shift(blob_frame): + a = blob_frame(seed=1).astype(np.float32) + b = np.roll(a, shift=(3, -5), axis=(0, 1)) + dx, dy = phase_correlation_shift(a, b) + assert (dx, dy) == (-5.0, 3.0) + + +def test_second_frame_of_quiet_sequence_is_not_interesting(blob_frame): + # Regression: with one baseline frame there is no noise estimate, and + # plain sensor noise used to read as a 3x event on frame 2. + det = ChangeDetector() + det.score_array(blob_frame(seed=0).astype(np.float32)) + sc = det.score_array(blob_frame(seed=1).astype(np.float32)) + assert sc.interesting is False, sc diff --git a/tests/test_context.py b/tests/test_context.py new file mode 100644 index 0000000..4b2aa67 --- /dev/null +++ b/tests/test_context.py @@ -0,0 +1,77 @@ +# harness/context.py: image pruning of a chat history between tool +# rounds. The rules that matter to a running harness: text always +# survives, the newest images survive, the caller's own history is never +# mutated (it keeps the full-fidelity copy), and non-image messages are +# passed through untouched, identity included. +import copy + +import pytest + +from harness.context import prune_context_images + + +def _img(label, kind="image"): + block = {"type": "image", "source": {"data": f"<{label}>"}} if kind == "image" \ + else {"type": "image_url", "image_url": {"url": f"data:{label}"}} + return {"role": "user", "content": [{"type": "text", "text": f"caption {label}"}, block]} + + +def _text(role, text): + return {"role": role, "content": [{"type": "text", "text": text}]} + + +def _has_image(msg): + return any(b["type"] in ("image", "image_url") for b in msg["content"]) + + +def test_keeps_only_last_n_images_and_all_text(): + history = [_text("user", "go"), _img("f1"), _text("assistant", "ok"), _img("f2"), _img("f3"), _img("f4")] + pruned = prune_context_images(history, keep_last_n=2) + assert [_has_image(m) for m in pruned] == [False, False, False, False, True, True] + assert pruned[1]["content"] == [{"type": "text", "text": "caption f1"}] + assert pruned[3]["content"][0]["text"] == "caption f2" + + +def test_keep_first_and_last_counts_image_messages_only(): + history = [_text("user", "a"), _img("f1"), _text("user", "b"), _img("f2"), _img("f3"), _text("user", "c")] + pruned = prune_context_images(history, keep_first_n=1, keep_last_n=1) + assert _has_image(pruned[1]) and _has_image(pruned[4]) + assert not _has_image(pruned[3]) + + +def test_does_not_mutate_caller_history(): + history = [_img("f1"), _img("f2"), _img("f3")] + snapshot = copy.deepcopy(history) + prune_context_images(history, keep_last_n=1) + assert history == snapshot + + +def test_non_image_messages_pass_through_by_identity(): + plain = _text("assistant", "thinking") + history = [_img("f1"), plain, _img("f2")] + pruned = prune_context_images(history, keep_last_n=1) + assert pruned[1] is plain + + +def test_string_content_and_openai_image_blocks(): + history = [{"role": "user", "content": "plain string"}, _img("f1", kind="image_url"), _img("f2", kind="image_url")] + pruned = prune_context_images(history, keep_last_n=1) + assert pruned[0] == {"role": "user", "content": "plain string"} + assert not _has_image(pruned[1]) and _has_image(pruned[2]) + + +def test_zero_keeps_strips_everything_and_defaults_are_zero(): + history = [_img("f1"), _img("f2")] + assert not any(_has_image(m) for m in prune_context_images(history)) + + +def test_keep_more_than_exist_is_a_no_op(): + history = [_img("f1"), _img("f2")] + assert prune_context_images(history, keep_last_n=10) == history + + +def test_negative_counts_rejected(): + with pytest.raises(ValueError): + prune_context_images([], keep_last_n=-1) + with pytest.raises(ValueError): + prune_context_images([], keep_first_n=-1) diff --git a/tests/test_frame_audit.py b/tests/test_frame_audit.py new file mode 100644 index 0000000..6341021 --- /dev/null +++ b/tests/test_frame_audit.py @@ -0,0 +1,76 @@ +import json + +from timelapse.frame_audit import audit, frames_from_dir, apply_timestamps, main, parse_duration + + +def _sequence(write_frame, tmp_path, n=12, event_at=8, gap_at=None, shift_at=None, dark_at=None): + """n frames 600s apart. Optional: blob shrinks at event_at, a 3x time + gap before gap_at, an XY shift at shift_at, a global dimming at dark_at.""" + frames, ts = [], [] + t = 1_700_000_000.0 + for i in range(n): + kw = {"seed": i} + if event_at is not None and i >= event_at: + kw["radius"] = 10.0 + if shift_at is not None and i >= shift_at: + kw["center"] = (78, 64) + if dark_at is not None and i >= dark_at: + kw["background"] = 120.0 + kw["foreground"] = 30.0 + frames.append(write_frame(f"frame_{i:03d}.png", **kw)) + if gap_at is not None and i == gap_at: + t += 1200.0 + ts.append(t) + t += 600.0 + ts_file = tmp_path / "timestamps.csv" + ts_file.write_text("\n".join(str(x) for x in ts) + "\n") + return frames, ts_file + + +def test_shrink_event_flagged_as_change_not_gap(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=8) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + by_index = {r.index: r for r in results} + assert "CHANGE" in by_index[8].flags + assert not any(f.startswith("GAP") for f in by_index[8].flags) + assert all(not r.flags for r in results if r.index not in (8,)), [r.flags for r in results] + + +def test_time_gap_flagged(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=None, gap_at=5) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + r = next(r for r in results if r.index == 5) # the gap is *before* frame 5 + assert r.dt_ratio > 2.5 + assert any(f.startswith("GAP") for f in r.flags) + + +def test_shift_and_dimming_flagged_as_non_biological(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=None, shift_at=6, dark_at=9) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + by_index = {r.index: r for r in results} + assert any(f.startswith("SHIFT") for f in by_index[6].flags) + assert any(f.startswith("INTENSITY") for f in by_index[9].flags) + assert "CHANGE" not in by_index[6].flags + assert "CHANGE" not in by_index[9].flags + + +def test_cli_around_window_and_json(write_frame, tmp_path, capsys): + _sequence(write_frame, tmp_path, event_at=8) + out_json = tmp_path / "audit.json" + rc = main([str(tmp_path), "--timestamps", str(tmp_path / "timestamps.csv"), + "--around", "80m", "--window", "10m", "--json", str(out_json)]) + assert rc == 0 + text = capsys.readouterr().out + assert "12 frames" in text + assert "CHANGE" in text + data = json.loads(out_json.read_text()) + assert len(data) == 11 + + +def test_parse_duration(): + assert parse_duration("25h") == 90000.0 + assert parse_duration("90m") == 5400.0 + assert parse_duration("30") == 30.0 diff --git a/tests/test_harness_gate.py b/tests/test_harness_gate.py new file mode 100644 index 0000000..9afaf90 --- /dev/null +++ b/tests/test_harness_gate.py @@ -0,0 +1,209 @@ +# The harnesses' real-hardware approval gate is the load-bearing safety +# property of harness/agent.py and harness/mcp_agent.py (see their header +# comments). These tests pin it down without a model or hardware: +# +# - the model never sees a "confirm" parameter in any tool schema +# - a real-hardware call (get_image on sdk, move on sdk) stops at the +# gate; a decline returns an error result and executes nothing; +# an approval injects confirm=True itself +# - a mock call never touches the gate at all +# +# The gate's input() prompt is monkeypatched with a recorder, and the +# underlying tool is replaced with a stub that records what it was called +# with, so the assertions are about the harness's decisions only. +import asyncio +import json + +import pytest + +from harness import agent, mcp_agent +from mcp_server import loop_tools + + +class FakeImage: + def to_image_content(self): + class C: + mime_type = "image/jpeg" + data = "AAAA" + return C() + + +@pytest.fixture +def gate(monkeypatch): + """Replace both harnesses' gate with a recorder whose answer we control.""" + calls = [] + state = {"answer": False} + + def sync_gate(name, tool_input): + calls.append((name, dict(tool_input))) + return state["answer"] + + async def async_gate(name, tool_input): + calls.append((name, dict(tool_input))) + return state["answer"] + + monkeypatch.setattr(agent, "_confirm_real_hardware_action", sync_gate) + monkeypatch.setattr(mcp_agent, "_confirm_real_hardware_action", async_gate) + state["calls"] = calls + return state + + +@pytest.fixture +def stub_tools(monkeypatch): + """Stub loop_tools so nothing real runs; record kwargs each was called with.""" + seen = {} + + def get_image(**kw): + seen["get_image"] = kw + return {"frame_id": 1, "image": "x.png", "backend": kw.get("backend", "sdk")}, FakeImage() + + def move(**kw): + seen["move"] = kw + return {"position": {"x": kw["x"], "y": kw["y"], "z": 0}} + + monkeypatch.setattr(loop_tools, "get_image", get_image) + monkeypatch.setattr(loop_tools, "move", move) + return seen + + +# -- schemas ----------------------------------------------------------------- + +def _all_property_names(schema: dict) -> set[str]: + names = set() + for key, sub in schema.get("properties", {}).items(): + names.add(key) + if isinstance(sub, dict): + names |= _all_property_names(sub) + return names + + +def test_agent_tool_schemas_never_expose_confirm(): + for tool in agent.TOOLS: + assert "confirm" not in _all_property_names(tool["input_schema"]), tool["name"] + + +def test_mcp_agent_strips_confirm_from_real_server_schemas(): + from mcp_server.server_loop import mcp + mcp_tools = asyncio.run(mcp.list_tools()) + assert any("confirm" in t.input_schema.get("properties", {}) for t in mcp_tools), \ + "precondition: the server itself does expose confirm (the harness must strip it)" + for t in mcp_agent._mcp_tools_to_anthropic_tools(mcp_tools): + assert "confirm" not in t["input_schema"]["properties"], t["name"] + + +def test_strip_confirm_does_not_mutate_input(): + schema = {"type": "object", "properties": {"confirm": {"type": "boolean"}, "x": {"type": "number"}}} + out = mcp_agent._strip_confirm(schema) + assert "confirm" not in out["properties"] + assert "confirm" in schema["properties"] + + +# -- harness/agent.py (in-process) -------------------------------------------- + +def test_agent_mock_capture_skips_gate(gate, stub_tools): + content, is_error = agent._execute_tool("get_image", {"backend": "mock"}) + assert gate["calls"] == [] + assert is_error is False + assert "confirm" not in stub_tools["get_image"] + assert [c["type"] for c in content] == ["text", "image"] + + +def test_agent_real_capture_declined_executes_nothing(gate, stub_tools): + gate["answer"] = False + content, is_error = agent._execute_tool("get_image", {}) + assert gate["calls"] == [("get_image", {})] + assert is_error is True + assert "get_image" not in stub_tools + assert "declined" in content[0]["text"].lower() + + +def test_agent_real_capture_approved_injects_confirm(gate, stub_tools): + gate["answer"] = True + _, is_error = agent._execute_tool("get_image", {"exposure_time_us": 5000}) + assert is_error is False + assert stub_tools["get_image"]["confirm"] is True + assert stub_tools["get_image"]["exposure_time_us"] == 5000 + + +def test_agent_model_cannot_smuggle_confirm_on_mock(gate, stub_tools): + # Even if the model somehow sent confirm=True, a mock call must not be + # promoted to real hardware: backend decides, not the flag. + agent._execute_tool("get_image", {"backend": "mock", "confirm": True}) + assert gate["calls"] == [] + assert stub_tools["get_image"]["backend"] == "mock" + + +def test_agent_move_gate_matches_backend(gate, stub_tools): + agent._execute_tool("move", {"x": 1, "y": 2}) # backend defaults to mock + assert gate["calls"] == [] and "confirm" not in stub_tools["move"] + + gate["answer"] = False + _, is_error = agent._execute_tool("move", {"x": 1, "y": 2, "backend": "sdk"}) + assert is_error is True and gate["calls"] == [("move", {"x": 1, "y": 2, "backend": "sdk"})] + + gate["answer"] = True + _, is_error = agent._execute_tool("move", {"x": 3, "y": 4, "backend": "sdk"}) + assert is_error is False and stub_tools["move"]["confirm"] is True + + +# -- harness/mcp_agent.py (over a fake MCP client) ----------------------------- + +class FakeMCPClient: + """Records call_tool arguments; returns a fixed text result.""" + + def __init__(self): + self.calls = [] + + async def call_tool(self, name, arguments): + self.calls.append((name, dict(arguments))) + + class Block: + type = "text" + text = json.dumps({"ok": True}) + + class Result: + content = [Block()] + is_error = False + return Result() + + +def _run(coro): + return asyncio.run(coro) + + +def test_mcp_agent_mock_capture_skips_gate(gate): + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {"backend": "mock"})) + assert gate["calls"] == [] + assert is_error is False + assert client.calls == [("get_image", {"backend": "mock"})] + + +def test_mcp_agent_real_capture_declined_never_reaches_server(gate): + gate["answer"] = False + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {})) + assert is_error is True + assert client.calls == [] + + +def test_mcp_agent_real_capture_approved_injects_confirm(gate): + gate["answer"] = True + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {"gain": 2.0})) + assert is_error is False + assert client.calls == [("get_image", {"gain": 2.0, "confirm": True})] + + +def test_mcp_agent_move_gate_matches_backend(gate): + client = FakeMCPClient() + _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "mock"})) + assert gate["calls"] == [] and client.calls[-1][1].get("confirm") is None + + gate["answer"] = False + _, is_error = _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "sdk"})) + assert is_error is True and len(client.calls) == 1 + + gate["answer"] = True + _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "sdk"})) + assert client.calls[-1] == ("move", {"x": 1, "y": 2, "backend": "sdk", "confirm": True}) diff --git a/tests/test_loop_tools_mock.py b/tests/test_loop_tools_mock.py new file mode 100644 index 0000000..6dc4ea3 --- /dev/null +++ b/tests/test_loop_tools_mock.py @@ -0,0 +1,35 @@ +import os +from pathlib import Path + +import pytest + +from acquisition.backends import nis_mock +from mcp_server import loop_tools + + +def test_get_image_sdk_default_requires_confirm(): + with pytest.raises(PermissionError): + loop_tools.get_image() + + +def test_get_image_mock_returns_metadata_and_preview(write_frame, monkeypatch): + frame = write_frame("sample.png") + monkeypatch.setattr(nis_mock, "SAMPLE_FRAME_PATH", frame) + metadata, preview = loop_tools.get_image(backend="mock", max_dimension=64) + assert metadata["backend"] == "mock" + assert Path(metadata["image"]).exists() + assert Path(metadata["image"]).resolve().is_relative_to(Path(os.environ["CONFOCAL_MCP_DATA_DIR"]).resolve()) + assert set(metadata["position"]) == {"x", "y", "z"} + assert metadata["frame_id"] >= 1 + assert preview.to_image_content().mime_type == "image/jpeg" + assert loop_tools.get_frame(metadata["frame_id"])["image"] == metadata["image"] + + +def test_get_image_mock_rejects_bad_crop(): + with pytest.raises(ValueError): + loop_tools.get_image(backend="mock", crop={"x": 0.8, "y": 0.0, "width": 0.5, "height": 0.5}) + + +def test_get_image_unknown_backend(): + with pytest.raises(ValueError): + loop_tools.get_image(backend="nope") diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..cb356c1 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,94 @@ +# End-to-end over the real protocol: spawn `python -m mcp_server.server_loop` +# as a subprocess (exactly what Claude Desktop/Code and harness/mcp_agent.py +# do), talk MCP to it over stdio, and exercise every tool against the mock +# backends. No hardware, no model. Guards two things the in-process tests +# cannot: that the server actually starts and serves on a core-only +# install, and that the model-facing tool surface is exactly the agreed +# set - nothing added by accident. +import asyncio +import json +import os +import sys +from pathlib import Path + +import pytest +from mcp.client import Client +from mcp.client.stdio import StdioServerParameters, stdio_client + +EXPECTED_TOOLS = {"get_image", "get_pos", "move", "get_move_history", "estop"} + + +def _server_params(data_dir: Path, frame: Path) -> StdioServerParameters: + env = dict(os.environ) + env.update({ + "CONFOCAL_MCP_DATA_DIR": str(data_dir), + "CONFOCAL_MOCK_FRAME_PATH": str(frame), + "CONFOCAL_NO_ESTOP_PANEL": "1", # headless: don't try to open the STOP window + }) + return StdioServerParameters(command=sys.executable, args=["-m", "mcp_server.server_loop"], env=env) + + +def _text(result) -> dict: + block = next(b for b in result.content if b.type == "text") + return json.loads(block.text) + + +async def _session(data_dir: Path, frame: Path): + async with Client(stdio_client(_server_params(data_dir, frame))) as client: + tools = (await client.list_tools()).tools + names = {t.name for t in tools} + + pos = _text(await client.call_tool("get_pos", {"backend": "mock"})) + moved = _text(await client.call_tool("move", {"x": 250.0, "y": -100.0, "backend": "mock"})) + history = _text(await client.call_tool("get_move_history", {"limit": 5})) + + img = await client.call_tool("get_image", {"backend": "mock", "max_dimension": 64}) + img_meta = _text(img) + img_types = [b.type for b in img.content] + + gated = await client.call_tool("get_image", {}) # backend defaults to sdk, no confirm + gated_move = await client.call_tool("move", {"x": 0, "y": 0, "backend": "sdk"}) + estop = _text(await client.call_tool("estop", {"action": "status"})) + + return dict(names=names, pos=pos, moved=moved, history=history, img_meta=img_meta, + img_types=img_types, gated=gated, gated_move=gated_move, estop=estop) + + +@pytest.fixture(scope="module") +def run(tmp_path_factory): + data_dir = tmp_path_factory.mktemp("mcp-data") + sys.path.insert(0, str(Path(__file__).parent)) + from conftest import make_blob_frame, save_frame + frame = save_frame(make_blob_frame(), data_dir / "frame.png") + return asyncio.run(_session(data_dir, frame)) + + +def test_tool_surface_is_exactly_the_agreed_set(run): + assert run["names"] == EXPECTED_TOOLS + + +def test_mock_stage_round_trip(run): + assert run["pos"]["backend"] == "mock" + assert run["moved"]["position"]["x"] == 250.0 + assert run["moved"]["position"]["y"] == -100.0 + assert run["history"]["total_moves"] >= 1 + last = run["history"]["history"][-1] + assert last["requested"] == {"x": 250.0, "y": -100.0} + assert last["position"]["x"] == 250.0 + + +def test_mock_capture_returns_metadata_and_embedded_image(run): + assert run["img_meta"]["backend"] == "mock" + assert Path(run["img_meta"]["image"]).exists() + assert run["img_meta"]["position"]["x"] == 250.0 # position is coupled to the capture + assert "image" in run["img_types"], run["img_types"] + + +def test_real_hardware_refused_without_confirm_over_protocol(run): + assert run["gated"].is_error is True + assert run["gated_move"].is_error is True + + +def test_estop_status_reports_over_protocol(run): + assert "engaged" in run["estop"] + assert run["estop"]["engaged"] is False diff --git a/tests/test_model_trigger.py b/tests/test_model_trigger.py new file mode 100644 index 0000000..8875a78 --- /dev/null +++ b/tests/test_model_trigger.py @@ -0,0 +1,126 @@ +# timelapse/model_trigger.py with a fake model: what the model is shown, +# how its answer is read, and that every failure path is a harmless None. +import json + +import numpy as np + +from timelapse.change_detector import ChangeScore +from timelapse.model_trigger import ModelTrigger, SYSTEM_PROMPT, build_user_content, parse_decision + + +def _score(reason="foreground area shrank by 5.7%"): + return ChangeScore(diff_score=6.2, area_delta=-0.057, shift_px=0.0, foreground_fraction=0.05, + baseline_frames=5, interesting=True, reason=reason) + + +def _event(write_frame, with_previous=True): + after = write_frame("after.png", radius=10.0, seed=2) + before = write_frame("before.png", seed=1) if with_previous else None + return {"event": "capture", "frame_id": 7, "image": str(after), + "previous_image": str(before) if before else None, "position": {"x": 1, "y": 2, "z": 3}} + + +class FakeAsk: + def __init__(self, reply): + self.reply = reply + self.received = [] + + def __call__(self, system_prompt, content): + self.received.append((system_prompt, content)) + if isinstance(self.reply, Exception): + raise self.reply + return self.reply + + +class FakeClock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +# -- what the model is shown --------------------------------------------------- + +def test_prompt_contains_before_and_after_images_and_detector_numbers(write_frame): + content = build_user_content(_event(write_frame), _score()) + kinds = [b["type"] for b in content] + assert kinds == ["text", "image", "text", "image", "text"] + for b in content: + if b["type"] == "image": + assert b["source"]["media_type"] == "image/jpeg" and len(b["source"]["data"]) > 100 + tail = content[-1]["text"] + assert "shrank" in tail and "6.2x" in tail and "-5.7%" in tail + + +def test_prompt_without_previous_frame_has_one_image(write_frame): + content = build_user_content(_event(write_frame, with_previous=False), _score()) + assert [b["type"] for b in content] == ["text", "image", "text"] + + +# -- how the answer is read ---------------------------------------------------- + +def test_parse_decision_variants(): + assert parse_decision('{"decision": "extend", "reason": "tube thickening"}') == ("extend", "tube thickening") + assert parse_decision('Sure. {"decision":"IGNORE","reason":"bubble"} ok') == ("ignore", "bubble") + assert parse_decision('{"decision": "maybe"}')[0] is None + assert parse_decision("no json here")[0] is None + assert parse_decision("{broken")[0] is None + assert parse_decision("")[0] is None + + +def test_trigger_returns_model_decision_and_records_reason(write_frame): + ask = FakeAsk(json.dumps({"decision": "extend", "reason": "front advancing"})) + trig = ModelTrigger(ask=ask, log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) == "extend" + assert ask.received[0][0] == SYSTEM_PROMPT + assert trig.decisions[-1]["decision"] == "extend" + assert trig.decisions[-1]["reason"] == "front advancing" + assert trig.calls == 1 + + +# -- every failure is a harmless None ----------------------------------------- + +def test_model_exception_is_swallowed(write_frame): + trig = ModelTrigger(ask=FakeAsk(RuntimeError("no network")), log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) is None + assert "no network" in trig.decisions[-1]["reason"] + + +def test_unparseable_reply_is_no_opinion(write_frame): + trig = ModelTrigger(ask=FakeAsk("I think it's interesting!"), log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) is None + + +def test_max_calls_cap_and_min_interval(write_frame): + clock = FakeClock() + ask = FakeAsk('{"decision": "extend", "reason": "x"}') + trig = ModelTrigger(ask=ask, max_calls=2, min_interval_s=30.0, clock=clock, log=lambda s: None) + ev = _event(write_frame) + assert trig(ev, {}, _score()) == "extend" # call 1 + assert trig(ev, {}, _score()) is None # too soon + clock.t = 31.0 + assert trig(ev, {}, _score()) == "extend" # call 2 + clock.t = 62.0 + assert trig(ev, {}, _score()) is None # cap reached + assert len(ask.received) == 2 + assert "max_calls" in trig.decisions[-1]["reason"] + + +# -- wired into the scheduler -------------------------------------------------- + +def test_scheduler_passes_previous_image_and_honours_ignore(write_frame, tmp_path): + from tests.test_scheduler import FakeClock as SchedClock, FrameSource, _config + from timelapse.scheduler import AdaptiveTimelapse + clock = SchedClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(6)] + src = FrameSource(clock, quiet + shrunk) + ask = FakeAsk('{"decision": "ignore", "reason": "bubble"}') + trig = ModelTrigger(ask=ask, min_interval_s=0.0, log=lambda s: None) + summary = AdaptiveTimelapse(_config(tmp_path, max_captures=12), capture=src, on_trigger=trig, + clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert trig.calls >= 1 + _, content = ask.received[0] + assert [b["type"] for b in content][:2] == ["text", "image"] # a "before" frame was available + assert summary.burst_captures == 0 # ignore ended the burst at once diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..0d73202 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,238 @@ +import json + +import pytest + +from timelapse.scheduler import AdaptiveTimelapse, TimelapseConfig + + +class FakeClock: + def __init__(self): + self.t = 1000.0 + + def __call__(self): + return self.t + + def sleep(self, s): + assert s >= 0 + self.t += s + + +class FrameSource: + """Serves frames in order; the scheduler's capture() pulls the next one. + After the list runs out, keeps serving the last frame.""" + + def __init__(self, clock, paths): + self.clock = clock + self.paths = list(paths) + self.i = 0 + self.captured_at = [] + + def __call__(self): + path = self.paths[min(self.i, len(self.paths) - 1)] + self.i += 1 + self.captured_at.append(self.clock()) + return {"image": str(path), "frame_id": self.i, "position": {"x": 0, "y": 0, "z": 0}} + + +def _config(tmp_path, **kw): + base = dict(backend="mock", slow_interval_s=60.0, burst_interval_s=5.0, burst_duration_s=30.0, + max_captures=40, max_runtime_s=10_000.0, events_path=tmp_path / "events.jsonl") + base.update(kw) + return TimelapseConfig(**base) + + +def _events(path): + return [json.loads(l) for l in path.read_text().splitlines()] + + +def test_quiet_run_stays_slow_and_stops_at_max_captures(write_frame, tmp_path): + clock = FakeClock() + frames = [write_frame(f"q{i}.png", seed=i) for i in range(12)] + src = FrameSource(clock, frames) + cfg = _config(tmp_path, max_captures=10) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.stop_reason == "max_captures" + assert summary.captures == 10 + assert summary.bursts == 0 + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert all(g == 60.0 for g in gaps), gaps + + +def test_change_starts_burst_then_returns_to_slow(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(20)] + src = FrameSource(clock, quiet + shrunk) + cfg = _config(tmp_path, max_captures=25) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + + kinds = [e["event"] for e in _events(cfg.events_path)] + assert "burst_start" in kinds and "burst_end" in kinds + assert summary.bursts >= 1 + # First burst starts at the 7th capture (first shrunk frame) - then 5s spacing. + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert gaps[5] == 60.0 # slow gap into the first shrunk frame + assert gaps[6] == 5.0 # burst spacing right after trigger + assert summary.burst_captures > 0 + # After the specimen settles at its new size the baseline catches up + # and the loop must be back on the slow interval by the end. + assert gaps[-1] == 60.0, gaps + + +def test_stage_shift_does_not_start_burst(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shifted = [write_frame(f"m{i}.png", center=(78, 64), seed=200 + i) for i in range(4)] + src = FrameSource(clock, quiet + shifted) + cfg = _config(tmp_path, max_captures=10) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.shifts >= 1 + assert summary.bursts == 0 + + +def test_on_trigger_ignore_ends_burst_immediately(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(6)] + src = FrameSource(clock, quiet + shrunk) + cfg = _config(tmp_path, max_captures=12) + seen = [] + + def on_trigger(event, metadata, score): + seen.append(score.reason) + return "ignore" + + summary = AdaptiveTimelapse(cfg, capture=src, on_trigger=on_trigger, + clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert seen, "hook never called" + assert summary.burst_captures == 0 + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert all(g == 60.0 for g in gaps), gaps + + +def test_max_runtime_stops_run(write_frame, tmp_path): + clock = FakeClock() + src = FrameSource(clock, [write_frame("q.png")]) + cfg = _config(tmp_path, max_captures=1000, max_runtime_s=300.0) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.stop_reason == "max_runtime" + assert summary.captures == 6 # t=0,60,...,300 -> the 300s check fires before a 7th + + +def test_config_validation(tmp_path): + with pytest.raises(ValueError): + _config(tmp_path, burst_interval_s=120.0).validate() + with pytest.raises(ValueError): + _config(tmp_path, backend="nope").validate() + + +def test_cli_dry_run_prints_plan(capsys): + from timelapse.scheduler import main + assert main(["--dry-run", "--backend", "sdk", "--slow", "30"]) == 0 + out = capsys.readouterr().out + assert "REAL HARDWARE" in out and "30s" in out + + +# -- background consults -------------------------------------------------------- + +class ManualExecutor: + """Executor whose futures complete only when the test says so.""" + + def __init__(self): + self.futures = [] + + def submit(self, fn, *args): + from concurrent.futures import Future + f = Future() + f._fn_args = (fn, args) + self.futures.append(f) + return f + + def resolve(self, i, value=None, exc=None): + f = self.futures[i] + if exc is not None: + f.set_exception(exc) + else: + f.set_result(value) + + def shutdown(self, wait=True, cancel_futures=False): + pass + + +def _shrink_run(write_frame, tmp_path, max_captures): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(30)] + src = FrameSource(clock, quiet + shrunk) + ex = ManualExecutor() + calls = [] + tl = AdaptiveTimelapse(_config(tmp_path, max_captures=max_captures), capture=src, + on_trigger=lambda e, m, s: calls.append(e["frame_id"]), + clock=clock, sleep=clock.sleep, log=lambda s: None, + consult_in_background=True, executor=ex) + return tl, src, ex, clock + + +def test_background_consult_does_not_block_burst_timing(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=9) + tl.run() + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert gaps[6] == 5.0 and gaps[7] == 5.0 # burst spacing held while the consult was pending + assert len(ex.futures) >= 1 # the hook was submitted, never awaited + + +def test_background_extend_is_applied_at_next_step(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() # 6 quiet + first shrunk -> burst_start, consult pending + assert tl.mode == "burst" and len(ex.futures) == 1 + until_before = tl.burst_until + ex.resolve(0, "extend") + clock.t += 5.0 + tl.step() # next frame also fires the detector - must not undo the extension + assert tl.burst_until == until_before + tl.config.burst_duration_s + kinds = [json.loads(l)["event"] for l in tl.config.events_path.read_text().splitlines()] + assert kinds.count("burst_extend") >= 1 + + +def test_background_ignore_ends_burst_and_late_reply_is_logged(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() + assert tl.mode == "burst" + ex.resolve(0, "ignore") + clock.t += 5.0 + tl.step() + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "burst_end" and "ignore" in e["detail"] for e in events) + + # Keep stepping until the specimen's new size is the baseline and the + # loop has settled back to slow on its own. + for _ in range(12): + clock.t += 5.0 + tl.step() + assert tl.mode == "slow" + # Consults still outstanding from those bursts now answer "extend" - + # too late to matter, and logged as such. + for i, f in enumerate(ex.futures): + if not f.done(): + ex.resolve(i, "extend") + clock.t += 60.0 + tl.step() + assert tl.mode == "slow" + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "consult_late" for e in events) + + +def test_background_consult_exception_is_logged_not_raised(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() + ex.resolve(0, exc=RuntimeError("model down")) + clock.t += 5.0 + tl.step() # must not raise + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "consult_failed" and "model down" in e["detail"] for e in events) diff --git a/timelapse/__init__.py b/timelapse/__init__.py new file mode 100644 index 0000000..c33957f --- /dev/null +++ b/timelapse/__init__.py @@ -0,0 +1,16 @@ +# timelapse/ +# ------------------------------------------------------------ +# Offline-testable pieces of the adaptive time-lapse idea: +# +# change_detector.py - scores a new frame against a rolling baseline +# (no model call, no hardware - numpy + Pillow) +# frame_audit.py - audits an existing frame sequence for acquisition +# gaps, global intensity jumps and XY shifts, to +# separate "filming error" from "biological change" +# scheduler.py - the slow-loop / burst-mode acquisition loop that +# drives loop_tools.get_image() using the detector +# +# None of this is an MCP tool. The model-facing surface stays at 5 tools +# (see mcp_server/loop_tools.py's header); this package sits *beside* the +# tools and calls get_image() itself. +# ------------------------------------------------------------ diff --git a/timelapse/change_detector.py b/timelapse/change_detector.py new file mode 100644 index 0000000..8a02cb8 --- /dev/null +++ b/timelapse/change_detector.py @@ -0,0 +1,225 @@ +# change_detector.py +# ------------------------------------------------------------ +# Cheap, model-free "did something happen?" score for one new frame. +# +# WHY NO LLM HERE: the fast loop of an adaptive time-lapse may run every +# few seconds for days. A vision-model call per frame is far too slow and +# expensive for that; it is also unnecessary - "a lot of pixels changed +# where the specimen is" is a numpy question. The model is only consulted +# when this detector fires (see scheduler.py), to decide what to do about +# it, not whether it happened. +# +# WHAT IT MEASURES (all on a small grayscale copy of the frame): +# diff_score - mean |frame - baseline| / baseline noise level, where +# the baseline is a running median of the last N frames. +# Robust to a single bad frame; drifts with slow change. +# area_delta - fractional change in the foreground-mask area (Otsu +# threshold on the baseline, applied to the new frame). +# A retraction/expansion of a specimen shows up here even +# when the per-pixel diff is spread thin. +# shift_px - global XY translation between baseline and frame (phase +# correlation). Large shift = the stage or the sample +# moved, which is NOT biology - the scheduler treats it as +# a separate event class. +# +# THRESHOLDS are configuration, not science - tune them on a recorded run +# (frame_audit.py prints the same quantities for an existing sequence, so +# the values seen at a known event give the starting point). +# ------------------------------------------------------------ + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +from PIL import Image + +ANALYSIS_MAX_DIMENSION = 256 + + +def load_gray(path: str | Path, max_dimension: int = ANALYSIS_MAX_DIMENSION) -> np.ndarray: + """Load an image as a float32 grayscale array, downscaled so its long + edge is at most `max_dimension`. Small on purpose - the metrics below + are about gross change, and a 256px thumbnail makes every one of them + sub-millisecond.""" + with Image.open(path) as img: + img = img.convert("L") + img.thumbnail((max_dimension, max_dimension)) + return np.asarray(img, dtype=np.float32) + + +def otsu_threshold(gray: np.ndarray) -> float: + """Otsu's threshold on a grayscale array (0-255 range). Pure numpy.""" + hist, edges = np.histogram(gray, bins=256, range=(0.0, 256.0)) + hist = hist.astype(np.float64) + total = hist.sum() + if total == 0: + return 0.0 + centers = (edges[:-1] + edges[1:]) / 2.0 + weight_bg = np.cumsum(hist) + weight_fg = total - weight_bg + sum_bg = np.cumsum(hist * centers) + mean_bg = np.divide(sum_bg, weight_bg, out=np.zeros_like(sum_bg), where=weight_bg > 0) + mean_fg = np.divide(sum_bg[-1] - sum_bg, weight_fg, out=np.zeros_like(sum_bg), where=weight_fg > 0) + between = weight_bg * weight_fg * (mean_bg - mean_fg) ** 2 + return float(centers[int(np.argmax(between))]) + + +def foreground_fraction(gray: np.ndarray, threshold: float) -> float: + """Fraction of pixels on the darker-than-threshold side. Works for + brightfield specimens on a bright background; for fluorescence + (bright specimen, dark background) callers pass `invert=True` to + ChangeDetector.""" + return float((gray < threshold).mean()) + + +def phase_correlation_shift(a: np.ndarray, b: np.ndarray) -> tuple[float, float]: + """(dx, dy) in pixels that shifts `a` onto `b`, via phase correlation. + Integer precision - good enough to tell "the stage moved" from "it + didn't". Both arrays must have the same shape.""" + if a.shape != b.shape: + raise ValueError(f"shape mismatch: {a.shape} vs {b.shape}") + fa = np.fft.fft2(a - a.mean()) + fb = np.fft.fft2(b - b.mean()) + cross = fa * np.conj(fb) + denom = np.abs(cross) + denom[denom == 0] = 1.0 + corr = np.fft.ifft2(cross / denom).real + peak = np.unravel_index(int(np.argmax(corr)), corr.shape) + dy, dx = peak + h, w = a.shape + if dy > h // 2: + dy -= h + if dx > w // 2: + dx -= w + return float(-dx), float(-dy) + + +def estimate_shift(a: np.ndarray, b: np.ndarray, min_improvement: float = 0.5) -> tuple[float, float]: + """Like phase_correlation_shift, but returns (0, 0) unless applying the + candidate shift actually makes `b` match `a` much better - specifically + unless it cuts mean|a - b| to below `min_improvement` of the unshifted + value. Phase correlation happily returns a peak for a specimen that + changed *shape* in place (a shrinking blob, say) with no real + translation; this check rejects those.""" + dx, dy = phase_correlation_shift(a, b) + if dx == 0.0 and dy == 0.0: + return 0.0, 0.0 + unshifted = float(np.mean(np.abs(a - b))) + if unshifted == 0.0: + return 0.0, 0.0 + b_back = np.roll(b, shift=(int(-dy), int(-dx)), axis=(0, 1)) + shifted = float(np.mean(np.abs(a - b_back))) + if shifted < unshifted * min_improvement: + return dx, dy + return 0.0, 0.0 + + +@dataclass +class ChangeScore: + diff_score: float + area_delta: float + shift_px: float + foreground_fraction: float + baseline_frames: int + interesting: bool + reason: str + + def as_dict(self) -> dict: + return { + "diff_score": round(self.diff_score, 3), + "area_delta": round(self.area_delta, 4), + "shift_px": round(self.shift_px, 1), + "foreground_fraction": round(self.foreground_fraction, 4), + "baseline_frames": self.baseline_frames, + "interesting": self.interesting, + "reason": self.reason, + } + + +@dataclass +class ChangeDetector: + """Scores each new frame against a running median of the previous + `baseline_frames` frames. Feed it frames in time order via score(). + + diff_threshold: diff_score above this is "interesting". Units are + multiples of the baseline's own frame-to-frame noise, so ~3 means + "three times noisier than a quiet stretch". + area_threshold: |area_delta| above this fraction is "interesting" + (0.05 = the specimen's footprint changed by 5%). + shift_threshold_px: shift above this (in analysis-thumbnail pixels) + is flagged as a stage/sample move rather than biology. + invert: True for bright-on-dark images (fluorescence). + """ + + baseline_frames: int = 5 + diff_threshold: float = 3.0 + area_threshold: float = 0.05 + shift_threshold_px: float = 4.0 + invert: bool = False + _history: deque = field(default_factory=deque, repr=False) + + def __post_init__(self) -> None: + self._history = deque(maxlen=self.baseline_frames) + + def reset(self) -> None: + self._history.clear() + + def score_array(self, gray: np.ndarray) -> ChangeScore: + if self.invert: + gray = 255.0 - gray + n = len(self._history) + if n == 0: + self._history.append(gray) + thr = otsu_threshold(gray) + return ChangeScore(0.0, 0.0, 0.0, foreground_fraction(gray, thr), 0, False, "first frame") + + stack = np.stack(self._history) + baseline = np.median(stack, axis=0) + # Baseline noise: how much the baseline frames disagree with each + # other. With a single baseline frame there is no estimate at all, + # so the pixel-difference criterion is held back (diff_ready) until + # a second frame is in - otherwise ordinary sensor noise on frame 2 + # reads as a 3x+ event. Area and shift criteria don't need a noise + # estimate and stay live from frame 2. + diff_ready = n >= 2 + noise = float(np.mean(np.abs(stack - baseline))) if diff_ready else 1.0 + noise = max(noise, 1.0) + + if gray.shape != baseline.shape: + # Camera settings changed mid-run (binning, ROI). Restart. + self.reset() + self._history.append(gray) + return ChangeScore(0.0, 0.0, 0.0, 0.0, 0, True, "frame size changed - baseline reset") + + diff_score = float(np.mean(np.abs(gray - baseline))) / noise + thr = otsu_threshold(baseline) + fg_base = foreground_fraction(baseline, thr) + fg_new = foreground_fraction(gray, thr) + area_delta = fg_new - fg_base + dx, dy = estimate_shift(baseline, gray) + shift_px = float(np.hypot(dx, dy)) + + reasons = [] + if shift_px > self.shift_threshold_px: + reasons.append(f"xy shift {shift_px:.1f}px (stage/sample moved?)") + if abs(area_delta) > self.area_threshold: + reasons.append(f"foreground area {'grew' if area_delta > 0 else 'shrank'} by {abs(area_delta):.1%}") + if diff_ready and diff_score > self.diff_threshold: + reasons.append(f"pixel change {diff_score:.1f}x baseline noise") + + self._history.append(gray) + return ChangeScore( + diff_score=diff_score, + area_delta=float(area_delta), + shift_px=shift_px, + foreground_fraction=fg_new, + baseline_frames=n, + interesting=bool(reasons), + reason="; ".join(reasons) if reasons else "quiet", + ) + + def score(self, path: str | Path) -> ChangeScore: + return self.score_array(load_gray(path)) diff --git a/timelapse/frame_audit.py b/timelapse/frame_audit.py new file mode 100644 index 0000000..9910830 --- /dev/null +++ b/timelapse/frame_audit.py @@ -0,0 +1,250 @@ +# frame_audit.py +# ------------------------------------------------------------ +# "Was that a filming error or biology?" - audit an existing frame +# sequence around a suspected event. +# +# For every consecutive pair of frames it reports: +# dt_s - seconds between the two frames, and its ratio to the +# median interval. A ratio well above 1 means the +# acquisition stalled: the "sudden" change in the video +# is really a missing stretch of time. (For an adaptive +# run from scheduler.py the median is the burst interval, +# so every slow-mode interval reads as a GAP - expected.) +# mean_delta - relative change in global mean intensity. A large +# value with little local change = illumination/exposure +# changed, not the specimen. +# shift_px - global XY translation (phase correlation). Non-zero = +# the stage or the sample moved. +# diff_score / area_delta - the same quantities change_detector.py +# uses live, so the values seen at a known event give the +# thresholds to configure the scheduler with. +# +# Timestamps come from, in order of preference: +# --history frame_history.jsonl (this repo's own capture log) +# --timestamps file.csv (one ISO-8601 or epoch-seconds per line, +# same order as the frames; e.g. exported +# from the ND2's metadata) +# file mtime (fallback - only trustworthy if the files +# were written as they were captured) +# +# Usage: +# python -m timelapse.frame_audit FRAME_DIR [--glob '*.png'] [--around 25h --window 1h] +# python -m timelapse.frame_audit --history logs/frame_history.jsonl +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path + +import numpy as np + +from timelapse.change_detector import estimate_shift, foreground_fraction, load_gray, otsu_threshold + +_DURATION_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*(s|m|h|d)?$") +_DURATION_UNIT_S = {"s": 1.0, "m": 60.0, "h": 3600.0, "d": 86400.0, None: 1.0} + + +def parse_duration(text: str) -> float: + """'25h' -> 90000.0, '90m' -> 5400.0, '30' -> 30.0 (seconds).""" + m = _DURATION_RE.match(text.strip()) + if not m: + raise ValueError(f"bad duration {text!r} (expected e.g. '25h', '90m', '30s')") + return float(m.group(1)) * _DURATION_UNIT_S[m.group(2)] + + +def parse_timestamp(text: str) -> float: + """ISO-8601 or epoch seconds -> epoch seconds.""" + text = text.strip() + try: + return float(text) + except ValueError: + return datetime.fromisoformat(text).timestamp() + + +@dataclass +class FrameRecord: + index: int + path: Path + t_s: float # epoch seconds + + +@dataclass +class PairAudit: + index: int # index of the later frame + t_rel_s: float # seconds since the first frame + dt_s: float + dt_ratio: float # dt / median dt + mean_delta: float # (mean_b - mean_a) / mean_a + shift_px: float + diff_score: float + area_delta: float + flags: list[str] + + +def frames_from_dir(frame_dir: Path, glob: str) -> list[FrameRecord]: + paths = sorted(frame_dir.glob(glob)) + if not paths: + raise FileNotFoundError(f"no files matching {glob!r} in {frame_dir}") + return [FrameRecord(i, p, p.stat().st_mtime) for i, p in enumerate(paths)] + + +def frames_from_history(history_path: Path) -> list[FrameRecord]: + records = [] + with open(history_path) as f: + for line in f: + line = line.strip() + if not line: + continue + rec = json.loads(line) + records.append(FrameRecord(len(records), Path(rec["image"]), parse_timestamp(rec["captured_at"]))) + if not records: + raise ValueError(f"{history_path} has no frame records") + return records + + +def apply_timestamps(frames: list[FrameRecord], timestamps_path: Path) -> list[FrameRecord]: + with open(timestamps_path) as f: + rows = [row[0] for row in csv.reader(f) if row and row[0].strip() and not row[0].startswith("#")] + if len(rows) != len(frames): + raise ValueError(f"{timestamps_path} has {len(rows)} timestamps but there are {len(frames)} frames") + return [FrameRecord(fr.index, fr.path, parse_timestamp(ts)) for fr, ts in zip(frames, rows)] + + +def audit( + frames: list[FrameRecord], + gap_ratio: float = 1.5, + mean_delta_threshold: float = 0.10, + shift_threshold_px: float = 4.0, + diff_threshold: float = 3.0, + area_threshold: float = 0.05, +) -> list[PairAudit]: + """Score every consecutive pair. `frames` must be in time order. + + Two passes: the first computes raw pairwise quantities, the second + normalizes the pixel-difference by the *median* pairwise difference of + the whole sequence (the quiet-stretch noise level) - so diff_score is + "how many times noisier than a typical interval", the same units the + live ChangeDetector reports. Frame-to-frame rather than + against-a-rolling-baseline on purpose: an audit wants to localize an + event to one interval, not smear it over the baseline window.""" + if len(frames) < 2: + raise ValueError("need at least 2 frames to audit") + times = np.array([fr.t_s for fr in frames]) + dts = np.diff(times) + median_dt = float(np.median(dts)) + + raw = [] # (dt, mean_delta, shift, raw_diff, area_delta, size_changed) + prev = load_gray(frames[0].path) + for i in range(1, len(frames)): + cur = load_gray(frames[i].path) + mean_a, mean_b = float(prev.mean()), float(cur.mean()) + mean_delta = (mean_b - mean_a) / mean_a if mean_a > 0 else 0.0 + size_changed = cur.shape != prev.shape + if size_changed: + shift, raw_diff, area_delta = float("nan"), float("nan"), float("nan") + else: + dx, dy = estimate_shift(prev, cur) + shift = float(np.hypot(dx, dy)) + raw_diff = float(np.mean(np.abs(cur - prev))) + thr = otsu_threshold(prev) + area_delta = foreground_fraction(cur, thr) - foreground_fraction(prev, thr) + raw.append((float(dts[i - 1]), mean_delta, shift, raw_diff, area_delta, size_changed)) + prev = cur + + diffs = np.array([r[3] for r in raw]) + noise = float(np.nanmedian(diffs)) if np.isfinite(diffs).any() else 1.0 + noise = max(noise, 1.0) # floor of one gray level - identical frames must not blow this up + + t0 = frames[0].t_s + results: list[PairAudit] = [] + for i, (dt, mean_delta, shift, raw_diff, area_delta, size_changed) in enumerate(raw, start=1): + dt_ratio = dt / median_dt if median_dt > 0 else 1.0 + diff_score = raw_diff / noise + flags = [] + if dt_ratio > gap_ratio: + flags.append(f"GAP x{dt_ratio:.1f}") + if dt <= 0: + flags.append("NON-MONOTONIC TIME") + if size_changed: + flags.append("SIZE CHANGED") + if abs(mean_delta) > mean_delta_threshold: + flags.append(f"INTENSITY {mean_delta:+.0%}") + if shift > shift_threshold_px: + flags.append(f"SHIFT {shift:.0f}px") + non_bio = any(f.startswith(("SHIFT", "INTENSITY", "SIZE")) for f in flags) + if not non_bio and (diff_score > diff_threshold or abs(area_delta) > area_threshold): + flags.append("CHANGE") + results.append(PairAudit( + index=i, t_rel_s=frames[i].t_s - t0, dt_s=dt, dt_ratio=dt_ratio, + mean_delta=mean_delta, shift_px=shift, diff_score=diff_score, + area_delta=float(area_delta), flags=flags, + )) + return results + + +def _fmt_rel(seconds: float) -> str: + h, rem = divmod(int(seconds), 3600) + m, s = divmod(rem, 60) + return f"{h:02d}:{m:02d}:{s:02d}" + + +def print_table(results: list[PairAudit], out=None) -> None: + out = out or sys.stdout + print(f"{'idx':>5} {'t_rel':>9} {'dt_s':>8} {'dt_x':>5} {'mean%':>7} {'shift':>6} {'diff':>6} {'area%':>7} flags", file=out) + for r in results: + print( + f"{r.index:>5} {_fmt_rel(r.t_rel_s):>9} {r.dt_s:>8.1f} {r.dt_ratio:>5.2f} " + f"{r.mean_delta * 100:>+7.1f} {r.shift_px:>6.1f} {r.diff_score:>6.2f} {r.area_delta * 100:>+7.2f} " + f"{', '.join(r.flags)}", + file=out, + ) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__ or "Audit a frame sequence for acquisition gaps and non-biological changes.") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("frame_dir", nargs="?", type=Path, help="directory of frames (sorted by filename)") + src.add_argument("--history", type=Path, help="this repo's logs/frame_history.jsonl") + ap.add_argument("--glob", default="*.png") + ap.add_argument("--timestamps", type=Path, help="CSV of one timestamp per frame (ISO-8601 or epoch seconds)") + ap.add_argument("--around", help="only report pairs near this offset from the first frame, e.g. 25h") + ap.add_argument("--window", default="1h", help="half-width of --around window (default 1h)") + ap.add_argument("--gap-ratio", type=float, default=1.5) + ap.add_argument("--mean-delta", type=float, default=0.10) + ap.add_argument("--shift-px", type=float, default=4.0) + ap.add_argument("--json", type=Path, help="also write full results as JSON here") + ap.add_argument("--flagged-only", action="store_true") + args = ap.parse_args(argv) + + frames = frames_from_history(args.history) if args.history else frames_from_dir(args.frame_dir, args.glob) + if args.timestamps: + frames = apply_timestamps(frames, args.timestamps) + results = audit(frames, args.gap_ratio, args.mean_delta, args.shift_px) + + dts = [r.dt_s for r in results] + print(f"{len(frames)} frames, {len(results)} intervals, median interval {np.median(dts):.1f}s, " + f"span {_fmt_rel(results[-1].t_rel_s)}") + + shown = results + if args.around: + center, half = parse_duration(args.around), parse_duration(args.window) + shown = [r for r in results if abs(r.t_rel_s - center) <= half] + if args.flagged_only: + shown = [r for r in shown if r.flags] + print_table(shown) + + if args.json: + args.json.write_text(json.dumps([asdict(r) for r in results], default=str, indent=1)) + print(f"wrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/timelapse/model_trigger.py b/timelapse/model_trigger.py new file mode 100644 index 0000000..fb1b6a8 --- /dev/null +++ b/timelapse/model_trigger.py @@ -0,0 +1,182 @@ +# model_trigger.py +# ------------------------------------------------------------ +# The "ask the model" seam for the adaptive time-lapse: an on_trigger +# hook for scheduler.AdaptiveTimelapse that shows Claude the frame before +# and the frame that fired the detector, plus the detector's own reason, +# and asks one question: is this worth a longer look? +# +# "extend" - keep burst-imaging for another window +# "ignore" - this is not interesting (dust, a bubble, a drift), end +# the burst now and go back to the slow interval +# None - no opinion; the scheduler carries on with its own timer +# +# WHY THE MODEL IS HERE AND NOWHERE ELSE: the detector runs on every +# frame and is free. The model runs only when the detector fires - a +# handful of times per run - so its cost and latency never touch the +# fast loop, and every decision it makes is written down with a reason +# (see .decisions and the scheduler's event log). +# +# WHY IT CAN NEVER MAKE THINGS WORSE: every failure path returns None. +# No key, no network, a refusal, an unparseable answer - the scheduler +# behaves exactly as it would with no hook. The model can shorten or +# lengthen a burst; it cannot capture, move, or refocus anything. +# +# THE API CALL IS ONE INJECTABLE FUNCTION (`ask`), so the whole class is +# testable without a key or a network: tests pass a fake `ask` and +# assert on what it was given and how its answer was interpreted. +# ------------------------------------------------------------ + +from __future__ import annotations + +import base64 +import io +import json +import re +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from PIL import Image + +from timelapse.change_detector import ChangeScore + +MODEL = "claude-opus-5" # same as harness/agent.py and harness/mcp_agent.py +MAX_TOKENS = 1024 +PREVIEW_MAX_DIMENSION = 512 # small on purpose: the question is "did the specimen change", not fine detail + +SYSTEM_PROMPT = """\ +You are watching a live-cell time-lapse on a microscope. A change detector has +just flagged the latest frame as different from the frames before it, and the +scheduler has switched to fast burst imaging. You see the frame from just before +the trigger and the frame that fired it, with the detector's numeric reason. + +Decide whether continuing fast imaging is worth the light exposure: +- "extend": the specimen itself is doing something (moving, growing, retracting, + changing shape or internal structure). Keep imaging fast. +- "ignore": the change is not the specimen - debris, a bubble, a focus drift, + an illumination change, noise, or a stage shift. Stop the burst. + +Reply with only a JSON object: {"decision": "extend" | "ignore", "reason": ""}. +""" + +AskFn = Callable[[str, list[dict]], str] # (system_prompt, user_content_blocks) -> model text + + +def _jpeg_b64(path: str | Path, max_dimension: int = PREVIEW_MAX_DIMENSION) -> str: + with Image.open(path) as img: + img = img.convert("RGB") + img.thumbnail((max_dimension, max_dimension)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=80) + return base64.standard_b64encode(buf.getvalue()).decode("ascii") + + +def _image_block(path: str | Path) -> dict: + return {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": _jpeg_b64(path)}} + + +def build_user_content(event: dict, score: ChangeScore) -> list[dict]: + """The user turn: before-frame (if there is one), trigger frame, and the + detector's numbers. Frames are downscaled JPEGs - the full-res files + stay on disk, exactly as get_image() does for its own preview.""" + blocks: list[dict] = [] + previous = event.get("previous_image") + if previous and Path(previous).exists(): + blocks.append({"type": "text", "text": "Frame before the trigger:"}) + blocks.append(_image_block(previous)) + blocks.append({"type": "text", "text": "Frame that fired the detector:"}) + blocks.append(_image_block(event["image"])) + blocks.append({"type": "text", "text": ( + f"Detector reason: {score.reason}. " + f"Pixel change {score.diff_score:.1f}x baseline noise; " + f"foreground area changed by {score.area_delta:+.1%}; " + f"whole-field shift {score.shift_px:.1f}px. " + f"Stage position: {event.get('position')}." + )}) + return blocks + + +def parse_decision(text: str) -> tuple[str | None, str]: + """Pull {"decision": ..., "reason": ...} out of the model's reply. + Tolerates prose around the JSON. Anything unrecognizable -> (None, text).""" + match = re.search(r"\{.*\}", text, re.S) + if not match: + return None, text.strip() + try: + obj = json.loads(match.group(0)) + except json.JSONDecodeError: + return None, text.strip() + decision = str(obj.get("decision", "")).strip().lower() + reason = str(obj.get("reason", "")).strip() + if decision not in ("extend", "ignore"): + return None, reason or text.strip() + return decision, reason + + +def anthropic_ask(system_prompt: str, content: list[dict]) -> str: + """The real call. Constructed lazily so importing this module never + needs the SDK or a key - only actually asking does. Credentials come + from the environment / .env / `ant auth login`, same as the harnesses.""" + import anthropic + from dotenv import load_dotenv + load_dotenv() + client = anthropic.Anthropic() + response = client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + thinking={"type": "adaptive"}, + system=system_prompt, + messages=[{"role": "user", "content": content}], + ) + if response.stop_reason == "refusal": + return "" + return "".join(block.text for block in response.content if block.type == "text") + + +@dataclass +class ModelTrigger: + """on_trigger hook for AdaptiveTimelapse. Call it like the scheduler + does: trigger(event, metadata, score) -> "extend" | "ignore" | None. + + max_calls: hard cap on model calls per run - after it, the hook is a + no-op (returns None). Cost control for an unattended run. + min_interval_s: don't ask again within this many seconds of the last + ask - a burst that keeps extending should not spam the model. + """ + + ask: AskFn = anthropic_ask + max_calls: int = 50 + min_interval_s: float = 30.0 + clock: Callable[[], float] = time.monotonic + log: Callable[[str], None] = print + calls: int = 0 + decisions: list[dict] = field(default_factory=list) + _last_ask_at: float | None = None + + def __call__(self, event: dict, metadata: dict, score: ChangeScore) -> str | None: + now = self.clock() + if self.calls >= self.max_calls: + self._record(event, None, f"skipped: max_calls={self.max_calls} reached") + return None + if self._last_ask_at is not None and now - self._last_ask_at < self.min_interval_s: + self._record(event, None, f"skipped: asked {now - self._last_ask_at:.0f}s ago") + return None + + self.calls += 1 + self._last_ask_at = now + try: + content = build_user_content(event, score) + text = self.ask(SYSTEM_PROMPT, content) + except Exception as exc: # noqa: BLE001 - any failure here must be non-fatal + self._record(event, None, f"model call failed: {type(exc).__name__}: {exc}") + return None + decision, reason = parse_decision(text) + self._record(event, decision, reason if decision else f"unparseable reply: {reason[:120]}") + return decision + + def _record(self, event: dict, decision: str | None, reason: str) -> None: + entry = {"frame_id": event.get("frame_id"), "image": event.get("image"), + "decision": decision, "reason": reason} + self.decisions.append(entry) + self.log(f"[model] frame {entry['frame_id']}: {decision or 'no opinion'} - {reason}") diff --git a/timelapse/scheduler.py b/timelapse/scheduler.py new file mode 100644 index 0000000..c59ccd0 --- /dev/null +++ b/timelapse/scheduler.py @@ -0,0 +1,380 @@ +# scheduler.py +# ------------------------------------------------------------ +# Adaptive time-lapse: image slowly, look for change cheaply, image fast +# when something is happening. +# +# slow mode - one capture every `slow_interval_s` +# burst mode - one capture every `burst_interval_s` for `burst_duration_s` +# after the ChangeDetector fires; each further "interesting" +# frame inside the burst extends it +# back to slow when the burst window runs out +# +# WHY: a fixed-interval time-lapse spends its light and disk budget evenly +# on the boring hours and the interesting minutes alike, and the event you +# care about (a retraction that looked instantaneous at a 20-minute +# interval, say) falls between two frames. This loop puts the fast frames +# where the change is. +# +# WHAT DECIDES: timelapse.change_detector - numpy, no model call, so the +# slow loop costs nothing per frame. An optional `on_trigger` hook runs +# when a burst starts or extends; that is the place to ask a vision +# model "is this worth a longer look?" (return "extend" / "ignore"), or +# to notify someone. The hook is off the fast path: it is called once per +# trigger, not once per frame. timelapse.model_trigger.ModelTrigger is +# the Claude implementation of that hook (--model-trigger on the CLI). +# +# WHAT IT DOES NOT DO: move the stage, refocus, or change exposure. It +# only calls get_image(). A stage/sample shift the detector sees is +# logged as a "shift" event and does NOT start a burst - that is not +# biology, and burst-imaging a drifted field is wasted light. +# +# THE HOOK CAN RUN IN THE BACKGROUND (consult_in_background=True, on by +# default from the CLI): a model call takes seconds, and a burst at a +# short interval must not stall while it thinks. The call is submitted to +# a single worker thread and the capture loop carries on; the answer is +# applied at the next step. "extend" then lengthens whatever burst is +# still running; "ignore" ends it; an answer that lands after the burst +# has already ended is logged as late and changes nothing. One worker, so +# consults are serialized and never pile up. +# +# HARD CAPS (max_captures, max_runtime_s) always end the run, whatever +# mode it is in. On real hardware the run is a single up-front approval +# of the whole plan (see main()): the per-call confirm=True that +# get_image() requires is supplied by this scheduler after that one +# approval, never by anything the model says. Keep it that way. +# +# Log: one JSON line per event in logs/timelapse_events.jsonl (under the +# CONFOCAL_MCP_DATA_DIR data root, same as frame_history.jsonl), so a run +# can be replayed or audited with frame_audit.py --history afterwards. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import json +import sys +import time +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass, field, asdict +from datetime import datetime +from pathlib import Path +from typing import Callable + +from acquisition.paths import logs_dir as _logs_dir +from timelapse.change_detector import ChangeDetector, ChangeScore, load_gray + +CaptureFn = Callable[[], dict] # -> get_image() metadata dict +TriggerFn = Callable[[dict, dict, ChangeScore], str | None] # (event, metadata, score) -> "extend" | "ignore" | None + + +@dataclass +class TimelapseConfig: + backend: str = "mock" + slow_interval_s: float = 60.0 + burst_interval_s: float = 5.0 + burst_duration_s: float = 120.0 + max_captures: int = 500 + max_runtime_s: float = 3600.0 + max_bursts: int | None = None + baseline_frames: int = 5 + diff_threshold: float = 3.0 + area_threshold: float = 0.05 + shift_threshold_px: float = 4.0 + invert: bool = False + preview_max_dimension: int = 256 # passed to get_image - small, the model isn't looking + events_path: Path | None = None # default: /logs/timelapse_events.jsonl + + def validate(self) -> None: + if self.backend not in ("mock", "sdk"): + raise ValueError(f"backend must be 'mock' or 'sdk', got {self.backend!r}") + if self.burst_interval_s <= 0 or self.slow_interval_s <= 0: + raise ValueError("intervals must be > 0") + if self.burst_interval_s > self.slow_interval_s: + raise ValueError("burst_interval_s must not exceed slow_interval_s") + if self.max_captures < 1 or self.max_runtime_s <= 0: + raise ValueError("max_captures and max_runtime_s must be positive") + + def detector(self) -> ChangeDetector: + return ChangeDetector( + baseline_frames=self.baseline_frames, + diff_threshold=self.diff_threshold, + area_threshold=self.area_threshold, + shift_threshold_px=self.shift_threshold_px, + invert=self.invert, + ) + + +def _default_capture(config: TimelapseConfig) -> CaptureFn: + """Capture via loop_tools.get_image(). For backend='sdk' this supplies + confirm=True - main() has already obtained the one-time human approval + for the whole run before this is ever constructed.""" + from mcp_server import loop_tools + + def capture() -> dict: + metadata, _preview = loop_tools.get_image( + backend=config.backend, + confirm=(config.backend == "sdk"), + max_dimension=config.preview_max_dimension, + ) + return metadata + return capture + + +@dataclass +class RunSummary: + captures: int = 0 + bursts: int = 0 + burst_captures: int = 0 + shifts: int = 0 + elapsed_s: float = 0.0 + stop_reason: str = "" + events_path: str = "" + frames: list[dict] = field(default_factory=list) # per-capture event dicts + + +class AdaptiveTimelapse: + def __init__( + self, + config: TimelapseConfig, + capture: CaptureFn | None = None, + on_trigger: TriggerFn | None = None, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + log: Callable[[str], None] = print, + consult_in_background: bool = False, + executor=None, + ): + config.validate() + self.config = config + self.capture = capture or _default_capture(config) + self.on_trigger = on_trigger + self.consult_in_background = consult_in_background + # Anything with .submit(fn, *args) -> Future and .shutdown(); tests + # inject a manual one so replies land exactly when they choose. + self._executor = executor + self._pending: list[tuple[Future, int | None]] = [] + self.clock = clock + self.sleep = sleep + self.log = log + self.detector = config.detector() + self.events_path = Path(config.events_path) if config.events_path else _logs_dir() / "timelapse_events.jsonl" + self.mode = "slow" + self.burst_until: float | None = None + self.summary = RunSummary(events_path=str(self.events_path)) + self._t0: float | None = None + self._previous_image: str | None = None + + # -- state helpers ------------------------------------------------- + def _elapsed(self) -> float: + return self.clock() - (self._t0 if self._t0 is not None else self.clock()) + + def _interval(self) -> float: + return self.config.burst_interval_s if self.mode == "burst" else self.config.slow_interval_s + + def _emit(self, event: dict) -> None: + event = {"at": datetime.now().astimezone().isoformat(timespec="milliseconds"), + "elapsed_s": round(self._elapsed(), 3), "mode": self.mode, **event} + self.events_path.parent.mkdir(parents=True, exist_ok=True) + with open(self.events_path, "a") as f: + f.write(json.dumps(event, default=str) + "\n") + if event["event"] != "capture": + self.log(f"[timelapse +{event['elapsed_s']:.0f}s] {event['event']}: {event.get('detail', '')}") + + # -- one iteration --------------------------------------------------- + def step(self) -> dict: + """Capture one frame, score it, update mode. Returns the capture event.""" + self._apply_pending_consults() + metadata = self.capture() + score = self.detector.score_array(load_gray(metadata["image"])) + now = self.clock() + self.summary.captures += 1 + if self.mode == "burst": + self.summary.burst_captures += 1 + + event = {"event": "capture", "frame_id": metadata.get("frame_id"), "image": metadata.get("image"), + "previous_image": self._previous_image, + "position": metadata.get("position"), "score": score.as_dict()} + self._emit(event) + self.summary.frames.append(event) + self._previous_image = metadata.get("image") + + is_shift = score.shift_px > self.config.shift_threshold_px + if is_shift: + self.summary.shifts += 1 + self._emit({"event": "shift", "frame_id": metadata.get("frame_id"), + "detail": f"{score.shift_px:.1f}px - stage/sample moved, not starting a burst"}) + elif score.interesting: + if self.mode == "slow": + if self.config.max_bursts is not None and self.summary.bursts >= self.config.max_bursts: + self._emit({"event": "burst_skipped", "detail": "max_bursts reached"}) + else: + self.mode = "burst" + self.summary.bursts += 1 + self.burst_until = now + self.config.burst_duration_s + self._emit({"event": "burst_start", "frame_id": metadata.get("frame_id"), "detail": score.reason}) + self._consult(event, metadata, score) + else: + # Never shorten a window the model has already lengthened. + self.burst_until = max(self.burst_until or 0.0, now + self.config.burst_duration_s) + self._emit({"event": "burst_extend", "frame_id": metadata.get("frame_id"), "detail": score.reason}) + self._consult(event, metadata, score) + + self._apply_pending_consults() + if self.mode == "burst" and self.burst_until is not None and now >= self.burst_until: + self._end_burst("burst window elapsed") + return event + + def _consult(self, event: dict, metadata: dict, score: ChangeScore) -> None: + if self.on_trigger is None: + return + if not self.consult_in_background: + self._apply_decision(self.on_trigger(event, metadata, score), event.get("frame_id"), late_ok=False) + return + if self._executor is None: + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="timelapse-consult") + future = self._executor.submit(self.on_trigger, event, metadata, score) + self._pending.append((future, event.get("frame_id"))) + + def _apply_pending_consults(self) -> None: + still_pending = [] + for future, frame_id in self._pending: + if not future.done(): + still_pending.append((future, frame_id)) + continue + exc = future.exception() + if exc is not None: + self._emit({"event": "consult_failed", "frame_id": frame_id, + "detail": f"{type(exc).__name__}: {exc}"}) + continue + self._apply_decision(future.result(), frame_id, late_ok=True) + self._pending = still_pending + + def _apply_decision(self, decision: str | None, frame_id: int | None, late_ok: bool) -> None: + if decision not in ("extend", "ignore"): + return + if self.mode != "burst": + if late_ok: + self._emit({"event": "consult_late", "frame_id": frame_id, + "detail": f"on_trigger said {decision} after the burst had ended - no change"}) + return + if decision == "ignore": + self._end_burst(f"on_trigger said ignore (frame {frame_id})") + elif self.burst_until is not None: + # A full extra window from now or from the current end, whichever is later. + self.burst_until = max(self.burst_until, self.clock()) + self.config.burst_duration_s + self._emit({"event": "burst_extend", "frame_id": frame_id, "detail": "on_trigger said extend"}) + + def _end_burst(self, why: str) -> None: + self.mode = "slow" + self.burst_until = None + self._emit({"event": "burst_end", "detail": why}) + + # -- the loop -------------------------------------------------------- + def run(self) -> RunSummary: + self._t0 = self.clock() + self._emit({"event": "start", "detail": json.dumps(asdict(self.config), default=str)}) + next_at = self._t0 + try: + while True: + if self.summary.captures >= self.config.max_captures: + self.summary.stop_reason = "max_captures" + break + if self._elapsed() >= self.config.max_runtime_s: + self.summary.stop_reason = "max_runtime" + break + wait = next_at - self.clock() + if wait > 0: + self.sleep(wait) + self.step() + # Schedule from the intended slot, not from "now", so slow + # captures don't drift; but never queue up a backlog. + next_at = max(next_at + self._interval(), self.clock()) + if self.mode == "burst" and self.burst_until is not None: + next_at = min(next_at, self.burst_until) + except KeyboardInterrupt: + self.summary.stop_reason = "interrupted" + finally: + if self._executor is not None: + # A reply that lands after the run is over is worthless; don't wait for it. + self._executor.shutdown(wait=False, cancel_futures=True) + if self._pending: + self._emit({"event": "consult_abandoned", "detail": f"{len(self._pending)} pending at stop"}) + self._pending = [] + if self.mode == "burst": + self._end_burst("run ended") + self.summary.elapsed_s = self._elapsed() + self._emit({"event": "stop", "detail": f"{self.summary.stop_reason}; {self.summary.captures} captures, " + f"{self.summary.bursts} bursts, {self.summary.shifts} shifts"}) + return self.summary + + +# -- CLI ---------------------------------------------------------------------- + +def _plan_text(config: TimelapseConfig) -> str: + slow_only = config.max_runtime_s / config.slow_interval_s + return ( + f" backend {config.backend}{' (REAL HARDWARE)' if config.backend == 'sdk' else ''}\n" + f" slow interval {config.slow_interval_s:g}s (~{slow_only:.0f} captures if nothing happens)\n" + f" burst every {config.burst_interval_s:g}s for {config.burst_duration_s:g}s per trigger" + f"{'' if config.max_bursts is None else f', at most {config.max_bursts} bursts'}\n" + f" hard caps {config.max_captures} captures, {config.max_runtime_s:g}s runtime\n" + f" thresholds diff>{config.diff_threshold:g}x noise, |area|>{config.area_threshold:.0%}, " + f"shift>{config.shift_threshold_px:g}px{', inverted (bright-on-dark)' if config.invert else ''}\n" + ) + + +def _approve_real_hardware(config: TimelapseConfig) -> bool: + print("\n[HARDWARE GATE] This run will fire the REAL camera repeatedly, unattended, with this plan:") + print(_plan_text(config)) + answer = input("Approve this whole acquisition plan? [y/N]: ").strip().lower() + return answer in ("y", "yes") + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Adaptive time-lapse: slow captures, burst mode on detected change.") + ap.add_argument("--backend", choices=["mock", "sdk"], default="mock") + ap.add_argument("--slow", type=float, default=60.0, help="slow-mode interval, seconds") + ap.add_argument("--burst", type=float, default=5.0, help="burst-mode interval, seconds") + ap.add_argument("--burst-duration", type=float, default=120.0, help="seconds of burst per trigger") + ap.add_argument("--max-captures", type=int, default=500) + ap.add_argument("--max-runtime", type=float, default=3600.0, help="seconds") + ap.add_argument("--max-bursts", type=int, default=None) + ap.add_argument("--baseline-frames", type=int, default=5) + ap.add_argument("--diff-threshold", type=float, default=3.0) + ap.add_argument("--area-threshold", type=float, default=0.05) + ap.add_argument("--shift-px", type=float, default=4.0) + ap.add_argument("--invert", action="store_true", help="bright specimen on dark background (fluorescence)") + ap.add_argument("--events", type=Path, default=None, help="events JSONL path (default: logs/timelapse_events.jsonl)") + ap.add_argument("--dry-run", action="store_true", help="print the plan and exit without capturing") + ap.add_argument("--model-trigger", action="store_true", + help="ask Claude at each trigger whether to extend or end the burst (needs API credentials)") + ap.add_argument("--model-max-calls", type=int, default=50, help="cap on model calls per run") + args = ap.parse_args(argv) + + config = TimelapseConfig( + backend=args.backend, slow_interval_s=args.slow, burst_interval_s=args.burst, + burst_duration_s=args.burst_duration, max_captures=args.max_captures, max_runtime_s=args.max_runtime, + max_bursts=args.max_bursts, baseline_frames=args.baseline_frames, diff_threshold=args.diff_threshold, + area_threshold=args.area_threshold, shift_threshold_px=args.shift_px, invert=args.invert, + events_path=args.events, + ) + config.validate() + if args.dry_run: + print(_plan_text(config)) + return 0 + if config.backend == "sdk" and not _approve_real_hardware(config): + print("Not approved. Nothing captured.") + return 2 + + on_trigger = None + if args.model_trigger: + from timelapse.model_trigger import ModelTrigger + on_trigger = ModelTrigger(max_calls=args.model_max_calls) + summary = AdaptiveTimelapse(config, on_trigger=on_trigger, consult_in_background=True).run() + print(f"done: {summary.stop_reason}; {summary.captures} captures ({summary.burst_captures} in {summary.bursts} bursts), " + f"{summary.shifts} shifts, {summary.elapsed_s:.0f}s. Events: {summary.events_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())