From 7121a2654e3a623c328f899b48afb09cb5a53390 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Sun, 13 Sep 2026 00:21:39 -0400 Subject: [PATCH 01/42] Capture illumination in optical config; add Ti2 inventory/config tools get_optical_configuration() never recorded the illumination state. It walked OPTICAL_CONFIG_PROPERTIES, a hardcoded list that omitted iDIA_LAMP_Switch and iDIA_LAMP_Pos, so a saved configuration did not say whether the transmitted lamp was on - the one setting that decides whether a camera on the camera port sees anything at all. The iDLED* names the list did carry are ignored by this microscope, whose D-LEDI is not driven through the Ti2 body, so every write to them vanished without an error. Both methods now use the SDK's own DataGet, which fills an INikonTi2AxData with all ~90 properties in one call, keyed by the writable i* names. The hardcoded list is deleted. Pass None as DataGet's first argument - it is declared [in,out] and win32com returns the filled object; the NikonTi2AxData coclass is not registered, so constructing one is a dead end. apply_optical_configuration() gains include_motion=False. The snapshot is now the full device set rather than a curated subset, so without that guard restoring a lamp setting would also drive the stage. Add acquisition/calibration/: - ti2_inventory.py lists every device with its range, unit and Control value, discovering field names from the COM type library and grouping by Control rather than any list written down here. Control is the practical guide to what can be driven: -1 devices (D-LEDI, DIA/EPI/AUX shutters, Intensilight, TIRF, LAPP) reliably ignore writes, >= 0 usually accepts, with measured exceptions (iDIC_PRISM, iTURRET2SHUTTER). Enabled is not a fitted-hardware flag despite the name - it reads True for all 88 devices. - ti2_config.py gives save/show/diff/apply from the command line. apply requires --confirm, skips motion devices unless --include-motion, and polls the read-back instead of reading once, because several devices report the old value for up to a second after a write that did take. Also fixes acquisition/paths.py: runtime data no longer falls back to an unwritable working directory. MCP clients choose that directory and Claude Desktop on Windows uses C:\WINDOWS\system32, so with CONFOCAL_MCP_DATA_DIR unset the first get_image() failed with an access denied error that read as a camera fault. --- CHANGELOG.md | 41 +++ README.md | 19 +- acquisition/backends/nis_sdk.py | 117 +++++---- acquisition/calibration/__init__.py | 0 acquisition/calibration/ti2_config.py | 317 +++++++++++++++++++++++ acquisition/calibration/ti2_inventory.py | 186 +++++++++++++ acquisition/paths.py | 89 ++++++- 7 files changed, 706 insertions(+), 63 deletions(-) create mode 100644 acquisition/calibration/__init__.py create mode 100644 acquisition/calibration/ti2_config.py create mode 100644 acquisition/calibration/ti2_inventory.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a54e519..76bb3ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,47 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow ## [Unreleased] +### Added +- `acquisition/calibration/ti2_inventory.py` - read-only inventory of every + device the Ti2 reports, with each one's valid range, unit, and `Control` + value. Field names come from the COM type library at runtime and rows are + grouped by the microscope's own `Control` value, so there is no device list + to keep in sync. `Control` is the practical guide to what can be driven: + `-1` devices (the D-LEDI channels, the DIA/EPI/AUX shutters, Intensilight, + TIRF, LAPP) reliably ignore writes; `>= 0` usually accepts, with measured + exceptions (`iDIC_PRISM`, `iTURRET2SHUTTER`). Note `Enabled` is *not* a + fitted-hardware flag - it reads True for all 88 devices. +- `acquisition/calibration/ti2_config.py` - `save` / `show` / `diff` / `apply` + for the full device configuration from the command line. `apply` requires + `--confirm`, skips stage/objective/TIRF unless `--include-motion`, and polls + the read-back rather than reading once (several devices report the old value + for up to a second after a successful write). + +### Fixed +- `get_optical_configuration()` recorded no illumination state. It walked + `OPTICAL_CONFIG_PROPERTIES`, a hardcoded list that omitted + `iDIA_LAMP_Switch`/`iDIA_LAMP_Pos` entirely - so a saved configuration never + captured whether the transmitted lamp was on, the one setting that decides + whether a camera on the camera port sees anything. The `iDLED*` names it did + list are ignored by this microscope, whose D-LEDI is not driven through the + Ti2 body. Both it and `apply_optical_configuration()` now use the SDK's own + `DataGet`, which returns all ~90 properties in one call, and the hardcoded + list is gone. `apply_optical_configuration()` gains `include_motion=False`: + the snapshot is now the full device set, so without that guard restoring a + lamp setting would also drive the stage. + +- Runtime data no longer falls back to an unwritable working directory. MCP + clients choose the working directory their servers are launched with, and + Claude Desktop on Windows uses `C:\WINDOWS\system32` - so with + `CONFOCAL_MCP_DATA_DIR` unset, the first `get_image()` failed with an + "access is denied" `OSError` that read as a camera fault, and under an + elevated process would instead have written captures into a system + directory. `acquisition/paths.py` now rejects a working directory that is + inside the Windows directory or that it cannot create a file in, falling + back to `~/.confocal-mcp` with a warning on stderr. Setting + `CONFOCAL_MCP_DATA_DIR` is still honoured as-is, and a normal source + checkout still resolves to the checkout directory. + ## [0.1.0] - 2026-08-31 First packaged release. diff --git a/README.md b/README.md index ec86f72..7159433 100644 --- a/README.md +++ b/README.md @@ -99,9 +99,22 @@ project-level `.mcp.json` for Claude Code: { "mcpServers": { "confocal": { "command": "confocal-mcp" } } } ``` -Data (captures, move/frame history logs) is written under the current working -directory when the server runs from an installed package - launch it from a -stable location. +Data (captures, move/frame history logs, saved positions) is written under the +current working directory - so launch the server from a stable location, or set +`CONFOCAL_MCP_DATA_DIR` to pin it explicitly: + +```json +{ "mcpServers": { "confocal": { + "command": "confocal-mcp", + "env": { "CONFOCAL_MCP_DATA_DIR": "D:\\path\\to\\your\\data" } +} } } +``` + +Setting it is worth doing for any MCP client, because the client picks the +working directory and the caller has no say in it - Claude Desktop on Windows +launches servers in `C:\WINDOWS\system32`. When the working directory is +unusable like that, the server falls back to `~/.confocal-mcp` and says so on +stderr rather than failing at the first capture. ## Harness loop (`harness/agent.py`) diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index 9cc17fc..9be91a5 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -67,28 +67,15 @@ MAX_XY_STEP_UM = 5000.0 MAX_Z_STEP_UM = 50.0 -# Direct scalar properties making up an "optical configuration" (objective, -# filters, light path, illumination) - confirmed present in NkTi2Ax.py's -# INikonTi2AxMicroscope interface, same source as the stage properties -# above. Used by get_optical_configuration()/apply_optical_configuration() -# below - see acquisition/orchestration/imaging_profile.py for the -# save/list/apply-by-experiment-name layer on top of these two methods. -OPTICAL_CONFIG_PROPERTIES = [ - "iNOSEPIECE", # objective (turret slot 1-6) - "iDIC_PRISM", - "iDIC_POLARIZER", - "iANALYZER_POS", - "iANALYZER_SLOT", - "iLIGHTPATH", - "iCONDENSER", - "iOPTZOOM", - "iTURRET1POS", "iTURRET1SHUTTER", - "iTURRET2POS", "iTURRET2SHUTTER", - "iDLED1_POS", "iDLED1_SWITCH", - "iDLED2_POS", "iDLED2_SWITCH", - "iDLED3_POS", "iDLED3_SWITCH", - "iDLED4_POS", "iDLED4_SWITCH", -] +# There is deliberately no hardcoded list of "optical configuration" +# properties here any more. One used to live at this spot +# (OPTICAL_CONFIG_PROPERTIES) and it silently omitted iDIA_LAMP_Switch and +# iDIA_LAMP_Pos, so get_optical_configuration() never recorded whether the +# transmitted lamp was on - and every write to the iDLED* names it did list +# was ignored, because this microscope's D-LEDI is not driven through the +# Ti2 body. Both methods now enumerate whatever DataGet returns. See +# acquisition/calibration/ti2_inventory.py for what the interface actually +# reports per device, including which ones accept writes. # PFS offset units aren't calibrated to microns (unlike XY/Z - see the # module docstring), so nudge_pfs_offset's cap is expressed as a fraction @@ -387,50 +374,76 @@ def do_nudge(m): # way an uncapped Z move is. def get_optical_configuration(self) -> dict: - """Snapshot the current optical/device configuration - objective - (nosepiece), DIC prism/polarizer, analyzer, light path, condenser, - zoom, both filter turrets, and the 4 D-LEDI illumination channels - - as a dict of raw SDK property values. Save this (e.g. to a JSON - file) and pass it to apply_optical_configuration() later to - reproduce the same setup. - - Property semantics (e.g. whether iDLED1_POS is an intensity - percent or something else) are NOT independently calibrated the - way XY/Z/PFS units were (see this module's docstring for that - methodology) - this assumes reading then writing back the same - raw value reproduces the same NIS-Elements UI state, which is - reasonable since these are exactly the properties NIS's own - "Optical Configuration" presets are built from, but it hasn't - been verified against a known physical reference the way stage - position was. + """Snapshot every device property the microscope reports, as a dict + of raw SDK values keyed by property name (iLIGHTPATH, + iDIA_LAMP_Switch, iNOSEPIECE, ...). Save this (e.g. to a JSON file) + and pass it to apply_optical_configuration() to reproduce the setup. + + Taken with one DataGet call, which fills an INikonTi2AxData object + with all ~90 properties at once. Pass None as its first argument - + it is declared [in,out] and win32com returns the filled object. + Constructing the object yourself does not work: the NikonTi2AxData + coclass is not registered (CoCreateInstance gives "Class not + registered"). + + This used to walk OPTICAL_CONFIG_PROPERTIES, a hardcoded list that + omitted iDIA_LAMP_Switch and iDIA_LAMP_Pos - so a saved config did + not record whether the transmitted lamp was on, which is the single + setting deciding whether a camera on the camera port sees anything + at all. Enumerating whatever DataGet returns removes the list, and + with it the chance of it drifting out of sync again. + + Property semantics are NOT independently calibrated the way XY/Z + units were (see this module's docstring) - this assumes reading then + writing back the same raw value reproduces the same state. """ def read(m): - return {name: getattr(m, name) for name in OPTICAL_CONFIG_PROPERTIES} + data = m.DataGet(None, 0) + values = {} + for name in sorted(getattr(type(data), "_prop_map_get_", {}) or {}): + try: + values[name] = getattr(data, name) + except Exception: + continue + return values return self._thread.call(read) - def apply_optical_configuration(self, config: dict) -> dict: + def apply_optical_configuration(self, config: dict, + include_motion: bool = False) -> dict: """Write back a config dict from get_optical_configuration(). - Only keys in OPTICAL_CONFIG_PROPERTIES are applied - unknown keys - (e.g. from a config saved by a newer version of this code) are - silently ignored rather than erroring. Returns the read-back - value of every property actually applied. - - Switching the objective (iNOSEPIECE) changes working distance, - which may invalidate prior Z-safety assumptions for whatever - position you're at - this method never touches Z itself, that's - on the caller to handle deliberately afterward if needed. + Returns the read-back value of every property actually applied. + Properties that raise are recorded as None rather than aborting the + rest, and a write the microscope declines to act on shows up as a + read-back that differs from the requested value - the Ti2 ignores + such writes silently rather than raising (the D-LEDI channels and + the DIA/EPI shutters do this consistently on this workstation). + + Motion devices - stage XY/Z, the nosepiece, TIRF - are skipped + unless include_motion=True. get_optical_configuration() now returns + the full device set rather than a curated subset, so without this + guard restoring a lamp setting from a file would also drive the + stage across the slide as a side effect. Switching the objective + also changes working distance, which can invalidate Z-safety + assumptions for the current position; this method never moves Z on + its own unless you opt in. """ def write(m): applied = {} for name, value in config.items(): - if name not in OPTICAL_CONFIG_PROPERTIES: + if not include_motion and any( + key in name.upper() + for key in ("XPOSITION", "YPOSITION", "ZPOSITION", + "NOSEPIECE", "TIRF", "ZESCAPE", "RESET")): continue - setattr(m, name, value) - applied[name] = getattr(m, name) + try: + setattr(m, name, value) + applied[name] = getattr(m, name) + except Exception: + applied[name] = None return applied return self._thread.call(write) diff --git a/acquisition/calibration/__init__.py b/acquisition/calibration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/acquisition/calibration/ti2_config.py b/acquisition/calibration/ti2_config.py new file mode 100644 index 0000000..41889f9 --- /dev/null +++ b/acquisition/calibration/ti2_config.py @@ -0,0 +1,317 @@ +# ti2_config.py +# ------------------------------------------------------------ +# Save and restore the Ti2's device configuration from the command line. +# +# python -m acquisition.calibration.ti2_config save +# python -m acquisition.calibration.ti2_config save --out protocols/brightfield.json +# python -m acquisition.calibration.ti2_config show protocols/brightfield.json +# python -m acquisition.calibration.ti2_config diff protocols/brightfield.json +# python -m acquisition.calibration.ti2_config apply protocols/brightfield.json --confirm +# +# HOW THE SNAPSHOT IS TAKEN: INikonTi2AxMicroscope.DataGet fills an +# INikonTi2AxData object with all 90 device properties in one COM call, and +# it uses the writable i* names (iLIGHTPATH, iDIA_LAMP_Switch, ...) - so a +# saved file can be written straight back with setattr, no name mapping. +# +# data = microscope.DataGet(None, 0) +# +# Pass None for the first argument: it is declared [in,out] (VT_BYREF| +# VT_DISPATCH) and win32com returns the filled object. Do not try to +# construct the object yourself - the NikonTi2AxData coclass is not +# registered on this workstation (CoCreateInstance gives "Class not +# registered"), which is a dead end DataGet sidesteps entirely. +# +# WHY NOT NISSdk.get_optical_configuration(): it loops over +# OPTICAL_CONFIG_PROPERTIES, a hand-maintained list in nis_sdk.py that +# omits iDIA_LAMP_Switch and iDIA_LAMP_Pos. A configuration saved through +# it does not record whether the transmitted lamp was on - the one setting +# that decides whether the camera sees anything at all. +# +# WHY NOT DataSet FOR RESTORE: DataSet(newVal, ...) writes the whole device +# set in one call, so restoring a lamp setting would also drive the stage, +# Z and the nosepiece as an unavoidable side effect. apply() writes +# per-property instead, which allows skipping motion devices and reporting +# each write individually. --bulk opts into DataSet when that is what you +# actually want. +# +# Control: each Setting reports a Control value. It is a useful filter but +# NOT a guarantee, and the asymmetry matters: +# +# Control = -1 reliably refuses. The D-LEDI channels, the DIA/EPI/AUX +# shutters, Intensilight, TIRF and LAPP all sit here, and +# every write to them was silently ignored. Worth skipping +# rather than issuing writes that vanish without an error. +# Control >= 0 usually accepts, but not always. Measured exceptions on +# this Ti2-E: iDIC_PRISM (Control=0) and iTURRET2SHUTTER +# (Control=1) both refuse writes that never take, even +# given seconds to settle. Something - an interlock, or +# NIS-Elements holding those devices - overrides them. +# +# So treat a Control >= 0 "not-applied" as real, and check the device +# rather than assuming the report is a timing artifact. +# +# SAFETY: save/show/diff are read-only. apply moves real hardware, needs +# --confirm, and skips the motion devices unless --include-motion. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import datetime +import json +import re +import time +from pathlib import Path +from typing import Any + +from acquisition.calibration.ti2_inventory import inventory + +#: Some devices do not report a new value the instant the write returns - +#: the DIC prism and turret shutters are mechanical, the DIA lamp settles. +#: Reading back immediately made apply report DiaLampPos, DicPrism and +#: Turret2Shutter as failures when all three had in fact applied. Poll +#: instead of trusting one immediate read. +SETTLE_SECONDS = 1.5 +SETTLE_POLL_SECONDS = 0.25 + +#: Devices that physically move something substantial; skipped by apply +#: unless --include-motion. Matched as substrings of the i* property name. +_MOTION_PROPERTIES = ("XPOSITION", "YPOSITION", "ZPOSITION", "ZEscape", + "ZReset", "XReset", "YReset", "NOSEPIECE", "Tirf") + + +def _is_motion(prop: str) -> bool: + return any(key.lower() in prop.lower() for key in _MOTION_PROPERTIES) + + +def _normalise(name: str) -> str: + """Fold a property name to a comparable key. + + Only needed to join DataGet's i* names to the friendly names Settings + reports (iDIA_LAMP_Switch <-> DiaLampSwitch), so that a snapshot can + carry each property's Control value and valid range alongside it. + """ + stripped = name[1:] if name.startswith("i") else name + return re.sub(r"[^a-z0-9]", "", stripped.lower()) + + +def _data_properties(data: Any) -> list[str]: + """Property names on the INikonTi2AxData object, from the type library.""" + return sorted(getattr(type(data), "_prop_map_get_", {}) or {}) + + +def save_config(sdk: Any = None) -> dict: + """Snapshot every device property via DataGet, plus Settings metadata. + + Read-only. Values come from one DataGet call; Control/range/unit are + joined on from INikonTi2AxMicroscope.Settings so the saved file records + what can be written back and within what limits. + """ + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + + def snapshot(microscope: Any) -> dict: + data = microscope.DataGet(None, 0) + values = {} + for prop in _data_properties(data): + try: + values[prop] = getattr(data, prop) + except Exception: + continue + return values + + values = sdk._thread.call(snapshot) + meta = {_normalise(str(row.get("Name"))): row for row in inventory(sdk) if row.get("Name")} + + properties = {} + for prop, value in sorted(values.items()): + row = meta.get(_normalise(prop), {}) + properties[prop] = { + "value": value, + "control": row.get("Control"), + "lower": row.get("Lower"), + "higher": row.get("Higher"), + "unit": row.get("Unit"), + "friendly_name": row.get("Name"), + } + return { + "saved_at": datetime.datetime.now().isoformat(timespec="seconds"), + "property_count": len(properties), + "properties": properties, + } + + +def _writable(entry: dict) -> bool: + control = entry.get("control") + return control is not None and control >= 0 + + +def format_config(config: dict, only_writable: bool = False) -> str: + props = config.get("properties", {}) + lines = ["saved_at: %s (%d properties)" % (config.get("saved_at", "?"), len(props)), ""] + lines.append(" %-24s %10s %8s %s" % ("PROPERTY", "VALUE", "CONTROL", "RANGE")) + for prop, entry in sorted(props.items()): + if only_writable and not _writable(entry): + continue + rng = "" if entry.get("lower") is None else "%s .. %s" % (entry["lower"], entry["higher"]) + lines.append(" %-24s %10s %8s %s" + % (prop, entry.get("value"), entry.get("control"), rng)) + return "\n".join(lines) + + +def diff_config(config: dict, sdk: Any = None) -> list[dict]: + """Properties whose live value differs from the saved one. Read-only.""" + live = save_config(sdk)["properties"] + rows = [] + for prop, entry in sorted(config.get("properties", {}).items()): + now = live.get(prop, {}).get("value") + if now != entry.get("value"): + rows.append({ + "property": prop, "saved": entry.get("value"), "now": now, + "control": entry.get("control"), "motion": _is_motion(prop), + }) + return rows + + +def _write_and_settle(microscope: Any, prop: str, value: Any) -> tuple[Any, Any]: + """setattr, then poll the read-back until it matches or SETTLE_SECONDS. + + Returns (before, readback). Polling rather than one immediate read is + the whole point - see SETTLE_SECONDS. + """ + before = getattr(microscope, prop) + setattr(microscope, prop, value) + deadline = time.monotonic() + SETTLE_SECONDS + readback = getattr(microscope, prop) + while readback != value and time.monotonic() < deadline: + time.sleep(SETTLE_POLL_SECONDS) + readback = getattr(microscope, prop) + return before, readback + + +def apply_config(config: dict, sdk: Any = None, include_motion: bool = False, + bulk: bool = False) -> dict: + """Write a saved config back. Real hardware - see this module's header.""" + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + + props = config.get("properties", {}) + + def run(microscope: Any) -> dict: + if bulk: + data = microscope.DataGet(None, 0) + for prop, entry in props.items(): + try: + setattr(data, prop, entry.get("value")) + except Exception: + continue + microscope.DataSet(data, 0, 1) + return {"_bulk": {"status": "DataSet issued", "properties": len(props)}} + + report: dict[str, dict] = {} + for prop, entry in sorted(props.items()): + value = entry.get("value") + if not _writable(entry): + report[prop] = {"status": "skipped", "why": "control=%s" % entry.get("control")} + continue + if not include_motion and _is_motion(prop): + report[prop] = {"status": "skipped", "why": "motion device"} + continue + try: + if getattr(microscope, prop) == value: + report[prop] = {"status": "unchanged", "value": value} + continue + before, readback = _write_and_settle(microscope, prop, value) + report[prop] = { + "status": "written" if readback == value else "not-applied", + "from": before, "to": value, "readback": readback, + } + except Exception as exc: + report[prop] = {"status": "error", + "error": "%s: %s" % (type(exc).__name__, exc)} + return report + + return sdk._thread.call(run) + + +def _default_out() -> Path: + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + return Path("protocols") / ("optical_config_%s.json" % stamp) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Save and restore the Ti2 device configuration.") + sub = parser.add_subparsers(dest="command", required=True) + + p_save = sub.add_parser("save", help="snapshot the configuration (read-only)") + p_save.add_argument("--out", type=Path, default=None) + + p_show = sub.add_parser("show", help="print a saved configuration file") + p_show.add_argument("path", type=Path) + p_show.add_argument("--writable", action="store_true", + help="only properties this SDK can drive (control >= 0)") + + p_diff = sub.add_parser("diff", help="compare a saved file against the live scope") + p_diff.add_argument("path", type=Path) + + p_apply = sub.add_parser("apply", help="write a saved configuration back") + p_apply.add_argument("path", type=Path) + p_apply.add_argument("--confirm", action="store_true", + help="required: this moves real hardware") + p_apply.add_argument("--include-motion", action="store_true", + help="also restore stage/objective/TIRF positions") + p_apply.add_argument("--bulk", action="store_true", + help="use DataSet to write everything in one call " + "(ignores --include-motion; moves the stage)") + + args = parser.parse_args() + + if args.command == "save": + config = save_config() + out = args.out or _default_out() + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(config, indent=2, default=str), encoding="utf-8") + print(format_config(config)) + print("\nwrote %s" % out) + + elif args.command == "show": + config = json.loads(args.path.read_text(encoding="utf-8")) + print(format_config(config, only_writable=args.writable)) + + elif args.command == "diff": + config = json.loads(args.path.read_text(encoding="utf-8")) + rows = diff_config(config) + if not rows: + print("no differences - live scope matches %s" % args.path) + else: + print(" %-24s %10s %10s %8s %s" + % ("PROPERTY", "SAVED", "NOW", "CONTROL", "RESTORABLE?")) + for row in rows: + if row["control"] is None or row["control"] < 0: + verdict = "no (control=%s)" % row["control"] + elif row["motion"]: + verdict = "only with --include-motion" + else: + verdict = "yes" + print(" %-24s %10s %10s %8s %s" + % (row["property"], row["saved"], row["now"], row["control"], verdict)) + print("\n%d propert%s differ" % (len(rows), "y" if len(rows) == 1 else "ies")) + + elif args.command == "apply": + if not args.confirm: + raise SystemExit("apply moves real hardware - re-run with --confirm") + config = json.loads(args.path.read_text(encoding="utf-8")) + report = apply_config(config, include_motion=args.include_motion, bulk=args.bulk) + counts: dict[str, int] = {} + for prop, result in sorted(report.items()): + counts[result["status"]] = counts.get(result["status"], 0) + 1 + if result["status"] in ("written", "not-applied", "error", "DataSet issued"): + print(" %-24s %s" % (prop, result)) + print("\n" + ", ".join("%s: %d" % kv for kv in sorted(counts.items()))) + + +if __name__ == "__main__": + main() diff --git a/acquisition/calibration/ti2_inventory.py b/acquisition/calibration/ti2_inventory.py new file mode 100644 index 0000000..a099799 --- /dev/null +++ b/acquisition/calibration/ti2_inventory.py @@ -0,0 +1,186 @@ +# ti2_inventory.py +# ------------------------------------------------------------ +# What this Ti2 reports about its own devices, straight from the SDK. +# +# python -m acquisition.calibration.ti2_inventory +# python -m acquisition.calibration.ti2_inventory --json out.json +# python -m acquisition.calibration.ti2_inventory --columns Name,Value,Control +# +# WHY THIS EXISTS: the Ti2 COM interface exposes ~88 writable i* properties +# covering every device a Ti2 *could* have - TIRF stages, LAPP branches, +# filter wheels, four D-LEDI channels, Intensilight, and so on. A write the +# microscope will not act on is silently ignored: no exception, no error, +# the read-back simply keeps the old value. So write-then-read-back cannot +# tell "device not under Ti2 control" from "value refused" from "write +# worked and something else reset it" - which is the hole a long dark-image +# hunt falls into. +# +# INikonTi2AxMicroscope.Settings is the metadata side of those properties: +# a tuple of INikonTi2AxSetting objects, one per device. +# +# Control The field that predicts whether a write will take. Measured +# against this Ti2-E: Control >= 0 accepted, Control = -1 was +# refused, for 7 of the 8 devices actually written to during +# the session that produced this module. The exception was +# Turret2Shutter (Control=1, refused at the time), so treat +# this as a strong signal rather than a guarantee. +# Enabled NOT a fitted-hardware flag, despite the name: it reads True +# for all 88 devices, including every one that refuses writes. +# Read it as "the SDK models this device". +# Lower/Higher valid range, so "which iLIGHTPATH values exist?" is a +# lookup (1-4) rather than a sweep, and "iDIA_LAMP_Pos = +# 2100 out of what?" is answerable (0-2100, i.e. maximum). +# Unit/Scale units where the SDK declares them (um for the stages). +# +# Field names are discovered from the COM type library at runtime, and rows +# are grouped by the Control value the microscope itself reports - nothing +# about the device set is written down here. That is deliberate: a +# hardcoded property list (OPTICAL_CONFIG_PROPERTIES in nis_sdk.py) is what +# silently dropped every illumination setting from +# get_optical_configuration() in the first place, and a hardcoded list here +# would rot the same way against a different Ti2 or a newer SDK. +# +# Read-only and side-effect free - it moves nothing and changes no state, +# so it is safe to run against live hardware mid-experiment. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import json +from typing import Any + +#: Zero-argument accessor methods on INikonTi2AxSetting that are safe to call. +#: An allowlist rather than "call everything callable", because the same +#: interface exposes mutators (SetLongName/SetShortName) and methods needing +#: arguments (ConvertDev2Phys, ConvertPhys2Dev, GetConvertParams). Plain +#: properties are not listed anywhere - those come from the type library. +_SAFE_ACCESSOR_METHODS = ("LongName", "ShortName") + +#: Shown by default. Any field present on the Setting can be asked for with +#: --columns; this is a display choice, not a claim about what exists. +_DEFAULT_COLUMNS = ("Name", "Value", "Lower", "Higher", "Unit", "Enabled") + + +def _setting_fields(setting: Any) -> list[str]: + """Field names for one Setting, read from the COM type library. + + win32com's generated wrapper records every readable property of the + interface in ``_prop_map_get_``, so reading that gives whatever this + SDK version actually exposes - including fields added after this module + was written. + """ + cls = type(setting) + names = sorted(getattr(cls, "_prop_map_get_", {}) or {}) + names += [m for m in _SAFE_ACCESSOR_METHODS if hasattr(cls, m)] + return names + + +def _read_field(setting: Any, name: str) -> Any: + """Read one field, calling it if it is one of the accessor methods. + + Returns None on failure - one field that raises should not cost us the + other eleven. + """ + try: + value = getattr(setting, name) + except Exception: + return None + if callable(value): + try: + value = value() + except Exception: + return None + return value + + +def _read_settings(microscope: Any) -> list[dict]: + """Snapshot every Setting. Runs on the COM thread - see inventory().""" + try: + settings = microscope.Settings + except Exception as exc: + return [{"_error": "%s: %s" % (type(exc).__name__, exc)}] + rows = [] + for index, setting in enumerate(settings): + row = {"index": index} + for name in _setting_fields(setting): + row[name] = _read_field(setting, name) + rows.append(row) + return rows + + +def inventory(sdk: Any = None) -> list[dict]: + """Every device Setting the microscope reports, as a list of dicts. + + Pass an existing NISSdk to reuse its COM thread; omit it to build one. + NISSdk is imported lazily so this module stays importable without + pywin32 or the Ti2 SDK installed (same pattern as the lazy backend + imports in mcp_server/loop_tools.py). + """ + if sdk is None: + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + return sdk._thread.call(_read_settings) + + +def _control_label(control: Any) -> str: + """Section heading for one Control value - see this module's header.""" + if control is None: + return "Control unknown" + if control < 0: + return "Control = %s (writes refused on every device tested)" % control + return "Control = %s (writes took effect)" % control + + +def format_inventory(rows: list[dict], columns: list[str] | None = None) -> str: + """Render inventory() grouped by the Control value the microscope reports.""" + if rows and "_error" in rows[0]: + return "could not read Settings: " + rows[0]["_error"] + + columns = list(columns or _DEFAULT_COLUMNS) + widths = { + col: max([len(col)] + [len(str(r.get(col, "") or "")) for r in rows]) + for col in columns + } + + def row_text(values: dict) -> str: + return " " + " ".join( + str(values.get(col, "") if values.get(col) is not None else "").ljust(widths[col]) + for col in columns + ) + + lines = [] + for control in sorted({r.get("Control") for r in rows}, key=lambda c: (c is None, c)): + group = [r for r in rows if r.get("Control") == control] + lines.append("[%s] %d devices" % (_control_label(control), len(group))) + lines.append(row_text({col: col for col in columns})) + for row in sorted(group, key=lambda r: str(r.get("Name") or "")): + lines.append(row_text(row)) + lines.append("") + + discovered = sorted(set(rows[0]) - {"index"}) if rows else [] + lines.append("%d devices. Fields discovered from the type library: %s" + % (len(rows), ", ".join(discovered))) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Inventory the Ti2's device settings (read-only).") + parser.add_argument("--json", metavar="PATH", + help="also write the full raw inventory to this JSON file") + parser.add_argument("--columns", metavar="A,B,C", + help="comma-separated fields to show instead of the default") + args = parser.parse_args() + + rows = inventory() + columns = args.columns.split(",") if args.columns else None + print(format_inventory(rows, columns=columns)) + if args.json: + with open(args.json, "w", encoding="utf-8") as handle: + json.dump(rows, handle, indent=2, default=str) + print("\nwrote %s" % args.json) + + +if __name__ == "__main__": + main() diff --git a/acquisition/paths.py b/acquisition/paths.py index b1001e1..1e673e6 100644 --- a/acquisition/paths.py +++ b/acquisition/paths.py @@ -1,25 +1,98 @@ # paths.py # ------------------------------------------------------------ # Where the server writes runtime data (captured frames, move/frame history -# logs). Anchored to the *working directory*, not to __file__ - once this is -# installed as a package, __file__ lives in site-packages and must never be -# written to. +# logs, saved stage positions). Anchored to the *working directory*, not to +# __file__ - once this is installed as a package, __file__ lives in +# site-packages and must never be written to. # -# CONFOCAL_MCP_DATA_DIR if set, the base directory to use -# (unset) the current working directory +# CONFOCAL_MCP_DATA_DIR if set, the base directory to use - always +# honoured as-is, no second-guessing +# (unset) the current working directory, if it is a usable +# place to write; otherwise FALLBACK_DATA_ROOT # -# Resolved once at import time, from wherever the server process was launched - -# so launch it from a stable location (or set the env var). +# WHY THE CWD IS NOT TRUSTED BLINDLY: an MCP client picks the working +# directory its servers are launched with, and the caller has no say in it. +# Claude Desktop on Windows launches them in C:\WINDOWS\system32, so a plain +# Path.cwd() sends captures to C:\WINDOWS\system32\data\captures - which +# fails with a bare "access is denied" OSError at the first capture, a long +# way from this module and looking for all the world like a camera fault. +# Running elevated is worse than the error: the write then *succeeds*, and +# scatters image files through a system directory. +# +# So an unset env var resolves the cwd through _usable_data_root(), which +# rejects anything under the Windows directory and anything it cannot +# actually create a file in, and falls back to a per-user directory with a +# warning on stderr (never stdout - that is the JSON-RPC channel). +# +# Resolved once per process and cached: every caller reads it into a +# module-level constant at import time anyway. Tests that manipulate the env +# var must call data_root.cache_clear() to see the change. # ------------------------------------------------------------ +import functools import os +import sys +import tempfile from pathlib import Path +#: Used when the working directory is not a usable data root. Under the +#: user's home, so it is writable without elevation and easy to find. +FALLBACK_DATA_ROOT = Path.home() / ".confocal-mcp" + + +def _is_system_dir(path: Path) -> bool: + """True if `path` is inside the Windows directory. + + Checked separately from writability because an elevated process *can* + write to the Windows system directory - that is precisely the case + worth refusing rather than permitting. + """ + system_root = os.environ.get("SystemRoot") + if not system_root: + return False + try: + return path == Path(system_root) or Path(system_root) in path.parents + except OSError: + return False + + +def _is_writable(path: Path) -> bool: + """True if a file can actually be created in `path`. + + Probes with a real file rather than os.access(), which on Windows + reports only the read-only attribute and happily returns True for + directories that ACLs deny. + """ + try: + path.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path, prefix=".confocal-write-test-"): + pass + return True + except OSError: + return False + + +def _usable_data_root(cwd: Path) -> Path: + """`cwd` if it is a sane place to write runtime data, else the fallback.""" + if not _is_system_dir(cwd) and _is_writable(cwd): + return cwd + + print( + f"[confocal-mcp] Working directory {cwd} is not usable for runtime data " + f"(system directory or not writable); using {FALLBACK_DATA_ROOT} instead. " + f"Set CONFOCAL_MCP_DATA_DIR to choose the location explicitly.", + file=sys.stderr, + ) + return FALLBACK_DATA_ROOT + +@functools.lru_cache(maxsize=1) def data_root() -> Path: """Base directory for all runtime data. See module docstring.""" env = os.environ.get("CONFOCAL_MCP_DATA_DIR") - return Path(env).expanduser().resolve() if env else Path.cwd() + if env: + return Path(env).expanduser().resolve() + return _usable_data_root(Path.cwd()) def captures_dir() -> Path: From d02e178d7e98fd903c225719e085f95ec47afa49 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:15 -0400 Subject: [PATCH 02/42] Fix Bayer decode order and report camera white-balance mode Decoding the VCXU-23C's BayerRG8 stream with COLOR_BayerRG2RGB inverted R/B; BayerBG2RGB matches the camera's own debayer. get_settings() now reports BalanceWhiteAuto, which persists on the camera across processes. --- acquisition/backends/baumer_genicam.py | 37 ++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/acquisition/backends/baumer_genicam.py b/acquisition/backends/baumer_genicam.py index 763465f..46182bb 100644 --- a/acquisition/backends/baumer_genicam.py +++ b/acquisition/backends/baumer_genicam.py @@ -119,13 +119,25 @@ def get_settings(self) -> dict: node_map = self._acquirer.remote_device.node_map exposure = node_map.ExposureTime gain = node_map.Gain - return { + settings = { "exposure_time_us": exposure.value, "exposure_time_range_us": [exposure.min, exposure.max], "gain": gain.value, "gain_range": [gain.min, gain.max], "pixel_format": node_map.PixelFormat.value, } + # REPORTED BECAUSE IT IS OTHERWISE INVISIBLE STATE. White balance + # persists on the camera across processes, is settable from any + # GenICam consumer, and silently rewrites every captured colour - + # a whole time-lapse was acquired on 2026-09-21 without anyone + # being able to tell from the saved metadata what the colour + # pipeline had done to it. BalanceRatio is not exposed on the + # VCXU-23C, so mode is all there is to record. + try: + settings["balance_white_auto"] = node_map.BalanceWhiteAuto.value + except Exception: + settings["balance_white_auto"] = None + return settings def set_settings(self, exposure_time_us: float | None = None, gain: float | None = None) -> dict: """Set exposure time (microseconds) and/or gain via the GenICam @@ -217,10 +229,25 @@ def capture(self) -> Path: elif pixel_format == "Mono8": rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_GRAY2RGB) elif pixel_format.startswith("Bayer"): - # Literal-name match to cv2's code; unverified against a - # known-colour target. If colours look swapped under real - # light, try COLOR_BayerGR/BG/GB2RGB instead. - rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_BayerRG2RGB) + # VERIFIED 2026-09-21 against the camera's OWN internal + # debayer, which is the ground truth available without a + # colour target: capturing the same scene in BGR8 and in + # RGB8 agreed exactly (organism hue 54 deg, saturation + # ~105), while decoding the BayerRG8 stream with + # COLOR_BayerRG2RGB - what this line used to do - gave hue + # 188 deg, very nearly the complement. That is R/B + # inversion, so the correct cv2 code for this body's + # "BayerRG8" is BayerBG2RGB, not BayerRG2RGB. + # + # PREFER BGR8 OVER ANY BAYER FORMAT ON THIS CAMERA. The + # Bayer stream also carries markedly less colour than the + # camera's own debayer of the same scene (channel spread + # 4.99 vs 11.93, saturation 56 vs 105), so it loses real + # information rather than just mislabelling it. The + # PixelFormat is whatever the camera was last left in by + # any GenICam consumer - if someone sets it back to Bayer, + # this branch at least renders the right hue. + rgb = cv2.cvtColor(data.reshape(h, w), cv2.COLOR_BayerBG2RGB) else: raise RuntimeError( f"Unsupported camera PixelFormat {pixel_format!r} - " From 9fe791f8ae4c6101bf2b40a905b66b235754f3d3 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:15 -0400 Subject: [PATCH 03/42] Add a file-based emergency stop enforced inside every motion primitive acquisition/estop.py holds the flag and CLI (engage/status/release). NISSdk.XY_Move, Z_Move and nudge_pfs_offset check it before moving, so no caller can move an axis while it is engaged. --- acquisition/backends/nis_sdk.py | 23 ++++ acquisition/estop.py | 191 ++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 acquisition/estop.py diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index 9be91a5..e01f87e 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -51,6 +51,11 @@ import win32com.client import NkTi2Ax +# Imported at module level so the guard cannot be skipped by an import +# failing lazily inside a move. estop imports THIS module only inside +# halt(), so there is no import cycle. +from acquisition import estop + # Raw-count-per-micron scale factors confirmed above. XY_COUNTS_PER_UM = 10.0 Z_COUNTS_PER_UM = 100.0 @@ -183,6 +188,12 @@ def XY_Move(self, x: float, y: float) -> None: than MAX_XY_STEP_UM, or if the target is outside the stage's own reported travel range (XPosition/YPosition .Lower/.Higher). """ + # E-STOP FIRST, before range checks, before anything. This is the + # lowest point every XY move passes through, which is the only + # place a stop can be effective against a caller that is already + # misbehaving - see acquisition/estop.py's header for the incident + # this exists because of. + estop.check() before = self.XY_GetPosition() x, y = to_plain_float(x), to_plain_float(y) step_um = ((x - before[0]) ** 2 + (y - before[1]) ** 2) ** 0.5 @@ -235,6 +246,10 @@ def Z_Move(self, z: float) -> None: nudge_pfs_offset() for routine focus adjustment when PFS is engaged, and reserve this for deliberate, small, checked steps. """ + # E-stop before anything else - Z is the axis that can drive the + # objective into the sample, so this is the most important guard + # in the file. + estop.check() before = self.Z_GetPosition() z = to_plain_float(z) step_um = abs(z - before) @@ -319,6 +334,13 @@ def nudge_pfs_offset(self, delta_counts: float) -> dict: capped to PFS_MAX_OFFSET_STEP_FRACTION of the SDK's own reported valid offset range. + TODO(unconfirmed, 2026-08-10): writing iPFS_OFFSET while PFS is + actively enabled/locked has been observed to silently no-op on + real hardware - offset_before == offset_after, no exception - on + Guarded by the e-stop like XY_Move/Z_Move: it is small and + relative, but it still moves focus, and "only a little" is not a + category the stop should recognise. + TODO(unconfirmed, 2026-08-10): writing iPFS_OFFSET while PFS is actively enabled/locked has been observed to silently no-op on real hardware - offset_before == offset_after, no exception - on @@ -334,6 +356,7 @@ def nudge_pfs_offset(self, delta_counts: float) -> dict: hardware. Failure mode observed so far is safe (no motion, no error) - not a functional feature yet, but not a hazard either. """ + estop.check() delta_counts = to_plain_float(delta_counts) def do_nudge(m): diff --git a/acquisition/estop.py b/acquisition/estop.py new file mode 100644 index 0000000..18f2bd3 --- /dev/null +++ b/acquisition/estop.py @@ -0,0 +1,191 @@ +# estop.py +# ------------------------------------------------------------ +# Emergency stop for every process that can move this microscope. +# +# python -m acquisition.estop engage # STOP EVERYTHING, NOW +# python -m acquisition.estop status +# python -m acquisition.estop release # deliberate, separate action +# +# WHY THIS EXISTS. On 2026-09-21 a mosaic raster with a bad focus plane +# began driving Z toward the sample. Stopping it took several minutes and +# three separate interventions, because every mechanism available was the +# wrong shape: +# +# * stopping the background SHELL left its python CHILD running, still +# issuing moves - the process list, not the task list, was the truth +# * a tool call the operator REJECTED had already spawned its process, +# which kept looping Z moves regardless of the rejection +# * a "STOP file" checked once per mosaic tile is not a stop; between +# checks it still issued dozens of moves +# * the Ti2 SDK has NO abort/halt command (searched: only ZESCAPE, which +# is a retract, not a stop), so there is nothing to "cancel" with +# +# The lesson is that an e-stop must sit BELOW whatever is misbehaving. So +# this is enforced inside nis_sdk's move primitives themselves: no caller, +# no matter how confused, can move an axis while the stop is engaged, +# because the check happens after the caller has already decided to move. +# +# WHY A FILE. It must work across processes that do not know about each +# other - that was the whole failure. A file is visible to every process, +# survives the death of whoever set it, needs no server or port, and can +# be set by a human from any shell (or by creating the file by hand) when +# the agent itself is the thing malfunctioning. +# +# WHY A FIXED PATH, NOT data_root(). data_root() resolves relative to the +# working directory, and an MCP server launched by a desktop client gets a +# different cwd than a terminal. Two processes would then disagree about +# where the e-stop lives, which is precisely the failure this must not +# have. ESTOP_PATH is absolute and identical for every process on the +# machine. +# +# FAIL SAFE: if the state cannot be determined (unreadable directory, odd +# filesystem error), check() treats the stop as ENGAGED. An e-stop that +# fails open is not a safety device. The cost of the opposite error is a +# refused move and a clear message, which is recoverable; the cost of +# failing open is a driven objective. +# +# WHAT IT DOES NOT DO: it cannot un-issue a setpoint the controller has +# already accepted. engage() therefore also calls halt(), which writes the +# CURRENT position back as the target - with setpoint-based motion that is +# the only available way to stop an axis already in motion. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import sys +from pathlib import Path + +#: Absolute and machine-wide. Never derive this from the working +#: directory - see the module header. +ESTOP_PATH = Path(os.environ.get("CONFOCAL_ESTOP_FILE", + Path.home() / ".confocal-mcp" / "ESTOP")) + + +class EStopEngaged(RuntimeError): + """Raised by any motion primitive while the e-stop is engaged.""" + + +def is_engaged() -> bool: + """True if motion is currently forbidden. + + Any error resolving the state counts as engaged - see FAIL SAFE. + """ + try: + return ESTOP_PATH.exists() + except OSError: + return True + + +def details() -> dict | None: + """Who engaged the stop, when, and why - or None if it is clear.""" + try: + if not ESTOP_PATH.exists(): + return None + return json.loads(ESTOP_PATH.read_text(encoding="utf-8")) + except (OSError, ValueError): + # Present but unreadable still means engaged; say so without a reason. + return {"reason": "(e-stop file present but unreadable)"} + + +def check() -> None: + """Raise EStopEngaged if motion is forbidden. Called by every move.""" + if is_engaged(): + info = details() or {} + raise EStopEngaged( + "E-STOP ENGAGED - motion refused. " + f"engaged_at={info.get('engaged_at', '?')} " + f"by={info.get('by', '?')} reason={info.get('reason', '?')}. " + f"Release with: python -m acquisition.estop release " + f"(or delete {ESTOP_PATH})" + ) + + +def engage(reason: str = "manual", halt_stage: bool = True) -> dict: + """Forbid all motion immediately, and try to halt anything in flight. + + The flag is written FIRST and the hardware halt attempted second: if + halting raises (SDK missing, COM busy, no hardware), the prohibition is + already in force. Doing it the other way round would leave a window + where a failed halt also meant no flag. + """ + info = { + "engaged_at": datetime.datetime.now().isoformat(timespec="seconds"), + "by": f"pid {os.getpid()}", + "reason": reason, + } + ESTOP_PATH.parent.mkdir(parents=True, exist_ok=True) + ESTOP_PATH.write_text(json.dumps(info, indent=2), encoding="utf-8") + + if halt_stage: + try: + info["halt"] = halt() + except Exception as exc: # hardware may be absent + info["halt"] = f"not halted ({type(exc).__name__}: {exc})" + return info + + +def release() -> None: + """Allow motion again. Deliberately a separate, explicit action.""" + try: + ESTOP_PATH.unlink() + except FileNotFoundError: + pass + + +def halt() -> str: + """Stop axes already in motion by re-commanding their current position. + + The Ti2 exposes no abort: a move is a setpoint write and the stage + servos to it. Writing the position it is at right now is therefore the + only way to stop an axis mid-travel. + + Deliberately bypasses nis_sdk's XY_Move/Z_Move - those now refuse to + run while the stop is engaged, and a stop primitive that the stop + itself blocks would be useless. + """ + from acquisition.backends.nis_sdk import NISSdk, XY_COUNTS_PER_UM, Z_COUNTS_PER_UM + + sdk = NISSdk() + + def freeze(m): + x, y, z = m.iXPOSITION, m.iYPOSITION, m.iZPOSITION + m.iXPOSITION, m.iYPOSITION, m.iZPOSITION = x, y, z + return (x / XY_COUNTS_PER_UM, y / XY_COUNTS_PER_UM, z / Z_COUNTS_PER_UM) + + x, y, z = sdk._thread.call(freeze) + return f"halted at x={x:.1f} y={y:.1f} z={z:.2f} um" + + +def main() -> None: + ap = argparse.ArgumentParser(description="Emergency stop for microscope motion.") + ap.add_argument("action", choices=("engage", "release", "status")) + ap.add_argument("--reason", default="manual") + ap.add_argument("--no-halt", action="store_true", + help="set the flag only; do not try to halt the stage") + args = ap.parse_args() + + if args.action == "engage": + info = engage(args.reason, halt_stage=not args.no_halt) + print("E-STOP ENGAGED") + for k, v in info.items(): + print(f" {k}: {v}") + print(f"\nflag file: {ESTOP_PATH}") + print("All motion is now refused. Release with: python -m acquisition.estop release") + elif args.action == "release": + release() + print(f"e-stop released - motion permitted again ({ESTOP_PATH} removed)") + else: + if is_engaged(): + print("ENGAGED - motion is refused") + for k, v in (details() or {}).items(): + print(f" {k}: {v}") + sys.exit(2) + print(f"clear - motion permitted (no {ESTOP_PATH})") + + +if __name__ == "__main__": + main() From fb1cb3a5e6475d77044e516efdc61a606979f80c Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:15 -0400 Subject: [PATCH 04/42] Add an always-on-top STOP panel and expose the e-stop over MCP The MCP tool can engage or report the stop but never release it. The server launches the panel on startup (CONFOCAL_NO_ESTOP_PANEL disables). --- acquisition/estop_panel.py | 152 +++++++++++++++++++++++++++++++++++++ mcp_server/loop_tools.py | 50 ++++++++++++ mcp_server/server_loop.py | 32 +++++++- 3 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 acquisition/estop_panel.py diff --git a/acquisition/estop_panel.py b/acquisition/estop_panel.py new file mode 100644 index 0000000..7f95682 --- /dev/null +++ b/acquisition/estop_panel.py @@ -0,0 +1,152 @@ +# estop_panel.py +# ------------------------------------------------------------ +# An always-on-top STOP button for the microscope. +# +# python -m acquisition.estop_panel +# +# Small, borderless, stays above other windows, drag it anywhere. One big +# button. Hitting it engages acquisition.estop, which every motion +# primitive checks before it moves - see that module's header. +# +# WHY A WINDOW AND NOT JUST THE CLI: during the 2026-09-21 incident the +# stage was moving while the operator hunted for a terminal, typed a +# command, and waited for python to start. Seconds matter and typing does +# not scale under stress. A button that is already on screen costs one +# click, with no target to find and nothing to remember. +# +# WHY IT POLLS THE FLAG FILE rather than tracking its own state: the stop +# can also be engaged from a terminal, from the MCP tool, by another +# process, or by a human creating the file. The panel must show what is +# actually true, not what it last did - a panel reading "CLEAR" while the +# stop is engaged elsewhere would be worse than no panel. +# +# WHY RELEASE IS DELIBERATELY AWKWARD: the release control is small, +# visually quiet, and asks for confirmation, while STOP is huge and +# instant. The asymmetry is the point - the cost of an accidental stop is +# a few seconds, the cost of an accidental release is an unguarded +# microscope. Engaging is also idempotent, so panic-clicking is harmless. +# +# TKINTER because it ships with CPython: a safety control that depends on +# `pip install` is a safety control that is missing on the day it matters. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import sys +import tkinter as tk +from tkinter import messagebox + +from acquisition import estop + +POLL_MS = 300 + +_RED, _RED_DARK = "#c62828", "#8e0000" +_GREEN, _GREY = "#2e7d32", "#9e9e9e" +_BG = "#1b1b1b" + + +class Panel: + def __init__(self, root: tk.Tk, topmost: bool = True, borderless: bool = True): + self.root = root + root.title("E-STOP") + root.configure(bg=_BG) + root.resizable(False, False) + if topmost: + root.attributes("-topmost", True) + if borderless: + # No title bar - it would be most of a window this small, and + # the drag handler below replaces the only thing it was for. + root.overrideredirect(True) + root.geometry("+40+40") + + self.button = tk.Button( + root, text="STOP", font=("Segoe UI", 26, "bold"), + bg=_RED, fg="white", activebackground=_RED_DARK, activeforeground="white", + relief="raised", bd=3, width=8, height=1, cursor="hand2", + command=self.on_stop) + self.button.pack(padx=8, pady=(8, 4)) + + self.status = tk.Label(root, text="checking...", font=("Segoe UI", 9, "bold"), + bg=_BG, fg=_GREY) + self.status.pack() + + bar = tk.Frame(root, bg=_BG) + bar.pack(fill="x", padx=8, pady=(2, 6)) + # Quiet, small, and confirmed - see the header on asymmetry. + self.release_btn = tk.Button(bar, text="release", font=("Segoe UI", 8), + bg="#2a2a2a", fg="#bbb", relief="flat", + cursor="hand2", command=self.on_release) + self.release_btn.pack(side="left") + tk.Button(bar, text="✕", font=("Segoe UI", 8), bg="#2a2a2a", fg="#bbb", + relief="flat", cursor="hand2", command=root.destroy).pack(side="right") + + # Drag from anywhere on the background, since there is no title bar. + for w in (root, self.status, bar): + w.bind("", self._press) + w.bind("", self._drag) + self._dx = self._dy = 0 + + self._last = None + self.tick() + + def _press(self, e) -> None: + self._dx, self._dy = e.x_root - self.root.winfo_x(), e.y_root - self.root.winfo_y() + + def _drag(self, e) -> None: + self.root.geometry(f"+{e.x_root - self._dx}+{e.y_root - self._dy}") + + def on_stop(self) -> None: + # Engage first and report afterwards: the flag write is what makes + # the microscope safe, and it must not wait behind a dialog. + try: + estop.engage("STOP button") + except Exception as exc: + messagebox.showerror("E-STOP", f"Could not engage:\n{exc}") + self.refresh(force=True) + + def on_release(self) -> None: + if not estop.is_engaged(): + return + if messagebox.askyesno("Release e-stop", + "Allow the microscope to move again?\n\n" + "Only do this once you know why it stopped."): + estop.release() + self.refresh(force=True) + + def refresh(self, force: bool = False) -> None: + engaged = estop.is_engaged() + if engaged == self._last and not force: + return + self._last = engaged + if engaged: + self.status.configure(text="ENGAGED - motion refused", fg=_RED) + self.button.configure(text="STOPPED", bg=_RED_DARK) + self.release_btn.configure(state="normal") + else: + self.status.configure(text="clear - motion permitted", fg=_GREEN) + self.button.configure(text="STOP", bg=_RED) + self.release_btn.configure(state="disabled") + + def tick(self) -> None: + self.refresh() + self.root.after(POLL_MS, self.tick) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--no-topmost", action="store_true") + ap.add_argument("--titlebar", action="store_true", help="keep a normal window frame") + args = ap.parse_args() + try: + root = tk.Tk() + except tk.TclError as exc: + print(f"no display available for the e-stop panel ({exc}). " + f"Use: python -m acquisition.estop engage", file=sys.stderr) + raise SystemExit(1) + Panel(root, topmost=not args.no_topmost, borderless=not args.titlebar) + root.mainloop() + + +if __name__ == "__main__": + main() diff --git a/mcp_server/loop_tools.py b/mcp_server/loop_tools.py index 9a030a1..da006f7 100644 --- a/mcp_server/loop_tools.py +++ b/mcp_server/loop_tools.py @@ -562,3 +562,53 @@ def get_move_history(limit: int = 50) -> dict: records = [json.loads(line) for line in lines[-limit:]] return {"history": records, "returned": len(records), "total_moves": len(lines)} + + +def estop(action: str = "status", reason: str = "requested via MCP") -> dict: + """EMERGENCY STOP: immediately forbid all microscope motion. + + Call this the moment anything looks wrong - an unexpected move, a + position that doesn't match what you asked for, a stage that seems to + be travelling when it shouldn't be, or an instruction from the + operator to stop. It is cheap, it is instant, and a needless stop + costs nothing but a release. Do not deliberate: stop first, diagnose + afterwards. + + Once engaged, EVERY motion primitive refuses - not just this session's, + but every process on the machine that drives this microscope, + including runs started by something else entirely. The flag lives in a + file, so it outlives whatever set it and cannot be lost by a process + dying. Engaging also re-commands the stage to its current position, + which is the only way to halt an axis already in flight: the Ti2 has + no abort command, and a move is a setpoint the controller servos to. + + action: "engage" to stop everything, or "status" to report the current + state. RELEASE IS DELIBERATELY NOT AVAILABLE HERE - a stop that the + agent can lift by itself is not a safety device. A human clears it + from a terminal with `python -m acquisition.estop release`. + + Returns the resulting state. This tool never needs confirm=True: a + stop is always safe to perform, and requiring a confirmation step for + an emergency control would defeat its purpose. + """ + from acquisition import estop as _estop + + if action == "engage": + info = _estop.engage(reason) + return { + "engaged": True, + "detail": info, + "release_with": "python -m acquisition.estop release", + "note": "All motion is now refused for every process on this machine.", + } + if action == "status": + return { + "engaged": _estop.is_engaged(), + "detail": _estop.details(), + "flag_file": str(_estop.ESTOP_PATH), + } + raise ValueError( + f"Unknown estop action {action!r}. Use 'engage' to stop motion, or " + "'status' to check. Releasing is a human action performed at a " + "terminal: python -m acquisition.estop release" + ) diff --git a/mcp_server/server_loop.py b/mcp_server/server_loop.py index 4e8250d..7d9d878 100644 --- a/mcp_server/server_loop.py +++ b/mcp_server/server_loop.py @@ -12,6 +12,10 @@ # { "mcpServers": { "confocal": { "command": "confocal-mcp" } } } # ------------------------------------------------------------ +import os +import subprocess +import sys + from mcp.server.mcpserver import MCPServer from mcp_server import loop_tools as tools @@ -22,10 +26,36 @@ mcp.add_tool(tools.get_pos) mcp.add_tool(tools.move) mcp.add_tool(tools.get_move_history) +mcp.add_tool(tools.estop) + + +def _launch_estop_panel() -> None: + """Put the STOP button on screen for as long as this server runs. + + Detached, not a child we wait on: the panel must survive this process + hanging, and a server that cannot draw a window (headless, no display) + must still serve tools. Any failure here is reported to stderr - never + stdout, which is the JSON-RPC channel - and never prevents startup. + + Launched by default because the one time it was needed, the operator + was hunting for a terminal while the stage was moving. A safety + control you have to remember to start is one you will not have. + """ + if os.environ.get("CONFOCAL_NO_ESTOP_PANEL"): + return + try: + flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) | getattr(subprocess, "DETACHED_PROCESS", 0) + subprocess.Popen([sys.executable, "-m", "acquisition.estop_panel"], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, creationflags=flags) + except Exception as exc: + print(f"[confocal-mcp] could not start the e-stop panel ({exc}). " + f"Stop manually with: python -m acquisition.estop engage", file=sys.stderr) def main() -> None: - """Console entry point (``confocal-mcp``): serve the 4 tools over stdio.""" + """Console entry point (``confocal-mcp``): serve the 5 tools over stdio.""" + _launch_estop_panel() mcp.run(transport="stdio") From 0fe4b79c76bfb7375c2efec19cd9a73a1416ddc0 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:15 -0400 Subject: [PATCH 05/42] Add move_xy: manual XY stage moves split into legal hops Walks long moves in 4 mm hops under nis_sdk's 5 mm step limit and asks before moves over 2 mm. XY only; Z stays on the focus knob. --- acquisition/move_xy.py | 115 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 acquisition/move_xy.py diff --git a/acquisition/move_xy.py b/acquisition/move_xy.py new file mode 100644 index 0000000..35715f1 --- /dev/null +++ b/acquisition/move_xy.py @@ -0,0 +1,115 @@ +# move_xy.py +# ------------------------------------------------------------ +# Drive the XY stage by hand, from a terminal. +# +# python -m acquisition.move_xy # just show where we are +# python -m acquisition.move_xy --by 500 0 # relative, microns +# python -m acquisition.move_xy --to -1848 -10021 # absolute, microns +# python -m acquisition.move_xy --by 20000 0 --yes # skip the confirmation +# +# XY ONLY, ON PURPOSE. Z is what drives the objective into the sample, and +# it is not reachable from here - use the focus knob. This mirrors the +# same choice in the MCP tools. +# +# LONG MOVES ARE SPLIT AUTOMATICALLY. nis_sdk refuses any single step over +# MAX_XY_STEP_UM (5 mm) so that a fat-fingered coordinate cannot become a +# stage-length dash. That guard is worth keeping, but it makes a legitimate +# 15 mm move fail halfway with the stage somewhere unintended - which is +# exactly what happened on 2026-09-21, leaving the stage 8 mm from where +# anyone thought it was. So this walks the distance in legal hops and +# reports where it actually ended up. +# +# ANYTHING OVER --confirm-above ASKS FIRST, because the difference between +# a 200 um nudge and a 20000 um traverse is one keystroke, and only one of +# them can put the objective under the dish holder. +# +# The e-stop is checked by nis_sdk on every hop, so engaging it mid-move +# stops this at the next hop rather than at the end of the journey. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import sys +import time + +import numpy as np + +from acquisition import estop + +#: Kept under nis_sdk's 5000 um hard limit so a rounding error at the +#: boundary cannot trip the very guard we are working within. +HOP_UM = 4000.0 + +#: Moves longer than this ask for confirmation first. +CONFIRM_ABOVE_UM = 2000.0 + + +def move_to(sdk, x: float, y: float, quiet: bool = False) -> tuple[float, float]: + """Absolute move in microns, split into legal hops.""" + while True: + cx, cy = sdk.XY_GetPosition() + dx, dy = x - cx, y - cy + dist = float(np.hypot(dx, dy)) + if dist < 1.0: + return cx, cy + f = min(1.0, HOP_UM / dist) + sdk.XY_Move(cx + dx * f, cy + dy * f) + time.sleep(0.4) + if not quiet and dist > HOP_UM: + nx, ny = sdk.XY_GetPosition() + print(f" ... at ({nx:.1f}, {ny:.1f}), {dist - HOP_UM:.0f} um to go", flush=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description="Move the XY stage manually (microns).") + g = ap.add_mutually_exclusive_group() + g.add_argument("--to", nargs=2, type=float, metavar=("X", "Y"), help="absolute target") + g.add_argument("--by", nargs=2, type=float, metavar=("DX", "DY"), help="relative offset") + ap.add_argument("--yes", action="store_true", help="do not ask about long moves") + ap.add_argument("--confirm-above", type=float, default=CONFIRM_ABOVE_UM) + args = ap.parse_args() + + if estop.is_engaged(): + info = estop.details() or {} + print("E-STOP IS ENGAGED - the stage will not move.") + print(f" reason: {info.get('reason', '?')} at {info.get('engaged_at', '?')}") + print(" release with: python -m acquisition.estop release") + raise SystemExit(2) + + from acquisition.backends.nis_sdk import NISSdk + sdk = NISSdk() + x0, y0 = sdk.XY_GetPosition() + z0 = sdk.Z_GetPosition() + print(f"current: X={x0:.2f} Y={y0:.2f} (Z={z0:.2f}, not touched)") + + if not args.to and not args.by: + return + + tx, ty = (args.to[0], args.to[1]) if args.to else (x0 + args.by[0], y0 + args.by[1]) + dist = float(np.hypot(tx - x0, ty - y0)) + print(f"target : X={tx:.2f} Y={ty:.2f} (moving {dist:.1f} um)") + + if dist > args.confirm_above and not args.yes: + # Interactive only: piped/automated use must pass --yes explicitly + # rather than have a prompt silently read EOF and proceed. + if not sys.stdin.isatty(): + print("long move needs --yes when not run interactively."); raise SystemExit(3) + if input(f" move {dist:.0f} um? [y/N] ").strip().lower() not in ("y", "yes"): + print(" cancelled - nothing moved."); return + + try: + x1, y1 = move_to(sdk, tx, ty) + except estop.EStopEngaged as exc: + cx, cy = sdk.XY_GetPosition() + print(f"\nSTOPPED BY E-STOP at ({cx:.2f}, {cy:.2f}) - target not reached.") + print(f" {exc}") + raise SystemExit(2) + + err = float(np.hypot(x1 - tx, y1 - ty)) + print(f"arrived: X={x1:.2f} Y={y1:.2f} (moved {np.hypot(x1-x0, y1-y0):.1f} um, " + f"{err:.2f} um from target)") + + +if __name__ == "__main__": + main() From 55f15c07ec620ebeed8cc23708d7edd240f42743 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:15 -0400 Subject: [PATCH 06/42] Add Ti2 stage probe scripts for halt and read-during-move behaviour --- acquisition/raw_move_xy.py | 55 ++++++++++++++++ acquisition/read_during_move_test.py | 93 ++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 acquisition/raw_move_xy.py create mode 100644 acquisition/read_during_move_test.py diff --git a/acquisition/raw_move_xy.py b/acquisition/raw_move_xy.py new file mode 100644 index 0000000..c95c770 --- /dev/null +++ b/acquisition/raw_move_xy.py @@ -0,0 +1,55 @@ +import os +import sys +import threading +import time + +# Multi-threaded COM, so the watchdog thread can use the same microscope +# object as the main thread. Must be set before pythoncom is imported. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + + +def log(msg): + now = time.time() + print(f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d} {msg}", flush=True) + + +def watchdog(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + log("watchdog: 500ms elapsed, halting") + x, y = m.iXPOSITION, m.iYPOSITION + m.iXPOSITION, m.iYPOSITION = x, y + log(f"watchdog: halted at ({x}, {y})") + + +log("dispatch start") +m = win32com.client.Dispatch(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID) +log("dispatch done") +log(f"start position ({m.iXPOSITION}, {m.iYPOSITION})") + +# Armed after the connect, so the halt always has a microscope to talk to. +timer = threading.Timer(0.5, watchdog) +timer.start() +log("watchdog armed (500ms)") + +log("set iXPOSITION start") +m.iXPOSITION = 0 +log("set iXPOSITION done") + +log("set iYPOSITION start") +m.iYPOSITION = 0 +log("set iYPOSITION done") + +# Track progress until the halt, then for 1s after to see where it settles. +halted_at = None +while halted_at is None or time.time() - halted_at < 1.0: + log(f"position ({m.iXPOSITION}, {m.iYPOSITION})") + if halted_at is None and not timer.is_alive(): + halted_at = time.time() + time.sleep(0.05) + +log("exit") +os._exit(0) diff --git a/acquisition/read_during_move_test.py b/acquisition/read_during_move_test.py new file mode 100644 index 0000000..e3e7693 --- /dev/null +++ b/acquisition/read_during_move_test.py @@ -0,0 +1,93 @@ +# Does reading iXPOSITION block while a move is in progress, or return a +# cached value? And does the SDK push position/moving events during a move? +# +# Moves XY to (0, 0) - X then Y, with blocking writes on the main thread - +# while a reader thread times every read and an event sink logs callbacks. +import os +import sys +import threading +import time + +# Multi-threaded COM: the reader thread shares the microscope object, and +# events fire on the SDK's own thread without needing a message loop. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + +TARGET = (0, 0) +XY_STAGE_MASK = 0x2 # MIC_ACCESSORY_MASK_XYSTAGE + +GENERAL_EVENTS = {0: "DataSet_PositionChanged", 1: "RemCtrl_PositionChanged", + 3: "XYStageLogicalLimitsReached", 20: "DataSetReady", + 21: "DataSetFinished", 22: "DataSetAborted"} +DEDICATED_EVENTS = {25: "Ti2_IsBusy", 38: "MovingStatus_Changed"} + + +def log(msg): + now = time.time() + stamp = f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d}" + print(f"{stamp} [{threading.current_thread().name:>6}] {msg}", flush=True) + + +class Events: + def OnGeneralCallback(self, event, data): + d = win32com.client.Dispatch(data, None, NkTi2Ax.INikonTi2AxData) + log(f"EVENT general {event} {GENERAL_EVENTS.get(event, '?')} " + f"mask=0x{d.uiDataUsageMask:x} data=({d.iXPOSITION}, {d.iYPOSITION})") + + def OnDedicatedCallback(self, event, data): + log(f"EVENT dedicated {event} {DEDICATED_EVENTS.get(event, '?')} data={data!r}") + + def OnMetaDataCallback(self, mask): + log(f"EVENT metadata 0x{mask:x}") + + def OnEnabledCallback(self, mask): + log(f"EVENT enabled 0x{mask:x}") + + +def reader(stop): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + while not stop.is_set(): + t0 = time.perf_counter() + log("read start") + x, y = m.iXPOSITION, m.iYPOSITION + log(f"read done ({x}, {y}) took {(time.perf_counter() - t0) * 1000:.1f} ms") + time.sleep(0.05) + + +# Hard stop in case anything hangs. +threading.Timer(20.0, lambda: (log("20s safety timeout"), os._exit(1))).start() + +threading.current_thread().name = "main" +log("dispatch start") +m = win32com.client.DispatchWithEvents(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID, Events) +log("dispatch done") + +for cmd, arg in (("SET_MOVING_STATE_NOTIFICATION", str(XY_STAGE_MASK)), + ("GET_MOVING_STATE_NOTIFICATION", "")): + try: + log(f"{cmd} -> {m.DedicatedCommand(cmd, arg)!r}") + except Exception as e: + log(f"{cmd} failed: {e}") + +log(f"start position ({m.iXPOSITION}, {m.iYPOSITION})") + +stop = threading.Event() +threading.Thread(target=reader, args=(stop,), name="reader").start() +time.sleep(0.2) # a few idle reads first, as a baseline + +log("set iXPOSITION start") +m.iXPOSITION = TARGET[0] +log("set iXPOSITION done") + +log("set iYPOSITION start") +m.iYPOSITION = TARGET[1] +log("set iYPOSITION done") + +time.sleep(0.5) # a few reads after the move +stop.set() +time.sleep(0.2) +log("exit") +os._exit(0) From 73b44f196f4abb100635711f4aa1db9af76d41a9 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:16 -0400 Subject: [PATCH 07/42] Add fixed-position brightfield time-lapse acquisition Schedules frames against a fixed start time so capture latency does not accumulate, keeps the camera open for the whole run, and logs per-frame exposure and focus stats to frames.csv. --- acquisition/orchestration/timelapse.py | 175 +++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 acquisition/orchestration/timelapse.py diff --git a/acquisition/orchestration/timelapse.py b/acquisition/orchestration/timelapse.py new file mode 100644 index 0000000..0af66ee --- /dev/null +++ b/acquisition/orchestration/timelapse.py @@ -0,0 +1,175 @@ +# timelapse.py +# ------------------------------------------------------------ +# Fixed-position brightfield time-lapse from the Baumer camera. +# +# python -m acquisition.orchestration.timelapse --minutes 60 --interval 10 +# python -m acquisition.orchestration.timelapse --frames 12 --interval 5 --tag test +# +# WHY 10 s BY DEFAULT: written for Physarum polycephalum, whose +# cytoplasmic shuttle streaming reverses on a ~100-130 s contraction +# rhythm. A 10 s interval puts ~10-13 samples in each cycle - enough to +# fit the oscillation - while an hour of it still spans ~30 cycles, so a +# periodogram has something to work with. Slower front advance falls out +# of the same series for free. +# +# THE CAMERA IS OPENED ONCE for the whole run, not per frame. Connecting +# costs seconds and a GenICam camera can only be held by one process at a +# time, so a per-frame open/close would both blow the interval budget and +# leave a window where anything else could grab the device mid-run. +# +# ILLUMINATION IS LEFT ALONE. The DIA lamp stays at whatever it was set +# to, steady, for the run's whole duration - shuttering or dimming +# between frames would save the sample some light but makes every frame's +# exposure history different, which is precisely what ruins a +# quantitative intensity series. Set the lamp before starting. +# +# DRIFT: frames are scheduled against a fixed start time, not by sleeping +# `interval` after each capture. Capture takes real time (exposure + +# fetch + PNG encode), so sleep-after-capture accumulates a lag that +# grows all run - fatal for a periodogram. A frame whose slot has already +# passed is taken immediately and flagged `late` in the log rather than +# skipped. +# +# ROBUSTNESS: one failed frame does not end the run - it is recorded with +# its error in the CSV and the series continues on schedule. A run that +# dies at frame 200 of 360 still leaves 200 usable frames plus their +# metadata. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import shutil +import sys +import time +from pathlib import Path + +import numpy as np +from PIL import Image + +from acquisition.paths import data_root + +#: Per-frame columns. `mean`/`p1`/`p50`/`p99`/`sat` are written at capture +#: time because they are what tells you mid-run whether the series is +#: still well exposed and in focus, without reopening 300 PNGs. +_FIELDS = ("frame", "t_seconds", "wall_clock", "filename", "exposure_us", + "gain", "mean", "p1", "p50", "p99", "sat_frac", "focus", "late_s", "error") + + +def _series_dir(tag: str | None) -> Path: + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + name = f"timelapse_{stamp}" + (f"_{tag}" if tag else "") + return data_root() / "data" / name + + +def _stats(path: Path) -> dict: + """Exposure and focus numbers for one frame. + + `focus` is the variance of the Laplacian - the standard cheap + focus proxy. Only ever compare it between frames of one series at one + illumination; it is not an absolute scale. + """ + a = np.asarray(Image.open(path)) + g = a.mean(axis=2) + lap = (g[:-2, 1:-1] + g[2:, 1:-1] + g[1:-1, :-2] + g[1:-1, 2:] - 4 * g[1:-1, 1:-1]) + return { + "mean": round(float(g.mean()), 3), + "p1": round(float(np.percentile(g, 1)), 2), + "p50": round(float(np.percentile(g, 50)), 2), + "p99": round(float(np.percentile(g, 99)), 2), + "sat_frac": round(float((a >= 254).mean()), 6), + "focus": round(float(lap.var()), 2), + } + + +def run(frames: int, interval: float, exposure_us: float, gain: float, + tag: str | None = None) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + + out = _series_dir(tag) + out.mkdir(parents=True, exist_ok=True) + log_path = out / "frames.csv" + + cam = BaumerGenICam() + try: + settings = cam.set_settings(exposure_time_us=exposure_us, gain=gain) + print(f"[timelapse] camera: {settings}", flush=True) + print(f"[timelapse] {frames} frames every {interval}s " + f"(~{frames * interval / 60:.1f} min) -> {out}", flush=True) + + # One throwaway grab so the first logged frame is taken under the + # exposure just set, not the one before it. + cam.capture().unlink(missing_ok=True) + + start = time.monotonic() + wall_start = datetime.datetime.now() + + with log_path.open("w", newline="", encoding="utf-8") as fh: + writer = csv.DictWriter(fh, fieldnames=_FIELDS) + writer.writeheader() + + for i in range(frames): + target = start + i * interval + late = time.monotonic() - target + if late < 0: + time.sleep(-late) + late = 0.0 + + row = {f: "" for f in _FIELDS} + row.update(frame=i, + t_seconds=round(time.monotonic() - start, 3), + wall_clock=datetime.datetime.now().isoformat(timespec="seconds"), + exposure_us=exposure_us, gain=gain, + late_s=round(late, 3)) + try: + src = cam.capture() + dest = out / f"f{i:05d}.png" + shutil.move(str(src), dest) + row["filename"] = dest.name + row.update(_stats(dest)) + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + + writer.writerow(row) + fh.flush() # so the CSV is readable mid-run, not just at the end + + if row["error"]: + print(f"[timelapse] frame {i}: {row['error']}", flush=True) + elif i % 10 == 0 or i == frames - 1: + print(f"[timelapse] {i + 1}/{frames} t={row['t_seconds']:.0f}s " + f"mean={row['mean']} focus={row['focus']} " + f"sat={row['sat_frac']}", flush=True) + + elapsed = (datetime.datetime.now() - wall_start).total_seconds() + print(f"[timelapse] done in {elapsed / 60:.1f} min -> {out}", flush=True) + finally: + cam.close() + + return out + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + g = p.add_mutually_exclusive_group() + g.add_argument("--minutes", type=float, help="run length; with --interval sets frame count") + g.add_argument("--frames", type=int, help="explicit frame count") + p.add_argument("--interval", type=float, default=10.0, help="seconds between frames (default 10)") + p.add_argument("--exposure-us", type=float, default=40_000.0) + p.add_argument("--gain", type=float, default=1.0) + p.add_argument("--tag", help="appended to the output directory name") + args = p.parse_args() + + if args.frames: + frames = args.frames + elif args.minutes: + frames = int(round(args.minutes * 60 / args.interval)) + 1 + else: + p.error("pass --minutes or --frames") + + run(frames, args.interval, args.exposure_us, args.gain, args.tag) + + +if __name__ == "__main__": + sys.exit(main()) From e4f9d8ebf943239ec4aa8720d9c18ddc35c4e0fb Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:16 -0400 Subject: [PATCH 08/42] Add tiled mosaic acquisition with stitching and focus-plane support --- acquisition/orchestration/mosaic.py | 455 ++++++++++++++++++++++++++++ 1 file changed, 455 insertions(+) create mode 100644 acquisition/orchestration/mosaic.py diff --git a/acquisition/orchestration/mosaic.py b/acquisition/orchestration/mosaic.py new file mode 100644 index 0000000..426d54f --- /dev/null +++ b/acquisition/orchestration/mosaic.py @@ -0,0 +1,455 @@ +# mosaic.py +# ------------------------------------------------------------ +# Whole-dish time-lapse by tiling: raster a grid of fields with the stage, +# stitch them into one large image, and repeat the grid on an interval. +# +# # plan only - prints the grid, moves nothing +# python -m acquisition.orchestration.mosaic --grid 6x6 --dry-run +# +# # one mosaic now, to check focus and framing before committing +# python -m acquisition.orchestration.mosaic --grid 6x6 --rounds 1 +# +# # overnight: a mosaic every 20 min for 12 h +# python -m acquisition.orchestration.mosaic --grid 6x6 --interval-min 20 --hours 12 +# +# WHY TILING RATHER THAN A SECOND CAMERA: the Baumer has no lens of its +# own - it is a bare sensor fed by the microscope optics, so it cannot +# image a whole dish directly. The stage, however, travels +/-57 mm in X +# and +/-37.5 mm in Y, far more than a 35 mm dish. So the wide view is +# assembled from many narrow ones. +# +# STITCHING IS BY STAGE COORDINATES, NOT FEATURE MATCHING. The stage is +# accurate (commanded 150.00 um -> reported 150.00) and the image scale +# was measured (1.4901 um/px), so every tile's position on the canvas is +# computed, not searched for. That matters for a living sample: feature +# matching on an organism that moves between tiles can align to the +# organism instead of the substrate and warp the mosaic. Geometry cannot. +# +# THE STAGE->IMAGE MAPPING IS MEASURED, NOT ASSUMED. calibrate_axes() +# moves in X and then in Y and phase-correlates each, recovering a full +# 2x2 matrix. This gets the SIGNS right - image Y usually points down +# while stage Y points up, so assuming a sign is a coin flip that +# silently mirrors the mosaic - and absorbs any small camera rotation. +# Pass --scale to skip it only if you already know the mapping. +# +# LIGHT: Physarum is negatively phototactic, and a tiled run illuminates +# any given spot only for the moment its tile is taken rather than +# continuously. That is gentler than parking on one field for hours - a +# real advantage of this approach for an overnight run, not just a +# side effect. +# +# FOCUS IS THE KNOWN WEAKNESS: Z is held fixed across the whole grid, so +# a non-level agar surface will drift out of focus toward the edges. +# Run --rounds 1 first and look at the corner tiles before trusting an +# overnight run. Each tile's focus score is written to tiles.csv so the +# drift is measurable rather than a surprise in the morning. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +import sys +import time +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from acquisition.paths import data_root + +#: Per-call stage limit in nis_sdk. Longer hops are broken into chunks. +_MAX_STEP_UM = 4500.0 + +#: Seconds to let the stage settle before grabbing. The stage reports the +#: move complete before vibration has died away; a frame taken too early +#: is motion-blurred in a way no later processing can undo. +SETTLE_S = 0.8 + + +def _move_safe(sdk, x: float, y: float) -> tuple[float, float]: + """Absolute move, split into legal-sized hops. + + nis_sdk refuses a single step over MAX_XY_STEP_UM, which the jump from + the end of one grid back to the start of the next will exceed. + """ + while True: + cx, cy = sdk.XY_GetPosition() + dx, dy = x - cx, y - cy + dist = float(np.hypot(dx, dy)) + if dist < 1.0: + return cx, cy + if dist <= _MAX_STEP_UM: + sdk.XY_Move(x, y) + return sdk.XY_GetPosition() + f = _MAX_STEP_UM / dist + sdk.XY_Move(cx + dx * f, cy + dy * f) + + +#: A computed focus Z may never depart from the reference focus by more +#: than this. The sample tilt actually measured across a 15.6 mm span was +#: 306 um, so 400 um is generous for any real surface while still being +#: far short of the objective's working distance. See FocusPlane. +MAX_FOCUS_DEVIATION_UM = 400.0 + + +class FocusPlane: + """z = z_ref + b*(x-x0) + c*(y-y0), fitted to measured focus points. + + THIS CLASS EXISTS BECAUSE A NAIVE FIT DROVE THE OBJECTIVE AT THE DISH. + On 2026-09-21 a plane was fitted to TWO points with + `lstsq` on [1, x, y]. Two points cannot determine three coefficients, + so lstsq returned its minimum-norm solution - and because it minimises + a^2+b^2+c^2, and x,y are ~10^4 while z is ~4.6x10^3, the cheapest way + to satisfy both equations was to collapse the intercept to ZERO and + use enormous gradients: z = 0 + 0.209*x + 1.470*y. At the far corner + of the grid that evaluates to 10734 um against a true focus of 4573 - + about 6 mm of objective travel toward the sample. The raster was + stopped at tile 15 of 48 before reaching it. + So this class refuses the conditions that produced that: + * at least THREE points, and not collinear - anything less cannot + define a plane, and pretending otherwise is what caused the fault + * coordinates are centred before fitting, so the free parameter is + the mean measured focus rather than an intercept at x=y=0 tens of + millimetres outside the sample + * every evaluated Z is hard-clamped to MAX_FOCUS_DEVIATION_UM around + the reference, so no arithmetic error downstream can command a + large move even if the fit is somehow still wrong + """ + + def __init__(self, points: list[tuple[float, float, float]], + max_dev: float = MAX_FOCUS_DEVIATION_UM): + if len(points) < 3: + raise ValueError( + f"a focus plane needs at least 3 measured points, got {len(points)}. " + "Two points cannot define a plane - fitting them is what drove the " + "objective 6 mm off focus on 2026-09-21. Use a fixed Z instead.") + pts = np.asarray(points, dtype=float) + self.x0, self.y0 = pts[:, 0].mean(), pts[:, 1].mean() + dx, dy = pts[:, 0] - self.x0, pts[:, 1] - self.y0 + # Collinear points leave the perpendicular gradient undetermined, + # which is the same degenerate case by another route. + spread = np.linalg.svd(np.column_stack([dx, dy]), compute_uv=False) + if spread.min() < 1e-6 * max(spread.max(), 1.0): + raise ValueError( + "focus points are collinear - they cannot determine tilt " + "perpendicular to the line joining them. Add a point off that line.") + A = np.column_stack([np.ones_like(dx), dx, dy]) + coef, *_ = np.linalg.lstsq(A, pts[:, 2], rcond=None) + self.z_ref, self.b, self.c = (float(v) for v in coef) + self.max_dev = float(max_dev) + self.residuals = [abs(self.z(px, py) - pz) for px, py, pz in points] + + def z(self, x: float, y: float) -> float: + raw = self.z_ref + self.b * (x - self.x0) + self.c * (y - self.y0) + return float(np.clip(raw, self.z_ref - self.max_dev, self.z_ref + self.max_dev)) + + def would_clamp(self, x: float, y: float) -> bool: + raw = self.z_ref + self.b * (x - self.x0) + self.c * (y - self.y0) + return abs(raw - self.z_ref) > self.max_dev + + def __str__(self) -> str: + return (f"z = {self.z_ref:.2f} {self.b:+.5f}*(x-{self.x0:.0f}) " + f"{self.c:+.5f}*(y-{self.y0:.0f}) clamped to +/-{self.max_dev:.0f} um") + + +def _move_z(sdk, z: float, limit: float = 45.0) -> float: + """Z in steps under nis_sdk's 50 um per-call cap.""" + for _ in range(200): + cz = sdk.Z_GetPosition() + d = z - cz + if abs(d) < 0.3: + return cz + sdk.Z_Move(cz + max(-limit, min(limit, d))) + time.sleep(0.2) + return sdk.Z_GetPosition() + + +def _grab_gray(cam) -> np.ndarray: + p = cam.capture() + a = np.asarray(Image.open(p)).mean(axis=2).astype(np.float32) + p.unlink(missing_ok=True) + return a + + +def calibrate_axes(cam, sdk, step_um: float = 150.0) -> np.ndarray: + """Measure M where (image shift in px) = M @ (stage delta in um). + + Returns a 2x2 matrix. Inverting the sign of M gives where a tile + belongs on the canvas: if moving the stage +X slides features left in + the image, then the tile taken at larger X shows sample further right. + """ + x0, y0 = sdk.XY_GetPosition() + base = _grab_gray(cam) + win = cv2.createHanningWindow((base.shape[1], base.shape[0]), cv2.CV_32F) + + cols = [] + for axis in (0, 1): + tx = x0 + (step_um if axis == 0 else 0.0) + ty = y0 + (step_um if axis == 1 else 0.0) + _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S + 0.7) + xa, ya = sdk.XY_GetPosition() + actual = (xa - x0) if axis == 0 else (ya - y0) + moved = _grab_gray(cam) + (dx, dy), resp = cv2.phaseCorrelate(base, moved, win) + if abs(actual) < 1.0 or resp < 0.05: + raise SystemExit( + f"axis {'XY'[axis]} calibration failed (moved {actual:.2f} um, " + f"confidence {resp:.3f}) - too little texture in view, or the stage did not move" + ) + cols.append([dx / actual, dy / actual]) + print(f" axis {'XY'[axis]}: moved {actual:+.2f} um -> image ({dx:+.2f}, {dy:+.2f}) px, " + f"confidence {resp:.3f}") + _move_safe(sdk, x0, y0) + time.sleep(SETTLE_S) + + M = np.array(cols).T # columns are the X and Y responses + scale = float(np.hypot(M[0, 0], M[1, 0])) + print(f" => {1.0 / scale:.4f} um/px" if scale else " => degenerate mapping") + return M + + +def grid_positions(cx: float, cy: float, nx: int, ny: int, + step_x: float, step_y: float) -> list[tuple[float, float]]: + """Serpentine raster - each row reverses, so the stage never makes a + long dash back to the start of the next row. Less travel, less + vibration, less time per mosaic.""" + xs = (np.arange(nx) - (nx - 1) / 2.0) * step_x + cx + ys = (np.arange(ny) - (ny - 1) / 2.0) * step_y + cy + out = [] + for j, y in enumerate(ys): + row = xs if j % 2 == 0 else xs[::-1] + out.extend((float(x), float(y)) for x in row) + return out + + +def stitch(tiles: list[tuple[np.ndarray, float, float]], M: np.ndarray, + ref: tuple[float, float]) -> np.ndarray: + """Paste tiles onto one canvas using their stage coordinates.""" + Minv = -M # canvas offset is the negated image shift + offs = [Minv @ np.array([sx - ref[0], sy - ref[1]]) for _, sx, sy in tiles] + h, w = tiles[0][0].shape[:2] + ox = np.array([o[0] for o in offs]); oy = np.array([o[1] for o in offs]) + ox -= ox.min(); oy -= oy.min() + # Size from the ROUNDED placements, not the raw maxima: a tile whose + # offset is x.6 is written at x+1, which overruns a canvas sized on + # int(x). Rounding first makes the bound exact. + xi_all = np.round(ox).astype(int); yi_all = np.round(oy).astype(int) + canvas = np.zeros((int(yi_all.max()) + h, int(xi_all.max()) + w, 3), np.uint8) + for (img, _, _), xi, yi in zip(tiles, xi_all, yi_all): + canvas[yi:yi + h, xi:xi + w] = img + return canvas + + +def run(grid: tuple[int, int], overlap: float, rounds: int, interval_s: float, + exposure_us: float, mosaic_scale: float, keep_tiles: bool, + scale_um_px: float | None, tag: str | None, + focus_points: list[tuple[float, float, float]] | None = None, + centre: tuple[float, float] | None = None) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / (f"mosaic_{stamp}" + (f"_{tag}" if tag else "")) + out.mkdir(parents=True, exist_ok=True) + + sdk = NISSdk() + cam = BaumerGenICam() + cx = cy = 0.0 + try: + cam.set_settings(exposure_time_us=exposure_us, gain=1.0) + # AUTO WHITE BALANCE MUST BE OFF FOR A MOSAIC. Left on Continuous, + # the camera re-neutralises every tile independently - a tile of + # bare agar and a tile of organism then get different corrections, + # and the stitched result has visible colour seams at every tile + # boundary that no amount of post-processing can unpick. It resets + # itself to Continuous across power cycles, so force it per run + # rather than trusting whatever it was left in. + try: + node = cam._acquirer.remote_device.node_map.BalanceWhiteAuto + if node.value != "Off": + print(f"BalanceWhiteAuto was {node.value!r} - forcing 'Off' for consistent tiles") + node.value = "Off" + except Exception as exc: + print(f"WARNING: could not force BalanceWhiteAuto off ({exc}); " + "tiles may not colour-match", file=sys.stderr) + print(f"camera: {cam.get_settings()}") + if centre is not None: + cx, cy = centre + _move_safe(sdk, cx, cy) + cx, cy = sdk.XY_GetPosition() + else: + cx, cy = sdk.XY_GetPosition() + print(f"grid centre: ({cx:.1f}, {cy:.1f}) um") + + plane = None + z_fixed = sdk.Z_GetPosition() + if focus_points: + plane = FocusPlane(focus_points) + print(f"focus plane {plane} (from {len(focus_points)} points, " + f"max residual {max(plane.residuals):.1f} um)") + else: + print(f"FIXED focus: Z held at {z_fixed:.2f} um for every tile " + f"(no --focus-plane given)") + + if scale_um_px: + M = np.array([[1.0 / scale_um_px, 0.0], [0.0, 1.0 / scale_um_px]]) + print(f"using supplied scale {scale_um_px} um/px (axes assumed square and unflipped)") + else: + print("calibrating stage->image mapping...") + M = calibrate_axes(cam, sdk) + + probe = _grab_gray(cam) + h, w = probe.shape + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + step_x = w * umpx * (1.0 - overlap) + step_y = h * umpx * (1.0 - overlap) + nx, ny = grid + pts = grid_positions(cx, cy, nx, ny, step_x, step_y) + print(f"{nx}x{ny} tiles, step {step_x:.0f} x {step_y:.0f} um " + f"-> covers {nx * step_x / 1000:.2f} x {ny * step_y / 1000:.2f} mm") + + (out / "mosaic.json").write_text(json.dumps({ + "grid": [nx, ny], "overlap": overlap, "um_per_px": round(umpx, 4), + "step_um": [round(step_x, 1), round(step_y, 1)], + "centre_um": [cx, cy], "exposure_us": exposure_us, + "stage_to_image_matrix": M.tolist(), + "covers_mm": [round(nx * step_x / 1000, 3), round(ny * step_y / 1000, 3)], + }, indent=2), encoding="utf-8") + + # An out-of-band kill switch. Stopping a background shell does NOT + # always take its python child with it - on 2026-09-21 a stopped + # raster kept driving the stage afterwards. Creating this file makes + # the run halt itself at the next tile boundary, without needing to + # find and kill a process. + stop_file = out / "STOP" + print(f"to halt this run at any point, create: {stop_file}") + + fields = ("round", "tile", "wall_clock", "x_um", "y_um", "z_um", + "mean", "focus", "error") + log = (out / "tiles.csv").open("w", newline="", encoding="utf-8") + writer = csv.DictWriter(log, fieldnames=fields) + writer.writeheader() + + t_start = time.monotonic() + for r in range(rounds): + target = t_start + r * interval_s + if time.monotonic() < target: + time.sleep(target - time.monotonic()) + r_dir = out / f"round_{r:03d}" + if keep_tiles: + r_dir.mkdir(exist_ok=True) + print(f"\n[round {r + 1}/{rounds}] {datetime.datetime.now():%H:%M:%S}", flush=True) + + tiles = [] + for i, (tx, ty) in enumerate(pts): + row = {k: "" for k in fields} + row.update(round=r, tile=i, + wall_clock=datetime.datetime.now().isoformat(timespec="seconds")) + try: + if stop_file.exists(): + print(f" STOP FILE {stop_file.name} present - halting", flush=True) + raise KeyboardInterrupt + ax, ay = _move_safe(sdk, tx, ty) + if plane is not None: + _move_z(sdk, plane.z(ax, ay)) + row["z_um"] = round(sdk.Z_GetPosition(), 2) + time.sleep(SETTLE_S) + p = cam.capture() + img = np.asarray(Image.open(p)) + if keep_tiles: + p.replace(r_dir / f"t{i:03d}.png") + else: + p.unlink(missing_ok=True) + g = img.mean(axis=2) + lap = (g[:-2, 1:-1] + g[2:, 1:-1] + g[1:-1, :-2] + g[1:-1, 2:] + - 4 * g[1:-1, 1:-1]) + row.update(x_um=round(ax, 1), y_um=round(ay, 1), + mean=round(float(g.mean()), 2), focus=round(float(lap.var()), 2)) + tiles.append((img, ax, ay)) + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + print(f" tile {i}: {row['error']}", flush=True) + writer.writerow(row); log.flush() + + if tiles: + canvas = stitch(tiles, M, (cx, cy)) + if mosaic_scale != 1.0: + canvas = cv2.resize( + canvas, (int(canvas.shape[1] * mosaic_scale), + int(canvas.shape[0] * mosaic_scale)), + interpolation=cv2.INTER_AREA) + dest = out / f"mosaic_{r:03d}.png" + Image.fromarray(canvas).save(dest) + print(f" {len(tiles)}/{len(pts)} tiles -> {dest.name} " + f"({canvas.shape[1]}x{canvas.shape[0]})", flush=True) + + log.close() + finally: + try: + _move_safe(sdk, cx, cy) + print(f"returned to grid centre ({cx:.1f}, {cy:.1f}) um") + except Exception as exc: + print(f"WARNING: could not return to centre: {exc}", file=sys.stderr) + cam.close() + print("released") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--grid", default="6x6", help="tiles as NXxNY (default 6x6)") + ap.add_argument("--overlap", type=float, default=0.10, help="tile overlap fraction") + ap.add_argument("--rounds", type=int, help="number of mosaics (default: from --hours)") + ap.add_argument("--hours", type=float, help="run length; with --interval-min sets rounds") + ap.add_argument("--interval-min", type=float, default=20.0) + ap.add_argument("--exposure-us", type=float, default=10000.0) + ap.add_argument("--mosaic-scale", type=float, default=0.5, + help="downsample the stitched mosaic (tiles are kept full-res)") + ap.add_argument("--no-tiles", action="store_true", help="do not keep individual tiles") + ap.add_argument("--scale", type=float, help="known um/px; skips axis calibration") + ap.add_argument("--tag") + ap.add_argument("--focus-plane", + help="measured focus references as 'x,y,z;x,y,z;...' (um). Z is " + "interpolated per tile from a least-squares plane through them") + ap.add_argument("--centre", help="grid centre as 'x,y' (um); default is the current position") + ap.add_argument("--dry-run", action="store_true", help="print the plan, move nothing") + args = ap.parse_args() + + fpts = None + if args.focus_plane: + fpts = [tuple(float(v) for v in grp.split(",")) + for grp in args.focus_plane.split(";") if grp.strip()] + if any(len(p) != 3 for p in fpts): + ap.error("--focus-plane entries must each be x,y,z") + ctr = tuple(float(v) for v in args.centre.split(",")) if args.centre else None + + nx, ny = (int(v) for v in args.grid.lower().split("x")) + if args.rounds: + rounds = args.rounds + elif args.hours: + rounds = max(1, int(round(args.hours * 60 / args.interval_min))) + else: + rounds = 1 + + if args.dry_run: + umpx = args.scale or 1.4901 + sx = 1920 * umpx * (1 - args.overlap) / 1000 + sy = 1200 * umpx * (1 - args.overlap) / 1000 + print(f"{nx}x{ny} = {nx * ny} tiles, step {sx:.3f} x {sy:.3f} mm") + print(f"covers {nx * sx:.2f} x {ny * sy:.2f} mm at {umpx} um/px") + print(f"{rounds} rounds every {args.interval_min} min " + f"= {rounds * args.interval_min / 60:.1f} h") + print(f"~{nx * ny * 3:.0f} MB of tiles per round, ~{rounds * nx * ny * 3 / 1000:.1f} GB total") + return + + run((nx, ny), args.overlap, rounds, args.interval_min * 60.0, args.exposure_us, + args.mosaic_scale, not args.no_tiles, args.scale, args.tag, fpts, ctr) + + +if __name__ == "__main__": + main() From e612905eeb467bf4efc2989b6e03f9ac2bc1665a Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:16 -0400 Subject: [PATCH 09/42] Add Physarum contraction-rhythm analysis for time-lapse series --- analysis/physarum_rhythm.py | 308 ++++++++++++++++++++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 analysis/physarum_rhythm.py diff --git a/analysis/physarum_rhythm.py b/analysis/physarum_rhythm.py new file mode 100644 index 0000000..3851c10 --- /dev/null +++ b/analysis/physarum_rhythm.py @@ -0,0 +1,308 @@ +# physarum_rhythm.py +# ------------------------------------------------------------ +# Quantify a fixed-position brightfield time-lapse of a Physarum +# plasmodium: the contraction rhythm, and how the organism's footprint +# changes over the run. +# +# python -m analysis.physarum_rhythm data/timelapse_20260921_150051_physarum +# python -m analysis.physarum_rhythm --json out.json +# +# THE MEASUREMENT: in transmitted light a plasmodium's local brightness +# tracks its local thickness - a thicker strand absorbs/scatters more, so +# it reads darker. Physarum drives cytoplasm through its network with a +# peristaltic contraction wave, so thickness at a fixed point rises and +# falls, and transmitted intensity oscillates with it. The period of that +# oscillation is the quantity of interest; the textbook figure for +# P. polycephalum is ~100-130 s at room temperature. +# +# WHY A PER-PIXEL PERIODOGRAM AND NOT JUST THE FRAME MEAN: neighbouring +# regions of one plasmodium oscillate at a common period but *out of +# phase* - that phase gradient is the peristaltic wave. Averaging the +# whole frame first lets antiphase regions cancel, which can bury a +# strong local rhythm in a flat global trace. So the period is taken from +# the median of per-pixel dominant frequencies inside the mask, and the +# global trace is reported alongside as a sanity check, not as the +# primary estimate. +# +# DETRENDING: the slow drift from the organism advancing across the field +# (and any lamp drift) is a large low-frequency component that would +# otherwise dominate every periodogram. Each pixel's series is linearly +# detrended and Hann-windowed before the FFT, and frequencies below +# MIN_PERIOD_S/MAX_PERIOD_S are excluded from the peak search. +# +# SCALE: areas are reported in mm^2 when the series folder carries a +# calibration.json with a measured `um_per_px`, and in PIXELS otherwise - +# never by assuming a magnification. acquisition/orchestration's runs +# since 2026-09-21 write that file; older series predate it and will +# report pixels, which is the honest answer for them. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +#: Period search window, seconds. Wide enough to contain the textbook +#: ~100-130 s without presupposing it - a result landing hard against +#: either edge should be read as "outside the searched band", not as a +#: measurement. +MIN_PERIOD_S = 30.0 +MAX_PERIOD_S = 600.0 + +#: Spatial downsample for the per-pixel analysis. The rhythm is a +#: large-scale thickness wave, not a fine texture, so full resolution buys +#: nothing here and costs ~25x the memory (a full-res float stack of a +#: 360-frame run is several GB). +DOWNSAMPLE = 5 + + +def _um_per_px(series_dir: Path) -> float | None: + """Measured pixel size for this series, or None if it was never measured. + + Returning None rather than a default is deliberate: a plausible-looking + wrong scale silently turns every area into a wrong physical number, + which is worse than an honest pixel count. + """ + path = series_dir / "calibration.json" + if not path.exists(): + return None + try: + value = json.loads(path.read_text(encoding="utf-8")).get("um_per_px") + return float(value) if value else None + except (ValueError, OSError): + return None + + +def _load_series(series_dir: Path) -> tuple[np.ndarray, np.ndarray, list[str]]: + """Return (times_s, small_stack, filenames) for the frames that exist. + + Frames whose row carries an `error`, or whose file is missing, are + skipped rather than faked - a gap in the series is better than an + interpolated frame in a periodogram. + """ + rows = list(csv.DictReader((series_dir / "frames.csv").open(encoding="utf-8"))) + times, frames, names = [], [], [] + for r in rows: + if r.get("error") or not r.get("filename"): + continue + path = series_dir / r["filename"] + if not path.exists(): + continue + g = np.asarray(Image.open(path)).mean(axis=2).astype(np.float32) + frames.append(g[::DOWNSAMPLE, ::DOWNSAMPLE]) + times.append(float(r["t_seconds"])) + names.append(r["filename"]) + if len(frames) < 8: + raise SystemExit(f"only {len(frames)} usable frames in {series_dir} - too few to analyse") + return np.asarray(times), np.stack(frames), names + + +#: Band a plasmodium contraction rhythm is expected to fall in. Used ONLY +#: to score which Otsu region is the organism (see _regions), never to +#: constrain the reported period - that is searched over the full +#: MIN/MAX_PERIOD_S window so a result outside this band can still come out. +_BIO_BAND_S = (60.0, 200.0) + + +def _regions(stack: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """The two Otsu classes of the time-averaged frame, as (bright, dark). + + Otsu on the mean image rather than per frame: a per-frame threshold + would itself flicker with the contraction rhythm, writing the signal + being measured into the very region it is measured over. + + WHICH ONE IS THE ORGANISM IS NOT DECIDED HERE, deliberately. A thick + pigmented plasmodium usually reads darker than bare agar in + transmitted light, but that depends on the illumination, the + substrate and how thin the organism has spread - and getting it + backwards would measure the substrate with full confidence. analyse() + scores both regions for rhythmicity instead and lets the oscillation + identify the living one. + """ + mean_img = stack.mean(axis=0) + u8 = cv2.normalize(mean_img, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + _, th = cv2.threshold(u8, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + k = np.ones((3, 3), np.uint8) + th = cv2.morphologyEx(th, cv2.MORPH_OPEN, k, iterations=2) + th = cv2.morphologyEx(th, cv2.MORPH_CLOSE, k, iterations=2) + bright = th > 0 + return bright, ~bright + + +def _dominant_period(series: np.ndarray, dt: float) -> tuple[np.ndarray, np.ndarray]: + """Per-pixel dominant period (s) and its spectral power fraction. + + `series` is (T, N) - time along axis 0. Returns (period, strength), + each length N. `strength` is the peak's share of total in-band power: + a clean oscillation concentrates power in one bin, noise spreads it, + so it is the natural weight for "how periodic is this pixel". + """ + T = series.shape[0] + t = np.arange(T, dtype=np.float64) + + # Linear detrend per pixel (closed form - avoids scipy). + tm = t.mean() + denom = ((t - tm) ** 2).sum() + slope = ((t - tm)[:, None] * (series - series.mean(axis=0))).sum(axis=0) / denom + detr = series - (slope[None, :] * (t - tm)[:, None] + series.mean(axis=0)) + + win = np.hanning(T)[:, None] + spec = np.abs(np.fft.rfft(detr * win, axis=0)) ** 2 + freqs = np.fft.rfftfreq(T, d=dt) + + band = (freqs >= 1.0 / MAX_PERIOD_S) & (freqs <= 1.0 / MIN_PERIOD_S) + if not band.any(): + raise SystemExit("period search band contains no FFT bins - run longer or sample faster") + + spec_b, freqs_b = spec[band], freqs[band] + idx = spec_b.argmax(axis=0) + peak = spec_b[idx, np.arange(spec_b.shape[1])] + total = spec_b.sum(axis=0) + strength = np.divide(peak, total, out=np.zeros_like(peak), where=total > 0) + return 1.0 / freqs_b[idx], strength + + +def _score_region(flat: np.ndarray, mflat: np.ndarray, dt: float) -> dict: + """Periodogram summary for one region, plus a rhythmicity score. + + `rhythmic_fraction` - the share of the region's pixels that both + oscillate cleanly (peak holds >=15% of in-band power) and do so + within _BIO_BAND_S - is what distinguishes a contracting organism + from substrate that merely drifts or flickers. + """ + period, strength = _dominant_period(flat[:, mflat], dt) + lo, hi = _BIO_BAND_S + rhythmic = (strength >= 0.15) & (period >= lo) & (period <= hi) + good = strength >= np.percentile(strength, 75) + return { + "pixels": int(mflat.sum()), + "mean_intensity": round(float(flat[:, mflat].mean()), 2), + "period_s": round(float(np.median(period[good])), 1), + "period_iqr_s": [round(float(np.percentile(period[good], 25)), 1), + round(float(np.percentile(period[good], 75)), 1)], + "rhythmic_fraction": round(float(rhythmic.mean()), 3), + "_period": period, "_strength": strength, "_good": good, + } + + +def analyse(series_dir: Path) -> dict: + times, stack, names = _load_series(series_dir) + dt = float(np.median(np.diff(times))) + umpx = _um_per_px(series_dir) + bright, dark = _regions(stack) + + T = stack.shape[0] + flat = stack.reshape(T, -1) + + scored = {name: _score_region(flat, m.reshape(-1), dt) + for name, m in (("bright", bright), ("dark", dark))} + + # The organism is whichever region actually oscillates in the + # biological band - not whichever is brighter. If the two scores are + # close, that is reported rather than papered over: it means the + # segmentation did not separate organism from substrate, and the + # period below should not be trusted without looking at the frames. + order = sorted(scored, key=lambda k: scored[k]["rhythmic_fraction"], reverse=True) + organism, other = order[0], order[1] + margin = scored[organism]["rhythmic_fraction"] - scored[other]["rhythmic_fraction"] + + sel = scored[organism] + mflat = (bright if organism == "bright" else dark).reshape(-1) + period, strength, good = sel["_period"], sel["_strength"], sel["_good"] + period_est = sel["period_s"] + + # Global trace (sanity check - see module header on why it is not primary). + trace = flat[:, mflat].mean(axis=1) + + # Footprint over time, from a fixed threshold so area changes reflect + # the organism, not a moving threshold. + u8m = cv2.normalize(stack.mean(axis=0), None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8) + thr, _ = cv2.threshold(u8m, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) + lo, hi = float(stack.min()), float(stack.max()) + level = lo + (thr / 255.0) * (hi - lo) + side = (stack > level) if organism == "bright" else (stack < level) + area = side.reshape(T, -1).sum(axis=1).astype(float) + area_px = area * (DOWNSAMPLE ** 2) + + return { + "series_dir": str(series_dir), + "frames_used": T, + "duration_s": float(times[-1] - times[0]), + "interval_s": dt, + "organism_region": organism, + "organism_identified_by": "rhythmicity", + "rhythmic_margin": round(float(margin), 3), + "region_scores": {k: {kk: vv for kk, vv in v.items() if not kk.startswith("_")} + for k, v in scored.items()}, + "mask_pixels_downsampled": int(mflat.sum()), + "period_s": round(period_est, 1), + "period_iqr_s": [round(float(np.percentile(period[good], 25)), 1), + round(float(np.percentile(period[good], 75)), 1)], + "cycles_observed": round(float(times[-1] - times[0]) / period_est, 1), + "oscillating_pixel_fraction": round(float((strength > 0.15).mean()), 3), + "area_start_px": round(area_px[0], 0), + "area_end_px": round(area_px[-1], 0), + "area_change_pct": round(100.0 * (area_px[-1] - area_px[0]) / area_px[0], 2), + "um_per_px": umpx, + "area_start_mm2": round(area_px[0] * umpx ** 2 / 1e6, 4) if umpx else None, + "area_end_mm2": round(area_px[-1] * umpx ** 2 / 1e6, 4) if umpx else None, + "area_change_mm2": (round((area_px[-1] - area_px[0]) * umpx ** 2 / 1e6, 4) + if umpx else None), + "growth_rate_mm2_per_h": ( + round((area_px[-1] - area_px[0]) * umpx ** 2 / 1e6 + / max((times[-1] - times[0]) / 3600.0, 1e-9), 4) if umpx else None), + "trace_t_s": [round(float(x), 1) for x in times], + "trace_intensity": [round(float(x), 4) for x in trace], + "trace_area_px": [round(float(x), 0) for x in area_px], + "period_map_percentiles": { + str(p): round(float(np.percentile(period[good], p)), 1) + for p in (5, 25, 50, 75, 95) + }, + } + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--json", type=Path, help="also write the full result here") + args = ap.parse_args() + + res = analyse(args.series_dir) + if args.json: + args.json.write_text(json.dumps(res, indent=2), encoding="utf-8") + + print(f"frames {res['frames_used']} over {res['duration_s'] / 60:.1f} min " + f"@ {res['interval_s']:.1f}s") + rs = res["region_scores"] + print(f"organism region {res['organism_region']} " + f"(rhythmic fraction {rs[res['organism_region']]['rhythmic_fraction']} vs " + f"{min(v['rhythmic_fraction'] for v in rs.values())}, " + f"margin {res['rhythmic_margin']})") + if res["rhythmic_margin"] < 0.05: + print(" WARNING: the two regions are near-equally rhythmic - segmentation") + print(" did not separate organism from substrate. Treat the period below") + print(" as unverified and look at the frames.") + print(f"contraction period {res['period_s']} s " + f"(IQR {res['period_iqr_s'][0]}-{res['period_iqr_s'][1]} s, " + f"{res['cycles_observed']} cycles observed)") + print(f"oscillating pixels {res['oscillating_pixel_fraction'] * 100:.1f}% of the plasmodium mask") + if res["um_per_px"]: + print(f"footprint {res['area_change_pct']:+.2f}% " + f"({res['area_start_mm2']:.4f} -> {res['area_end_mm2']:.4f} mm2, " + f"{res['area_change_mm2']:+.4f} mm2)") + print(f"growth rate {res['growth_rate_mm2_per_h']:+.4f} mm2/h " + f"(scale {res['um_per_px']} um/px, measured)") + else: + print(f"footprint {res['area_change_pct']:+.2f}% " + f"({res['area_start_px']:.0f} -> {res['area_end_px']:.0f} px) " + f"- no calibration.json, so pixels only") + + +if __name__ == "__main__": + main() From dd669d44e1cf0ffd2a7ea52b33303f65df6e527e Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:16 -0400 Subject: [PATCH 10/42] Add make_movie: render a time-lapse series to MP4 --- analysis/make_movie.py | 183 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 analysis/make_movie.py diff --git a/analysis/make_movie.py b/analysis/make_movie.py new file mode 100644 index 0000000..6caaff9 --- /dev/null +++ b/analysis/make_movie.py @@ -0,0 +1,183 @@ +# make_movie.py +# ------------------------------------------------------------ +# Turn a timelapse series into a watchable MP4. +# +# python -m analysis.make_movie data/timelapse_20260921_163741_oats +# python -m analysis.make_movie --fps 24 --out front.mp4 +# python -m analysis.make_movie --scale 0.5 --no-overlay +# +# A folder of 500 PNGs is data; a 20-second movie is something a person +# can actually look at and see an organism move. This is the step that +# turns one into the other. +# +# SCALE BAR: the pixel size is not guessable from the image - it depends +# on the objective and the C-mount adapter. UM_PER_PX below was MEASURED +# on 2026-09-21 by moving the stage a known 150.00 um and phase- +# correlating the before/after frames (shift 100.66 px, confidence 0.95, +# dy 0.18 px so the camera is square to the stage axes). It is valid ONLY +# for nosepiece position 1 with that adapter - change the objective and +# it is wrong. Pass --um-per-px to override, or --no-overlay to omit the +# bar rather than draw a scale you do not trust. +# +# TIMESTAMPS come from frames.csv's t_seconds, not from frame index x +# interval: a run that dropped or lagged a frame would otherwise show a +# time that drifts from reality, and the whole point of the overlay is to +# let a viewer judge rate. +# +# WHY mp4v: cv2's ffmpeg build on Windows reliably has it. H.264 gives +# smaller files but is not always present in the wheel, and a movie that +# fails to write is worse than one that is a few MB larger. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import cv2 +import numpy as np + +#: Measured, not assumed - see module header. Objective-specific. +UM_PER_PX = 1.4901 + + +def _rows(series_dir: Path) -> list[dict]: + with (series_dir / "frames.csv").open(encoding="utf-8") as fh: + rows = [r for r in csv.DictReader(fh) if r.get("filename") and not r.get("error")] + return [r for r in rows if (series_dir / r["filename"]).exists()] + + +def wb_gains(img: np.ndarray) -> np.ndarray: + """Per-channel gains that make the brightest UNCLIPPED pixels neutral. + + In transmitted brightfield the bright background is light that missed + the specimen, so it is a legitimate white reference - but only where + it is not clipped. Clipped pixels are (255,255,255) by definition, so + including them forces the gains to 1.0 and silently reports "already + balanced" no matter how strong the cast really is. That mistake was + made once on this data; hence the explicit mask. + + Measured on this rig 2026-09-21: the camera's native response with + BalanceWhiteAuto off is markedly blue (background R=145 G=168 B=237), + needing roughly 1.26/1.09/0.77. + """ + chans = [img[:, :, i].astype(np.float32) for i in range(3)] + g = img.mean(axis=2) + unclipped = (img < 250).all(axis=2) + if unclipped.sum() < 1000: + return np.ones(3, dtype=np.float32) + ref = unclipped & (g > np.percentile(g[unclipped], 92)) + if ref.sum() < 200: + return np.ones(3, dtype=np.float32) + means = np.array([c[ref].mean() for c in chans], dtype=np.float32) + if (means <= 1).any(): + return np.ones(3, dtype=np.float32) + return means.mean() / means + + +def _scale_bar(img: np.ndarray, um_per_px: float) -> None: + """Draw a scale bar whose length is a round number of microns. + + Picks the largest 'nice' length that stays under a quarter of the + frame width, so the bar is meaningful at any magnification instead of + a fixed pixel count that means something different on every objective. + """ + h, w = img.shape[:2] + max_um = (w * 0.25) * um_per_px + nice = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + target = max([n for n in nice if n <= max_um], default=nice[0]) + px = int(round(target / um_per_px)) + + x1, y2 = w - 40, h - 40 + x0 = x1 - px + cv2.rectangle(img, (x0, y2 - 9), (x1, y2), (255, 255, 255), -1) + cv2.rectangle(img, (x0, y2 - 9), (x1, y2), (0, 0, 0), 1) + label = f"{target} um" if target < 1000 else f"{target / 1000:g} mm" + (tw, _), _ = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2) + cv2.putText(img, label, (x0 + (px - tw) // 2, y2 - 16), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 3) + cv2.putText(img, label, (x0 + (px - tw) // 2, y2 - 16), + cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 255), 1) + + +def _stamp(img: np.ndarray, t_s: float, i: int, n: int) -> None: + hms = f"{int(t_s) // 3600:d}:{(int(t_s) % 3600) // 60:02d}:{int(t_s) % 60:02d}" + txt = f"t + {hms} ({i + 1}/{n})" + cv2.putText(img, txt, (24, 44), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 0), 4) + cv2.putText(img, txt, (24, 44), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2) + + +def build(series_dir: Path, out: Path, fps: int, scale: float, + um_per_px: float, overlay: bool, white_balance: bool = True) -> Path: + rows = _rows(series_dir) + if not rows: + raise SystemExit(f"no usable frames in {series_dir}") + + first = cv2.imread(str(series_dir / rows[0]["filename"])) + if first is None: + raise SystemExit(f"could not read {rows[0]['filename']}") + + # ONE set of gains for the whole movie, from the first frame. Computing + # them per frame would re-neutralise every frame independently, so the + # colour would shift as the organism covers more or less of the field - + # visible as flicker, and fatal to any comparison across time. + gains = wb_gains(first) if white_balance else np.ones(3, dtype=np.float32) + if white_balance: + print(f"white balance gains (B,G,R) = " + f"{gains[0]:.3f}, {gains[1]:.3f}, {gains[2]:.3f}") + h, w = first.shape[:2] + if scale != 1.0: + w, h = int(w * scale), int(h * scale) + w -= w % 2; h -= h % 2 # some encoders reject odd dimensions + + writer = cv2.VideoWriter(str(out), cv2.VideoWriter_fourcc(*"mp4v"), fps, (w, h)) + if not writer.isOpened(): + raise SystemExit("could not open the video writer - is the mp4v codec available?") + + n = len(rows) + try: + for i, r in enumerate(rows): + img = cv2.imread(str(series_dir / r["filename"])) + if img is None: + continue # skip, do not abort the whole movie + if white_balance: + img = np.clip(img.astype(np.float32) * gains[None, None, :], + 0, 255).astype(np.uint8) + if (img.shape[1], img.shape[0]) != (w, h): + img = cv2.resize(img, (w, h), interpolation=cv2.INTER_AREA) + if overlay: + _stamp(img, float(r["t_seconds"]), i, n) + _scale_bar(img, um_per_px / scale) + writer.write(img) + if i % 50 == 0: + print(f" {i + 1}/{n}", flush=True) + finally: + writer.release() + + span = float(rows[-1]["t_seconds"]) - float(rows[0]["t_seconds"]) + print(f"\n{n} frames spanning {span / 3600:.2f} h -> {n / fps:.1f} s of video") + print(f"time compression: {span / (n / fps):.0f}x") + print(f"wrote {out} ({out.stat().st_size / 1e6:.1f} MB)") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--out", type=Path, help="default: /movie.mp4") + ap.add_argument("--fps", type=int, default=30) + ap.add_argument("--scale", type=float, default=1.0, help="resize factor (0.5 = half size)") + ap.add_argument("--um-per-px", type=float, default=UM_PER_PX) + ap.add_argument("--no-overlay", action="store_true", help="omit timestamp and scale bar") + ap.add_argument("--no-white-balance", action="store_true", + help="keep the camera's raw (blue-biased) colour") + args = ap.parse_args() + + out = args.out or (args.series_dir / "movie.mp4") + build(args.series_dir, out, args.fps, args.scale, args.um_per_px, + not args.no_overlay, not args.no_white_balance) + + +if __name__ == "__main__": + main() From 1de814cd800271dcabc0f68ab98362e87175e918 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:53:16 -0400 Subject: [PATCH 11/42] Add live_view: browser preview of a running acquisition's files --- analysis/live_view.py | 150 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 analysis/live_view.py diff --git a/analysis/live_view.py b/analysis/live_view.py new file mode 100644 index 0000000..d5704c6 --- /dev/null +++ b/analysis/live_view.py @@ -0,0 +1,150 @@ +# live_view.py +# ------------------------------------------------------------ +# Watch an acquisition as it happens, without touching the camera. +# +# python -m analysis.live_view # newest run, auto-detected +# python -m analysis.live_view data/mosaic_20260921_194543_region +# python -m analysis.live_view --fps 2 --width 1100 +# +# Then open the printed file:// link in a browser and leave it there. +# +# WHY NOT AN ACTUAL LIVE FEED: a GenICam camera can be held by exactly one +# process. Anything that opened the camera to stream preview frames would +# take it away from the run that is acquiring - the failure this repo hits +# repeatedly (AccessDeniedException) whenever two things want the device. +# +# So this never opens the camera. It watches the FILES the running +# acquisition is already writing and republishes the newest one as a fixed +# filename that a browser can poll. The acquisition does not know this +# exists and cannot be affected by it: worst case, the viewer shows a +# slightly stale frame. +# +# WHY A COPY RATHER THAN POINTING THE PAGE AT THE TILE ITSELF: the newest +# file changes name constantly, and a browser cannot discover that from a +# file:// page. Republishing to one stable name (latest.jpg) makes the +# page a two-line poll. JPEG rather than PNG because it is a preview - +# it re-encodes in milliseconds where a 3 MB PNG does not. +# +# PARTIAL WRITES: a file picked up mid-write decodes as a truncated or +# corrupt image. Each candidate is therefore only published once its size +# has stopped changing between polls, and any decode error is swallowed +# and retried rather than killing the viewer. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import time +from pathlib import Path + +from PIL import Image + +from acquisition.paths import data_root + +_PAGE = """ +microscope live view + +
+ watching __DIR__ + frame - + updated - + poll __MS__ ms +
+
waiting for the first frame...
+ +""" + + +def newest_image(root: Path) -> Path | None: + best, best_m = None, -1.0 + for p in root.rglob("*.png"): + if p.name == "latest.jpg": + continue + try: + m = p.stat().st_mtime + except OSError: + continue + if m > best_m: + best, best_m = p, m + return best + + +def newest_run(base: Path) -> Path | None: + runs = [p for p in base.glob("*") if p.is_dir() + and (p.name.startswith("mosaic_") or p.name.startswith("timelapse_"))] + return max(runs, key=lambda p: p.stat().st_mtime) if runs else None + + +def serve(watch: Path, out: Path, fps: float, width: int) -> None: + out.mkdir(parents=True, exist_ok=True) + period = max(0.2, 1.0 / max(fps, 0.1)) + page = out / "index.html" + page.write_text( + _PAGE.replace("__DIR__", watch.name).replace("__MS__", str(int(period * 1000))), + encoding="utf-8") + print(f"watching : {watch}") + print(f"OPEN THIS: {page.resolve().as_uri()}") + print("(Ctrl-C to stop - this never touches the camera)") + + published, last_size = None, -1 + while True: + try: + src = newest_image(watch) + if src is not None and src != published: + size = src.stat().st_size + if size == last_size and size > 0: + try: + im = Image.open(src) + im.thumbnail((width, width)) + im.convert("RGB").save(out / "latest.jpg", quality=85) + published = src + print(f" {time.strftime('%H:%M:%S')} {src.name}", flush=True) + except Exception: + last_size = -1 # truncated - wait and retry + else: + last_size = size + except Exception: + pass + time.sleep(period) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("watch", nargs="?", type=Path, + help="run folder to watch (default: newest under data/)") + ap.add_argument("--out", type=Path, help="where to write the viewer (default: /_live)") + ap.add_argument("--fps", type=float, default=1.0) + ap.add_argument("--width", type=int, default=1200) + args = ap.parse_args() + + watch = args.watch or newest_run(data_root() / "data") + if watch is None or not watch.exists(): + raise SystemExit("no run folder found to watch - pass one explicitly") + serve(watch, args.out or (watch / "_live"), args.fps, args.width) + + +if __name__ == "__main__": + main() From 01e81b83304e97574877d8b4ac31d2b80bd0b6ad Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 13:56:20 -0400 Subject: [PATCH 12/42] Remove duplicated TODO fragment from nudge_pfs_offset docstring The e-stop note was inserted mid-sentence into the existing TODO, leaving a truncated copy of its first three lines above the full one. --- acquisition/backends/nis_sdk.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index e01f87e..053ebf3 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -334,9 +334,6 @@ def nudge_pfs_offset(self, delta_counts: float) -> dict: capped to PFS_MAX_OFFSET_STEP_FRACTION of the SDK's own reported valid offset range. - TODO(unconfirmed, 2026-08-10): writing iPFS_OFFSET while PFS is - actively enabled/locked has been observed to silently no-op on - real hardware - offset_before == offset_after, no exception - on Guarded by the e-stop like XY_Move/Z_Move: it is small and relative, but it still moves focus, and "only a little" is not a category the stop should recognise. From 6021c3a67a09f73b36e7549622b234a2d71738fb Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 14:10:00 -0400 Subject: [PATCH 13/42] Keep Nikon Ti2 SDK documentation out of the repository The docs, original CHM/PDF files and sample scripts come from Nikon's installers and are licensed with SDK access, so they stay local only. --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 9384881..6d24c61 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,7 @@ eaa_integration/memory/ eaa_integration/*.sqlite eaa_integration/*.sqlite-shm eaa_integration/*.sqlite-wal + +# Nikon Ti2 SDK documentation, originals and samples extracted from the +# Nikon installers. Licensed with SDK access - kept local, not published. +docs/nikon-sdk/ From f190ea88039c34bfca6876cd779ef8688b148316 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 17:08:39 -0400 Subject: [PATCH 14/42] make_movie: render mosaic runs, one frame per stitched round mosaic.json now records mosaic_scale so the scale bar is true on the resized stitched image; older runs fall back to the 0.5 default. --- acquisition/orchestration/mosaic.py | 4 +++ analysis/make_movie.py | 48 +++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/acquisition/orchestration/mosaic.py b/acquisition/orchestration/mosaic.py index 426d54f..643b768 100644 --- a/acquisition/orchestration/mosaic.py +++ b/acquisition/orchestration/mosaic.py @@ -319,6 +319,10 @@ def run(grid: tuple[int, int], overlap: float, rounds: int, interval_s: float, "centre_um": [cx, cy], "exposure_us": exposure_us, "stage_to_image_matrix": M.tolist(), "covers_mm": [round(nx * step_x / 1000, 3), round(ny * step_y / 1000, 3)], + # The saved mosaic_NNN.png is resized by this factor, so its + # pixels are um_per_px / mosaic_scale - make_movie needs it + # to draw a scale bar that is true on the stitched image. + "mosaic_scale": mosaic_scale, }, indent=2), encoding="utf-8") # An out-of-band kill switch. Stopping a background shell does NOT diff --git a/analysis/make_movie.py b/analysis/make_movie.py index 6caaff9..04bf98f 100644 --- a/analysis/make_movie.py +++ b/analysis/make_movie.py @@ -5,6 +5,7 @@ # python -m analysis.make_movie data/timelapse_20260921_163741_oats # python -m analysis.make_movie --fps 24 --out front.mp4 # python -m analysis.make_movie --scale 0.5 --no-overlay +# python -m analysis.make_movie data/mosaic_ # one frame per round # # A folder of 500 PNGs is data; a 20-second movie is something a person # can actually look at and see an organism move. This is the step that @@ -33,6 +34,8 @@ import argparse import csv +import datetime +import json from pathlib import Path import cv2 @@ -42,7 +45,46 @@ UM_PER_PX = 1.4901 +def _mosaic_rows(series_dir: Path) -> list[dict]: + """One row per stitched round of a mosaic run, shaped like frames.csv. + + A round's time is the wall clock of its first tile, relative to round + 0's first tile - the same "time since start" meaning t_seconds has in + a fixed-position series. + """ + starts: dict[int, datetime.datetime] = {} + with (series_dir / "tiles.csv").open(encoding="utf-8") as fh: + for r in csv.DictReader(fh): + rnd = int(r["round"]) + if rnd not in starts: + starts[rnd] = datetime.datetime.fromisoformat(r["wall_clock"]) + if not starts: + return [] + t0 = starts[min(starts)] + rows = [] + for rnd in sorted(starts): + name = f"mosaic_{rnd:03d}.png" + if (series_dir / name).exists(): + rows.append({"filename": name, + "t_seconds": (starts[rnd] - t0).total_seconds()}) + return rows + + +def mosaic_um_per_px(series_dir: Path) -> float | None: + """Pixel size of a mosaic run's stitched images, or None if not a mosaic. + + Runs from before mosaic_scale was recorded used the 0.5 default. + """ + meta_path = series_dir / "mosaic.json" + if not meta_path.exists(): + return None + meta = json.loads(meta_path.read_text(encoding="utf-8")) + return meta["um_per_px"] / meta.get("mosaic_scale", 0.5) + + def _rows(series_dir: Path) -> list[dict]: + if (series_dir / "mosaic.json").exists(): + return _mosaic_rows(series_dir) with (series_dir / "frames.csv").open(encoding="utf-8") as fh: rows = [r for r in csv.DictReader(fh) if r.get("filename") and not r.get("error")] return [r for r in rows if (series_dir / r["filename"]).exists()] @@ -168,14 +210,16 @@ def main() -> None: ap.add_argument("--out", type=Path, help="default: /movie.mp4") ap.add_argument("--fps", type=int, default=30) ap.add_argument("--scale", type=float, default=1.0, help="resize factor (0.5 = half size)") - ap.add_argument("--um-per-px", type=float, default=UM_PER_PX) + ap.add_argument("--um-per-px", type=float, + help=f"default: from mosaic.json for a mosaic run, else {UM_PER_PX}") ap.add_argument("--no-overlay", action="store_true", help="omit timestamp and scale bar") ap.add_argument("--no-white-balance", action="store_true", help="keep the camera's raw (blue-biased) colour") args = ap.parse_args() out = args.out or (args.series_dir / "movie.mp4") - build(args.series_dir, out, args.fps, args.scale, args.um_per_px, + um_per_px = args.um_per_px or mosaic_um_per_px(args.series_dir) or UM_PER_PX + build(args.series_dir, out, args.fps, args.scale, um_per_px, not args.no_overlay, not args.no_white_balance) From 725c268f75947c53f37f9852ca79b70652870939 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 23 Sep 2026 18:26:21 -0400 Subject: [PATCH 15/42] Add oat_approach: measure plasmodium approach to food across mosaic rounds Per stitched round, segments plasmodium on the agar block and reports area in distance bands from a hand-located oat plus the area-weighted mean distance, so movement toward food is a number, not an impression. --- analysis/oat_approach.py | 171 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 analysis/oat_approach.py diff --git a/analysis/oat_approach.py b/analysis/oat_approach.py new file mode 100644 index 0000000..694631d --- /dev/null +++ b/analysis/oat_approach.py @@ -0,0 +1,171 @@ +# oat_approach.py +# ------------------------------------------------------------ +# Is the plasmodium moving toward the food? Measure it from a mosaic run. +# +# python -m analysis.oat_approach data/mosaic_ --oat-box 0,3350,1200,7300 +# +# For every stitched round (mosaic_NNN.png) this segments plasmodium and +# reports how much of it lies within each distance band of the oat, plus +# the area-weighted mean distance to the oat. A plasmodium heading for the +# food shows up as area moving into the near bands and the mean distance +# falling, which is a number rather than an impression from the movie. +# +# THE OAT IS LOCATED BY HAND, ONCE. It is a flake that does not move, and +# in brightfield it is the only thing that is truly black (grey ~0-2 on a +# 0-255 scale; plasmodium bottoms out around 10). But the dish rim is just +# as black, so --oat-box says where to look: pixels <= OAT_MAX_GREY inside +# that box and inside the dish, in round 0, are the oat for every round. +# +# THRESHOLDS WERE MEASURED on 2026-09-23 (16.7 ms, gamma 1.0, neutral +# gains; quarter-size mosaic): agar ~219, plasmodium fans and trunk +# 10-40, an out-of-focus smear 110-180, oat 0-1, bare glass 250-255. +# PLASMODIUM_MAX_GREY sits in the gap so the smear is not counted as +# organism. Re-check them if the exposure or lamp changes. +# +# ONLY THE AGAR BLOCK IS MEASURED - organism cannot be anywhere else, so +# glass, pen marks and the rim are excluded by construction. What is NOT +# excluded is the dark shadow along the block's top edge near the oat, +# which is as dark and as textured as real fans. It does not change +# between rounds, so it offsets the near bands but not their trend: read +# this output as change over time, not as absolute coverage. +# +# DISTANCES are straight-line over the agar, from the oat's edge. Most of +# this dish's oat lies beyond the glass window; only its visible strip is +# the reference, so distances run from where the oat enters the view. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +from pathlib import Path + +import cv2 +import numpy as np + +from analysis.make_movie import _mosaic_rows + +#: Work at this fraction of the stitched size - ~12 um/px, plenty for mm-scale fronts. +WORK_SCALE = 0.25 +OAT_MAX_GREY = 4 +PLASMODIUM_MAX_GREY = 90 +#: Bare glass beside the agar block reads 250-255; agar ~220. +GLASS_MIN_GREY = 246 +#: Band edges in mm from the oat's edge. +BANDS_MM = (0, 3, 6, 9, 12, 15, 20, 30) +#: Ordinal blue ramp, near = dark (BGR for cv2). +BAND_COLOURS = [(0x6b, 0x36, 0x0d), (0x95, 0x4f, 0x18), (0xbf, 0x6a, 0x25), + (0xe5, 0x87, 0x39), (0xe7, 0x98, 0x55), (0xec, 0xa7, 0x6d), + (0xef, 0xb6, 0x86)] + + +def _grey(path: Path) -> np.ndarray: + g = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) + return cv2.resize(g, None, fx=WORK_SCALE, fy=WORK_SCALE, interpolation=cv2.INTER_AREA) + + +def _hull(mask: np.ndarray) -> np.ndarray: + pts = np.column_stack(np.nonzero(mask)[::-1]).astype(np.int32) + out = np.zeros(mask.shape, dtype=np.uint8) + cv2.fillConvexPoly(out, cv2.convexHull(pts), 1) + return out + + +def dish_mask(g: np.ndarray) -> np.ndarray: + """The glass window: convex hull of all bare-glass pixels, pulled in so + the dark gradient at the rim is not counted as plasmodium.""" + return cv2.erode(_hull(g >= GLASS_MIN_GREY), np.ones((41, 41), np.uint8)) + + +def agar_mask(g: np.ndarray, dish: np.ndarray) -> np.ndarray: + """The agar block. Plasmodium only grows on agar, so everything outside + it - bare glass, the block's shadowed edge, pen marks on the dish - is + excluded. Everything in the dish that is not bare glass is agar, + organism or oat; the largest such region, as a convex outline, is the + block (the network cuts the visible agar into pieces, hence the hull).""" + m = ((g < GLASS_MIN_GREY) & (dish > 0)).astype(np.uint8) + m = cv2.morphologyEx(m, cv2.MORPH_OPEN, np.ones((9, 9), np.uint8)) + n, lab, st, _ = cv2.connectedComponentsWithStats(m) + big = 1 + int(np.argmax(st[1:, cv2.CC_STAT_AREA])) + return cv2.erode(_hull(lab == big), np.ones((21, 21), np.uint8)) + + +def oat_mask(g: np.ndarray, dish: np.ndarray, box: tuple[int, int, int, int]) -> np.ndarray: + x0, y0, x1, y1 = (int(v * WORK_SCALE) for v in box) + m = np.zeros_like(g, dtype=np.uint8) + m[y0:y1, x0:x1] = (g[y0:y1, x0:x1] <= OAT_MAX_GREY) & (dish[y0:y1, x0:x1] > 0) + n, lab, st, _ = cv2.connectedComponentsWithStats(m) + if n < 2: + raise SystemExit("no black object found inside --oat-box") + big = 1 + int(np.argmax(st[1:, cv2.CC_STAT_AREA])) + return (lab == big).astype(np.uint8) + + +def run(series_dir: Path, box: tuple[int, int, int, int]) -> Path: + meta = json.loads((series_dir / "mosaic.json").read_text(encoding="utf-8")) + um_px = meta["um_per_px"] / meta.get("mosaic_scale", 0.5) / WORK_SCALE + rows = _mosaic_rows(series_dir) + if not rows: + raise SystemExit(f"no stitched rounds in {series_dir}") + + g0 = _grey(series_dir / rows[0]["filename"]) + dish = dish_mask(g0) + oat = oat_mask(g0, dish, box) + agar = agar_mask(g0, dish) + # distance (mm) of every pixel from the oat's edge + dist_mm = cv2.distanceTransform((1 - oat).astype(np.uint8), cv2.DIST_L2, 5) * um_px / 1000 + valid = (agar > 0) & (oat == 0) + px_mm2 = (um_px / 1000) ** 2 + print(f"oat area {oat.sum() * px_mm2:.2f} mm^2, agar block {agar.sum() * px_mm2:.1f} mm^2, " + f"{um_px:.1f} um/px") + + edges = list(zip(BANDS_MM[:-1], BANDS_MM[1:])) + fields = (["round", "t_min", "total_mm2"] + [f"band_{a}_{b}mm_mm2" for a, b in edges] + + ["mean_dist_mm", "nearest_mm"]) + out = series_dir / "oat_approach.csv" + with out.open("w", newline="", encoding="utf-8") as fh: + w = csv.DictWriter(fh, fieldnames=fields) + w.writeheader() + for i, r in enumerate(rows): + g = g0 if i == 0 else _grey(series_dir / r["filename"]) + if g.shape != g0.shape: + g = cv2.resize(g, g0.shape[::-1], interpolation=cv2.INTER_AREA) + plas = (g <= PLASMODIUM_MAX_GREY) & (g > OAT_MAX_GREY) & valid + d = dist_mm[plas] + row = {"round": i, "t_min": round(r["t_seconds"] / 60, 1), + "total_mm2": round(plas.sum() * px_mm2, 2), + "mean_dist_mm": round(float(d.mean()), 3) if d.size else "", + "nearest_mm": round(float(d.min()), 3) if d.size else ""} + for a, b in edges: + row[f"band_{a}_{b}mm_mm2"] = round(((d >= a) & (d < b)).sum() * px_mm2, 2) + w.writerow(row) + print(f"round {i:3d} t={row['t_min']:6.1f} min total {row['total_mm2']:7.2f} mm^2 " + f"mean dist {row['mean_dist_mm']} mm") + + # One check image, so the oat, dish and bands can be verified by eye. + check = cv2.cvtColor(g0, cv2.COLOR_GRAY2BGR) + for (a, b), col in zip(edges, BAND_COLOURS): + ring = (dist_mm >= a) & (dist_mm < b) & valid & (g0 <= PLASMODIUM_MAX_GREY) & (g0 > OAT_MAX_GREY) + check[ring] = col + check[oat > 0] = (0, 0, 220) + for m, col in ((dish, (0, 180, 0)), (agar, (0, 200, 255))): + cnt, _ = cv2.findContours(m, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + cv2.drawContours(check, cnt, -1, col, 3) + cv2.imwrite(str(series_dir / "oat_approach_check.png"), check) + print(f"wrote {out} and oat_approach_check.png") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("series_dir", type=Path) + ap.add_argument("--oat-box", required=True, + help="x0,y0,x1,y1 in round-0 stitched-mosaic pixels bounding the oat") + args = ap.parse_args() + box = tuple(int(v) for v in args.oat_box.split(",")) + run(args.series_dir, box) + + +if __name__ == "__main__": + main() From 4d3ffbd14f54c3fcfd0708bc8a45ddc84653d4da Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Fri, 25 Sep 2026 18:09:54 -0400 Subject: [PATCH 16/42] Add estop_inflight_test: can the e-stop stop a move already running? Test A (Z) sends a halt write-back 0.2 s into a 1 mm downward move; Test B (XY) engages the flag during a multi-hop move_xy move. Results on the Ti2-E, 2026-09-25, at 4X: - Z: iZPOSITION reads stayed at the start value for the whole move. The halt read that stale value; its write queued behind the move, which completed all 1000 um, then drove Z back to start. A halt write-back never stops Z - it adds a second, reversed move. - XY: the flag was engaged mid-hop; the next hop ran to its end and the one after was refused. Stopped 0.54 s after the engage, having travelled 12 of 20 mm. Worst case is one hop (HOP_UM = 4 mm). --- acquisition/estop_inflight_test.py | 162 +++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 acquisition/estop_inflight_test.py diff --git a/acquisition/estop_inflight_test.py b/acquisition/estop_inflight_test.py new file mode 100644 index 0000000..1ac1708 --- /dev/null +++ b/acquisition/estop_inflight_test.py @@ -0,0 +1,162 @@ +# estop_inflight_test.py +# ------------------------------------------------------------ +# Can the e-stop stop a move that is ALREADY running? +# +# .venv/Scripts/python -m acquisition.estop_inflight_test z # Test A +# .venv/Scripts/python -m acquisition.estop_inflight_test xy # Test B +# +# The e-stop flag reliably refuses NEW moves. What is unknown is what +# happens to a move the controller has already accepted. XY reads are +# cached for the whole move (read_during_move_test.py), so writing the +# "current" XY back reverses the stage instead of freezing it. +# +# TEST A (Z). The Ti2 pushed Z-position events every ~500 ms during a +# move, which XY never did, so Z reads may be live. One raw Z move of +# Z_TEST_DROP_UM DOWNWARD - away from the sample, the only direction this +# test ever moves on its own - with a reader thread logging Z every 50 ms +# and a halt that writes the current Z back after HALT_AFTER_S. If Z reads +# are live, Z should stop partway. If they are cached, the write-back +# commands the start position, which is where we came from: the worst +# case of this test is going back up to where it began, never beyond. +# Afterwards Z is left where it stopped; restore focus by hand. +# +# TEST B (XY). No halt is possible, so the stop is the flag checked +# between move_xy's hops. A multi-hop move of XY_TEST_BY_UM in +X runs on +# a worker thread; the flag is engaged (no halt) after ENGAGE_AFTER_S. +# Reported: how long the stage kept moving after the engage, and how far +# it travelled in total. The stop is left ENGAGED - a human releases it. +# +# Raw COM, not NISSdk, for Test A: NISSdk caps a Z step at 50 um, which +# is over before any halt could land. Test B goes through NISSdk and +# move_xy on purpose, since that is the path real moves take. +# ------------------------------------------------------------ + +import os +import sys +import threading +import time + +# Multi-threaded COM, so the reader and halt threads share the microscope +# object with the thread that is blocked in the move. +sys.coinit_flags = 0 # COINIT_MULTITHREADED + +import pythoncom +import win32com.client +import NkTi2Ax + +from acquisition import estop + +Z_COUNTS_PER_UM = 100.0 +Z_TEST_DROP_UM = 1000.0 +HALT_AFTER_S = 0.2 + +XY_TEST_BY_UM = 20000.0 +ENGAGE_AFTER_S = 1.5 + + +def log(msg): + now = time.time() + stamp = f"{time.strftime('%H:%M:%S', time.localtime(now))}.{int(now % 1 * 1e6):06d}" + print(f"{stamp} [{threading.current_thread().name:>6}] {msg}", flush=True) + + +def test_z(): + m = win32com.client.Dispatch(NkTi2Ax.NikonTi2AxAutoConnectMicroscope.CLSID) + z0 = m.iZPOSITION + lo = m.ZPosition.Lower + target = z0 - round(Z_TEST_DROP_UM * Z_COUNTS_PER_UM) + log(f"start Z {z0 / Z_COUNTS_PER_UM:.2f} um, target {target / Z_COUNTS_PER_UM:.2f} um (down)") + if target < lo: + log(f"target below travel range ({lo / Z_COUNTS_PER_UM:.1f} um) - not moving") + return + estop.check() + + stop = threading.Event() + + def reader(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + while not stop.is_set(): + log(f"Z read {m.iZPOSITION / Z_COUNTS_PER_UM:.2f}") + time.sleep(0.05) + + def halter(): + pythoncom.CoInitializeEx(pythoncom.COINIT_MULTITHREADED) + z = m.iZPOSITION + log(f"HALT: read Z {z / Z_COUNTS_PER_UM:.2f}, writing it back") + try: + m.iZPOSITION = z + log("HALT: write-back accepted") + except Exception as e: + log(f"HALT: write-back failed: {e}") + + threading.Thread(target=reader, name="reader", daemon=True).start() + time.sleep(0.2) # a few idle reads first, as a baseline + threading.Timer(HALT_AFTER_S, halter).start() + + t0 = time.perf_counter() + log("set iZPOSITION start") + try: + m.iZPOSITION = target + log(f"set iZPOSITION done after {time.perf_counter() - t0:.2f} s") + except Exception as e: + log(f"set iZPOSITION failed after {time.perf_counter() - t0:.2f} s: {e}") + + time.sleep(1.5) # let whatever is still moving settle + stop.set() + z1 = m.iZPOSITION + travelled = (z0 - z1) / Z_COUNTS_PER_UM + log(f"final Z {z1 / Z_COUNTS_PER_UM:.2f} um - moved {travelled:.2f} of {Z_TEST_DROP_UM:.0f} um down") + if abs(travelled) < 1: + log("RESULT: Z back at start - halt read a stale position (or the move never ran)") + elif travelled < Z_TEST_DROP_UM - 1: + log("RESULT: Z stopped partway - halt works on Z") + else: + log("RESULT: Z reached target - halt did not stop it") + + +def test_xy(): + from acquisition.backends.nis_sdk import NISSdk + from acquisition.move_xy import move_to + + sdk = NISSdk() + x0, y0 = sdk.XY_GetPosition() + log(f"start ({x0:.1f}, {y0:.1f}), moving +{XY_TEST_BY_UM:.0f} um in X") + estop.check() + + at_engage = {} + + def engager(): + # Flag first: NISSdk has one COM thread, busy for the whole hop in + # flight, so reading the position before engaging would delay the + # engage by up to a hop - the very latency being measured. + estop.engage("estop_inflight_test", halt_stage=False) + at_engage["t"] = time.perf_counter() + log("ENGAGED (flag only)") + + threading.Timer(ENGAGE_AFTER_S, engager).start() + t0 = time.perf_counter() + try: + move_to(sdk, x0 + XY_TEST_BY_UM, y0) + log("move finished WITHOUT being stopped") + except estop.EStopEngaged: + log("move refused its next hop - stopped by e-stop") + t_end = time.perf_counter() + + x1, y1 = sdk.XY_GetPosition() + log(f"final ({x1:.1f}, {y1:.1f}) after {t_end - t0:.2f} s") + if at_engage: + log(f"RESULT: kept moving {t_end - at_engage['t']:.2f} s after engage " + f"(travelled {x1 - x0:.0f} of {XY_TEST_BY_UM:.0f} um in total)") + log("e-stop left ENGAGED - release by hand: python -m acquisition.estop release") + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ("z", "xy"): + print(__doc__ or "usage: python -m acquisition.estop_inflight_test z|xy") + raise SystemExit(2) + threading.current_thread().name = "main" + # Hard stop in case anything hangs. + threading.Timer(60.0, lambda: (log("60s safety timeout"), os._exit(1))).start() + (test_z if sys.argv[1] == "z" else test_xy)() + log("exit") + os._exit(0) From c028cee597c0696587e292d6bb18f469c9c54964 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Fri, 25 Sep 2026 18:21:08 -0400 Subject: [PATCH 17/42] Make the e-stop flag-only: remove the halt write-back estop.engage() also wrote the current position back as the target, to freeze an axis mid-move. estop_inflight_test showed it cannot: position reads are cached for the whole move (XY and Z), so the halt commands the START position, and its write queues behind the move. A 1 mm Z move ran to its end and the halt then drove Z all the way back - a second move issued by the stop itself. engage() now only sets the flag. A move in flight runs to its end and the next is refused; move_xy's hops bound the XY overshoot to one hop (measured 0.54 s, <= 4 mm). halt() and the CLI's --no-halt are gone. The STOP panel and the MCP estop tool both go through engage(). --- acquisition/backends/nis_sdk.py | 4 +-- acquisition/estop.py | 57 ++++++++---------------------- acquisition/estop_inflight_test.py | 6 ++-- mcp_server/loop_tools.py | 6 ++-- 4 files changed, 23 insertions(+), 50 deletions(-) diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index 053ebf3..bbbd15f 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -52,8 +52,8 @@ import NkTi2Ax # Imported at module level so the guard cannot be skipped by an import -# failing lazily inside a move. estop imports THIS module only inside -# halt(), so there is no import cycle. +# failing lazily inside a move. estop does not import this module, so +# there is no import cycle. from acquisition import estop # Raw-count-per-micron scale factors confirmed above. diff --git a/acquisition/estop.py b/acquisition/estop.py index 18f2bd3..cc1ca73 100644 --- a/acquisition/estop.py +++ b/acquisition/estop.py @@ -44,10 +44,17 @@ # refused move and a clear message, which is recoverable; the cost of # failing open is a driven objective. # -# WHAT IT DOES NOT DO: it cannot un-issue a setpoint the controller has -# already accepted. engage() therefore also calls halt(), which writes the -# CURRENT position back as the target - with setpoint-based motion that is -# the only available way to stop an axis already in motion. +# WHAT IT DOES NOT DO: it cannot stop a move the controller has already +# accepted. A move in flight runs to its end; the stop takes effect at the +# next move. move_xy splits long moves into HOP_UM hops, so an XY move +# stops within one hop (measured: 0.54 s, <= 4 mm). +# +# WHY NO HALT. engage() used to also write the current position back as +# the target, to freeze an axis mid-travel. estop_inflight_test showed +# that is worse than nothing: position reads are cached for the whole move +# (XY and Z), so the "current" position is the START, and the write queues +# behind the move. A 1 mm Z move ran to its end, then the halt drove Z all +# the way back - a second move, issued by the stop. Never add it back. # ------------------------------------------------------------ from __future__ import annotations @@ -104,13 +111,10 @@ def check() -> None: ) -def engage(reason: str = "manual", halt_stage: bool = True) -> dict: - """Forbid all motion immediately, and try to halt anything in flight. +def engage(reason: str = "manual") -> dict: + """Forbid all motion immediately. Sets the flag and nothing else. - The flag is written FIRST and the hardware halt attempted second: if - halting raises (SDK missing, COM busy, no hardware), the prohibition is - already in force. Doing it the other way round would leave a window - where a failed halt also meant no flag. + It never commands the stage - see WHY NO HALT in the module header. """ info = { "engaged_at": datetime.datetime.now().isoformat(timespec="seconds"), @@ -119,12 +123,6 @@ def engage(reason: str = "manual", halt_stage: bool = True) -> dict: } ESTOP_PATH.parent.mkdir(parents=True, exist_ok=True) ESTOP_PATH.write_text(json.dumps(info, indent=2), encoding="utf-8") - - if halt_stage: - try: - info["halt"] = halt() - except Exception as exc: # hardware may be absent - info["halt"] = f"not halted ({type(exc).__name__}: {exc})" return info @@ -136,40 +134,15 @@ def release() -> None: pass -def halt() -> str: - """Stop axes already in motion by re-commanding their current position. - - The Ti2 exposes no abort: a move is a setpoint write and the stage - servos to it. Writing the position it is at right now is therefore the - only way to stop an axis mid-travel. - - Deliberately bypasses nis_sdk's XY_Move/Z_Move - those now refuse to - run while the stop is engaged, and a stop primitive that the stop - itself blocks would be useless. - """ - from acquisition.backends.nis_sdk import NISSdk, XY_COUNTS_PER_UM, Z_COUNTS_PER_UM - - sdk = NISSdk() - - def freeze(m): - x, y, z = m.iXPOSITION, m.iYPOSITION, m.iZPOSITION - m.iXPOSITION, m.iYPOSITION, m.iZPOSITION = x, y, z - return (x / XY_COUNTS_PER_UM, y / XY_COUNTS_PER_UM, z / Z_COUNTS_PER_UM) - - x, y, z = sdk._thread.call(freeze) - return f"halted at x={x:.1f} y={y:.1f} z={z:.2f} um" - def main() -> None: ap = argparse.ArgumentParser(description="Emergency stop for microscope motion.") ap.add_argument("action", choices=("engage", "release", "status")) ap.add_argument("--reason", default="manual") - ap.add_argument("--no-halt", action="store_true", - help="set the flag only; do not try to halt the stage") args = ap.parse_args() if args.action == "engage": - info = engage(args.reason, halt_stage=not args.no_halt) + info = engage(args.reason) print("E-STOP ENGAGED") for k, v in info.items(): print(f" {k}: {v}") diff --git a/acquisition/estop_inflight_test.py b/acquisition/estop_inflight_test.py index 1ac1708..75caa2a 100644 --- a/acquisition/estop_inflight_test.py +++ b/acquisition/estop_inflight_test.py @@ -22,7 +22,7 @@ # # TEST B (XY). No halt is possible, so the stop is the flag checked # between move_xy's hops. A multi-hop move of XY_TEST_BY_UM in +X runs on -# a worker thread; the flag is engaged (no halt) after ENGAGE_AFTER_S. +# a worker thread; the flag is engaged after ENGAGE_AFTER_S. # Reported: how long the stage kept moving after the engage, and how far # it travelled in total. The stop is left ENGAGED - a human releases it. # @@ -129,9 +129,9 @@ def engager(): # Flag first: NISSdk has one COM thread, busy for the whole hop in # flight, so reading the position before engaging would delay the # engage by up to a hop - the very latency being measured. - estop.engage("estop_inflight_test", halt_stage=False) + estop.engage("estop_inflight_test") at_engage["t"] = time.perf_counter() - log("ENGAGED (flag only)") + log("ENGAGED") threading.Timer(ENGAGE_AFTER_S, engager).start() t0 = time.perf_counter() diff --git a/mcp_server/loop_tools.py b/mcp_server/loop_tools.py index da006f7..81bd40f 100644 --- a/mcp_server/loop_tools.py +++ b/mcp_server/loop_tools.py @@ -578,9 +578,9 @@ def estop(action: str = "status", reason: str = "requested via MCP") -> dict: but every process on the machine that drives this microscope, including runs started by something else entirely. The flag lives in a file, so it outlives whatever set it and cannot be lost by a process - dying. Engaging also re-commands the stage to its current position, - which is the only way to halt an axis already in flight: the Ti2 has - no abort command, and a move is a setpoint the controller servos to. + dying. It cannot stop a move already in flight (the Ti2 has no abort): + that move runs to its end, and every move after it is refused. Long XY + moves are split into hops, so the stage stops within one hop. action: "engage" to stop everything, or "status" to report the current state. RELEASE IS DELIBERATELY NOT AVAILABLE HERE - a stop that the From ff37bd850a375d1cf9cdffdb06132bd1abe698c5 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:39:47 -0400 Subject: [PATCH 18/42] get_image(): add backend='mock' path via MockNIS.capture(), keep sdk default gated Lets the capture-analyze-decide loop run off the microscope PC. CONFOCAL_MOCK_FRAME_PATH chooses the PNG the mock serves. Both harnesses skip the hardware gate for backend='mock'. --- CHANGELOG.md | 8 ++++++ README.md | 10 +++++--- acquisition/backends/nis_mock.py | 12 +++++++-- harness/agent.py | 24 ++++++++++++------ harness/mcp_agent.py | 4 +-- mcp_server/loop_tools.py | 43 +++++++++++++++++++------------- 6 files changed, 68 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 76bb3ff..37750bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,14 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow back to `~/.confocal-mcp` with a warning on stderr. Setting `CONFOCAL_MCP_DATA_DIR` is still honoured as-is, and a normal source checkout still resolves to the checkout directory. +- `get_image(backend=...)`: `backend="mock"` returns a simulated frame via + `MockNIS.capture()` with no camera attached and no `confirm`, so capture + logic can be developed off the microscope PC. `backend="sdk"` (the + default) is unchanged and still requires `confirm=True`. The metadata dict + now carries `backend`. Both harnesses skip the hardware gate for mock + captures. +- `CONFOCAL_MOCK_FRAME_PATH`: PNG the mock capture serves (defaults to the + old `data/analysis/nd2_sample/frame_0.png` location). ## [0.1.0] - 2026-08-31 diff --git a/README.md b/README.md index 7159433..6aed8ac 100644 --- a/README.md +++ b/README.md @@ -14,9 +14,11 @@ Exactly 4, by design - kept deliberately minimal: - **`move(x, y, backend, confirm)`** - move the XY stage to an absolute position, returns the *actual* resulting position. XY-only, on purpose (no absolute Z move is reachable from chat - crash risk into the sample). -- **`get_image(confirm, exposure_time_us, gain, crop, max_dimension)`** - - capture a real frame from the Baumer camera, paired with the exact - stage position it was taken at. Returns both the metadata and an +- **`get_image(confirm, exposure_time_us, gain, crop, max_dimension, backend)`** - + capture a frame from the Baumer camera (`backend="sdk"`, the default) + or a simulated one from the mock (`backend="mock"`, no hardware, no + `confirm` needed - set `CONFOCAL_MOCK_FRAME_PATH` to choose the PNG it + serves), paired with the exact stage position it was taken at. Returns both the metadata and an embedded image preview, so the model can actually see the picture, not just a file path. `crop` (optional `{"x","y","width","height"}` fractions of the full frame) restricts the embedded preview to a @@ -129,7 +131,7 @@ Requires `ANTHROPIC_API_KEY` set - either in a `.env` file at the repo root (copy the commented-out line in `.env`, fill in your real key; this file is gitignored and loaded automatically by `harness/agent.py`), as a regular environment variable, or via `ant auth login`. Real hardware -(`backend="sdk"`, or any `get_image` call) always pauses for a live +(`backend="sdk"` on `move` or `get_image`) always pauses for a live "y/N" approval at the terminal before executing, regardless of what the model requests - see `harness/agent.py`'s header comment for why. Old captured images are pruned from the model's context after a couple of diff --git a/acquisition/backends/nis_mock.py b/acquisition/backends/nis_mock.py index 3ee2d89..bd4ab50 100644 --- a/acquisition/backends/nis_mock.py +++ b/acquisition/backends/nis_mock.py @@ -15,6 +15,7 @@ # nis = MockNIS() # ------------------------------------------------------------ +import os import shutil import sys import time @@ -48,8 +49,15 @@ # Sample frame copied by capture() to simulate a real image capture. Falls # back to a generated placeholder if this isn't present (it usually isn't - -# it's a dev-only asset under the gitignored data/ tree). -SAMPLE_FRAME_PATH = _data_root() / "data" / "analysis" / "nd2_sample" / "frame_0.png" +# it's a dev-only asset under the gitignored data/ tree). Set +# CONFOCAL_MOCK_FRAME_PATH to point at any PNG instead - e.g. a real +# time-lapse frame - so mock captures return something worth analyzing. +# Read at call time (not cached) so a test or a dev script can swap the +# file, or reassign this module attribute, between captures. +SAMPLE_FRAME_PATH = Path( + os.environ.get("CONFOCAL_MOCK_FRAME_PATH") + or _data_root() / "data" / "analysis" / "nd2_sample" / "frame_0.png" +) CAPTURE_DIR = _captures_dir() diff --git a/harness/agent.py b/harness/agent.py index cba83c9..f38ed7d 100644 --- a/harness/agent.py +++ b/harness/agent.py @@ -79,7 +79,7 @@ - Always use backend="mock" (the default) unless the user has clearly \ asked you to control the real, physical microscope. Only pass \ backend="sdk" when real hardware action is actually intended. -- Every real-hardware action (get_image always; move when \ +- Every real-hardware action (get_image and move when \ backend="sdk") pauses for a live human approval at the terminal before \ it executes - expect that pause, and explain to the user what you're \ about to do and why before calling it, so the approval makes sense to \ @@ -100,14 +100,20 @@ { "name": "get_image", "description": ( - "Capture one frame from the real camera, paired with the exact " - "stage position it was taken at. Always touches real hardware - " - "pauses for human approval before executing. Optionally crop and/or " - "resize the embedded preview." + "Capture one frame from the camera, paired with the exact " + "stage position it was taken at. backend=\"sdk\" (default) is the " + "real camera and pauses for human approval before executing; " + "backend=\"mock\" returns a simulated frame with no hardware. " + "Optionally crop and/or resize the embedded preview." ), "input_schema": { "type": "object", "properties": { + "backend": { + "type": "string", + "enum": ["mock", "sdk"], + "description": "\"sdk\" for the real camera (default), \"mock\" for a simulated frame.", + }, "exposure_time_us": { "type": ["number", "null"], "description": "Camera exposure time in microseconds. Omit to keep the current setting.", @@ -213,9 +219,11 @@ def _execute_tool(name: str, tool_input: dict) -> tuple[list[dict], bool]: tool_input = dict(tool_input) try: if name == "get_image": - if not _confirm_real_hardware_action(name, tool_input): - return _text_content("User declined this real-hardware capture. Not executed."), True - metadata, image = loop_tools.get_image(confirm=True, **tool_input) + if tool_input.get("backend", "sdk") == "sdk": + if not _confirm_real_hardware_action(name, tool_input): + return _text_content("User declined this real-hardware capture. Not executed."), True + tool_input["confirm"] = True + metadata, image = loop_tools.get_image(**tool_input) image_content = image.to_image_content() return [ {"type": "text", "text": json.dumps(metadata)}, diff --git a/harness/mcp_agent.py b/harness/mcp_agent.py index feafa6b..aeb394b 100644 --- a/harness/mcp_agent.py +++ b/harness/mcp_agent.py @@ -92,7 +92,7 @@ - Always use backend="mock" (the default) unless the user has clearly \ asked you to control the real, physical microscope. Only pass \ backend="sdk" when real hardware action is actually intended. -- Every real-hardware action (get_image always; move when \ +- Every real-hardware action (get_image and move when \ backend="sdk") pauses for a live human approval at the terminal before \ it executes - expect that pause, and explain to the user what you're \ about to do and why before calling it, so the approval makes sense to \ @@ -178,7 +178,7 @@ async def _execute_tool(mcp_client: Client, name: str, tool_input: dict) -> tupl """ tool_input = dict(tool_input) try: - if name == "get_image": + if name == "get_image" and tool_input.get("backend", "sdk") == "sdk": if not await _confirm_real_hardware_action(name, tool_input): return _text_content("User declined this real-hardware capture. Not executed."), True tool_input["confirm"] = True diff --git a/mcp_server/loop_tools.py b/mcp_server/loop_tools.py index 81bd40f..227767f 100644 --- a/mcp_server/loop_tools.py +++ b/mcp_server/loop_tools.py @@ -84,8 +84,11 @@ # # WHY get_image() REQUIRES confirm=True: unlike get_pos()/move(), it # fires a real camera - same safety-gate pattern used for anything that -# touches real hardware, even when (as here) there's no backend="mock" -# equivalent to fall back to. +# touches real hardware. backend="sdk" (the default - a capture is real +# hardware unless the caller says otherwise) requires confirm=True; +# backend="mock" routes to nis_mock.MockNIS.capture() (copies a sample +# frame to data/captures/) so the capture-analyze-decide loop can be +# developed and tested off the microscope PC with no camera attached. # # WHY get_image() RETURNS [metadata, image] INSTEAD OF JUST A PATH: MCP # tool results can embed real image content (base64 + mime type), not @@ -449,6 +452,7 @@ def get_image( gain: float | None = None, crop: dict | None = None, max_dimension: int | None = None, + backend: str = "sdk", ) -> list: """Grab one frame from the Baumer GenICam camera (see acquisition.backends.baumer_genicam.BaumerGenICam) and return it @@ -473,11 +477,16 @@ def get_image( No NIS-Elements involved - connects directly to the camera via its GenTL producer (see BaumerGenICam/find_cti_files), independent of any - NIS-Elements process. Real hardware only (no mock equivalent - there's - nothing to simulate a camera trigger against) - requires confirm=True, - same safety-gate pattern as every other real-hardware-touching tool in - this repo. Position is read via backend="sdk" (Ti2 ActiveX SDK), the - same NIS-Elements-independent hardware path move()/get_pos() use. + NIS-Elements process. backend="sdk" (default) is real hardware and + requires confirm=True, same safety-gate pattern as every other + real-hardware-touching tool in this repo; position is then read via + the Ti2 ActiveX SDK, the same NIS-Elements-independent path + move()/get_pos() use. backend="mock" needs no confirm: it copies a + sample frame via nis_mock.MockNIS.capture() (override the source + file with CONFOCAL_MOCK_FRAME_PATH) and reads the mock stage's + position, so everything downstream of a capture - preview, crop, + frame_id, frame history - can be exercised with no camera attached. + exposure_time_us/gain are accepted and ignored on the mock. exposure_time_us, gain: optional - if given, applied via BaumerGenICam.set_settings() before capturing (raises ValueError if @@ -509,19 +518,18 @@ def get_image( already-small region can afford more pixels). Omit to use the default. """ - if not confirm: - raise PermissionError( - "get_image fires a real camera and requires confirm=True. " - "Refusing to proceed without explicit confirmation." - ) + _require_confirm_for_sdk(backend, confirm) if crop is not None: _validate_crop(crop) - camera = _get_camera() - if exposure_time_us is not None or gain is not None: - camera.set_settings(exposure_time_us=exposure_time_us, gain=gain) - image_path = camera.capture() - pos = get_pos(backend="sdk") + if backend == "mock": + image_path = _get_backend("mock").capture() + else: + camera = _get_camera() + if exposure_time_us is not None or gain is not None: + camera.set_settings(exposure_time_us=exposure_time_us, gain=gain) + image_path = camera.capture() + pos = get_pos(backend=backend) metadata = { "image": str(image_path), @@ -531,6 +539,7 @@ def get_image( "captured_at": pos["measured_at"], "monotonic_ms": pos["monotonic_ms"], "crop": crop, + "backend": backend, } _append_frame_history(metadata) preview = _make_preview_image( From 7b8c53df4e00ba8d8014ee153ed9d9e0a6f0cdea Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:43:36 -0400 Subject: [PATCH 19/42] Add timelapse/ package: change_detector, frame_audit, first test suite - change_detector.ChangeDetector: model-free per-frame score against a rolling median baseline (pixel diff vs noise, Otsu foreground area delta, validated phase-correlation XY shift). - frame_audit: CLI to audit an existing frame sequence for acquisition gaps, intensity jumps, stage shifts vs. real specimen change - for the 'filming error or biology?' question around the 25h event. - tests/: synthetic-frame tests for both, plus get_image(backend='mock'). - numpy added to core deps; 'test' extra with pytest. --- pyproject.toml | 9 +- tests/conftest.py | 50 +++++++ tests/test_change_detector.py | 62 +++++++++ tests/test_frame_audit.py | 76 +++++++++++ tests/test_loop_tools_mock.py | 35 +++++ timelapse/__init__.py | 16 +++ timelapse/change_detector.py | 220 ++++++++++++++++++++++++++++++ timelapse/frame_audit.py | 248 ++++++++++++++++++++++++++++++++++ 8 files changed, 714 insertions(+), 2 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/test_change_detector.py create mode 100644 tests/test_frame_audit.py create mode 100644 tests/test_loop_tools_mock.py create mode 100644 timelapse/__init__.py create mode 100644 timelapse/change_detector.py create mode 100644 timelapse/frame_audit.py diff --git a/pyproject.toml b/pyproject.toml index 5a7cf24..d27d507 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "mcp==2.0.0", "Pillow==12.3.0", "PyYAML==6.0.3", + "numpy==2.5.3", ] [project.optional-dependencies] @@ -35,6 +36,10 @@ camera = [ sdk = [ "pywin32==312", ] +# Running the test suite (tests/ - mock backend only, no hardware). +test = [ + "pytest==9.1.1", +] # The standalone Claude harness loops in harness/ (not the MCP server). harness = [ "anthropic==1.0.0", @@ -42,7 +47,7 @@ harness = [ ] # Everything, for a full workstation install. all = [ - "confocal-mcp[camera,sdk,harness]", + "confocal-mcp[camera,sdk,harness,test]", ] [project.scripts] @@ -53,4 +58,4 @@ Homepage = "https://github.com/BioNanomics/microscope-agent" Repository = "https://github.com/BioNanomics/microscope-agent" [tool.hatch.build.targets.wheel] -packages = ["mcp_server", "acquisition", "harness"] +packages = ["mcp_server", "acquisition", "harness", "timelapse"] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f9ed12b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,50 @@ +# Runtime data (captures, history logs) is anchored to CONFOCAL_MCP_DATA_DIR +# at *import* time (acquisition/paths.py), so it must be set before any +# test module imports mcp_server.loop_tools or acquisition.backends.nis_mock. +import os +import tempfile +from pathlib import Path + +import numpy as np +import pytest +from PIL import Image + +_DATA_DIR = tempfile.mkdtemp(prefix="confocal-mcp-tests-") +os.environ["CONFOCAL_MCP_DATA_DIR"] = _DATA_DIR + + +def make_blob_frame( + size: int = 128, + center: tuple[float, float] = (64, 64), + radius: float = 20.0, + background: float = 200.0, + foreground: float = 60.0, + noise: float = 3.0, + seed: int = 0, +) -> np.ndarray: + """Brightfield-like frame: a dark disc on a bright background, plus + Gaussian noise. uint8 array of shape (size, size).""" + rng = np.random.default_rng(seed) + yy, xx = np.mgrid[0:size, 0:size] + disc = (xx - center[0]) ** 2 + (yy - center[1]) ** 2 <= radius ** 2 + frame = np.full((size, size), background, dtype=np.float32) + frame[disc] = foreground + frame += rng.normal(0.0, noise, frame.shape).astype(np.float32) + return np.clip(frame, 0, 255).astype(np.uint8) + + +def save_frame(array: np.ndarray, path: Path) -> Path: + Image.fromarray(array, mode="L").convert("RGB").save(path) + return path + + +@pytest.fixture +def blob_frame(): + return make_blob_frame + + +@pytest.fixture +def write_frame(tmp_path): + def _write(name: str, **kwargs) -> Path: + return save_frame(make_blob_frame(**kwargs), tmp_path / name) + return _write diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py new file mode 100644 index 0000000..ae3f9ce --- /dev/null +++ b/tests/test_change_detector.py @@ -0,0 +1,62 @@ +import numpy as np + +from timelapse.change_detector import ChangeDetector, otsu_threshold, phase_correlation_shift + + +def _feed_quiet(det, blob_frame, n=6): + for i in range(n): + det.score_array(blob_frame(seed=i).astype(np.float32)) + + +def test_first_frame_is_never_interesting(blob_frame): + det = ChangeDetector() + sc = det.score_array(blob_frame().astype(np.float32)) + assert sc.interesting is False + assert sc.baseline_frames == 0 + + +def test_quiet_sequence_stays_quiet(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(seed=99).astype(np.float32)) + assert sc.interesting is False, sc + assert sc.diff_score < 2.0 + + +def test_shrinking_blob_triggers_area_delta(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(radius=10.0, seed=7).astype(np.float32)) + assert sc.interesting is True + assert sc.area_delta < -0.04 + assert "shrank" in sc.reason + + +def test_stage_shift_is_reported_as_shift(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(center=(76, 64), seed=7).astype(np.float32)) + assert sc.interesting is True + assert sc.shift_px > 8 + assert "shift" in sc.reason + + +def test_frame_size_change_resets_baseline(blob_frame): + det = ChangeDetector() + _feed_quiet(det, blob_frame) + sc = det.score_array(blob_frame(size=64).astype(np.float32)) + assert "reset" in sc.reason + assert sc.baseline_frames == 0 + + +def test_otsu_splits_bimodal(): + arr = np.concatenate([np.full(500, 50.0), np.full(500, 200.0)]) + thr = otsu_threshold(arr) + assert 50 < thr < 200 + + +def test_phase_correlation_recovers_integer_shift(blob_frame): + a = blob_frame(seed=1).astype(np.float32) + b = np.roll(a, shift=(3, -5), axis=(0, 1)) + dx, dy = phase_correlation_shift(a, b) + assert (dx, dy) == (-5.0, 3.0) diff --git a/tests/test_frame_audit.py b/tests/test_frame_audit.py new file mode 100644 index 0000000..6341021 --- /dev/null +++ b/tests/test_frame_audit.py @@ -0,0 +1,76 @@ +import json + +from timelapse.frame_audit import audit, frames_from_dir, apply_timestamps, main, parse_duration + + +def _sequence(write_frame, tmp_path, n=12, event_at=8, gap_at=None, shift_at=None, dark_at=None): + """n frames 600s apart. Optional: blob shrinks at event_at, a 3x time + gap before gap_at, an XY shift at shift_at, a global dimming at dark_at.""" + frames, ts = [], [] + t = 1_700_000_000.0 + for i in range(n): + kw = {"seed": i} + if event_at is not None and i >= event_at: + kw["radius"] = 10.0 + if shift_at is not None and i >= shift_at: + kw["center"] = (78, 64) + if dark_at is not None and i >= dark_at: + kw["background"] = 120.0 + kw["foreground"] = 30.0 + frames.append(write_frame(f"frame_{i:03d}.png", **kw)) + if gap_at is not None and i == gap_at: + t += 1200.0 + ts.append(t) + t += 600.0 + ts_file = tmp_path / "timestamps.csv" + ts_file.write_text("\n".join(str(x) for x in ts) + "\n") + return frames, ts_file + + +def test_shrink_event_flagged_as_change_not_gap(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=8) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + by_index = {r.index: r for r in results} + assert "CHANGE" in by_index[8].flags + assert not any(f.startswith("GAP") for f in by_index[8].flags) + assert all(not r.flags for r in results if r.index not in (8,)), [r.flags for r in results] + + +def test_time_gap_flagged(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=None, gap_at=5) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + r = next(r for r in results if r.index == 5) # the gap is *before* frame 5 + assert r.dt_ratio > 2.5 + assert any(f.startswith("GAP") for f in r.flags) + + +def test_shift_and_dimming_flagged_as_non_biological(write_frame, tmp_path): + _sequence(write_frame, tmp_path, event_at=None, shift_at=6, dark_at=9) + frames = apply_timestamps(frames_from_dir(tmp_path, "*.png"), tmp_path / "timestamps.csv") + results = audit(frames) + by_index = {r.index: r for r in results} + assert any(f.startswith("SHIFT") for f in by_index[6].flags) + assert any(f.startswith("INTENSITY") for f in by_index[9].flags) + assert "CHANGE" not in by_index[6].flags + assert "CHANGE" not in by_index[9].flags + + +def test_cli_around_window_and_json(write_frame, tmp_path, capsys): + _sequence(write_frame, tmp_path, event_at=8) + out_json = tmp_path / "audit.json" + rc = main([str(tmp_path), "--timestamps", str(tmp_path / "timestamps.csv"), + "--around", "80m", "--window", "10m", "--json", str(out_json)]) + assert rc == 0 + text = capsys.readouterr().out + assert "12 frames" in text + assert "CHANGE" in text + data = json.loads(out_json.read_text()) + assert len(data) == 11 + + +def test_parse_duration(): + assert parse_duration("25h") == 90000.0 + assert parse_duration("90m") == 5400.0 + assert parse_duration("30") == 30.0 diff --git a/tests/test_loop_tools_mock.py b/tests/test_loop_tools_mock.py new file mode 100644 index 0000000..bda313f --- /dev/null +++ b/tests/test_loop_tools_mock.py @@ -0,0 +1,35 @@ +import os +from pathlib import Path + +import pytest + +from acquisition.backends import nis_mock +from mcp_server import loop_tools + + +def test_get_image_sdk_default_requires_confirm(): + with pytest.raises(PermissionError): + loop_tools.get_image() + + +def test_get_image_mock_returns_metadata_and_preview(write_frame, monkeypatch): + frame = write_frame("sample.png") + monkeypatch.setattr(nis_mock, "SAMPLE_FRAME_PATH", frame) + metadata, preview = loop_tools.get_image(backend="mock", max_dimension=64) + assert metadata["backend"] == "mock" + assert Path(metadata["image"]).exists() + assert Path(metadata["image"]).is_relative_to(Path(os.environ["CONFOCAL_MCP_DATA_DIR"])) + assert set(metadata["position"]) == {"x", "y", "z"} + assert metadata["frame_id"] >= 1 + assert preview.format == "jpeg" + assert loop_tools.get_frame(metadata["frame_id"])["image"] == metadata["image"] + + +def test_get_image_mock_rejects_bad_crop(): + with pytest.raises(ValueError): + loop_tools.get_image(backend="mock", crop={"x": 0.8, "y": 0.0, "width": 0.5, "height": 0.5}) + + +def test_get_image_unknown_backend(): + with pytest.raises(ValueError): + loop_tools.get_image(backend="nope") diff --git a/timelapse/__init__.py b/timelapse/__init__.py new file mode 100644 index 0000000..2ab03dc --- /dev/null +++ b/timelapse/__init__.py @@ -0,0 +1,16 @@ +# timelapse/ +# ------------------------------------------------------------ +# Offline-testable pieces of the adaptive time-lapse idea: +# +# change_detector.py - scores a new frame against a rolling baseline +# (no model call, no hardware - numpy + Pillow) +# frame_audit.py - audits an existing frame sequence for acquisition +# gaps, global intensity jumps and XY shifts, to +# separate "filming error" from "biological change" +# scheduler.py - the slow-loop / burst-mode acquisition loop that +# drives loop_tools.get_image() using the detector +# +# None of this is an MCP tool. The model-facing surface stays at 4 tools +# (see mcp_server/loop_tools.py's header); this package sits *beside* the +# tools and calls get_image() itself. +# ------------------------------------------------------------ diff --git a/timelapse/change_detector.py b/timelapse/change_detector.py new file mode 100644 index 0000000..33fa176 --- /dev/null +++ b/timelapse/change_detector.py @@ -0,0 +1,220 @@ +# change_detector.py +# ------------------------------------------------------------ +# Cheap, model-free "did something happen?" score for one new frame. +# +# WHY NO LLM HERE: the fast loop of an adaptive time-lapse may run every +# few seconds for days. A vision-model call per frame is far too slow and +# expensive for that; it is also unnecessary - "a lot of pixels changed +# where the specimen is" is a numpy question. The model is only consulted +# when this detector fires (see scheduler.py), to decide what to do about +# it, not whether it happened. +# +# WHAT IT MEASURES (all on a small grayscale copy of the frame): +# diff_score - mean |frame - baseline| / baseline noise level, where +# the baseline is a running median of the last N frames. +# Robust to a single bad frame; drifts with slow change. +# area_delta - fractional change in the foreground-mask area (Otsu +# threshold on the baseline, applied to the new frame). +# A retraction/expansion of a specimen shows up here even +# when the per-pixel diff is spread thin. +# shift_px - global XY translation between baseline and frame (phase +# correlation). Large shift = the stage or the sample +# moved, which is NOT biology - the scheduler treats it as +# a separate event class. +# +# THRESHOLDS are configuration, not science - tune them on a recorded run +# (frame_audit.py prints the same quantities for an existing sequence, so +# the values seen at a known event give the starting point). +# ------------------------------------------------------------ + +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +from PIL import Image + +ANALYSIS_MAX_DIMENSION = 256 + + +def load_gray(path: str | Path, max_dimension: int = ANALYSIS_MAX_DIMENSION) -> np.ndarray: + """Load an image as a float32 grayscale array, downscaled so its long + edge is at most `max_dimension`. Small on purpose - the metrics below + are about gross change, and a 256px thumbnail makes every one of them + sub-millisecond.""" + with Image.open(path) as img: + img = img.convert("L") + img.thumbnail((max_dimension, max_dimension)) + return np.asarray(img, dtype=np.float32) + + +def otsu_threshold(gray: np.ndarray) -> float: + """Otsu's threshold on a grayscale array (0-255 range). Pure numpy.""" + hist, edges = np.histogram(gray, bins=256, range=(0.0, 256.0)) + hist = hist.astype(np.float64) + total = hist.sum() + if total == 0: + return 0.0 + centers = (edges[:-1] + edges[1:]) / 2.0 + weight_bg = np.cumsum(hist) + weight_fg = total - weight_bg + sum_bg = np.cumsum(hist * centers) + mean_bg = np.divide(sum_bg, weight_bg, out=np.zeros_like(sum_bg), where=weight_bg > 0) + mean_fg = np.divide(sum_bg[-1] - sum_bg, weight_fg, out=np.zeros_like(sum_bg), where=weight_fg > 0) + between = weight_bg * weight_fg * (mean_bg - mean_fg) ** 2 + return float(centers[int(np.argmax(between))]) + + +def foreground_fraction(gray: np.ndarray, threshold: float) -> float: + """Fraction of pixels on the darker-than-threshold side. Works for + brightfield specimens on a bright background; for fluorescence + (bright specimen, dark background) callers pass `invert=True` to + ChangeDetector.""" + return float((gray < threshold).mean()) + + +def phase_correlation_shift(a: np.ndarray, b: np.ndarray) -> tuple[float, float]: + """(dx, dy) in pixels that shifts `a` onto `b`, via phase correlation. + Integer precision - good enough to tell "the stage moved" from "it + didn't". Both arrays must have the same shape.""" + if a.shape != b.shape: + raise ValueError(f"shape mismatch: {a.shape} vs {b.shape}") + fa = np.fft.fft2(a - a.mean()) + fb = np.fft.fft2(b - b.mean()) + cross = fa * np.conj(fb) + denom = np.abs(cross) + denom[denom == 0] = 1.0 + corr = np.fft.ifft2(cross / denom).real + peak = np.unravel_index(int(np.argmax(corr)), corr.shape) + dy, dx = peak + h, w = a.shape + if dy > h // 2: + dy -= h + if dx > w // 2: + dx -= w + return float(-dx), float(-dy) + + +def estimate_shift(a: np.ndarray, b: np.ndarray, min_improvement: float = 0.5) -> tuple[float, float]: + """Like phase_correlation_shift, but returns (0, 0) unless applying the + candidate shift actually makes `b` match `a` much better - specifically + unless it cuts mean|a - b| to below `min_improvement` of the unshifted + value. Phase correlation happily returns a peak for a specimen that + changed *shape* in place (a shrinking blob, say) with no real + translation; this check rejects those.""" + dx, dy = phase_correlation_shift(a, b) + if dx == 0.0 and dy == 0.0: + return 0.0, 0.0 + unshifted = float(np.mean(np.abs(a - b))) + if unshifted == 0.0: + return 0.0, 0.0 + b_back = np.roll(b, shift=(int(-dy), int(-dx)), axis=(0, 1)) + shifted = float(np.mean(np.abs(a - b_back))) + if shifted < unshifted * min_improvement: + return dx, dy + return 0.0, 0.0 + + +@dataclass +class ChangeScore: + diff_score: float + area_delta: float + shift_px: float + foreground_fraction: float + baseline_frames: int + interesting: bool + reason: str + + def as_dict(self) -> dict: + return { + "diff_score": round(self.diff_score, 3), + "area_delta": round(self.area_delta, 4), + "shift_px": round(self.shift_px, 1), + "foreground_fraction": round(self.foreground_fraction, 4), + "baseline_frames": self.baseline_frames, + "interesting": self.interesting, + "reason": self.reason, + } + + +@dataclass +class ChangeDetector: + """Scores each new frame against a running median of the previous + `baseline_frames` frames. Feed it frames in time order via score(). + + diff_threshold: diff_score above this is "interesting". Units are + multiples of the baseline's own frame-to-frame noise, so ~3 means + "three times noisier than a quiet stretch". + area_threshold: |area_delta| above this fraction is "interesting" + (0.05 = the specimen's footprint changed by 5%). + shift_threshold_px: shift above this (in analysis-thumbnail pixels) + is flagged as a stage/sample move rather than biology. + invert: True for bright-on-dark images (fluorescence). + """ + + baseline_frames: int = 5 + diff_threshold: float = 3.0 + area_threshold: float = 0.05 + shift_threshold_px: float = 4.0 + invert: bool = False + _history: deque = field(default_factory=deque, repr=False) + + def __post_init__(self) -> None: + self._history = deque(maxlen=self.baseline_frames) + + def reset(self) -> None: + self._history.clear() + + def score_array(self, gray: np.ndarray) -> ChangeScore: + if self.invert: + gray = 255.0 - gray + n = len(self._history) + if n == 0: + self._history.append(gray) + thr = otsu_threshold(gray) + return ChangeScore(0.0, 0.0, 0.0, foreground_fraction(gray, thr), 0, False, "first frame") + + stack = np.stack(self._history) + baseline = np.median(stack, axis=0) + # Baseline noise: how much the baseline frames disagree with each + # other. With a single frame there is no estimate; use a floor. + noise = float(np.mean(np.abs(stack - baseline))) if n > 1 else 0.0 + noise = max(noise, 1.0) + + if gray.shape != baseline.shape: + # Camera settings changed mid-run (binning, ROI). Restart. + self.reset() + self._history.append(gray) + return ChangeScore(0.0, 0.0, 0.0, 0.0, 0, True, "frame size changed - baseline reset") + + diff_score = float(np.mean(np.abs(gray - baseline))) / noise + thr = otsu_threshold(baseline) + fg_base = foreground_fraction(baseline, thr) + fg_new = foreground_fraction(gray, thr) + area_delta = fg_new - fg_base + dx, dy = estimate_shift(baseline, gray) + shift_px = float(np.hypot(dx, dy)) + + reasons = [] + if shift_px > self.shift_threshold_px: + reasons.append(f"xy shift {shift_px:.1f}px (stage/sample moved?)") + if abs(area_delta) > self.area_threshold: + reasons.append(f"foreground area {'grew' if area_delta > 0 else 'shrank'} by {abs(area_delta):.1%}") + if diff_score > self.diff_threshold: + reasons.append(f"pixel change {diff_score:.1f}x baseline noise") + + self._history.append(gray) + return ChangeScore( + diff_score=diff_score, + area_delta=float(area_delta), + shift_px=shift_px, + foreground_fraction=fg_new, + baseline_frames=n, + interesting=bool(reasons), + reason="; ".join(reasons) if reasons else "quiet", + ) + + def score(self, path: str | Path) -> ChangeScore: + return self.score_array(load_gray(path)) diff --git a/timelapse/frame_audit.py b/timelapse/frame_audit.py new file mode 100644 index 0000000..6af2190 --- /dev/null +++ b/timelapse/frame_audit.py @@ -0,0 +1,248 @@ +# frame_audit.py +# ------------------------------------------------------------ +# "Was that a filming error or biology?" - audit an existing frame +# sequence around a suspected event. +# +# For every consecutive pair of frames it reports: +# dt_s - seconds between the two frames, and its ratio to the +# median interval. A ratio well above 1 means the +# acquisition stalled: the "sudden" change in the video +# is really a missing stretch of time. +# mean_delta - relative change in global mean intensity. A large +# value with little local change = illumination/exposure +# changed, not the specimen. +# shift_px - global XY translation (phase correlation). Non-zero = +# the stage or the sample moved. +# diff_score / area_delta - the same quantities change_detector.py +# uses live, so the values seen at a known event give the +# thresholds to configure the scheduler with. +# +# Timestamps come from, in order of preference: +# --history frame_history.jsonl (this repo's own capture log) +# --timestamps file.csv (one ISO-8601 or epoch-seconds per line, +# same order as the frames; e.g. exported +# from the ND2's metadata) +# file mtime (fallback - only trustworthy if the files +# were written as they were captured) +# +# Usage: +# python -m timelapse.frame_audit FRAME_DIR [--glob '*.png'] [--around 25h --window 1h] +# python -m timelapse.frame_audit --history logs/frame_history.jsonl +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +from dataclasses import dataclass, asdict +from datetime import datetime +from pathlib import Path + +import numpy as np + +from timelapse.change_detector import estimate_shift, foreground_fraction, load_gray, otsu_threshold + +_DURATION_RE = re.compile(r"^(\d+(?:\.\d+)?)\s*(s|m|h|d)?$") +_DURATION_UNIT_S = {"s": 1.0, "m": 60.0, "h": 3600.0, "d": 86400.0, None: 1.0} + + +def parse_duration(text: str) -> float: + """'25h' -> 90000.0, '90m' -> 5400.0, '30' -> 30.0 (seconds).""" + m = _DURATION_RE.match(text.strip()) + if not m: + raise ValueError(f"bad duration {text!r} (expected e.g. '25h', '90m', '30s')") + return float(m.group(1)) * _DURATION_UNIT_S[m.group(2)] + + +def parse_timestamp(text: str) -> float: + """ISO-8601 or epoch seconds -> epoch seconds.""" + text = text.strip() + try: + return float(text) + except ValueError: + return datetime.fromisoformat(text).timestamp() + + +@dataclass +class FrameRecord: + index: int + path: Path + t_s: float # epoch seconds + + +@dataclass +class PairAudit: + index: int # index of the later frame + t_rel_s: float # seconds since the first frame + dt_s: float + dt_ratio: float # dt / median dt + mean_delta: float # (mean_b - mean_a) / mean_a + shift_px: float + diff_score: float + area_delta: float + flags: list[str] + + +def frames_from_dir(frame_dir: Path, glob: str) -> list[FrameRecord]: + paths = sorted(frame_dir.glob(glob)) + if not paths: + raise FileNotFoundError(f"no files matching {glob!r} in {frame_dir}") + return [FrameRecord(i, p, p.stat().st_mtime) for i, p in enumerate(paths)] + + +def frames_from_history(history_path: Path) -> list[FrameRecord]: + records = [] + with open(history_path) as f: + for line in f: + line = line.strip() + if not line: + continue + rec = json.loads(line) + records.append(FrameRecord(len(records), Path(rec["image"]), parse_timestamp(rec["captured_at"]))) + if not records: + raise ValueError(f"{history_path} has no frame records") + return records + + +def apply_timestamps(frames: list[FrameRecord], timestamps_path: Path) -> list[FrameRecord]: + with open(timestamps_path) as f: + rows = [row[0] for row in csv.reader(f) if row and row[0].strip() and not row[0].startswith("#")] + if len(rows) != len(frames): + raise ValueError(f"{timestamps_path} has {len(rows)} timestamps but there are {len(frames)} frames") + return [FrameRecord(fr.index, fr.path, parse_timestamp(ts)) for fr, ts in zip(frames, rows)] + + +def audit( + frames: list[FrameRecord], + gap_ratio: float = 1.5, + mean_delta_threshold: float = 0.10, + shift_threshold_px: float = 4.0, + diff_threshold: float = 3.0, + area_threshold: float = 0.05, +) -> list[PairAudit]: + """Score every consecutive pair. `frames` must be in time order. + + Two passes: the first computes raw pairwise quantities, the second + normalizes the pixel-difference by the *median* pairwise difference of + the whole sequence (the quiet-stretch noise level) - so diff_score is + "how many times noisier than a typical interval", the same units the + live ChangeDetector reports. Frame-to-frame rather than + against-a-rolling-baseline on purpose: an audit wants to localize an + event to one interval, not smear it over the baseline window.""" + if len(frames) < 2: + raise ValueError("need at least 2 frames to audit") + times = np.array([fr.t_s for fr in frames]) + dts = np.diff(times) + median_dt = float(np.median(dts)) + + raw = [] # (dt, mean_delta, shift, raw_diff, area_delta, size_changed) + prev = load_gray(frames[0].path) + for i in range(1, len(frames)): + cur = load_gray(frames[i].path) + mean_a, mean_b = float(prev.mean()), float(cur.mean()) + mean_delta = (mean_b - mean_a) / mean_a if mean_a > 0 else 0.0 + size_changed = cur.shape != prev.shape + if size_changed: + shift, raw_diff, area_delta = float("nan"), float("nan"), float("nan") + else: + dx, dy = estimate_shift(prev, cur) + shift = float(np.hypot(dx, dy)) + raw_diff = float(np.mean(np.abs(cur - prev))) + thr = otsu_threshold(prev) + area_delta = foreground_fraction(cur, thr) - foreground_fraction(prev, thr) + raw.append((float(dts[i - 1]), mean_delta, shift, raw_diff, area_delta, size_changed)) + prev = cur + + diffs = np.array([r[3] for r in raw]) + noise = float(np.nanmedian(diffs)) if np.isfinite(diffs).any() else 1.0 + noise = max(noise, 1e-6) + + t0 = frames[0].t_s + results: list[PairAudit] = [] + for i, (dt, mean_delta, shift, raw_diff, area_delta, size_changed) in enumerate(raw, start=1): + dt_ratio = dt / median_dt if median_dt > 0 else 1.0 + diff_score = raw_diff / noise + flags = [] + if dt_ratio > gap_ratio: + flags.append(f"GAP x{dt_ratio:.1f}") + if dt <= 0: + flags.append("NON-MONOTONIC TIME") + if size_changed: + flags.append("SIZE CHANGED") + if abs(mean_delta) > mean_delta_threshold: + flags.append(f"INTENSITY {mean_delta:+.0%}") + if shift > shift_threshold_px: + flags.append(f"SHIFT {shift:.0f}px") + non_bio = any(f.startswith(("SHIFT", "INTENSITY", "SIZE")) for f in flags) + if not non_bio and (diff_score > diff_threshold or abs(area_delta) > area_threshold): + flags.append("CHANGE") + results.append(PairAudit( + index=i, t_rel_s=frames[i].t_s - t0, dt_s=dt, dt_ratio=dt_ratio, + mean_delta=mean_delta, shift_px=shift, diff_score=diff_score, + area_delta=float(area_delta), flags=flags, + )) + return results + + +def _fmt_rel(seconds: float) -> str: + h, rem = divmod(int(seconds), 3600) + m, s = divmod(rem, 60) + return f"{h:02d}:{m:02d}:{s:02d}" + + +def print_table(results: list[PairAudit], out=None) -> None: + out = out or sys.stdout + print(f"{'idx':>5} {'t_rel':>9} {'dt_s':>8} {'dt_x':>5} {'mean%':>7} {'shift':>6} {'diff':>6} {'area%':>7} flags", file=out) + for r in results: + print( + f"{r.index:>5} {_fmt_rel(r.t_rel_s):>9} {r.dt_s:>8.1f} {r.dt_ratio:>5.2f} " + f"{r.mean_delta * 100:>+7.1f} {r.shift_px:>6.1f} {r.diff_score:>6.2f} {r.area_delta * 100:>+7.2f} " + f"{', '.join(r.flags)}", + file=out, + ) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__ or "Audit a frame sequence for acquisition gaps and non-biological changes.") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("frame_dir", nargs="?", type=Path, help="directory of frames (sorted by filename)") + src.add_argument("--history", type=Path, help="this repo's logs/frame_history.jsonl") + ap.add_argument("--glob", default="*.png") + ap.add_argument("--timestamps", type=Path, help="CSV of one timestamp per frame (ISO-8601 or epoch seconds)") + ap.add_argument("--around", help="only report pairs near this offset from the first frame, e.g. 25h") + ap.add_argument("--window", default="1h", help="half-width of --around window (default 1h)") + ap.add_argument("--gap-ratio", type=float, default=1.5) + ap.add_argument("--mean-delta", type=float, default=0.10) + ap.add_argument("--shift-px", type=float, default=4.0) + ap.add_argument("--json", type=Path, help="also write full results as JSON here") + ap.add_argument("--flagged-only", action="store_true") + args = ap.parse_args(argv) + + frames = frames_from_history(args.history) if args.history else frames_from_dir(args.frame_dir, args.glob) + if args.timestamps: + frames = apply_timestamps(frames, args.timestamps) + results = audit(frames, args.gap_ratio, args.mean_delta, args.shift_px) + + dts = [r.dt_s for r in results] + print(f"{len(frames)} frames, {len(results)} intervals, median interval {np.median(dts):.1f}s, " + f"span {_fmt_rel(results[-1].t_rel_s)}") + + shown = results + if args.around: + center, half = parse_duration(args.around), parse_duration(args.window) + shown = [r for r in results if abs(r.t_rel_s - center) <= half] + if args.flagged_only: + shown = [r for r in shown if r.flags] + print_table(shown) + + if args.json: + args.json.write_text(json.dumps([asdict(r) for r in results], default=str, indent=1)) + print(f"wrote {args.json}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 45dbd54cef10927ddbd9d3ace22954c2bec6f612 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:44:05 -0400 Subject: [PATCH 20/42] tests: fix mock get_image assertions (resolve symlinked tmp path, MCP Image API) --- tests/test_loop_tools_mock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_loop_tools_mock.py b/tests/test_loop_tools_mock.py index bda313f..6dc4ea3 100644 --- a/tests/test_loop_tools_mock.py +++ b/tests/test_loop_tools_mock.py @@ -18,10 +18,10 @@ def test_get_image_mock_returns_metadata_and_preview(write_frame, monkeypatch): metadata, preview = loop_tools.get_image(backend="mock", max_dimension=64) assert metadata["backend"] == "mock" assert Path(metadata["image"]).exists() - assert Path(metadata["image"]).is_relative_to(Path(os.environ["CONFOCAL_MCP_DATA_DIR"])) + assert Path(metadata["image"]).resolve().is_relative_to(Path(os.environ["CONFOCAL_MCP_DATA_DIR"]).resolve()) assert set(metadata["position"]) == {"x", "y", "z"} assert metadata["frame_id"] >= 1 - assert preview.format == "jpeg" + assert preview.to_image_content().mime_type == "image/jpeg" assert loop_tools.get_frame(metadata["frame_id"])["image"] == metadata["image"] From e7173792973799047faf648eeebdbd61831b501e Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:46:59 -0400 Subject: [PATCH 21/42] Add timelapse/scheduler.py (adaptive slow/burst loop) and CI workflow AdaptiveTimelapse drives get_image() on a slow interval, switches to burst mode when ChangeDetector fires, extends on further change, and returns to slow. Stage/sample shifts are logged and never start a burst. Hard caps on captures and runtime. on_trigger hook is the seam for a later model consult. backend='sdk' needs one up-front approval of the whole plan at the CLI; confirm=True is supplied by the scheduler after that, never by a model. Verified live against the mock: swapping the served frame mid-run started a burst within one slow interval, then returned to slow. CI: pytest on 3.11/3.13 with a core+test install, plus an import check. --- .github/workflows/ci.yml | 25 +++ tests/test_change_detector.py | 9 + tests/test_scheduler.py | 133 +++++++++++++++ timelapse/change_detector.py | 11 +- timelapse/frame_audit.py | 6 +- timelapse/scheduler.py | 313 ++++++++++++++++++++++++++++++++++ 6 files changed, 492 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 tests/test_scheduler.py create mode 100644 timelapse/scheduler.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..53aa208 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Core + test only: the mock stage and mock camera need no hardware + # dependencies, so the whole suite runs on a plain runner. + - run: pip install -e ".[test]" + - run: python -m pytest -q tests + # The MCP server must import on a core-only install (the camera and + # SDK stacks are lazy imports) - fail the build if that regresses. + - run: python -c "import mcp_server.server_loop, timelapse.scheduler" diff --git a/tests/test_change_detector.py b/tests/test_change_detector.py index ae3f9ce..c2a9bfa 100644 --- a/tests/test_change_detector.py +++ b/tests/test_change_detector.py @@ -60,3 +60,12 @@ def test_phase_correlation_recovers_integer_shift(blob_frame): b = np.roll(a, shift=(3, -5), axis=(0, 1)) dx, dy = phase_correlation_shift(a, b) assert (dx, dy) == (-5.0, 3.0) + + +def test_second_frame_of_quiet_sequence_is_not_interesting(blob_frame): + # Regression: with one baseline frame there is no noise estimate, and + # plain sensor noise used to read as a 3x event on frame 2. + det = ChangeDetector() + det.score_array(blob_frame(seed=0).astype(np.float32)) + sc = det.score_array(blob_frame(seed=1).astype(np.float32)) + assert sc.interesting is False, sc diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..76280eb --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,133 @@ +import json + +import pytest + +from timelapse.scheduler import AdaptiveTimelapse, TimelapseConfig + + +class FakeClock: + def __init__(self): + self.t = 1000.0 + + def __call__(self): + return self.t + + def sleep(self, s): + assert s >= 0 + self.t += s + + +class FrameSource: + """Serves frames in order; the scheduler's capture() pulls the next one. + After the list runs out, keeps serving the last frame.""" + + def __init__(self, clock, paths): + self.clock = clock + self.paths = list(paths) + self.i = 0 + self.captured_at = [] + + def __call__(self): + path = self.paths[min(self.i, len(self.paths) - 1)] + self.i += 1 + self.captured_at.append(self.clock()) + return {"image": str(path), "frame_id": self.i, "position": {"x": 0, "y": 0, "z": 0}} + + +def _config(tmp_path, **kw): + base = dict(backend="mock", slow_interval_s=60.0, burst_interval_s=5.0, burst_duration_s=30.0, + max_captures=40, max_runtime_s=10_000.0, events_path=tmp_path / "events.jsonl") + base.update(kw) + return TimelapseConfig(**base) + + +def _events(path): + return [json.loads(l) for l in path.read_text().splitlines()] + + +def test_quiet_run_stays_slow_and_stops_at_max_captures(write_frame, tmp_path): + clock = FakeClock() + frames = [write_frame(f"q{i}.png", seed=i) for i in range(12)] + src = FrameSource(clock, frames) + cfg = _config(tmp_path, max_captures=10) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.stop_reason == "max_captures" + assert summary.captures == 10 + assert summary.bursts == 0 + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert all(g == 60.0 for g in gaps), gaps + + +def test_change_starts_burst_then_returns_to_slow(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(20)] + src = FrameSource(clock, quiet + shrunk) + cfg = _config(tmp_path, max_captures=25) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + + kinds = [e["event"] for e in _events(cfg.events_path)] + assert "burst_start" in kinds and "burst_end" in kinds + assert summary.bursts >= 1 + # First burst starts at the 7th capture (first shrunk frame) - then 5s spacing. + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert gaps[5] == 60.0 # slow gap into the first shrunk frame + assert gaps[6] == 5.0 # burst spacing right after trigger + assert summary.burst_captures > 0 + # After the specimen settles at its new size the baseline catches up + # and the loop must be back on the slow interval by the end. + assert gaps[-1] == 60.0, gaps + + +def test_stage_shift_does_not_start_burst(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shifted = [write_frame(f"m{i}.png", center=(78, 64), seed=200 + i) for i in range(4)] + src = FrameSource(clock, quiet + shifted) + cfg = _config(tmp_path, max_captures=10) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.shifts >= 1 + assert summary.bursts == 0 + + +def test_on_trigger_ignore_ends_burst_immediately(write_frame, tmp_path): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(6)] + src = FrameSource(clock, quiet + shrunk) + cfg = _config(tmp_path, max_captures=12) + seen = [] + + def on_trigger(event, metadata, score): + seen.append(score.reason) + return "ignore" + + summary = AdaptiveTimelapse(cfg, capture=src, on_trigger=on_trigger, + clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert seen, "hook never called" + assert summary.burst_captures == 0 + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert all(g == 60.0 for g in gaps), gaps + + +def test_max_runtime_stops_run(write_frame, tmp_path): + clock = FakeClock() + src = FrameSource(clock, [write_frame("q.png")]) + cfg = _config(tmp_path, max_captures=1000, max_runtime_s=300.0) + summary = AdaptiveTimelapse(cfg, capture=src, clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert summary.stop_reason == "max_runtime" + assert summary.captures == 6 # t=0,60,...,300 -> the 300s check fires before a 7th + + +def test_config_validation(tmp_path): + with pytest.raises(ValueError): + _config(tmp_path, burst_interval_s=120.0).validate() + with pytest.raises(ValueError): + _config(tmp_path, backend="nope").validate() + + +def test_cli_dry_run_prints_plan(capsys): + from timelapse.scheduler import main + assert main(["--dry-run", "--backend", "sdk", "--slow", "30"]) == 0 + out = capsys.readouterr().out + assert "REAL HARDWARE" in out and "30s" in out diff --git a/timelapse/change_detector.py b/timelapse/change_detector.py index 33fa176..8a02cb8 100644 --- a/timelapse/change_detector.py +++ b/timelapse/change_detector.py @@ -179,8 +179,13 @@ def score_array(self, gray: np.ndarray) -> ChangeScore: stack = np.stack(self._history) baseline = np.median(stack, axis=0) # Baseline noise: how much the baseline frames disagree with each - # other. With a single frame there is no estimate; use a floor. - noise = float(np.mean(np.abs(stack - baseline))) if n > 1 else 0.0 + # other. With a single baseline frame there is no estimate at all, + # so the pixel-difference criterion is held back (diff_ready) until + # a second frame is in - otherwise ordinary sensor noise on frame 2 + # reads as a 3x+ event. Area and shift criteria don't need a noise + # estimate and stay live from frame 2. + diff_ready = n >= 2 + noise = float(np.mean(np.abs(stack - baseline))) if diff_ready else 1.0 noise = max(noise, 1.0) if gray.shape != baseline.shape: @@ -202,7 +207,7 @@ def score_array(self, gray: np.ndarray) -> ChangeScore: reasons.append(f"xy shift {shift_px:.1f}px (stage/sample moved?)") if abs(area_delta) > self.area_threshold: reasons.append(f"foreground area {'grew' if area_delta > 0 else 'shrank'} by {abs(area_delta):.1%}") - if diff_score > self.diff_threshold: + if diff_ready and diff_score > self.diff_threshold: reasons.append(f"pixel change {diff_score:.1f}x baseline noise") self._history.append(gray) diff --git a/timelapse/frame_audit.py b/timelapse/frame_audit.py index 6af2190..9910830 100644 --- a/timelapse/frame_audit.py +++ b/timelapse/frame_audit.py @@ -7,7 +7,9 @@ # dt_s - seconds between the two frames, and its ratio to the # median interval. A ratio well above 1 means the # acquisition stalled: the "sudden" change in the video -# is really a missing stretch of time. +# is really a missing stretch of time. (For an adaptive +# run from scheduler.py the median is the burst interval, +# so every slow-mode interval reads as a GAP - expected.) # mean_delta - relative change in global mean intensity. A large # value with little local change = illumination/exposure # changed, not the specimen. @@ -158,7 +160,7 @@ def audit( diffs = np.array([r[3] for r in raw]) noise = float(np.nanmedian(diffs)) if np.isfinite(diffs).any() else 1.0 - noise = max(noise, 1e-6) + noise = max(noise, 1.0) # floor of one gray level - identical frames must not blow this up t0 = frames[0].t_s results: list[PairAudit] = [] diff --git a/timelapse/scheduler.py b/timelapse/scheduler.py new file mode 100644 index 0000000..7d349c5 --- /dev/null +++ b/timelapse/scheduler.py @@ -0,0 +1,313 @@ +# scheduler.py +# ------------------------------------------------------------ +# Adaptive time-lapse: image slowly, look for change cheaply, image fast +# when something is happening. +# +# slow mode - one capture every `slow_interval_s` +# burst mode - one capture every `burst_interval_s` for `burst_duration_s` +# after the ChangeDetector fires; each further "interesting" +# frame inside the burst extends it +# back to slow when the burst window runs out +# +# WHY: a fixed-interval time-lapse spends its light and disk budget evenly +# on the boring hours and the interesting minutes alike, and the event you +# care about (a retraction that looked instantaneous at a 20-minute +# interval, say) falls between two frames. This loop puts the fast frames +# where the change is. +# +# WHAT DECIDES: timelapse.change_detector - numpy, no model call, so the +# slow loop costs nothing per frame. An optional `on_trigger` hook runs +# when a burst starts or extends; that is the place to ask a vision +# model "is this worth a longer look?" (return "extend" / "ignore"), or +# to notify someone. The hook is off the fast path: it is called once per +# trigger, not once per frame. +# +# WHAT IT DOES NOT DO: move the stage, refocus, or change exposure. It +# only calls get_image(). A stage/sample shift the detector sees is +# logged as a "shift" event and does NOT start a burst - that is not +# biology, and burst-imaging a drifted field is wasted light. +# +# HARD CAPS (max_captures, max_runtime_s) always end the run, whatever +# mode it is in. On real hardware the run is a single up-front approval +# of the whole plan (see main()): the per-call confirm=True that +# get_image() requires is supplied by this scheduler after that one +# approval, never by anything the model says. Keep it that way. +# +# Log: one JSON line per event in logs/timelapse_events.jsonl (under the +# CONFOCAL_MCP_DATA_DIR data root, same as frame_history.jsonl), so a run +# can be replayed or audited with frame_audit.py --history afterwards. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import json +import sys +import time +from dataclasses import dataclass, field, asdict +from datetime import datetime +from pathlib import Path +from typing import Callable + +from acquisition.paths import logs_dir as _logs_dir +from timelapse.change_detector import ChangeDetector, ChangeScore, load_gray + +CaptureFn = Callable[[], dict] # -> get_image() metadata dict +TriggerFn = Callable[[dict, dict, ChangeScore], str | None] # (event, metadata, score) -> "extend" | "ignore" | None + + +@dataclass +class TimelapseConfig: + backend: str = "mock" + slow_interval_s: float = 60.0 + burst_interval_s: float = 5.0 + burst_duration_s: float = 120.0 + max_captures: int = 500 + max_runtime_s: float = 3600.0 + max_bursts: int | None = None + baseline_frames: int = 5 + diff_threshold: float = 3.0 + area_threshold: float = 0.05 + shift_threshold_px: float = 4.0 + invert: bool = False + preview_max_dimension: int = 256 # passed to get_image - small, the model isn't looking + events_path: Path | None = None # default: /logs/timelapse_events.jsonl + + def validate(self) -> None: + if self.backend not in ("mock", "sdk"): + raise ValueError(f"backend must be 'mock' or 'sdk', got {self.backend!r}") + if self.burst_interval_s <= 0 or self.slow_interval_s <= 0: + raise ValueError("intervals must be > 0") + if self.burst_interval_s > self.slow_interval_s: + raise ValueError("burst_interval_s must not exceed slow_interval_s") + if self.max_captures < 1 or self.max_runtime_s <= 0: + raise ValueError("max_captures and max_runtime_s must be positive") + + def detector(self) -> ChangeDetector: + return ChangeDetector( + baseline_frames=self.baseline_frames, + diff_threshold=self.diff_threshold, + area_threshold=self.area_threshold, + shift_threshold_px=self.shift_threshold_px, + invert=self.invert, + ) + + +def _default_capture(config: TimelapseConfig) -> CaptureFn: + """Capture via loop_tools.get_image(). For backend='sdk' this supplies + confirm=True - main() has already obtained the one-time human approval + for the whole run before this is ever constructed.""" + from mcp_server import loop_tools + + def capture() -> dict: + metadata, _preview = loop_tools.get_image( + backend=config.backend, + confirm=(config.backend == "sdk"), + max_dimension=config.preview_max_dimension, + ) + return metadata + return capture + + +@dataclass +class RunSummary: + captures: int = 0 + bursts: int = 0 + burst_captures: int = 0 + shifts: int = 0 + elapsed_s: float = 0.0 + stop_reason: str = "" + events_path: str = "" + frames: list[dict] = field(default_factory=list) # per-capture event dicts + + +class AdaptiveTimelapse: + def __init__( + self, + config: TimelapseConfig, + capture: CaptureFn | None = None, + on_trigger: TriggerFn | None = None, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + log: Callable[[str], None] = print, + ): + config.validate() + self.config = config + self.capture = capture or _default_capture(config) + self.on_trigger = on_trigger + self.clock = clock + self.sleep = sleep + self.log = log + self.detector = config.detector() + self.events_path = Path(config.events_path) if config.events_path else _logs_dir() / "timelapse_events.jsonl" + self.mode = "slow" + self.burst_until: float | None = None + self.summary = RunSummary(events_path=str(self.events_path)) + self._t0: float | None = None + + # -- state helpers ------------------------------------------------- + def _elapsed(self) -> float: + return self.clock() - (self._t0 if self._t0 is not None else self.clock()) + + def _interval(self) -> float: + return self.config.burst_interval_s if self.mode == "burst" else self.config.slow_interval_s + + def _emit(self, event: dict) -> None: + event = {"at": datetime.now().astimezone().isoformat(timespec="milliseconds"), + "elapsed_s": round(self._elapsed(), 3), "mode": self.mode, **event} + self.events_path.parent.mkdir(parents=True, exist_ok=True) + with open(self.events_path, "a") as f: + f.write(json.dumps(event, default=str) + "\n") + if event["event"] != "capture": + self.log(f"[timelapse +{event['elapsed_s']:.0f}s] {event['event']}: {event.get('detail', '')}") + + # -- one iteration --------------------------------------------------- + def step(self) -> dict: + """Capture one frame, score it, update mode. Returns the capture event.""" + metadata = self.capture() + score = self.detector.score_array(load_gray(metadata["image"])) + now = self.clock() + self.summary.captures += 1 + if self.mode == "burst": + self.summary.burst_captures += 1 + + event = {"event": "capture", "frame_id": metadata.get("frame_id"), "image": metadata.get("image"), + "position": metadata.get("position"), "score": score.as_dict()} + self._emit(event) + self.summary.frames.append(event) + + is_shift = score.shift_px > self.config.shift_threshold_px + if is_shift: + self.summary.shifts += 1 + self._emit({"event": "shift", "frame_id": metadata.get("frame_id"), + "detail": f"{score.shift_px:.1f}px - stage/sample moved, not starting a burst"}) + elif score.interesting: + if self.mode == "slow": + if self.config.max_bursts is not None and self.summary.bursts >= self.config.max_bursts: + self._emit({"event": "burst_skipped", "detail": "max_bursts reached"}) + else: + self.mode = "burst" + self.summary.bursts += 1 + self.burst_until = now + self.config.burst_duration_s + self._emit({"event": "burst_start", "frame_id": metadata.get("frame_id"), "detail": score.reason}) + self._consult(event, metadata, score) + else: + self.burst_until = now + self.config.burst_duration_s + self._emit({"event": "burst_extend", "frame_id": metadata.get("frame_id"), "detail": score.reason}) + self._consult(event, metadata, score) + + if self.mode == "burst" and self.burst_until is not None and now >= self.burst_until: + self._end_burst("burst window elapsed") + return event + + def _consult(self, event: dict, metadata: dict, score: ChangeScore) -> None: + if self.on_trigger is None: + return + decision = self.on_trigger(event, metadata, score) + if decision == "ignore": + self._end_burst("on_trigger said ignore") + elif decision == "extend" and self.burst_until is not None: + self.burst_until += self.config.burst_duration_s + self._emit({"event": "burst_extend", "detail": "on_trigger said extend"}) + + def _end_burst(self, why: str) -> None: + self.mode = "slow" + self.burst_until = None + self._emit({"event": "burst_end", "detail": why}) + + # -- the loop -------------------------------------------------------- + def run(self) -> RunSummary: + self._t0 = self.clock() + self._emit({"event": "start", "detail": json.dumps(asdict(self.config), default=str)}) + next_at = self._t0 + try: + while True: + if self.summary.captures >= self.config.max_captures: + self.summary.stop_reason = "max_captures" + break + if self._elapsed() >= self.config.max_runtime_s: + self.summary.stop_reason = "max_runtime" + break + wait = next_at - self.clock() + if wait > 0: + self.sleep(wait) + self.step() + # Schedule from the intended slot, not from "now", so slow + # captures don't drift; but never queue up a backlog. + next_at = max(next_at + self._interval(), self.clock()) + if self.mode == "burst" and self.burst_until is not None: + next_at = min(next_at, self.burst_until) + except KeyboardInterrupt: + self.summary.stop_reason = "interrupted" + finally: + if self.mode == "burst": + self._end_burst("run ended") + self.summary.elapsed_s = self._elapsed() + self._emit({"event": "stop", "detail": f"{self.summary.stop_reason}; {self.summary.captures} captures, " + f"{self.summary.bursts} bursts, {self.summary.shifts} shifts"}) + return self.summary + + +# -- CLI ---------------------------------------------------------------------- + +def _plan_text(config: TimelapseConfig) -> str: + slow_only = config.max_runtime_s / config.slow_interval_s + return ( + f" backend {config.backend}{' (REAL HARDWARE)' if config.backend == 'sdk' else ''}\n" + f" slow interval {config.slow_interval_s:g}s (~{slow_only:.0f} captures if nothing happens)\n" + f" burst every {config.burst_interval_s:g}s for {config.burst_duration_s:g}s per trigger" + f"{'' if config.max_bursts is None else f', at most {config.max_bursts} bursts'}\n" + f" hard caps {config.max_captures} captures, {config.max_runtime_s:g}s runtime\n" + f" thresholds diff>{config.diff_threshold:g}x noise, |area|>{config.area_threshold:.0%}, " + f"shift>{config.shift_threshold_px:g}px{', inverted (bright-on-dark)' if config.invert else ''}\n" + ) + + +def _approve_real_hardware(config: TimelapseConfig) -> bool: + print("\n[HARDWARE GATE] This run will fire the REAL camera repeatedly, unattended, with this plan:") + print(_plan_text(config)) + answer = input("Approve this whole acquisition plan? [y/N]: ").strip().lower() + return answer in ("y", "yes") + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Adaptive time-lapse: slow captures, burst mode on detected change.") + ap.add_argument("--backend", choices=["mock", "sdk"], default="mock") + ap.add_argument("--slow", type=float, default=60.0, help="slow-mode interval, seconds") + ap.add_argument("--burst", type=float, default=5.0, help="burst-mode interval, seconds") + ap.add_argument("--burst-duration", type=float, default=120.0, help="seconds of burst per trigger") + ap.add_argument("--max-captures", type=int, default=500) + ap.add_argument("--max-runtime", type=float, default=3600.0, help="seconds") + ap.add_argument("--max-bursts", type=int, default=None) + ap.add_argument("--baseline-frames", type=int, default=5) + ap.add_argument("--diff-threshold", type=float, default=3.0) + ap.add_argument("--area-threshold", type=float, default=0.05) + ap.add_argument("--shift-px", type=float, default=4.0) + ap.add_argument("--invert", action="store_true", help="bright specimen on dark background (fluorescence)") + ap.add_argument("--events", type=Path, default=None, help="events JSONL path (default: logs/timelapse_events.jsonl)") + ap.add_argument("--dry-run", action="store_true", help="print the plan and exit without capturing") + args = ap.parse_args(argv) + + config = TimelapseConfig( + backend=args.backend, slow_interval_s=args.slow, burst_interval_s=args.burst, + burst_duration_s=args.burst_duration, max_captures=args.max_captures, max_runtime_s=args.max_runtime, + max_bursts=args.max_bursts, baseline_frames=args.baseline_frames, diff_threshold=args.diff_threshold, + area_threshold=args.area_threshold, shift_threshold_px=args.shift_px, invert=args.invert, + events_path=args.events, + ) + config.validate() + if args.dry_run: + print(_plan_text(config)) + return 0 + if config.backend == "sdk" and not _approve_real_hardware(config): + print("Not approved. Nothing captured.") + return 2 + + summary = AdaptiveTimelapse(config).run() + print(f"done: {summary.stop_reason}; {summary.captures} captures ({summary.burst_captures} in {summary.bursts} bursts), " + f"{summary.shifts} shifts, {summary.elapsed_s:.0f}s. Events: {summary.events_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From d6726a562940864dc027f83b37fd58e65178ad32 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:47:48 -0400 Subject: [PATCH 22/42] docs: adaptive time-lapse proposal, README sections, changelog --- CHANGELOG.md | 7 ++ README.md | 24 +++++++ docs/adaptive_timelapse.md | 127 +++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 docs/adaptive_timelapse.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 37750bd..35b1de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,13 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow captures. - `CONFOCAL_MOCK_FRAME_PATH`: PNG the mock capture serves (defaults to the old `data/analysis/nd2_sample/frame_0.png` location). +- `timelapse/` package (not MCP tools): `change_detector` (model-free + per-frame change score), `frame_audit` (CLI: gaps / intensity jumps / + stage shifts vs. specimen change in an existing sequence), `scheduler` + (adaptive slow/burst acquisition loop with hard caps and a one-time + real-hardware approval). See `docs/adaptive_timelapse.md`. +- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. +- `numpy` is now a core dependency; new `test` optional group (pytest). ## [0.1.0] - 2026-08-31 diff --git a/README.md b/README.md index 6aed8ac..25a73f4 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,30 @@ No NIS-Elements involved anywhere - the camera is reached directly via GenICam/GenTL, and the stage via the Ti2 ActiveX SDK, both independent of whether NIS-Elements software is even running. +## Adaptive time-lapse (`timelapse/`) + +Not an MCP tool - a loop that sits beside the tools and calls `get_image()` +itself. It captures on a slow interval, scores each frame for change with +plain numpy (no model call), and switches to a fast burst when something +happens. `timelapse/frame_audit.py` applies the same scores to an existing +frame sequence to tell an acquisition gap, an illumination change or a +stage bump from real specimen change. Design, safety model and status: +[docs/adaptive_timelapse.md](docs/adaptive_timelapse.md). + +``` +python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-duration 5 --max-runtime 30 +python -m timelapse.frame_audit FRAME_DIR --timestamps times.csv --around 25h --window 1h +``` + +## Tests + +``` +pip install -e ".[test]" +python -m pytest -q tests +``` + +Mock backend only - nothing touches hardware. CI runs the same on every push. + ## Install This is a normal installable package (`confocal-mcp`) with a console entry diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md new file mode 100644 index 0000000..d8fd85d --- /dev/null +++ b/docs/adaptive_timelapse.md @@ -0,0 +1,127 @@ +# Adaptive time-lapse: proposal and status + +## The question behind this + +In the long time-lapse, the specimen pulled back from one region at about +the 25 hour mark, and it looked instantaneous. Two readings are possible: + +- **Biology.** One cell, no nutrients in that region, so it withdraws its + material and redirects it. A retraction moves mass toward the main body, + so the connecting tubes and body should thicken in the frames just after. +- **Filming artefact.** A stalled acquisition, a focus loss, an + illumination change, or a stage bump can all make a region "vanish" + between two frames. + +The rendered video cannot settle this. It may drop or duplicate frames, +and it carries no timestamps. The raw frames can. + +## How to answer it (when at the microscope) + +`timelapse/frame_audit.py` scores every consecutive pair of frames in a +sequence and flags, per interval: + +| Flag | Meaning | +|---|---| +| `GAP x3.2` | the interval was 3.2x the median: the acquisition stalled, the change is a missing stretch of time | +| `INTENSITY +25%` | the whole field got brighter or darker: illumination or exposure, not the specimen | +| `SHIFT 12px` | the whole field translated: stage or sample moved | +| `CHANGE` | the specimen's footprint or pixels changed with none of the above: biology | + +Run it on the frames around the event with the real timestamps: + +``` +python -m timelapse.frame_audit RAW_FRAME_DIR --timestamps frame_times.csv --around 25h --window 1h +``` + +`frame_times.csv` is one timestamp per frame, in frame order, exported from +the raw dataset's metadata. Without it the tool falls back to file +modification times, which are only trustworthy if the files were written +as they were captured. + +**Finer time slices exist only if the raw dataset has more frames than +the video.** Compare the two frame counts first. If the run was a fixed +interval and the video used every frame, there is nothing finer to +recover, and the honest answer is that the interval was too coarse. + +## The proposal: image slowly, look cheaply, image fast when it matters + +A fixed interval spends light, disk and time evenly on the boring hours +and the interesting minutes alike. The adaptive loop puts the fast +frames where the change is: + +1. **Slow mode.** One capture per slow interval, previewed at low + resolution. +2. **Cheap change score, no model.** Each frame is compared with a + running median of the last few frames: pixel change relative to the + baseline's own noise, change in the specimen's footprint area, and a + whole-field shift check. This is numpy on a 256 pixel thumbnail and + costs nothing per frame, so it can run for days. +3. **Burst mode.** When the score crosses the threshold, capture at the + fast interval for a fixed window. Further change inside the window + extends it. When the window lapses, return to slow. +4. **Model only at the trigger.** A hook fires once per trigger, not + once per frame. That is where a vision model looks at the before and + after frames and says "extend", "ignore", or where a notification + goes out. This keeps model calls rare and gives a written reason per + event. +5. **Shifts never start a burst.** A whole-field translation is the + stage or the sample moving. It is logged as its own event. Burst + imaging a drifted field is wasted light. + +### Safety and budget + +- **Hard caps.** Maximum captures and maximum runtime end the run in any + mode. +- **One approval per run, not per frame.** The unattended loop cannot + stop for a human on every capture. On real hardware the whole plan + (intervals, burst length, caps, thresholds) is printed and approved + once at the terminal before the first frame. The scheduler then + supplies the per-call hardware confirmation itself. No model ever + supplies it. +- **Light budget.** For brightfield the slow loop is nearly free. For + fluorescence the slow loop adds phototoxicity, so the preview channel + should be the least damaging one, and the burst window should be + short. +- **Nothing else moves.** The loop only captures. It does not move the + stage, refocus, or change exposure. Those stay under the existing + human-gated tools. + +### What is built (branch `adaptive-timelapse`) + +- `get_image(backend="mock")`: a simulated capture with no camera, so all + of this runs off the microscope PC. The real path is unchanged and + still gated. +- `timelapse/change_detector.py`: the change score. +- `timelapse/frame_audit.py`: the audit tool above. +- `timelapse/scheduler.py`: the slow/burst loop, with the caps, the + one-time approval, and the trigger hook. Verified against the mock: + swapping the served frame mid-run started a burst within one slow + interval and returned to slow afterwards. +- A test suite (24 tests, synthetic frames with known answers) and a CI + workflow that runs it on every push. + +Try it on any machine: + +``` +python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-duration 5 --max-runtime 30 +``` + +### What is next + +1. **Tune thresholds on a real recording.** Run the audit on the existing + time-lapse. The scores at the known 25 hour event and during quiet + stretches give the starting thresholds for the scheduler. +2. **First real run, attended.** Short slow interval, low caps, someone + watching, brightfield only. +3. **Wire the model into the trigger hook.** Before and after previews + go to Claude with the detector's reason, and its answer extends or + ends the burst. +4. **Decide on focus.** Long runs drift. Either the Perfect Focus System + holds it, or the loop needs a bounded refocus step, which would be a + new, gated capability. + +### One request + +Export the per-frame timestamps from the raw 25 hour dataset, or just +report the raw frame count next to the video frame count. That settles +whether finer slices exist before anyone spends time looking for them. From cbfe0e689b2ded207cee3a154157cffe9ad58b68 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:51:41 -0400 Subject: [PATCH 23/42] changelog: file adaptive time-lapse entries under Added; note the fixed-interval recorder in the proposal --- CHANGELOG.md | 30 +++++++++++++++--------------- docs/adaptive_timelapse.md | 6 ++++++ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35b1de8..dea22ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,21 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow `--confirm`, skips stage/objective/TIRF unless `--include-motion`, and polls the read-back rather than reading once (several devices report the old value for up to a second after a successful write). +- `get_image(backend=...)`: `backend="mock"` returns a simulated frame via + `MockNIS.capture()` with no camera attached and no `confirm`, so capture + logic can be developed off the microscope PC. `backend="sdk"` (the + default) is unchanged and still requires `confirm=True`. The metadata dict + now carries `backend`. Both harnesses skip the hardware gate for mock + captures. +- `CONFOCAL_MOCK_FRAME_PATH`: PNG the mock capture serves (defaults to the + old `data/analysis/nd2_sample/frame_0.png` location). +- `timelapse/` package (not MCP tools): `change_detector` (model-free + per-frame change score), `frame_audit` (CLI: gaps / intensity jumps / + stage shifts vs. specimen change in an existing sequence), `scheduler` + (adaptive slow/burst acquisition loop with hard caps and a one-time + real-hardware approval). See `docs/adaptive_timelapse.md`. +- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. +- `numpy` is now a core dependency; new `test` optional group (pytest). ### Fixed - `get_optical_configuration()` recorded no illumination state. It walked @@ -46,21 +61,6 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow back to `~/.confocal-mcp` with a warning on stderr. Setting `CONFOCAL_MCP_DATA_DIR` is still honoured as-is, and a normal source checkout still resolves to the checkout directory. -- `get_image(backend=...)`: `backend="mock"` returns a simulated frame via - `MockNIS.capture()` with no camera attached and no `confirm`, so capture - logic can be developed off the microscope PC. `backend="sdk"` (the - default) is unchanged and still requires `confirm=True`. The metadata dict - now carries `backend`. Both harnesses skip the hardware gate for mock - captures. -- `CONFOCAL_MOCK_FRAME_PATH`: PNG the mock capture serves (defaults to the - old `data/analysis/nd2_sample/frame_0.png` location). -- `timelapse/` package (not MCP tools): `change_detector` (model-free - per-frame change score), `frame_audit` (CLI: gaps / intensity jumps / - stage shifts vs. specimen change in an existing sequence), `scheduler` - (adaptive slow/burst acquisition loop with hard caps and a one-time - real-hardware approval). See `docs/adaptive_timelapse.md`. -- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. -- `numpy` is now a core dependency; new `test` optional group (pytest). ## [0.1.0] - 2026-08-31 diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md index d8fd85d..964d556 100644 --- a/docs/adaptive_timelapse.md +++ b/docs/adaptive_timelapse.md @@ -45,6 +45,12 @@ recover, and the honest answer is that the interval was too coarse. ## The proposal: image slowly, look cheaply, image fast when it matters +The fixed-interval recorder on this branch, +`acquisition/orchestration/timelapse.py`, stays as it is: a steady 10 s +series is the right tool for the shuttle-streaming rhythm, and its +drift-free scheduling is reused here. The adaptive loop is for the long, +slow runs where the interesting minutes are rare and unpredictable. + A fixed interval spends light, disk and time evenly on the boring hours and the interesting minutes alike. The adaptive loop puts the fast frames where the change is: From a3dff0be709355d69b4bc99aff4e88a04481219b Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 15:59:26 -0400 Subject: [PATCH 24/42] Pin numpy to 2.4.6: 2.5.x requires Python 3.12, project supports 3.11 CI on the pull request failed at pip install on the 3.11 job. --- CHANGELOG.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dea22ae..dc3ef0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow (adaptive slow/burst acquisition loop with hard caps and a one-time real-hardware approval). See `docs/adaptive_timelapse.md`. - `tests/` (pytest, mock only) and a GitHub Actions CI workflow. -- `numpy` is now a core dependency; new `test` optional group (pytest). +- `numpy` (2.4.x, the last line that supports Python 3.11) is now a core dependency; new `test` optional group (pytest). ### Fixed - `get_optical_configuration()` recorded no illumination state. It walked diff --git a/pyproject.toml b/pyproject.toml index d27d507..abb28dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ dependencies = [ "mcp==2.0.0", "Pillow==12.3.0", "PyYAML==6.0.3", - "numpy==2.5.3", + "numpy==2.4.6", ] [project.optional-dependencies] From e848f0a8e6b32af221a49995b9433a46aaa0e288 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 16:11:53 -0400 Subject: [PATCH 25/42] Add protocol-level MCP server test: subprocess server, exact tool set, every tool over stdio Spawns python -m mcp_server.server_loop (with the e-stop panel suppressed for headless CI), lists tools and asserts the set is exactly {get_image, get_pos, move, get_move_history, estop}, then drives the mock stage and mock capture and confirms the sdk paths are refused without confirm - all over real MCP stdio, the path Claude Desktop uses. --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 4 +- tests/test_mcp_server.py | 94 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 tests/test_mcp_server.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53aa208..f78d1c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,4 +22,5 @@ jobs: - run: python -m pytest -q tests # The MCP server must import on a core-only install (the camera and # SDK stacks are lazy imports) - fail the build if that regresses. + # (tests/test_mcp_server.py goes further and actually runs it.) - run: python -c "import mcp_server.server_loop, timelapse.scheduler" diff --git a/CHANGELOG.md b/CHANGELOG.md index dc3ef0d..0c6f150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,9 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow stage shifts vs. specimen change in an existing sequence), `scheduler` (adaptive slow/burst acquisition loop with hard caps and a one-time real-hardware approval). See `docs/adaptive_timelapse.md`. -- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. +- `tests/` (pytest, mock only) and a GitHub Actions CI workflow. Includes a + protocol-level test that spawns `mcp_server.server_loop` as a subprocess, + asserts the exact tool set, and drives every tool over MCP stdio. - `numpy` (2.4.x, the last line that supports Python 3.11) is now a core dependency; new `test` optional group (pytest). ### Fixed diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..cb356c1 --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,94 @@ +# End-to-end over the real protocol: spawn `python -m mcp_server.server_loop` +# as a subprocess (exactly what Claude Desktop/Code and harness/mcp_agent.py +# do), talk MCP to it over stdio, and exercise every tool against the mock +# backends. No hardware, no model. Guards two things the in-process tests +# cannot: that the server actually starts and serves on a core-only +# install, and that the model-facing tool surface is exactly the agreed +# set - nothing added by accident. +import asyncio +import json +import os +import sys +from pathlib import Path + +import pytest +from mcp.client import Client +from mcp.client.stdio import StdioServerParameters, stdio_client + +EXPECTED_TOOLS = {"get_image", "get_pos", "move", "get_move_history", "estop"} + + +def _server_params(data_dir: Path, frame: Path) -> StdioServerParameters: + env = dict(os.environ) + env.update({ + "CONFOCAL_MCP_DATA_DIR": str(data_dir), + "CONFOCAL_MOCK_FRAME_PATH": str(frame), + "CONFOCAL_NO_ESTOP_PANEL": "1", # headless: don't try to open the STOP window + }) + return StdioServerParameters(command=sys.executable, args=["-m", "mcp_server.server_loop"], env=env) + + +def _text(result) -> dict: + block = next(b for b in result.content if b.type == "text") + return json.loads(block.text) + + +async def _session(data_dir: Path, frame: Path): + async with Client(stdio_client(_server_params(data_dir, frame))) as client: + tools = (await client.list_tools()).tools + names = {t.name for t in tools} + + pos = _text(await client.call_tool("get_pos", {"backend": "mock"})) + moved = _text(await client.call_tool("move", {"x": 250.0, "y": -100.0, "backend": "mock"})) + history = _text(await client.call_tool("get_move_history", {"limit": 5})) + + img = await client.call_tool("get_image", {"backend": "mock", "max_dimension": 64}) + img_meta = _text(img) + img_types = [b.type for b in img.content] + + gated = await client.call_tool("get_image", {}) # backend defaults to sdk, no confirm + gated_move = await client.call_tool("move", {"x": 0, "y": 0, "backend": "sdk"}) + estop = _text(await client.call_tool("estop", {"action": "status"})) + + return dict(names=names, pos=pos, moved=moved, history=history, img_meta=img_meta, + img_types=img_types, gated=gated, gated_move=gated_move, estop=estop) + + +@pytest.fixture(scope="module") +def run(tmp_path_factory): + data_dir = tmp_path_factory.mktemp("mcp-data") + sys.path.insert(0, str(Path(__file__).parent)) + from conftest import make_blob_frame, save_frame + frame = save_frame(make_blob_frame(), data_dir / "frame.png") + return asyncio.run(_session(data_dir, frame)) + + +def test_tool_surface_is_exactly_the_agreed_set(run): + assert run["names"] == EXPECTED_TOOLS + + +def test_mock_stage_round_trip(run): + assert run["pos"]["backend"] == "mock" + assert run["moved"]["position"]["x"] == 250.0 + assert run["moved"]["position"]["y"] == -100.0 + assert run["history"]["total_moves"] >= 1 + last = run["history"]["history"][-1] + assert last["requested"] == {"x": 250.0, "y": -100.0} + assert last["position"]["x"] == 250.0 + + +def test_mock_capture_returns_metadata_and_embedded_image(run): + assert run["img_meta"]["backend"] == "mock" + assert Path(run["img_meta"]["image"]).exists() + assert run["img_meta"]["position"]["x"] == 250.0 # position is coupled to the capture + assert "image" in run["img_types"], run["img_types"] + + +def test_real_hardware_refused_without_confirm_over_protocol(run): + assert run["gated"].is_error is True + assert run["gated_move"].is_error is True + + +def test_estop_status_reports_over_protocol(run): + assert "engaged" in run["estop"] + assert run["estop"]["engaged"] is False From ff9350d35db969ea41704d075d9be2daa6138d11 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 16:22:04 -0400 Subject: [PATCH 26/42] Test the harness real-hardware approval gate in both harnesses Pins the safety property from harness/agent.py and harness/mcp_agent.py: the model never sees 'confirm' in a tool schema; get_image/move on backend='sdk' stop at the gate, a decline returns an error result and executes nothing, an approval injects confirm=True; backend='mock' never prompts and cannot be promoted to real hardware by a smuggled flag. The gate prompt and the tools are stubbed, so no model and no hardware. CI now installs the 'harness' extra so the loops import on the runner. --- .github/workflows/ci.yml | 8 +- CHANGELOG.md | 5 +- tests/test_harness_gate.py | 209 +++++++++++++++++++++++++++++++++++++ 3 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 tests/test_harness_gate.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f78d1c9..d183d2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - # Core + test only: the mock stage and mock camera need no hardware - # dependencies, so the whole suite runs on a plain runner. - - run: pip install -e ".[test]" + # No hardware groups: the mock stage and mock camera need none, so the + # whole suite runs on a plain runner. 'harness' is pure Python and is + # needed to import the Claude loops for the approval-gate tests (no + # API key required - no model is ever called). + - run: pip install -e ".[harness,test]" - run: python -m pytest -q tests # The MCP server must import on a core-only install (the camera and # SDK stacks are lazy imports) - fail the build if that regresses. diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c6f150..0ed2705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,7 +36,10 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow real-hardware approval). See `docs/adaptive_timelapse.md`. - `tests/` (pytest, mock only) and a GitHub Actions CI workflow. Includes a protocol-level test that spawns `mcp_server.server_loop` as a subprocess, - asserts the exact tool set, and drives every tool over MCP stdio. + asserts the exact tool set, and drives every tool over MCP stdio. Also + tests of both harnesses' real-hardware approval gate: no `confirm` in any + model-facing schema, sdk calls stop at the gate and a decline executes + nothing, mock calls never prompt. - `numpy` (2.4.x, the last line that supports Python 3.11) is now a core dependency; new `test` optional group (pytest). ### Fixed diff --git a/tests/test_harness_gate.py b/tests/test_harness_gate.py new file mode 100644 index 0000000..9afaf90 --- /dev/null +++ b/tests/test_harness_gate.py @@ -0,0 +1,209 @@ +# The harnesses' real-hardware approval gate is the load-bearing safety +# property of harness/agent.py and harness/mcp_agent.py (see their header +# comments). These tests pin it down without a model or hardware: +# +# - the model never sees a "confirm" parameter in any tool schema +# - a real-hardware call (get_image on sdk, move on sdk) stops at the +# gate; a decline returns an error result and executes nothing; +# an approval injects confirm=True itself +# - a mock call never touches the gate at all +# +# The gate's input() prompt is monkeypatched with a recorder, and the +# underlying tool is replaced with a stub that records what it was called +# with, so the assertions are about the harness's decisions only. +import asyncio +import json + +import pytest + +from harness import agent, mcp_agent +from mcp_server import loop_tools + + +class FakeImage: + def to_image_content(self): + class C: + mime_type = "image/jpeg" + data = "AAAA" + return C() + + +@pytest.fixture +def gate(monkeypatch): + """Replace both harnesses' gate with a recorder whose answer we control.""" + calls = [] + state = {"answer": False} + + def sync_gate(name, tool_input): + calls.append((name, dict(tool_input))) + return state["answer"] + + async def async_gate(name, tool_input): + calls.append((name, dict(tool_input))) + return state["answer"] + + monkeypatch.setattr(agent, "_confirm_real_hardware_action", sync_gate) + monkeypatch.setattr(mcp_agent, "_confirm_real_hardware_action", async_gate) + state["calls"] = calls + return state + + +@pytest.fixture +def stub_tools(monkeypatch): + """Stub loop_tools so nothing real runs; record kwargs each was called with.""" + seen = {} + + def get_image(**kw): + seen["get_image"] = kw + return {"frame_id": 1, "image": "x.png", "backend": kw.get("backend", "sdk")}, FakeImage() + + def move(**kw): + seen["move"] = kw + return {"position": {"x": kw["x"], "y": kw["y"], "z": 0}} + + monkeypatch.setattr(loop_tools, "get_image", get_image) + monkeypatch.setattr(loop_tools, "move", move) + return seen + + +# -- schemas ----------------------------------------------------------------- + +def _all_property_names(schema: dict) -> set[str]: + names = set() + for key, sub in schema.get("properties", {}).items(): + names.add(key) + if isinstance(sub, dict): + names |= _all_property_names(sub) + return names + + +def test_agent_tool_schemas_never_expose_confirm(): + for tool in agent.TOOLS: + assert "confirm" not in _all_property_names(tool["input_schema"]), tool["name"] + + +def test_mcp_agent_strips_confirm_from_real_server_schemas(): + from mcp_server.server_loop import mcp + mcp_tools = asyncio.run(mcp.list_tools()) + assert any("confirm" in t.input_schema.get("properties", {}) for t in mcp_tools), \ + "precondition: the server itself does expose confirm (the harness must strip it)" + for t in mcp_agent._mcp_tools_to_anthropic_tools(mcp_tools): + assert "confirm" not in t["input_schema"]["properties"], t["name"] + + +def test_strip_confirm_does_not_mutate_input(): + schema = {"type": "object", "properties": {"confirm": {"type": "boolean"}, "x": {"type": "number"}}} + out = mcp_agent._strip_confirm(schema) + assert "confirm" not in out["properties"] + assert "confirm" in schema["properties"] + + +# -- harness/agent.py (in-process) -------------------------------------------- + +def test_agent_mock_capture_skips_gate(gate, stub_tools): + content, is_error = agent._execute_tool("get_image", {"backend": "mock"}) + assert gate["calls"] == [] + assert is_error is False + assert "confirm" not in stub_tools["get_image"] + assert [c["type"] for c in content] == ["text", "image"] + + +def test_agent_real_capture_declined_executes_nothing(gate, stub_tools): + gate["answer"] = False + content, is_error = agent._execute_tool("get_image", {}) + assert gate["calls"] == [("get_image", {})] + assert is_error is True + assert "get_image" not in stub_tools + assert "declined" in content[0]["text"].lower() + + +def test_agent_real_capture_approved_injects_confirm(gate, stub_tools): + gate["answer"] = True + _, is_error = agent._execute_tool("get_image", {"exposure_time_us": 5000}) + assert is_error is False + assert stub_tools["get_image"]["confirm"] is True + assert stub_tools["get_image"]["exposure_time_us"] == 5000 + + +def test_agent_model_cannot_smuggle_confirm_on_mock(gate, stub_tools): + # Even if the model somehow sent confirm=True, a mock call must not be + # promoted to real hardware: backend decides, not the flag. + agent._execute_tool("get_image", {"backend": "mock", "confirm": True}) + assert gate["calls"] == [] + assert stub_tools["get_image"]["backend"] == "mock" + + +def test_agent_move_gate_matches_backend(gate, stub_tools): + agent._execute_tool("move", {"x": 1, "y": 2}) # backend defaults to mock + assert gate["calls"] == [] and "confirm" not in stub_tools["move"] + + gate["answer"] = False + _, is_error = agent._execute_tool("move", {"x": 1, "y": 2, "backend": "sdk"}) + assert is_error is True and gate["calls"] == [("move", {"x": 1, "y": 2, "backend": "sdk"})] + + gate["answer"] = True + _, is_error = agent._execute_tool("move", {"x": 3, "y": 4, "backend": "sdk"}) + assert is_error is False and stub_tools["move"]["confirm"] is True + + +# -- harness/mcp_agent.py (over a fake MCP client) ----------------------------- + +class FakeMCPClient: + """Records call_tool arguments; returns a fixed text result.""" + + def __init__(self): + self.calls = [] + + async def call_tool(self, name, arguments): + self.calls.append((name, dict(arguments))) + + class Block: + type = "text" + text = json.dumps({"ok": True}) + + class Result: + content = [Block()] + is_error = False + return Result() + + +def _run(coro): + return asyncio.run(coro) + + +def test_mcp_agent_mock_capture_skips_gate(gate): + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {"backend": "mock"})) + assert gate["calls"] == [] + assert is_error is False + assert client.calls == [("get_image", {"backend": "mock"})] + + +def test_mcp_agent_real_capture_declined_never_reaches_server(gate): + gate["answer"] = False + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {})) + assert is_error is True + assert client.calls == [] + + +def test_mcp_agent_real_capture_approved_injects_confirm(gate): + gate["answer"] = True + client = FakeMCPClient() + _, is_error = _run(mcp_agent._execute_tool(client, "get_image", {"gain": 2.0})) + assert is_error is False + assert client.calls == [("get_image", {"gain": 2.0, "confirm": True})] + + +def test_mcp_agent_move_gate_matches_backend(gate): + client = FakeMCPClient() + _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "mock"})) + assert gate["calls"] == [] and client.calls[-1][1].get("confirm") is None + + gate["answer"] = False + _, is_error = _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "sdk"})) + assert is_error is True and len(client.calls) == 1 + + gate["answer"] = True + _run(mcp_agent._execute_tool(client, "move", {"x": 1, "y": 2, "backend": "sdk"})) + assert client.calls[-1] == ("move", {"x": 1, "y": 2, "backend": "sdk", "confirm": True}) From c7cd4063d2e7670e61b20ebe728c319595865f2f Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 16:28:02 -0400 Subject: [PATCH 27/42] Test harness/context.py image pruning; fix stale '4 tools' wording, list estop in README harness/agent.py's 'four tools' line is left as is: that in-process harness really does expose only four (it has no estop entry). --- CHANGELOG.md | 5 ++- README.md | 8 +++- docs/mcp_harness.md | 4 +- eaa_integration/run_session.py | 2 +- harness/mcp_agent.py | 2 +- mcp_server/loop_tools.py | 7 ++-- mcp_server/server_loop.py | 4 +- tests/test_context.py | 77 ++++++++++++++++++++++++++++++++++ timelapse/__init__.py | 2 +- 9 files changed, 99 insertions(+), 12 deletions(-) create mode 100644 tests/test_context.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed2705..03c3a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,10 +39,13 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow asserts the exact tool set, and drives every tool over MCP stdio. Also tests of both harnesses' real-hardware approval gate: no `confirm` in any model-facing schema, sdk calls stop at the gate and a decline executes - nothing, mock calls never prompt. + nothing, mock calls never prompt. And tests of `harness/context.py`'s + image pruning. - `numpy` (2.4.x, the last line that supports Python 3.11) is now a core dependency; new `test` optional group (pytest). ### Fixed +- README and code comments said the MCP surface was 4 tools; it has been 5 + since `estop` was added. The README's tool list now includes `estop`. - `get_optical_configuration()` recorded no illumination state. It walked `OPTICAL_CONFIG_PROPERTIES`, a hardcoded list that omitted `iDIA_LAMP_Switch`/`iDIA_LAMP_Pos` entirely - so a saved configuration never diff --git a/README.md b/README.md index 25a73f4..24ae646 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ MCP client). ## Tools -Exactly 4, by design - kept deliberately minimal: +Exactly 5, by design - kept deliberately minimal: - **`get_pos(backend)`** - current stage (x, y, z) position, in microns. A cheap, on-demand sync primitive, not something to call before every @@ -37,6 +37,12 @@ identifying that specific capture. the stage to this session, so "where have we already been" doesn't need to be re-derived from conversation history. +- **`estop(action)`** - emergency stop. `"engage"` forbids all stage motion + for every process on the machine via a flag file; `"status"` reports it. + Needs no `confirm`: a stop is always safe. Release is deliberately not + reachable from chat - a human clears it with + `python -m acquisition.estop release`. + `backend` is `"mock"` (default, safe, simulated) or `"sdk"` (real hardware - `move()`/`get_image()` require `confirm=True` for anything that touches real hardware). See `mcp_server/loop_tools.py`'s header diff --git a/docs/mcp_harness.md b/docs/mcp_harness.md index bf40889..66b28ba 100644 --- a/docs/mcp_harness.md +++ b/docs/mcp_harness.md @@ -165,7 +165,7 @@ MCP tool results (`CallToolResult.content`) are typed content blocks - |---|---| | `TextContent(text=...)` | `{"type": "text", "text": ...}` | | `ImageContent(data=..., mime_type=...)` | `{"type": "image", "source": {"type": "base64", "media_type": mime_type, "data": data}}` | -| anything else (audio, resource links, ...) | a text block noting the unsupported type - `loop_tools.py`'s 4 tools never actually produce these, so this is a defensive fallback, not a real code path | +| anything else (audio, resource links, ...) | a text block noting the unsupported type - `loop_tools.py`'s 5 tools never actually produce these, so this is a defensive fallback, not a real code path | Conveniently, MCP's `Tool.input_schema` is already plain JSON Schema in the same shape Anthropic's tool `input_schema` expects, so @@ -209,7 +209,7 @@ approval, same as `agent.py`. Both with the mock backend (no real hardware needed) and real Claude API calls: -- `list_tools()` correctly discovers all 4 tools, with `confirm` absent +- `list_tools()` correctly discovers all 5 tools, with `confirm` absent from every schema Claude sees. - `call_tool()` round-trips correctly for `get_pos`, `move`, and `get_move_history` - including error cases (unknown tool name, a diff --git a/eaa_integration/run_session.py b/eaa_integration/run_session.py index 1672e61..c0c5228 100644 --- a/eaa_integration/run_session.py +++ b/eaa_integration/run_session.py @@ -291,7 +291,7 @@ def main() -> None: # also fixes a real crash: several of those built-ins use dotted # names (e.g. "simple_python_eval_tool.evaluate_python_expression"), # which OpenAI's function-calling API rejects outright (names must - # match ^[a-zA-Z0-9_-]+$, no dots) - our own 4 tools are named + # match ^[a-zA-Z0-9_-]+$, no dots) - our own 5 tools are named # cleanly and were never the problem. task_manager.tool_manager.disable_bash_coding_tool() task_manager.tool_manager.disable_python_coding_tool() diff --git a/harness/mcp_agent.py b/harness/mcp_agent.py index aeb394b..2666326 100644 --- a/harness/mcp_agent.py +++ b/harness/mcp_agent.py @@ -155,7 +155,7 @@ async def _confirm_real_hardware_action(tool_name: str, tool_input: dict) -> boo def _mcp_content_to_anthropic_content(mcp_content: list) -> list[dict]: """Convert a CallToolResult's content blocks (TextContent/ImageContent/ ...) to Anthropic tool_result content blocks. Only text and image are - handled - loop_tools.py's 4 tools never return audio/resource blocks. + handled - loop_tools.py's 5 tools never return audio/resource blocks. """ blocks = [] for block in mcp_content: diff --git a/mcp_server/loop_tools.py b/mcp_server/loop_tools.py index 227767f..6b92ef0 100644 --- a/mcp_server/loop_tools.py +++ b/mcp_server/loop_tools.py @@ -5,8 +5,9 @@ # get_pos() - a lightweight, on-demand position sync primitive # move() - move XY, return the ACTUAL resulting position # get_move_history() - every point move() has visited this session +# estop() - emergency stop: forbid all motion, machine-wide # -# Exactly 4 tools, by explicit direction - do not add more without +# Exactly 5 tools, by explicit direction - do not add more without # checking first. Calibration, historical-frame lookup, etc. should be # done by the model reasoning over get_image()'s embedded picture, not # by adding a dedicated tool per capability. @@ -75,7 +76,7 @@ # _append_frame_history), and get_frame(frame_id) resolves a frame_id # back to that record - but get_frame() is a plain Python function, NOT # a 5th MCP tool: team direction is to keep the model-facing surface at -# exactly 4 tools, so history/lookup logic can grow underneath without +# exactly 5 tools, so history/lookup logic can grow underneath without # growing what the model itself can call. # # WHY move() IS XY-ONLY (not x/y/z): a blind absolute Z move must never @@ -315,7 +316,7 @@ def get_frame(frame_id: int) -> dict: returned for it. NOT registered as an MCP tool (see server_loop.py) - by explicit team - direction the chat-facing surface stays at 4 tools. This is a plain + direction the chat-facing surface stays at 5 tools. This is a plain importable function for harness/analysis code that needs to resolve "frame 42" to its full-res file and the position it was captured at, without re-reading FRAME_HISTORY_PATH by hand. diff --git a/mcp_server/server_loop.py b/mcp_server/server_loop.py index 7d9d878..4c2b891 100644 --- a/mcp_server/server_loop.py +++ b/mcp_server/server_loop.py @@ -1,8 +1,8 @@ # server_loop.py # ------------------------------------------------------------ # Minimal MCP server entry point: registers ONLY loop_tools.py's -# functions (get_image, get_pos, move, get_move_history) - kept -# deliberately minimal, 4 tools total, by explicit team direction. +# functions (get_image, get_pos, move, get_move_history, estop) - kept +# deliberately minimal, 5 tools total, by explicit team direction. # # Run, either way: # confocal-mcp (installed console script - see pyproject.toml) diff --git a/tests/test_context.py b/tests/test_context.py new file mode 100644 index 0000000..4b2aa67 --- /dev/null +++ b/tests/test_context.py @@ -0,0 +1,77 @@ +# harness/context.py: image pruning of a chat history between tool +# rounds. The rules that matter to a running harness: text always +# survives, the newest images survive, the caller's own history is never +# mutated (it keeps the full-fidelity copy), and non-image messages are +# passed through untouched, identity included. +import copy + +import pytest + +from harness.context import prune_context_images + + +def _img(label, kind="image"): + block = {"type": "image", "source": {"data": f"<{label}>"}} if kind == "image" \ + else {"type": "image_url", "image_url": {"url": f"data:{label}"}} + return {"role": "user", "content": [{"type": "text", "text": f"caption {label}"}, block]} + + +def _text(role, text): + return {"role": role, "content": [{"type": "text", "text": text}]} + + +def _has_image(msg): + return any(b["type"] in ("image", "image_url") for b in msg["content"]) + + +def test_keeps_only_last_n_images_and_all_text(): + history = [_text("user", "go"), _img("f1"), _text("assistant", "ok"), _img("f2"), _img("f3"), _img("f4")] + pruned = prune_context_images(history, keep_last_n=2) + assert [_has_image(m) for m in pruned] == [False, False, False, False, True, True] + assert pruned[1]["content"] == [{"type": "text", "text": "caption f1"}] + assert pruned[3]["content"][0]["text"] == "caption f2" + + +def test_keep_first_and_last_counts_image_messages_only(): + history = [_text("user", "a"), _img("f1"), _text("user", "b"), _img("f2"), _img("f3"), _text("user", "c")] + pruned = prune_context_images(history, keep_first_n=1, keep_last_n=1) + assert _has_image(pruned[1]) and _has_image(pruned[4]) + assert not _has_image(pruned[3]) + + +def test_does_not_mutate_caller_history(): + history = [_img("f1"), _img("f2"), _img("f3")] + snapshot = copy.deepcopy(history) + prune_context_images(history, keep_last_n=1) + assert history == snapshot + + +def test_non_image_messages_pass_through_by_identity(): + plain = _text("assistant", "thinking") + history = [_img("f1"), plain, _img("f2")] + pruned = prune_context_images(history, keep_last_n=1) + assert pruned[1] is plain + + +def test_string_content_and_openai_image_blocks(): + history = [{"role": "user", "content": "plain string"}, _img("f1", kind="image_url"), _img("f2", kind="image_url")] + pruned = prune_context_images(history, keep_last_n=1) + assert pruned[0] == {"role": "user", "content": "plain string"} + assert not _has_image(pruned[1]) and _has_image(pruned[2]) + + +def test_zero_keeps_strips_everything_and_defaults_are_zero(): + history = [_img("f1"), _img("f2")] + assert not any(_has_image(m) for m in prune_context_images(history)) + + +def test_keep_more_than_exist_is_a_no_op(): + history = [_img("f1"), _img("f2")] + assert prune_context_images(history, keep_last_n=10) == history + + +def test_negative_counts_rejected(): + with pytest.raises(ValueError): + prune_context_images([], keep_last_n=-1) + with pytest.raises(ValueError): + prune_context_images([], keep_first_n=-1) diff --git a/timelapse/__init__.py b/timelapse/__init__.py index 2ab03dc..c33957f 100644 --- a/timelapse/__init__.py +++ b/timelapse/__init__.py @@ -10,7 +10,7 @@ # scheduler.py - the slow-loop / burst-mode acquisition loop that # drives loop_tools.get_image() using the detector # -# None of this is an MCP tool. The model-facing surface stays at 4 tools +# None of this is an MCP tool. The model-facing surface stays at 5 tools # (see mcp_server/loop_tools.py's header); this package sits *beside* the # tools and calls get_image() itself. # ------------------------------------------------------------ From fdf25432deb580bdcd33baa570df5dac92e5a29a Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 16:43:06 -0400 Subject: [PATCH 28/42] Add timelapse/model_trigger.py: Claude behind the scheduler's trigger hook At each burst start/extend the hook shows Claude the frame before, the frame that fired the detector, and the detector's numbers, and asks for {decision: extend|ignore, reason}. Extend lengthens the burst, ignore ends it; anything else is no opinion. Per-run call cap and a minimum interval between asks. Every failure path returns None so the scheduler is never worse off than with no hook. The API call is one injectable function; tests use a fake model. Enabled with --model-trigger. Scheduler capture events now carry previous_image. --- CHANGELOG.md | 4 +- README.md | 1 + docs/adaptive_timelapse.md | 15 ++- tests/__init__.py | 0 tests/test_model_trigger.py | 126 +++++++++++++++++++++++++ timelapse/model_trigger.py | 182 ++++++++++++++++++++++++++++++++++++ timelapse/scheduler.py | 15 ++- 7 files changed, 336 insertions(+), 7 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_model_trigger.py create mode 100644 timelapse/model_trigger.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 03c3a60..84cdeb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,9 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow per-frame change score), `frame_audit` (CLI: gaps / intensity jumps / stage shifts vs. specimen change in an existing sequence), `scheduler` (adaptive slow/burst acquisition loop with hard caps and a one-time - real-hardware approval). See `docs/adaptive_timelapse.md`. + real-hardware approval), `model_trigger` (Claude behind the scheduler's + trigger hook: extend or end a burst from the before/after frames, with + call caps and fail-safe "no opinion"). See `docs/adaptive_timelapse.md`. - `tests/` (pytest, mock only) and a GitHub Actions CI workflow. Includes a protocol-level test that spawns `mcp_server.server_loop` as a subprocess, asserts the exact tool set, and drives every tool over MCP stdio. Also diff --git a/README.md b/README.md index 24ae646..a5b8fa9 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ stage bump from real specimen change. Design, safety model and status: ``` python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-duration 5 --max-runtime 30 +python -m timelapse.scheduler --backend mock --model-trigger ... # Claude judges each trigger (needs API credentials) python -m timelapse.frame_audit FRAME_DIR --timestamps times.csv --around 25h --window 1h ``` diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md index 964d556..d80437e 100644 --- a/docs/adaptive_timelapse.md +++ b/docs/adaptive_timelapse.md @@ -100,7 +100,14 @@ frames where the change is: - `timelapse/change_detector.py`: the change score. - `timelapse/frame_audit.py`: the audit tool above. - `timelapse/scheduler.py`: the slow/burst loop, with the caps, the - one-time approval, and the trigger hook. Verified against the mock: + one-time approval, and the trigger hook. +- `timelapse/model_trigger.py`: Claude behind that hook. At each trigger + it sees the frame before, the frame that fired, and the detector's + numbers, and answers "extend" or "ignore" with a one-sentence reason + that is logged. Capped calls per run, never more than one ask per 30 s, + and every failure (no key, no network, refusal, odd reply) is a + harmless "no opinion". Enabled with `--model-trigger`. Tested with a + fake model; the live call has not yet been run. Verified against the mock: swapping the served frame mid-run started a burst within one slow interval and returned to slow afterwards. - A test suite (24 tests, synthetic frames with known answers) and a CI @@ -119,9 +126,9 @@ python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-durati stretches give the starting thresholds for the scheduler. 2. **First real run, attended.** Short slow interval, low caps, someone watching, brightfield only. -3. **Wire the model into the trigger hook.** Before and after previews - go to Claude with the detector's reason, and its answer extends or - ends the burst. +3. **Run the model trigger live once.** The code is written and tested + against a fake; one real run with `--model-trigger` on the mock, with + an API key, confirms the call and shows what its reasons look like. 4. **Decide on focus.** Long runs drift. Either the Perfect Focus System holds it, or the loop needs a bounded refocus step, which would be a new, gated capability. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_model_trigger.py b/tests/test_model_trigger.py new file mode 100644 index 0000000..8875a78 --- /dev/null +++ b/tests/test_model_trigger.py @@ -0,0 +1,126 @@ +# timelapse/model_trigger.py with a fake model: what the model is shown, +# how its answer is read, and that every failure path is a harmless None. +import json + +import numpy as np + +from timelapse.change_detector import ChangeScore +from timelapse.model_trigger import ModelTrigger, SYSTEM_PROMPT, build_user_content, parse_decision + + +def _score(reason="foreground area shrank by 5.7%"): + return ChangeScore(diff_score=6.2, area_delta=-0.057, shift_px=0.0, foreground_fraction=0.05, + baseline_frames=5, interesting=True, reason=reason) + + +def _event(write_frame, with_previous=True): + after = write_frame("after.png", radius=10.0, seed=2) + before = write_frame("before.png", seed=1) if with_previous else None + return {"event": "capture", "frame_id": 7, "image": str(after), + "previous_image": str(before) if before else None, "position": {"x": 1, "y": 2, "z": 3}} + + +class FakeAsk: + def __init__(self, reply): + self.reply = reply + self.received = [] + + def __call__(self, system_prompt, content): + self.received.append((system_prompt, content)) + if isinstance(self.reply, Exception): + raise self.reply + return self.reply + + +class FakeClock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +# -- what the model is shown --------------------------------------------------- + +def test_prompt_contains_before_and_after_images_and_detector_numbers(write_frame): + content = build_user_content(_event(write_frame), _score()) + kinds = [b["type"] for b in content] + assert kinds == ["text", "image", "text", "image", "text"] + for b in content: + if b["type"] == "image": + assert b["source"]["media_type"] == "image/jpeg" and len(b["source"]["data"]) > 100 + tail = content[-1]["text"] + assert "shrank" in tail and "6.2x" in tail and "-5.7%" in tail + + +def test_prompt_without_previous_frame_has_one_image(write_frame): + content = build_user_content(_event(write_frame, with_previous=False), _score()) + assert [b["type"] for b in content] == ["text", "image", "text"] + + +# -- how the answer is read ---------------------------------------------------- + +def test_parse_decision_variants(): + assert parse_decision('{"decision": "extend", "reason": "tube thickening"}') == ("extend", "tube thickening") + assert parse_decision('Sure. {"decision":"IGNORE","reason":"bubble"} ok') == ("ignore", "bubble") + assert parse_decision('{"decision": "maybe"}')[0] is None + assert parse_decision("no json here")[0] is None + assert parse_decision("{broken")[0] is None + assert parse_decision("")[0] is None + + +def test_trigger_returns_model_decision_and_records_reason(write_frame): + ask = FakeAsk(json.dumps({"decision": "extend", "reason": "front advancing"})) + trig = ModelTrigger(ask=ask, log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) == "extend" + assert ask.received[0][0] == SYSTEM_PROMPT + assert trig.decisions[-1]["decision"] == "extend" + assert trig.decisions[-1]["reason"] == "front advancing" + assert trig.calls == 1 + + +# -- every failure is a harmless None ----------------------------------------- + +def test_model_exception_is_swallowed(write_frame): + trig = ModelTrigger(ask=FakeAsk(RuntimeError("no network")), log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) is None + assert "no network" in trig.decisions[-1]["reason"] + + +def test_unparseable_reply_is_no_opinion(write_frame): + trig = ModelTrigger(ask=FakeAsk("I think it's interesting!"), log=lambda s: None) + assert trig(_event(write_frame), {}, _score()) is None + + +def test_max_calls_cap_and_min_interval(write_frame): + clock = FakeClock() + ask = FakeAsk('{"decision": "extend", "reason": "x"}') + trig = ModelTrigger(ask=ask, max_calls=2, min_interval_s=30.0, clock=clock, log=lambda s: None) + ev = _event(write_frame) + assert trig(ev, {}, _score()) == "extend" # call 1 + assert trig(ev, {}, _score()) is None # too soon + clock.t = 31.0 + assert trig(ev, {}, _score()) == "extend" # call 2 + clock.t = 62.0 + assert trig(ev, {}, _score()) is None # cap reached + assert len(ask.received) == 2 + assert "max_calls" in trig.decisions[-1]["reason"] + + +# -- wired into the scheduler -------------------------------------------------- + +def test_scheduler_passes_previous_image_and_honours_ignore(write_frame, tmp_path): + from tests.test_scheduler import FakeClock as SchedClock, FrameSource, _config + from timelapse.scheduler import AdaptiveTimelapse + clock = SchedClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(6)] + src = FrameSource(clock, quiet + shrunk) + ask = FakeAsk('{"decision": "ignore", "reason": "bubble"}') + trig = ModelTrigger(ask=ask, min_interval_s=0.0, log=lambda s: None) + summary = AdaptiveTimelapse(_config(tmp_path, max_captures=12), capture=src, on_trigger=trig, + clock=clock, sleep=clock.sleep, log=lambda s: None).run() + assert trig.calls >= 1 + _, content = ask.received[0] + assert [b["type"] for b in content][:2] == ["text", "image"] # a "before" frame was available + assert summary.burst_captures == 0 # ignore ended the burst at once diff --git a/timelapse/model_trigger.py b/timelapse/model_trigger.py new file mode 100644 index 0000000..fb1b6a8 --- /dev/null +++ b/timelapse/model_trigger.py @@ -0,0 +1,182 @@ +# model_trigger.py +# ------------------------------------------------------------ +# The "ask the model" seam for the adaptive time-lapse: an on_trigger +# hook for scheduler.AdaptiveTimelapse that shows Claude the frame before +# and the frame that fired the detector, plus the detector's own reason, +# and asks one question: is this worth a longer look? +# +# "extend" - keep burst-imaging for another window +# "ignore" - this is not interesting (dust, a bubble, a drift), end +# the burst now and go back to the slow interval +# None - no opinion; the scheduler carries on with its own timer +# +# WHY THE MODEL IS HERE AND NOWHERE ELSE: the detector runs on every +# frame and is free. The model runs only when the detector fires - a +# handful of times per run - so its cost and latency never touch the +# fast loop, and every decision it makes is written down with a reason +# (see .decisions and the scheduler's event log). +# +# WHY IT CAN NEVER MAKE THINGS WORSE: every failure path returns None. +# No key, no network, a refusal, an unparseable answer - the scheduler +# behaves exactly as it would with no hook. The model can shorten or +# lengthen a burst; it cannot capture, move, or refocus anything. +# +# THE API CALL IS ONE INJECTABLE FUNCTION (`ask`), so the whole class is +# testable without a key or a network: tests pass a fake `ask` and +# assert on what it was given and how its answer was interpreted. +# ------------------------------------------------------------ + +from __future__ import annotations + +import base64 +import io +import json +import re +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable + +from PIL import Image + +from timelapse.change_detector import ChangeScore + +MODEL = "claude-opus-5" # same as harness/agent.py and harness/mcp_agent.py +MAX_TOKENS = 1024 +PREVIEW_MAX_DIMENSION = 512 # small on purpose: the question is "did the specimen change", not fine detail + +SYSTEM_PROMPT = """\ +You are watching a live-cell time-lapse on a microscope. A change detector has +just flagged the latest frame as different from the frames before it, and the +scheduler has switched to fast burst imaging. You see the frame from just before +the trigger and the frame that fired it, with the detector's numeric reason. + +Decide whether continuing fast imaging is worth the light exposure: +- "extend": the specimen itself is doing something (moving, growing, retracting, + changing shape or internal structure). Keep imaging fast. +- "ignore": the change is not the specimen - debris, a bubble, a focus drift, + an illumination change, noise, or a stage shift. Stop the burst. + +Reply with only a JSON object: {"decision": "extend" | "ignore", "reason": ""}. +""" + +AskFn = Callable[[str, list[dict]], str] # (system_prompt, user_content_blocks) -> model text + + +def _jpeg_b64(path: str | Path, max_dimension: int = PREVIEW_MAX_DIMENSION) -> str: + with Image.open(path) as img: + img = img.convert("RGB") + img.thumbnail((max_dimension, max_dimension)) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=80) + return base64.standard_b64encode(buf.getvalue()).decode("ascii") + + +def _image_block(path: str | Path) -> dict: + return {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": _jpeg_b64(path)}} + + +def build_user_content(event: dict, score: ChangeScore) -> list[dict]: + """The user turn: before-frame (if there is one), trigger frame, and the + detector's numbers. Frames are downscaled JPEGs - the full-res files + stay on disk, exactly as get_image() does for its own preview.""" + blocks: list[dict] = [] + previous = event.get("previous_image") + if previous and Path(previous).exists(): + blocks.append({"type": "text", "text": "Frame before the trigger:"}) + blocks.append(_image_block(previous)) + blocks.append({"type": "text", "text": "Frame that fired the detector:"}) + blocks.append(_image_block(event["image"])) + blocks.append({"type": "text", "text": ( + f"Detector reason: {score.reason}. " + f"Pixel change {score.diff_score:.1f}x baseline noise; " + f"foreground area changed by {score.area_delta:+.1%}; " + f"whole-field shift {score.shift_px:.1f}px. " + f"Stage position: {event.get('position')}." + )}) + return blocks + + +def parse_decision(text: str) -> tuple[str | None, str]: + """Pull {"decision": ..., "reason": ...} out of the model's reply. + Tolerates prose around the JSON. Anything unrecognizable -> (None, text).""" + match = re.search(r"\{.*\}", text, re.S) + if not match: + return None, text.strip() + try: + obj = json.loads(match.group(0)) + except json.JSONDecodeError: + return None, text.strip() + decision = str(obj.get("decision", "")).strip().lower() + reason = str(obj.get("reason", "")).strip() + if decision not in ("extend", "ignore"): + return None, reason or text.strip() + return decision, reason + + +def anthropic_ask(system_prompt: str, content: list[dict]) -> str: + """The real call. Constructed lazily so importing this module never + needs the SDK or a key - only actually asking does. Credentials come + from the environment / .env / `ant auth login`, same as the harnesses.""" + import anthropic + from dotenv import load_dotenv + load_dotenv() + client = anthropic.Anthropic() + response = client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + thinking={"type": "adaptive"}, + system=system_prompt, + messages=[{"role": "user", "content": content}], + ) + if response.stop_reason == "refusal": + return "" + return "".join(block.text for block in response.content if block.type == "text") + + +@dataclass +class ModelTrigger: + """on_trigger hook for AdaptiveTimelapse. Call it like the scheduler + does: trigger(event, metadata, score) -> "extend" | "ignore" | None. + + max_calls: hard cap on model calls per run - after it, the hook is a + no-op (returns None). Cost control for an unattended run. + min_interval_s: don't ask again within this many seconds of the last + ask - a burst that keeps extending should not spam the model. + """ + + ask: AskFn = anthropic_ask + max_calls: int = 50 + min_interval_s: float = 30.0 + clock: Callable[[], float] = time.monotonic + log: Callable[[str], None] = print + calls: int = 0 + decisions: list[dict] = field(default_factory=list) + _last_ask_at: float | None = None + + def __call__(self, event: dict, metadata: dict, score: ChangeScore) -> str | None: + now = self.clock() + if self.calls >= self.max_calls: + self._record(event, None, f"skipped: max_calls={self.max_calls} reached") + return None + if self._last_ask_at is not None and now - self._last_ask_at < self.min_interval_s: + self._record(event, None, f"skipped: asked {now - self._last_ask_at:.0f}s ago") + return None + + self.calls += 1 + self._last_ask_at = now + try: + content = build_user_content(event, score) + text = self.ask(SYSTEM_PROMPT, content) + except Exception as exc: # noqa: BLE001 - any failure here must be non-fatal + self._record(event, None, f"model call failed: {type(exc).__name__}: {exc}") + return None + decision, reason = parse_decision(text) + self._record(event, decision, reason if decision else f"unparseable reply: {reason[:120]}") + return decision + + def _record(self, event: dict, decision: str | None, reason: str) -> None: + entry = {"frame_id": event.get("frame_id"), "image": event.get("image"), + "decision": decision, "reason": reason} + self.decisions.append(entry) + self.log(f"[model] frame {entry['frame_id']}: {decision or 'no opinion'} - {reason}") diff --git a/timelapse/scheduler.py b/timelapse/scheduler.py index 7d349c5..7a3170f 100644 --- a/timelapse/scheduler.py +++ b/timelapse/scheduler.py @@ -20,7 +20,8 @@ # when a burst starts or extends; that is the place to ask a vision # model "is this worth a longer look?" (return "extend" / "ignore"), or # to notify someone. The hook is off the fast path: it is called once per -# trigger, not once per frame. +# trigger, not once per frame. timelapse.model_trigger.ModelTrigger is +# the Claude implementation of that hook (--model-trigger on the CLI). # # WHAT IT DOES NOT DO: move the stage, refocus, or change exposure. It # only calls get_image(). A stage/sample shift the detector sees is @@ -144,6 +145,7 @@ def __init__( self.burst_until: float | None = None self.summary = RunSummary(events_path=str(self.events_path)) self._t0: float | None = None + self._previous_image: str | None = None # -- state helpers ------------------------------------------------- def _elapsed(self) -> float: @@ -172,9 +174,11 @@ def step(self) -> dict: self.summary.burst_captures += 1 event = {"event": "capture", "frame_id": metadata.get("frame_id"), "image": metadata.get("image"), + "previous_image": self._previous_image, "position": metadata.get("position"), "score": score.as_dict()} self._emit(event) self.summary.frames.append(event) + self._previous_image = metadata.get("image") is_shift = score.shift_px > self.config.shift_threshold_px if is_shift: @@ -286,6 +290,9 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument("--invert", action="store_true", help="bright specimen on dark background (fluorescence)") ap.add_argument("--events", type=Path, default=None, help="events JSONL path (default: logs/timelapse_events.jsonl)") ap.add_argument("--dry-run", action="store_true", help="print the plan and exit without capturing") + ap.add_argument("--model-trigger", action="store_true", + help="ask Claude at each trigger whether to extend or end the burst (needs API credentials)") + ap.add_argument("--model-max-calls", type=int, default=50, help="cap on model calls per run") args = ap.parse_args(argv) config = TimelapseConfig( @@ -303,7 +310,11 @@ def main(argv: list[str] | None = None) -> int: print("Not approved. Nothing captured.") return 2 - summary = AdaptiveTimelapse(config).run() + on_trigger = None + if args.model_trigger: + from timelapse.model_trigger import ModelTrigger + on_trigger = ModelTrigger(max_calls=args.model_max_calls) + summary = AdaptiveTimelapse(config, on_trigger=on_trigger).run() print(f"done: {summary.stop_reason}; {summary.captures} captures ({summary.burst_captures} in {summary.bursts} bursts), " f"{summary.shifts} shifts, {summary.elapsed_s:.0f}s. Events: {summary.events_path}") return 0 From a737c9824afb3d382853e2fb7369e0f6b3f279a4 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 16:45:34 -0400 Subject: [PATCH 29/42] Add .env.example template; README points at it for the API key --- .env.example | 13 +++++++++++++ README.md | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ab4cd76 --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +# Copy this file to .env (gitignored) and fill in your real key: +# cp .env.example .env +# +# Used by harness/agent.py, harness/mcp_agent.py and +# timelapse/model_trigger.py (loaded via python-dotenv). The MCP server +# itself and the mock-only tests never need it. +ANTHROPIC_API_KEY=sk-ant-... + +# Optional: where captures and logs are written (default: current directory). +# CONFOCAL_MCP_DATA_DIR=/path/to/data + +# Optional: PNG the mock camera serves for backend="mock" captures. +# CONFOCAL_MOCK_FRAME_PATH=/path/to/frame.png diff --git a/README.md b/README.md index a5b8fa9..9e70adb 100644 --- a/README.md +++ b/README.md @@ -159,8 +159,8 @@ unaffected either way - use whichever fits: MCP for Claude Desktop/Code, this loop for a standalone script. Requires `ANTHROPIC_API_KEY` set - either in a `.env` file at the repo -root (copy the commented-out line in `.env`, fill in your real key; this -file is gitignored and loaded automatically by `harness/agent.py`), as a +root (`cp .env.example .env`, fill in your real key; `.env` is gitignored +and loaded automatically by the harnesses and the model trigger), as a regular environment variable, or via `ant auth login`. Real hardware (`backend="sdk"` on `move` or `get_image`) always pauses for a live "y/N" approval at the terminal before executing, regardless of what the From 2a89c8b40fc8a41237890d118155f87f71dc6c1f Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 17:24:19 -0400 Subject: [PATCH 30/42] docs: model trigger verified live on the mock; note the call blocks the burst loop --- docs/adaptive_timelapse.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md index d80437e..49d9af6 100644 --- a/docs/adaptive_timelapse.md +++ b/docs/adaptive_timelapse.md @@ -106,8 +106,12 @@ frames where the change is: numbers, and answers "extend" or "ignore" with a one-sentence reason that is logged. Capped calls per run, never more than one ask per 30 s, and every failure (no key, no network, refusal, odd reply) is a - harmless "no opinion". Enabled with `--model-trigger`. Tested with a - fake model; the live call has not yet been run. Verified against the mock: + harmless "no opinion". Enabled with `--model-trigger`. Verified live on + the mock on 2026-09-28: at a synthetic shrink event the model answered + "extend" and gave the right reason (specimen contracted, no field shift + or illumination change). One call took about 7 s, during which the + burst loop waits - fine at a 5 s burst interval, but the call should + move to a background thread before bursts get faster than that. Verified against the mock: swapping the served frame mid-run started a burst within one slow interval and returned to slow afterwards. - A test suite (24 tests, synthetic frames with known answers) and a CI @@ -126,9 +130,9 @@ python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-durati stretches give the starting thresholds for the scheduler. 2. **First real run, attended.** Short slow interval, low caps, someone watching, brightfield only. -3. **Run the model trigger live once.** The code is written and tested - against a fake; one real run with `--model-trigger` on the mock, with - an API key, confirms the call and shows what its reasons look like. +3. **Move the model call off the capture thread.** It blocks the burst + loop for the seconds it takes; a background thread that applies the + answer when it arrives keeps burst timing exact. 4. **Decide on focus.** Long runs drift. Either the Perfect Focus System holds it, or the loop needs a bounded refocus step, which would be a new, gated capability. From cef573d2fd395abac8c4bc71db9aa0cffd68c385 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 17:51:18 -0400 Subject: [PATCH 31/42] ci: run on every branch push; verbose test names in the log --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d183d2d..3a26a48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,7 @@ name: CI on: - push: - branches: [main] + push: # every branch, so a push runs even before a pull request exists pull_request: jobs: @@ -21,7 +20,7 @@ jobs: # needed to import the Claude loops for the approval-gate tests (no # API key required - no model is ever called). - run: pip install -e ".[harness,test]" - - run: python -m pytest -q tests + - run: python -m pytest -v tests # -v names each test in the log # The MCP server must import on a core-only install (the camera and # SDK stacks are lazy imports) - fail the build if that regresses. # (tests/test_mcp_server.py goes further and actually runs it.) From 8f9f701f1a17f5b13984b3d6a327bd0566cd1679 Mon Sep 17 00:00:00 2001 From: Qurratul Quais Date: Mon, 28 Sep 2026 17:54:08 -0400 Subject: [PATCH 32/42] Scheduler: run the trigger consult on a background thread The model call no longer stalls the burst loop. Replies are applied at the next step; extend lengthens the running burst, ignore ends it, a reply after the burst has ended is logged as consult_late. Extensions are monotonic so a detector re-trigger never shortens a window the model lengthened. Verified live on the mock: 0.1 s burst spacing held through a 9 s model call. --- CHANGELOG.md | 3 +- docs/adaptive_timelapse.md | 12 ++--- tests/test_scheduler.py | 105 +++++++++++++++++++++++++++++++++++++ timelapse/scheduler.py | 70 ++++++++++++++++++++++--- 4 files changed, 175 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84cdeb9..5b9edf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,8 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow (adaptive slow/burst acquisition loop with hard caps and a one-time real-hardware approval), `model_trigger` (Claude behind the scheduler's trigger hook: extend or end a burst from the before/after frames, with - call caps and fail-safe "no opinion"). See `docs/adaptive_timelapse.md`. + call caps and fail-safe "no opinion"; runs on a background thread so + burst timing never waits on the model). See `docs/adaptive_timelapse.md`. - `tests/` (pytest, mock only) and a GitHub Actions CI workflow. Includes a protocol-level test that spawns `mcp_server.server_loop` as a subprocess, asserts the exact tool set, and drives every tool over MCP stdio. Also diff --git a/docs/adaptive_timelapse.md b/docs/adaptive_timelapse.md index 49d9af6..4f80a38 100644 --- a/docs/adaptive_timelapse.md +++ b/docs/adaptive_timelapse.md @@ -109,9 +109,10 @@ frames where the change is: harmless "no opinion". Enabled with `--model-trigger`. Verified live on the mock on 2026-09-28: at a synthetic shrink event the model answered "extend" and gave the right reason (specimen contracted, no field shift - or illumination change). One call took about 7 s, during which the - burst loop waits - fine at a 5 s burst interval, but the call should - move to a background thread before bursts get faster than that. Verified against the mock: + or illumination change). The call runs on a background thread: a + second live run showed burst spacing held exactly while the model took + 9 s to answer. A reply that lands after its burst has ended is logged + as late and changes nothing. Verified against the mock: swapping the served frame mid-run started a burst within one slow interval and returned to slow afterwards. - A test suite (24 tests, synthetic frames with known answers) and a CI @@ -130,10 +131,7 @@ python -m timelapse.scheduler --backend mock --slow 2 --burst 0.5 --burst-durati stretches give the starting thresholds for the scheduler. 2. **First real run, attended.** Short slow interval, low caps, someone watching, brightfield only. -3. **Move the model call off the capture thread.** It blocks the burst - loop for the seconds it takes; a background thread that applies the - answer when it arrives keeps burst timing exact. -4. **Decide on focus.** Long runs drift. Either the Perfect Focus System +3. **Decide on focus.** Long runs drift. Either the Perfect Focus System holds it, or the loop needs a bounded refocus step, which would be a new, gated capability. diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 76280eb..0d73202 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -131,3 +131,108 @@ def test_cli_dry_run_prints_plan(capsys): assert main(["--dry-run", "--backend", "sdk", "--slow", "30"]) == 0 out = capsys.readouterr().out assert "REAL HARDWARE" in out and "30s" in out + + +# -- background consults -------------------------------------------------------- + +class ManualExecutor: + """Executor whose futures complete only when the test says so.""" + + def __init__(self): + self.futures = [] + + def submit(self, fn, *args): + from concurrent.futures import Future + f = Future() + f._fn_args = (fn, args) + self.futures.append(f) + return f + + def resolve(self, i, value=None, exc=None): + f = self.futures[i] + if exc is not None: + f.set_exception(exc) + else: + f.set_result(value) + + def shutdown(self, wait=True, cancel_futures=False): + pass + + +def _shrink_run(write_frame, tmp_path, max_captures): + clock = FakeClock() + quiet = [write_frame(f"q{i}.png", seed=i) for i in range(6)] + shrunk = [write_frame(f"s{i}.png", radius=10.0, seed=100 + i) for i in range(30)] + src = FrameSource(clock, quiet + shrunk) + ex = ManualExecutor() + calls = [] + tl = AdaptiveTimelapse(_config(tmp_path, max_captures=max_captures), capture=src, + on_trigger=lambda e, m, s: calls.append(e["frame_id"]), + clock=clock, sleep=clock.sleep, log=lambda s: None, + consult_in_background=True, executor=ex) + return tl, src, ex, clock + + +def test_background_consult_does_not_block_burst_timing(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=9) + tl.run() + gaps = [b - a for a, b in zip(src.captured_at, src.captured_at[1:])] + assert gaps[6] == 5.0 and gaps[7] == 5.0 # burst spacing held while the consult was pending + assert len(ex.futures) >= 1 # the hook was submitted, never awaited + + +def test_background_extend_is_applied_at_next_step(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() # 6 quiet + first shrunk -> burst_start, consult pending + assert tl.mode == "burst" and len(ex.futures) == 1 + until_before = tl.burst_until + ex.resolve(0, "extend") + clock.t += 5.0 + tl.step() # next frame also fires the detector - must not undo the extension + assert tl.burst_until == until_before + tl.config.burst_duration_s + kinds = [json.loads(l)["event"] for l in tl.config.events_path.read_text().splitlines()] + assert kinds.count("burst_extend") >= 1 + + +def test_background_ignore_ends_burst_and_late_reply_is_logged(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() + assert tl.mode == "burst" + ex.resolve(0, "ignore") + clock.t += 5.0 + tl.step() + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "burst_end" and "ignore" in e["detail"] for e in events) + + # Keep stepping until the specimen's new size is the baseline and the + # loop has settled back to slow on its own. + for _ in range(12): + clock.t += 5.0 + tl.step() + assert tl.mode == "slow" + # Consults still outstanding from those bursts now answer "extend" - + # too late to matter, and logged as such. + for i, f in enumerate(ex.futures): + if not f.done(): + ex.resolve(i, "extend") + clock.t += 60.0 + tl.step() + assert tl.mode == "slow" + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "consult_late" for e in events) + + +def test_background_consult_exception_is_logged_not_raised(write_frame, tmp_path): + tl, src, ex, clock = _shrink_run(write_frame, tmp_path, max_captures=100) + tl._t0 = clock() + for _ in range(7): + tl.step() + ex.resolve(0, exc=RuntimeError("model down")) + clock.t += 5.0 + tl.step() # must not raise + events = [json.loads(l) for l in tl.config.events_path.read_text().splitlines()] + assert any(e["event"] == "consult_failed" and "model down" in e["detail"] for e in events) diff --git a/timelapse/scheduler.py b/timelapse/scheduler.py index 7a3170f..c59ccd0 100644 --- a/timelapse/scheduler.py +++ b/timelapse/scheduler.py @@ -28,6 +28,15 @@ # logged as a "shift" event and does NOT start a burst - that is not # biology, and burst-imaging a drifted field is wasted light. # +# THE HOOK CAN RUN IN THE BACKGROUND (consult_in_background=True, on by +# default from the CLI): a model call takes seconds, and a burst at a +# short interval must not stall while it thinks. The call is submitted to +# a single worker thread and the capture loop carries on; the answer is +# applied at the next step. "extend" then lengthens whatever burst is +# still running; "ignore" ends it; an answer that lands after the burst +# has already ended is logged as late and changes nothing. One worker, so +# consults are serialized and never pile up. +# # HARD CAPS (max_captures, max_runtime_s) always end the run, whatever # mode it is in. On real hardware the run is a single up-front approval # of the whole plan (see main()): the per-call confirm=True that @@ -45,6 +54,7 @@ import json import sys import time +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass, field, asdict from datetime import datetime from pathlib import Path @@ -131,11 +141,18 @@ def __init__( clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep, log: Callable[[str], None] = print, + consult_in_background: bool = False, + executor=None, ): config.validate() self.config = config self.capture = capture or _default_capture(config) self.on_trigger = on_trigger + self.consult_in_background = consult_in_background + # Anything with .submit(fn, *args) -> Future and .shutdown(); tests + # inject a manual one so replies land exactly when they choose. + self._executor = executor + self._pending: list[tuple[Future, int | None]] = [] self.clock = clock self.sleep = sleep self.log = log @@ -166,6 +183,7 @@ def _emit(self, event: dict) -> None: # -- one iteration --------------------------------------------------- def step(self) -> dict: """Capture one frame, score it, update mode. Returns the capture event.""" + self._apply_pending_consults() metadata = self.capture() score = self.detector.score_array(load_gray(metadata["image"])) now = self.clock() @@ -196,10 +214,12 @@ def step(self) -> dict: self._emit({"event": "burst_start", "frame_id": metadata.get("frame_id"), "detail": score.reason}) self._consult(event, metadata, score) else: - self.burst_until = now + self.config.burst_duration_s + # Never shorten a window the model has already lengthened. + self.burst_until = max(self.burst_until or 0.0, now + self.config.burst_duration_s) self._emit({"event": "burst_extend", "frame_id": metadata.get("frame_id"), "detail": score.reason}) self._consult(event, metadata, score) + self._apply_pending_consults() if self.mode == "burst" and self.burst_until is not None and now >= self.burst_until: self._end_burst("burst window elapsed") return event @@ -207,12 +227,42 @@ def step(self) -> dict: def _consult(self, event: dict, metadata: dict, score: ChangeScore) -> None: if self.on_trigger is None: return - decision = self.on_trigger(event, metadata, score) + if not self.consult_in_background: + self._apply_decision(self.on_trigger(event, metadata, score), event.get("frame_id"), late_ok=False) + return + if self._executor is None: + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="timelapse-consult") + future = self._executor.submit(self.on_trigger, event, metadata, score) + self._pending.append((future, event.get("frame_id"))) + + def _apply_pending_consults(self) -> None: + still_pending = [] + for future, frame_id in self._pending: + if not future.done(): + still_pending.append((future, frame_id)) + continue + exc = future.exception() + if exc is not None: + self._emit({"event": "consult_failed", "frame_id": frame_id, + "detail": f"{type(exc).__name__}: {exc}"}) + continue + self._apply_decision(future.result(), frame_id, late_ok=True) + self._pending = still_pending + + def _apply_decision(self, decision: str | None, frame_id: int | None, late_ok: bool) -> None: + if decision not in ("extend", "ignore"): + return + if self.mode != "burst": + if late_ok: + self._emit({"event": "consult_late", "frame_id": frame_id, + "detail": f"on_trigger said {decision} after the burst had ended - no change"}) + return if decision == "ignore": - self._end_burst("on_trigger said ignore") - elif decision == "extend" and self.burst_until is not None: - self.burst_until += self.config.burst_duration_s - self._emit({"event": "burst_extend", "detail": "on_trigger said extend"}) + self._end_burst(f"on_trigger said ignore (frame {frame_id})") + elif self.burst_until is not None: + # A full extra window from now or from the current end, whichever is later. + self.burst_until = max(self.burst_until, self.clock()) + self.config.burst_duration_s + self._emit({"event": "burst_extend", "frame_id": frame_id, "detail": "on_trigger said extend"}) def _end_burst(self, why: str) -> None: self.mode = "slow" @@ -244,6 +294,12 @@ def run(self) -> RunSummary: except KeyboardInterrupt: self.summary.stop_reason = "interrupted" finally: + if self._executor is not None: + # A reply that lands after the run is over is worthless; don't wait for it. + self._executor.shutdown(wait=False, cancel_futures=True) + if self._pending: + self._emit({"event": "consult_abandoned", "detail": f"{len(self._pending)} pending at stop"}) + self._pending = [] if self.mode == "burst": self._end_burst("run ended") self.summary.elapsed_s = self._elapsed() @@ -314,7 +370,7 @@ def main(argv: list[str] | None = None) -> int: if args.model_trigger: from timelapse.model_trigger import ModelTrigger on_trigger = ModelTrigger(max_calls=args.model_max_calls) - summary = AdaptiveTimelapse(config, on_trigger=on_trigger).run() + summary = AdaptiveTimelapse(config, on_trigger=on_trigger, consult_in_background=True).run() print(f"done: {summary.stop_reason}; {summary.captures} captures ({summary.burst_captures} in {summary.bursts} bursts), " f"{summary.shifts} shifts, {summary.elapsed_s:.0f}s. Events: {summary.events_path}") return 0 From 4067e53bd22bbcc821fb17fb78e1d3d703c8a8e3 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Fri, 25 Sep 2026 20:14:31 -0400 Subject: [PATCH 33/42] Open the STOP panel whenever a process connects to the stage The panel only appeared when the MCP server started. A mosaic, timelapse or move_xy run started from a terminal - how the 2026-09-21 run was started - had no STOP button on screen. NISSdk's first stage connection in a process now calls estop.launch_panel(), so the button is up before anything can move, however the run was started. The MCP server uses the same launcher. The panel holds a named mutex for its life; launch_panel() skips spawning when it exists, and a second copy started any other way exits without drawing. CONFOCAL_NO_ESTOP_PANEL=1 still turns it off. --- acquisition/backends/nis_sdk.py | 3 +++ acquisition/estop.py | 46 +++++++++++++++++++++++++++++++++ acquisition/estop_panel.py | 22 ++++++++++++++++ mcp_server/server_loop.py | 33 +++-------------------- 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/acquisition/backends/nis_sdk.py b/acquisition/backends/nis_sdk.py index bbbd15f..9866be6 100644 --- a/acquisition/backends/nis_sdk.py +++ b/acquisition/backends/nis_sdk.py @@ -156,6 +156,9 @@ def _get_com_thread() -> _ComThread: with _com_thread_lock: if _com_thread is None: _com_thread = _ComThread() + # First stage connection in this process: make sure the STOP + # button is on screen before anything can move. + estop.launch_panel() return _com_thread diff --git a/acquisition/estop.py b/acquisition/estop.py index cc1ca73..425d472 100644 --- a/acquisition/estop.py +++ b/acquisition/estop.py @@ -63,6 +63,7 @@ import datetime import json import os +import subprocess import sys from pathlib import Path @@ -72,6 +73,11 @@ Path.home() / ".confocal-mcp" / "ESTOP")) +#: Held by the running STOP panel for its whole life, so every process can +#: tell whether one is already on screen. Windows frees it if the panel dies. +PANEL_MUTEX = "Local\confocal-mcp-estop-panel" + + class EStopEngaged(RuntimeError): """Raised by any motion primitive while the e-stop is engaged.""" @@ -135,6 +141,46 @@ def release() -> None: +def panel_running() -> bool: + """True if a STOP panel is already on screen (Windows only).""" + if sys.platform != "win32": + return False + import ctypes + k32 = ctypes.windll.kernel32 + h = k32.OpenMutexW(0x00100000, False, PANEL_MUTEX) # SYNCHRONIZE + if h: + k32.CloseHandle(h) + return True + return False + + +def launch_panel() -> None: + """Put the STOP panel on screen unless one is already there. + + Called when a process first connects to the real stage, so the button + appears however a run was started - MCP server, mosaic, timelapse, a + one-liner. The 2026-09-21 run was started from a terminal, where no + panel existed; a safety control you have to remember to start is one + you will not have. + + Detached, not a child: the panel must outlive this process. Failures + go to stderr - never stdout, which is the MCP JSON-RPC channel - and + never stop the caller. CONFOCAL_NO_ESTOP_PANEL=1 turns it off. + """ + if os.environ.get("CONFOCAL_NO_ESTOP_PANEL"): + return + try: + if panel_running(): + return + flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) | getattr(subprocess, "DETACHED_PROCESS", 0) + subprocess.Popen([sys.executable, "-m", "acquisition.estop_panel"], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, creationflags=flags) + except Exception as exc: + print(f"[estop] could not start the STOP panel ({exc}). " + f"Stop manually with: python -m acquisition.estop engage", file=sys.stderr) + + def main() -> None: ap = argparse.ArgumentParser(description="Emergency stop for microscope motion.") ap.add_argument("action", choices=("engage", "release", "status")) diff --git a/acquisition/estop_panel.py b/acquisition/estop_panel.py index 7f95682..0718b49 100644 --- a/acquisition/estop_panel.py +++ b/acquisition/estop_panel.py @@ -26,6 +26,10 @@ # a few seconds, the cost of an accidental release is an unguarded # microscope. Engaging is also idempotent, so panic-clicking is harmless. # +# ONE PANEL. It opens by itself whenever a process connects to the stage +# (estop.launch_panel), so many processes ask for it. The first takes the +# PANEL_MUTEX; any later copy sees it and exits without drawing. +# # TKINTER because it ships with CPython: a safety control that depends on # `pip install` is a safety control that is missing on the day it matters. # ------------------------------------------------------------ @@ -133,11 +137,29 @@ def tick(self) -> None: self.root.after(POLL_MS, self.tick) +_mutex = None # kept for the life of the process; Windows frees it on exit + + +def _claim_single_instance() -> bool: + """Take the panel mutex; False if another panel already holds it.""" + global _mutex + if sys.platform != "win32": + return True + import ctypes + k32 = ctypes.windll.kernel32 + k32.CreateMutexW.restype = ctypes.c_void_p + _mutex = k32.CreateMutexW(None, False, estop.PANEL_MUTEX) + return k32.GetLastError() != 183 # ERROR_ALREADY_EXISTS + + def main() -> None: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--no-topmost", action="store_true") ap.add_argument("--titlebar", action="store_true", help="keep a normal window frame") args = ap.parse_args() + if not _claim_single_instance(): + print("a STOP panel is already open", file=sys.stderr) + return try: root = tk.Tk() except tk.TclError as exc: diff --git a/mcp_server/server_loop.py b/mcp_server/server_loop.py index 4c2b891..4845bb9 100644 --- a/mcp_server/server_loop.py +++ b/mcp_server/server_loop.py @@ -12,12 +12,10 @@ # { "mcpServers": { "confocal": { "command": "confocal-mcp" } } } # ------------------------------------------------------------ -import os -import subprocess -import sys - from mcp.server.mcpserver import MCPServer +from acquisition import estop + from mcp_server import loop_tools as tools mcp = MCPServer("ConfocalOrchestrator-Loop") @@ -29,33 +27,10 @@ mcp.add_tool(tools.estop) -def _launch_estop_panel() -> None: - """Put the STOP button on screen for as long as this server runs. - - Detached, not a child we wait on: the panel must survive this process - hanging, and a server that cannot draw a window (headless, no display) - must still serve tools. Any failure here is reported to stderr - never - stdout, which is the JSON-RPC channel - and never prevents startup. - - Launched by default because the one time it was needed, the operator - was hunting for a terminal while the stage was moving. A safety - control you have to remember to start is one you will not have. - """ - if os.environ.get("CONFOCAL_NO_ESTOP_PANEL"): - return - try: - flags = getattr(subprocess, "CREATE_NO_WINDOW", 0) | getattr(subprocess, "DETACHED_PROCESS", 0) - subprocess.Popen([sys.executable, "-m", "acquisition.estop_panel"], - stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, creationflags=flags) - except Exception as exc: - print(f"[confocal-mcp] could not start the e-stop panel ({exc}). " - f"Stop manually with: python -m acquisition.estop engage", file=sys.stderr) - - def main() -> None: """Console entry point (``confocal-mcp``): serve the 5 tools over stdio.""" - _launch_estop_panel() + # The STOP panel, on screen for as long as this server runs. + estop.launch_panel() mcp.run(transport="stdio") From 189cfb1644871491eb7e450c7188a5fde6c2bbf8 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 13:08:35 -0400 Subject: [PATCH 34/42] Add worm_follow: track one C. elegans across the dish with re-centred 3x3 mosaics Setup mode switches to 4x, restores the camera's linear neutral colour state (Camera Explorer had left Gamma 1.45, an adapted colour matrix and R/B gains of 5.9/4.4, saturating every frame), sets exposure, runs a capped focus sweep and centres the worm. The run stitches a 3x3 block each round, finds the worm by size and proximity, and re-centres past a deadband, clamped to a radius around the dish centre. --- acquisition/orchestration/worm_follow.py | 557 +++++++++++++++++++++++ 1 file changed, 557 insertions(+) create mode 100644 acquisition/orchestration/worm_follow.py diff --git a/acquisition/orchestration/worm_follow.py b/acquisition/orchestration/worm_follow.py new file mode 100644 index 0000000..72e8dca --- /dev/null +++ b/acquisition/orchestration/worm_follow.py @@ -0,0 +1,557 @@ +# worm_follow.py +# ------------------------------------------------------------ +# Follow one crawling worm across a dish: take a small mosaic (3x3 by +# default) around it, find the worm in the stitched image, re-centre the +# next mosaic on it, and repeat back-to-back. The result is the worm's +# path across the dish, one point every ~15 s, plus a mosaic per round. +# +# # detection check on a saved image, moves nothing +# python -m acquisition.orchestration.worm_follow --detect-only img.png --um-per-px 1.49 +# +# # one round, to check focus, exposure and that the worm is found +# python -m acquisition.orchestration.worm_follow --rounds 1 +# +# # follow for an hour +# python -m acquisition.orchestration.worm_follow --minutes 60 --tag celegans +# +# START WITH THE WORM IN THE MIDDLE OF THE FIELD. The first round takes +# the qualifying object nearest the grid centre; after that each round +# takes the one nearest the last position. Debris the size of an adult +# worm is rare, so size plus proximity is enough to stay locked on. +# +# WHY A SMALL MOSAIC AND NOT THE WHOLE DISH. A crawling adult covers +# 0.1-0.2 mm/s. A whole-dish round takes ~4.5 min, in which the worm can +# cross the dish, and tiles one row apart are ~25 s apart, so it is cut +# at seams or appears twice. A 3x3 round is ~13 s: the worm moves at most +# ~2 mm, so it stays inside the 7.7 x 4.8 mm block and is found again. +# +# WHY A DEADBAND ON RE-CENTRING. Worms react to vibration (tap response). +# The stage only jumps when the worm has moved more than --deadband-mm +# from the block centre, so a dwelling worm is not shaken every round. +# +# SAFETY: every re-centre is clamped to --limit-mm around the dish centre +# (default: where the run started), so a misdetection cannot walk the +# stage off the glass window. Z is fixed - the worm crawls on the agar +# surface, and a 3x3 block is small enough that tilt barely matters. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +import datetime +import json +import sys +import time +from pathlib import Path + +import cv2 +import numpy as np +from PIL import Image + +from acquisition.orchestration.mosaic import (SETTLE_S, _move_safe, calibrate_axes, + grid_positions) +from acquisition.paths import data_root + +#: Stage->image mapping measured at 4x on 2026-09-24 (night-2 Physarum +#: run, 1.4877 um/px). Same camera, same objective, so reused rather than +#: re-measured - calibration needs texture, and clean agar has little. +M_4X = np.array([[-0.6721844559295096, 0.0007000999870820124], + [0.0007312652036724406, 0.6711263094676038]]) + +#: Detection runs on the stitched block shrunk by this factor (~6 um/px +#: at 4x). An adult is ~50 um wide, so still ~8 px across. +DETECT_SCALE = 0.25 + + +def force_camera_autos_off(cam) -> None: + """Any auto colour/exposure function drifts a series mid-run + (2026-09-23: ColorTransformationAuto turned mosaics green). Camera + Explorer can switch them back on, so force them per run.""" + nm = cam._acquirer.remote_device.node_map + for name in ("BalanceWhiteAuto", "ColorTransformationAuto", "ExposureAuto", "GainAuto"): + try: + node = getattr(nm, name) + if node.value != "Off": + print(f"{name} was {node.value!r} - forcing 'Off'") + node.value = "Off" + except Exception: + pass + # Explorer also leaves its own colour pipeline behind: on 2026-09-29 it + # had Gamma 1.45, an adapted colour matrix that zeroed green, and red/ + # blue gains of 5.9/4.4 - every 4x frame was solid 255. Restore the + # linear, neutral state measured on clear agar on 2026-09-23. + try: + nm.Gamma.value = 1.0 + for s in nm.ColorTransformationValueSelector.symbolics: + nm.ColorTransformationValueSelector.value = s + nm.ColorTransformationValue.value = 1.0 if s[-2] == s[-1] else 0.0 + for s, v in (("Red", 1.689), ("GreenRed", 1.0), ("GreenBlue", 1.0), ("Blue", 2.884)): + nm.GainSelector.value = s + nm.Gain.value = v + nm.GainSelector.value = "All" + print("camera colour: Gamma 1.0, identity matrix, gains R 1.689 / G 1.0 / B 2.884") + except Exception as exc: + print(f"WARNING: could not restore camera colour state ({exc})", file=sys.stderr) + + +def stitch_with_origin(tiles, M, ref): + """mosaic.stitch, but also returns the canvas origin offset so canvas + pixels can be mapped back to stage coordinates.""" + offs = np.array([-M @ np.array([sx - ref[0], sy - ref[1]]) for _, sx, sy in tiles]) + omin = offs.min(axis=0) + pos = np.round(offs - omin).astype(int) + h, w = tiles[0][0].shape[:2] + canvas = np.zeros((pos[:, 1].max() + h, pos[:, 0].max() + w, 3), np.uint8) + for (img, _, _), (xi, yi) in zip(tiles, pos): + canvas[yi:yi + h, xi:xi + w] = img + return canvas, omin, (w, h) + + +def canvas_to_stage(px, py, M, ref, omin, tile_wh) -> tuple[float, float]: + """Stage position that would put canvas pixel (px, py) at the image + centre. A feature at stage-centre S appears in a tile taken at stage s + at c + M(s - S); with the tile's canvas offset -M(s-ref) - omin this + reduces to S = ref - M^-1 (P + omin - c), independent of the tile.""" + c = np.array(tile_wh, float) / 2.0 + S = np.array(ref) - np.linalg.solve(M, np.array([px, py]) + omin - c) + return float(S[0]), float(S[1]) + + +def find_worms(rgb: np.ndarray, um_per_px: float, min_area_mm2: float, + max_area_mm2: float, dark_ratio: float) -> tuple[list[dict], np.ndarray]: + """Dark objects of worm size against a locally-estimated background. + + Returns candidates (full-resolution centroid, area, length) and the + shrunk mask for inspection. The background is a morphological CLOSE + with a kernel wider than a worm: that fills in anything dark and thin + while following slow illumination falloff and vignetting, so the + threshold is relative to the local agar rather than a global level. + """ + small = cv2.resize(rgb, None, fx=DETECT_SCALE, fy=DETECT_SCALE, interpolation=cv2.INTER_AREA) + g = small.mean(axis=2).astype(np.float32) + # Canvas outside every tile is zero. The camera is rotated ~0.06 deg + # to the stage, so the stitched block has zero slivers a few px wide + # along its edges; shrunk, they turn into thin grey lines that read as + # dark objects. Judge validity at full resolution (a 2 px sliver is + # only half a shrunk pixel, never dark enough to test as empty), keep + # only shrunk pixels that were entirely inside tiles, then erode. + full = rgb.any(axis=2).astype(np.float32) + valid = cv2.resize(full, (g.shape[1], g.shape[0]), interpolation=cv2.INTER_AREA) > 0.999 + valid = cv2.erode(valid.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 + upx = um_per_px / DETECT_SCALE + k = int(round(150 / upx)) | 1 # 150 um: 3x a worm's width + bg = cv2.morphologyEx(g, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))) + bg = cv2.GaussianBlur(bg, (0, 0), k / 2) + mask = ((g < dark_ratio * np.maximum(bg, 1)) & valid).astype(np.uint8) + mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, np.ones((2, 2), np.uint8)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8)) + n, lab, stats, cents = cv2.connectedComponentsWithStats(mask, connectivity=8) + px_mm2 = (upx / 1000.0) ** 2 + out = [] + for i in range(1, n): + area = stats[i, cv2.CC_STAT_AREA] * px_mm2 + if not (min_area_mm2 <= area <= max_area_mm2): + continue + comp = (lab == i).astype(np.uint8) + # Skeleton-free length estimate: area / mean width, with width from + # the distance transform. Curled worms still read as long. + dt = cv2.distanceTransform(comp, cv2.DIST_L2, 3) + width_um = max(2.0 * float(dt.max()) * upx, 1.0) + length_mm = area / (width_um / 1000.0) + out.append({"x": cents[i][0] / DETECT_SCALE, "y": cents[i][1] / DETECT_SCALE, + "area_mm2": area, "width_um": width_um, "length_mm": length_mm}) + return out, mask * 255 + + +def pick(cands: list[dict], near_px: tuple[float, float], max_px: float, + area_mm2: float | None = None) -> dict | None: + """Nearest candidate within max_px. With area_mm2 (the tracked worm's + last area), candidates more than 2x bigger or smaller are ignored, so + the track does not hop onto a larva or another adult passing close by + - plates are crowded (2026-09-29: a second adult 2.5 mm away).""" + best = None + for c in cands: + if area_mm2 and not (0.5 <= c["area_mm2"] / area_mm2 <= 2.0): + continue + d = float(np.hypot(c["x"] - near_px[0], c["y"] - near_px[1])) + if d <= max_px and (best is None or d < best["dist_px"]): + best = {**c, "dist_px": d} + return best + + +def detect_only(path: Path, umpx: float, args) -> None: + rgb = np.asarray(Image.open(path).convert("RGB")) + cands, mask = find_worms(rgb, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + h, w = rgb.shape[:2] + for c in cands: + print(f" ({c['x']:.0f}, {c['y']:.0f}) px area {c['area_mm2']:.4f} mm2 " + f"width {c['width_um']:.0f} um length ~{c['length_mm']:.2f} mm") + best = pick(cands, (w / 2, h / 2), float("inf")) + print(f"{len(cands)} candidate(s); picked: {best and (round(best['x']), round(best['y']))}") + Image.fromarray(mask).save(path.with_name(path.stem + "_mask.png")) + + +#: Setup focus sweep never leaves this band around the starting Z. The 4x +#: has a 20 mm working distance, so this is far from any contact; it only +#: has to cover the parfocal offset from the 10x (a few tens of um). +SETUP_Z_RANGE_UM = 150.0 + + +def _focus_score(rgb: np.ndarray) -> float: + g = cv2.resize(rgb, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA).mean(axis=2) + return float(cv2.Laplacian(g.astype(np.float32), cv2.CV_32F).var()) + + +def setup(args) -> Path: + """Put the microscope in the state a run starts from: 4x objective, + exposure set once from the agar, focus found by a capped Z sweep, and + the worm moved to the middle of the field. Saves every sweep frame and + a final frame so the result can be checked by eye before a run.""" + from acquisition import estop + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + from acquisition.orchestration.mosaic import _move_z + + estop.check() # the nosepiece write below has no guard of its own + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / f"wormsetup_{stamp}" + out.mkdir(parents=True, exist_ok=True) + sdk = NISSdk() + cam = BaumerGenICam() + + def grab() -> np.ndarray: + p = cam.capture() + a = np.asarray(Image.open(p)); p.unlink(missing_ok=True) + return a + + try: + z_start = sdk.Z_GetPosition() + nose = sdk._thread.call(lambda m: int(m.iNOSEPIECE)) + if nose != args.nosepiece: + print(f"nosepiece {nose} -> {args.nosepiece} (Z {z_start:.2f} um before)", flush=True) + estop.check() + sdk._thread.call(lambda m: setattr(m, "iNOSEPIECE", args.nosepiece)) + deadline = time.monotonic() + 20 + while sdk._thread.call(lambda m: int(m.iNOSEPIECE)) != args.nosepiece: + if time.monotonic() > deadline: + raise SystemExit("nosepiece did not reach the requested position in 20 s") + time.sleep(0.25) + time.sleep(1.0) + z0 = sdk.Z_GetPosition() + print(f"nosepiece {args.nosepiece}, Z {z0:.2f} um", flush=True) + + force_camera_autos_off(cam) + exp = args.exposure_us + for _ in range(10): + cam.set_settings(exposure_time_us=exp, gain=1.0) + grab() # first frame after a change can be stale + p90 = float(np.percentile(grab().mean(axis=2), 90)) + print(f"exposure {exp / 1000:.2f} ms -> agar 90th percentile {p90:.0f}/255", flush=True) + if 170 <= p90 <= 230: + break + # A clipped frame says nothing about how far over it is, so + # halve until it reads; the response is linear after that. + exp = exp / 2 if p90 >= 250 else exp * 200.0 / max(p90, 1.0) + exp = float(np.clip(exp, 100.0, 60000.0)) + + def sweep(centre: float, half: float, step: float, tag: str) -> float: + scores = [] + for z in np.arange(centre - half, centre + half + 0.1, step): + z = float(np.clip(z, z0 - SETUP_Z_RANGE_UM, z0 + SETUP_Z_RANGE_UM)) + _move_z(sdk, z) + time.sleep(0.4) + img = grab() + s = _focus_score(img) + print(f" {tag} Z {sdk.Z_GetPosition():.1f} um focus {s:.1f}", flush=True) + Image.fromarray(cv2.resize(img, None, fx=0.4, fy=0.4)).save( + out / f"{tag}_z{z:.0f}.jpg", quality=85) + scores.append((s, z)) + # A flat curve (e.g. every frame clipped - 2026-09-29 sent Z to + # the top of the range on all-zero scores) means no information: + # stay where the sweep was centred rather than at an end. + vals = [s for s, _ in scores] + if max(vals) < 1.0 or max(vals) < 1.2 * min(vals): + print(f" {tag} sweep is flat - keeping Z {centre:.1f} um", flush=True) + return centre + return max(scores)[1] + + z_best = sweep(z0, SETUP_Z_RANGE_UM, 25.0, "coarse") + _move_z(sdk, z_best) + + M = M_4X + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + img = grab() + h, w = img.shape[:2] + cands, _ = find_worms(img, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + best = pick(cands, (w / 2, h / 2), float("inf")) + if best: + # A feature at image pixel q is centred by moving the stage by + # -M^-1 (q - c) - same geometry as canvas_to_stage. + x, y = sdk.XY_GetPosition() + d = np.linalg.solve(M, np.array([best["x"] - w / 2, best["y"] - h / 2])) + tx, ty = x - d[0], y - d[1] + print(f"worm at ({best['x']:.0f}, {best['y']:.0f}) px, area {best['area_mm2']:.3f} mm2 " + f"-> centring: stage ({x:.1f}, {y:.1f}) -> ({tx:.1f}, {ty:.1f}) um", flush=True) + _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S) + else: + print(f"NO WORM FOUND in the 4x field ({len(cands)} candidates) - stage not moved", + flush=True) + + z_fine = sweep(z_best, 20.0, 5.0, "fine") + _move_z(sdk, z_fine) + time.sleep(0.4) + final = grab() + cands, _ = find_worms(final, umpx, args.min_area_mm2, args.max_area_mm2, args.dark_ratio) + b = pick(cands, (w / 2, h / 2), float("inf")) + mark = final.copy() + if b: + cv2.circle(mark, (int(b["x"]), int(b["y"])), int(700 / umpx), (255, 0, 0), 4) + Image.fromarray(final).save(out / "final.png") + Image.fromarray(cv2.resize(mark, None, fx=0.5, fy=0.5)).save(out / "final_marked.jpg", quality=90) + x, y = sdk.XY_GetPosition() + summary = {"nosepiece": args.nosepiece, "z_um": sdk.Z_GetPosition(), "z_before_um": z_start, + "xy_um": [x, y], "exposure_us": exp, "worm_found": bool(b), + "worm_px": [b["x"], b["y"]] if b else None} + (out / "setup.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(f"READY: {summary}\nimages in {out}", flush=True) + finally: + cam.close() + return out + + +def run(args) -> Path: + from acquisition.backends.baumer_genicam import BaumerGenICam + from acquisition.backends.nis_sdk import NISSdk + + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + out = data_root() / "data" / (f"wormfollow_{stamp}" + (f"_{args.tag}" if args.tag else "")) + (out / "crops").mkdir(parents=True, exist_ok=True) + stop_file = out / "STOP" + + sdk = NISSdk() + cam = BaumerGenICam() + cx = cy = 0.0 + try: + force_camera_autos_off(cam) + cam.set_settings(exposure_time_us=args.exposure_us, gain=1.0) + cx, cy = sdk.XY_GetPosition() + dish = tuple(float(v) for v in args.dish_centre.split(",")) if args.dish_centre else (cx, cy) + z = sdk.Z_GetPosition() + print(f"start ({cx:.1f}, {cy:.1f}) um, Z fixed at {z:.2f} um; " + f"stage confined to {args.limit_mm} mm around ({dish[0]:.0f}, {dish[1]:.0f})") + + if args.calibrate: + M = calibrate_axes(cam, sdk) + else: + M = M_4X + print("using the 4x stage->image mapping measured 2026-09-24 (pass --calibrate to re-measure)") + umpx = 1.0 / float(np.hypot(M[0, 0], M[1, 0])) + + probe = cam.capture() + img = np.asarray(Image.open(probe)); probe.unlink(missing_ok=True) + h, w = img.shape[:2] + bright = float(np.percentile(img.mean(axis=2), 90)) + print(f"camera {cam.get_settings()} - probe frame 90th percentile {bright:.0f}/255") + if bright > 245 or bright < 60: + print("WARNING: agar is " + ("clipped" if bright > 245 else "dark") + + " - adjust --exposure-us or the lamp", file=sys.stderr) + + step = (w * umpx * (1 - args.overlap), h * umpx * (1 - args.overlap)) + (out / "run.json").write_text(json.dumps({ + "grid": args.grid, "search_grid": args.search_grid, "overlap": args.overlap, + "um_per_px": round(umpx, 4), "stage_to_image_matrix": M.tolist(), + "step_um": [round(s, 1) for s in step], "z_um": z, "exposure_us": args.exposure_us, + "dish_centre_um": dish, "limit_mm": args.limit_mm, "deadband_mm": args.deadband_mm, + "mosaic_scale": args.mosaic_scale, "min_area_mm2": args.min_area_mm2, + "max_area_mm2": args.max_area_mm2, "dark_ratio": args.dark_ratio, + }, indent=2), encoding="utf-8") + print(f"to halt this run at any point, create: {stop_file}") + + fields = ("round", "wall_clock", "t_s", "grid", "centre_x_um", "centre_y_um", + "found", "worm_x_um", "worm_y_um", "area_mm2", "length_mm", + "n_candidates", "recentred", "error") + log = (out / "track.csv").open("w", newline="", encoding="utf-8") + writer = csv.DictWriter(log, fieldnames=fields) + writer.writeheader() + + last = None # last worm position, stage um + last_area = None + misses = 0 + t0 = time.monotonic() + r = 0 + while (args.rounds is None or r < args.rounds) and \ + (args.minutes is None or time.monotonic() - t0 < args.minutes * 60): + if args.interval_s: + target = t0 + r * args.interval_s + if time.monotonic() < target: + time.sleep(target - time.monotonic()) + gspec = args.search_grid if misses else args.grid + nx, ny = (int(v) for v in gspec.lower().split("x")) + row = {k: "" for k in fields} + row.update(round=r, wall_clock=datetime.datetime.now().isoformat(timespec="seconds"), + t_s=round(time.monotonic() - t0, 1), grid=gspec, + centre_x_um=round(cx, 1), centre_y_um=round(cy, 1), found=0) + try: + tiles = [] + for tx, ty in grid_positions(cx, cy, nx, ny, *step): + if stop_file.exists(): + print("STOP file present - halting", flush=True) + raise KeyboardInterrupt + ax, ay = _move_safe(sdk, tx, ty) + time.sleep(SETTLE_S) + p = cam.capture() + tiles.append((np.asarray(Image.open(p)), ax, ay)) + p.unlink(missing_ok=True) + canvas, omin, twh = stitch_with_origin(tiles, M, (cx, cy)) + cands, _ = find_worms(canvas, umpx, args.min_area_mm2, args.max_area_mm2, + args.dark_ratio) + row["n_candidates"] = len(cands) + # Where the previous worm position (or, first round, the + # block centre) sits on this canvas. Gate generously: a + # worm reversing and sprinting can do ~0.3 mm/s. + anchor = last or (cx, cy) + ap = -M @ (np.array(anchor) - np.array((cx, cy))) - omin + np.array(twh) / 2 + gate_px = (args.gate_mm * 1000 if last else args.first_gate_mm * 1000) / umpx + if misses: + gate_px *= 2 + best = pick(cands, (ap[0], ap[1]), gate_px, last_area) + + if best: + wx, wy = canvas_to_stage(best["x"], best["y"], M, (cx, cy), omin, twh) + last, misses, last_area = (wx, wy), 0, best["area_mm2"] + row.update(found=1, worm_x_um=round(wx, 1), worm_y_um=round(wy, 1), + area_mm2=round(best["area_mm2"], 4), + length_mm=round(best["length_mm"], 2)) + half = int(args.crop_mm * 1000 / umpx / 2) + bx, by = int(best["x"]), int(best["y"]) + crop = canvas[max(0, by - half):by + half, max(0, bx - half):bx + half] + Image.fromarray(crop).save(out / "crops" / f"crop_{r:04d}.jpg", quality=92) + else: + misses += 1 + + small = cv2.resize(canvas, None, fx=args.mosaic_scale, fy=args.mosaic_scale, + interpolation=cv2.INTER_AREA) + if best: + cv2.circle(small, (int(best["x"] * args.mosaic_scale), + int(best["y"] * args.mosaic_scale)), + int(0.7 * 1000 / umpx * args.mosaic_scale), (255, 0, 0), 3) + Image.fromarray(small).save(out / f"round_{r:04d}.jpg", quality=90) + + # Re-centre, only past the deadband, clamped to the dish. + if last and np.hypot(last[0] - cx, last[1] - cy) > args.deadband_mm * 1000: + nxp, nyp = last + dx, dy = nxp - dish[0], nyp - dish[1] + rad = float(np.hypot(dx, dy)); lim = args.limit_mm * 1000 + if rad > lim: + nxp, nyp = dish[0] + dx * lim / rad, dish[1] + dy * lim / rad + print(f" re-centre clamped to the {args.limit_mm} mm limit", flush=True) + cx, cy = nxp, nyp + row["recentred"] = 1 + state = (f"worm at ({row['worm_x_um']}, {row['worm_y_um']}) um, " + f"{row['area_mm2']} mm2" if best else f"NOT FOUND (miss {misses}, " + f"{len(cands)} candidates) - next round searches {args.search_grid}") + print(f"[{r:04d}] {datetime.datetime.now():%H:%M:%S} {gspec} {state}" + + (" -> re-centred" if row["recentred"] else ""), flush=True) + except KeyboardInterrupt: + writer.writerow(row); log.flush() + break + except Exception as exc: + row["error"] = f"{type(exc).__name__}: {exc}" + print(f"[{r:04d}] ERROR {row['error']}", flush=True) + if "e-stop" in str(exc).lower() or "estop" in str(exc).lower(): + writer.writerow(row); log.flush() + break + writer.writerow(row); log.flush() + r += 1 + log.close() + plot_track(out) + finally: + try: + _move_safe(sdk, cx, cy) + except Exception as exc: + print(f"WARNING: could not park at the last centre: {exc}", file=sys.stderr) + cam.close() + print(f"released - output in {out}") + return out + + +def plot_track(out: Path) -> None: + """track.png: the path drawn the way the mosaics show it (stage +X + moves features left, +Y moves them down, so X runs right-to-left and + Y bottom-to-top), coloured blue -> yellow by time, with a 1 mm bar.""" + with (out / "track.csv").open(encoding="utf-8") as fh: + rows = [r for r in csv.DictReader(fh) if r["found"] == "1"] + if not rows: + return + x = np.array([float(r["worm_x_um"]) for r in rows]) / 1000 + y = np.array([float(r["worm_y_um"]) for r in rows]) / 1000 + t = np.array([float(r["t_s"]) for r in rows]) / 60 + size, pad = 900, 70 + span = max(np.ptp(x), np.ptp(y), 2.0) + s = (size - 2 * pad) / span + cxm, cym = (x.max() + x.min()) / 2, (y.max() + y.min()) / 2 + pts = np.column_stack([size / 2 - (x - cxm) * s, size / 2 - (y - cym) * s]).astype(np.int32) + img = np.full((size, size, 3), 255, np.uint8) + cols = cv2.applyColorMap(np.linspace(0, 255, len(pts)).astype(np.uint8)[:, None], + cv2.COLORMAP_VIRIDIS)[:, 0] + for i in range(1, len(pts)): + cv2.line(img, tuple(pts[i - 1]), tuple(pts[i]), [int(v) for v in cols[i]], 2, cv2.LINE_AA) + cv2.circle(img, tuple(pts[0]), 9, (0, 0, 0), 2, cv2.LINE_AA) + cv2.circle(img, tuple(pts[-1]), 7, [int(v) for v in cols[-1]], -1, cv2.LINE_AA) + cv2.line(img, (pad, size - 30), (pad + int(s), size - 30), (0, 0, 0), 3) + cv2.putText(img, "1 mm", (pad, size - 40), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 0), 1, cv2.LINE_AA) + d = np.hypot(np.diff(x), np.diff(y)).sum() + cv2.putText(img, f"{len(rows)} points, {t[-1]:.1f} min, {d:.1f} mm travelled " + f"(o = start, colour = time)", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, + 0.6, (0, 0, 0), 1, cv2.LINE_AA) + cv2.imwrite(str(out / "track.png"), img) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--grid", default="3x3", help="block while locked on (default 3x3)") + ap.add_argument("--search-grid", default="5x5", help="block after a miss (default 5x5)") + ap.add_argument("--overlap", type=float, default=0.10) + ap.add_argument("--rounds", type=int) + ap.add_argument("--minutes", type=float) + ap.add_argument("--interval-s", type=float, default=0.0, + help="seconds between round starts; 0 = back to back (default)") + ap.add_argument("--exposure-us", type=float, default=1600.0, + help="4x at lamp 1072: 1.6 ms puts agar at ~200/255 (2026-09-29)") + ap.add_argument("--deadband-mm", type=float, default=0.5) + ap.add_argument("--gate-mm", type=float, default=3.0, + help="max distance from the last position to accept a detection") + ap.add_argument("--first-gate-mm", type=float, default=2.0, + help="first round: max distance from the block centre") + ap.add_argument("--limit-mm", type=float, default=10.0, + help="never centre farther than this from the dish centre") + ap.add_argument("--dish-centre", help="'x,y' um; default is the start position") + ap.add_argument("--min-area-mm2", type=float, default=0.012) + ap.add_argument("--max-area-mm2", type=float, default=0.25) + ap.add_argument("--dark-ratio", type=float, default=0.85, + help="a pixel is worm if darker than this fraction of local agar") + ap.add_argument("--crop-mm", type=float, default=1.8) + ap.add_argument("--mosaic-scale", type=float, default=0.5) + ap.add_argument("--calibrate", action="store_true", help="re-measure the stage->image mapping") + ap.add_argument("--tag") + ap.add_argument("--detect-only", type=Path, help="run detection on an image and exit") + ap.add_argument("--um-per-px", type=float, default=1.4877, help="for --detect-only") + ap.add_argument("--setup", action="store_true", + help="switch objective, set exposure, focus and centre the worm, then exit") + ap.add_argument("--nosepiece", type=int, default=1, help="for --setup (1 = 4x)") + args = ap.parse_args() + if args.setup: + setup(args) + return + if args.detect_only: + detect_only(args.detect_only, args.um_per_px, args) + return + if args.rounds is None and args.minutes is None: + args.rounds = 1 + run(args) + + +if __name__ == "__main__": + main() From b1a24b234f4f1f31c6f6a87c44746cebec0c16a6 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 15:41:31 -0400 Subject: [PATCH 35/42] Add aml18_survey: find worms, stage and head in an AML18 .nd2 frame Segments worms in the TD channel (eggs rejected by shape), measures body length along the skeleton for a rough life stage, and calls the head as the end with more neuronal fluorescence. Defaults to RFP because on the AX it is read out with TD, while a sequentially scanned GFP shows a crawling worm somewhere else. --- analysis/aml18_survey.py | 221 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 analysis/aml18_survey.py diff --git a/analysis/aml18_survey.py b/analysis/aml18_survey.py new file mode 100644 index 0000000..8696798 --- /dev/null +++ b/analysis/aml18_survey.py @@ -0,0 +1,221 @@ +# aml18_survey.py +# ------------------------------------------------------------ +# Population survey of AML18 worms (pan-neuronal nuclear GFP + tagRFP) +# from a single NIS-Elements .nd2 frame: find every worm in the +# transmitted-light (TD) channel, measure its length along the body, +# estimate its life stage, and use the neuronal fluorescence to tell the +# head (nerve ring = brightest neuron cluster) from the tail. +# +# python -m analysis.aml18_survey D:\...\CelegansAML-18\Test1.nd2 +# python -m analysis.aml18_survey file.nd2 --neuron-channel GFP +# +# Writes _survey/: worms.csv, survey.png (annotated overlay). +# +# WHICH FLUORESCENCE CHANNEL. The neuron channel must be captured at the +# same instant as TD, or it shows where a crawling worm used to be. On +# the AX, TD is read out with the 561 nm laser, so RFP lines up with TD +# and a sequentially-scanned GFP does not (2026-09-29, Test1.nd2: 11.7 s +# frames, GFP offset from every worm). Default is RFP for that reason; +# use --neuron-channel GFP only for simultaneous acquisitions. +# +# WORMS VS EGGS. Both are dark in TD. Eggs are compact (~50 x 30 um); +# worms are long and thin. Objects are kept only if their body length +# (longest path through the skeleton) is > --min-length-um and at least +# 4x their width. +# +# LIFE STAGE is a rough guide from body length alone (N2 at 20 C: +# L1 ~0.25, L2 ~0.36, L3 ~0.49, L4 ~0.62-0.9, adult ~1.0-1.3 mm). A +# curled or partly hidden worm reads short, so treat it as indicative. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import cv2 +import nd2 +import numpy as np +from scipy import ndimage as ndi +from scipy.sparse import coo_matrix +from scipy.sparse.csgraph import dijkstra +from skimage.morphology import remove_small_objects, skeletonize + +STAGES = [(0.30, "L1"), (0.42, "L2"), (0.56, "L3"), (0.90, "L4"), (99.0, "adult")] + + +def load(path: Path) -> tuple[dict[str, np.ndarray], float]: + with nd2.ND2File(path) as f: + if set(f.sizes) - {"C", "Y", "X"}: + raise SystemExit(f"expected one frame (C, Y, X), got {f.sizes} - " + "time-lapse / Z / multi-position files are not handled yet") + arr = f.asarray().astype(np.float32) + names = [c.channel.name for c in f.metadata.channels] + um_px = float(f.voxel_size().x) + return dict(zip(names, arr)), um_px + + +def segment_worms(td: np.ndarray, um_px: float, dark_ratio: float) -> np.ndarray: + """Dark, thin objects relative to local background (closing removes + anything narrower than the kernel, leaving the agar).""" + k = int(round(160 / um_px)) | 1 # 160 um: 3x an adult's width + bg = cv2.morphologyEx(td, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (k, k))) + bg = cv2.GaussianBlur(bg, (0, 0), k / 3) + mask = td < dark_ratio * bg + mask = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_OPEN, np.ones((2, 2), np.uint8)) + mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8)).astype(bool) + return remove_small_objects(mask, max_size=int(2000 / um_px ** 2)) # <= 2000 um2 is debris + + +def longest_path(skel: np.ndarray) -> tuple[np.ndarray, float]: + """Pixels of the longest geodesic path through a skeleton, and its + length in pixels (diagonal steps count sqrt 2).""" + ys, xs = np.nonzero(skel) + idx = -np.ones(skel.shape, int) + idx[ys, xs] = np.arange(len(ys)) + rows, cols, w = [], [], [] + for dy, dx in ((0, 1), (1, 0), (1, 1), (1, -1)): + y2, x2 = ys + dy, xs + dx + ok = (y2 >= 0) & (y2 < skel.shape[0]) & (x2 >= 0) & (x2 < skel.shape[1]) + ok[ok] = skel[y2[ok], x2[ok]] + rows += list(idx[ys[ok], xs[ok]]); cols += list(idx[y2[ok], x2[ok]]) + w += [np.hypot(dy, dx)] * int(ok.sum()) + n = len(ys) + g = coo_matrix((w, (rows, cols)), shape=(n, n)).tocsr() + d0 = dijkstra(g, directed=False, indices=0) + a = int(np.nanargmax(np.where(np.isinf(d0), -1, d0))) + da, pred = dijkstra(g, directed=False, indices=a, return_predecessors=True) + b = int(np.nanargmax(np.where(np.isinf(da), -1, da))) + path = [b] + while path[-1] != a: + path.append(pred[path[-1]]) + return np.column_stack([ys[path], xs[path]]), float(da[b]) + + +def neuron_mask(ch: np.ndarray, um_px: float) -> tuple[np.ndarray, float]: + """Fluorescent neurons: well above the (mostly zero) background.""" + bg = cv2.medianBlur(ch.astype(np.float32), 5) if ch.max() > 0 else ch + noise = float(np.percentile(ch, 99)) # >99% of the frame is empty agar + thr = max(3.0 * noise, noise + 5.0) + return (cv2.GaussianBlur(ch, (0, 0), 1.0) > thr), thr + + +def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float) -> Path: + ch, um_px = load(path) + if "TD" not in ch or neuron_channel not in ch: + raise SystemExit(f"need TD and {neuron_channel}; file has {list(ch)}") + td, neu = ch["TD"], ch[neuron_channel] + out = path.with_name(path.stem + "_survey") + out.mkdir(exist_ok=True) + + worms = segment_worms(td, um_px, dark_ratio) + nmask, thr = neuron_mask(neu, um_px) + lab, n = ndi.label(worms) + dist = ndi.distance_transform_edt(worms) + near = ndi.distance_transform_edt(~worms) <= 25 / um_px # neurons may sit just outside the TD edge + + rows = [] + for i in range(1, n + 1): + obj = lab == i + skel = skeletonize(obj) + if skel.sum() < 3: + continue + path_px, length_px = longest_path(skel) + length_um = length_px * um_px + width_um = 2 * float(np.median(dist[skel])) * um_px + if length_um < min_length_um or length_um < 4 * width_um: + continue # egg or debris + ys, xs = np.nonzero(obj) + edge = bool(ys.min() == 0 or xs.min() == 0 or ys.max() == td.shape[0] - 1 + or xs.max() == td.shape[1] - 1) + # Neuron signal belonging to this worm: inside the body or within + # 25 um of it, and not closer to another worm. + zone = ndi.binary_dilation(obj, iterations=int(25 / um_px)) & near + sig = np.where(zone & nmask, neu, 0) + total = float(sig.sum()) + # Head = the end whose first 15% of body length holds more signal. + seg = max(3, int(0.15 * len(path_px))) + r = int(60 / um_px) + + def end_signal(pts): + m = np.zeros_like(obj) + for y, x in pts: + cv2.circle(m.view(np.uint8), (int(x), int(y)), r, 1, -1) + return float(sig[m.astype(bool)].sum()) + + s_a, s_b = end_signal(path_px[:seg]), end_signal(path_px[-seg:]) + if total == 0 or max(s_a, s_b) < 1.5 * min(s_a, s_b) + 1: + head = None # no clear winner + else: + head = tuple(path_px[0] if s_a > s_b else path_px[-1]) + # An end at the image border is where the frame cut the worm, + # not a real end - its neurons may be off-frame, so the + # comparison is meaningless (Test1 worm 6 was called at the cut). + hy, hx = head + if min(hy, hx) <= 3 or hy >= td.shape[0] - 4 or hx >= td.shape[1] - 4: + head = None + stage = next(name for lim, name in STAGES if length_um / 1000 < lim) + cy, cx = ndi.center_of_mass(obj) + rows.append({ + "worm": len(rows) + 1, "x_px": round(cx), "y_px": round(cy), + "length_mm": round(length_um / 1000, 3), "width_um": round(width_um, 1), + "stage_estimate": stage + (" (cut by edge)" if edge else ""), + "neuron_signal": round(total), "neuron_px": int((sig > 0).sum()), + "head_x_px": head[1] if head else "", "head_y_px": head[0] if head else "", + "head_confidence": round(max(s_a, s_b) / (min(s_a, s_b) + 1), 1) if total else 0, + "_path": path_px, "_obj": obj, + }) + + # Annotated overlay: TD grey, neuron channel magenta, worm outlines cyan. + g = np.clip((td - np.percentile(td, 0.5)) / (np.percentile(td, 99.5) - np.percentile(td, 0.5)), 0, 1) + f = np.clip(neu / max(np.percentile(neu[nmask], 95) if nmask.any() else 1, 1), 0, 1) + img = np.dstack([g * 0.7 + f, g * 0.7, g * 0.7 + f]) + img = (np.clip(img, 0, 1) * 255).astype(np.uint8)[..., ::-1].copy() # BGR + img = cv2.resize(img, None, fx=2, fy=2, interpolation=cv2.INTER_NEAREST) + for w in rows: + cnts, _ = cv2.findContours(w["_obj"].astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) + cv2.drawContours(img, [c * 2 for c in cnts], -1, (255, 255, 0), 1) + if w["head_x_px"] != "": + cv2.circle(img, (2 * w["head_x_px"], 2 * w["head_y_px"]), 14, (0, 255, 255), 3) + lbl = f"{w['worm']}: {w['stage_estimate'].split()[0]} {w['length_mm']:.2f}mm" + p = (2 * w["x_px"] + 30, 2 * w["y_px"]) + cv2.putText(img, lbl, p, cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 0), 5, cv2.LINE_AA) + cv2.putText(img, lbl, p, cv2.FONT_HERSHEY_SIMPLEX, 0.9, (255, 255, 255), 2, cv2.LINE_AA) + px = int(round(500 / um_px)) * 2 + h, wd = img.shape[:2] + cv2.rectangle(img, (wd - 40 - px, h - 50), (wd - 40, h - 38), (255, 255, 255), -1) + cv2.putText(img, "500 um", (wd - 40 - px, h - 60), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2) + cv2.putText(img, f"{path.name}: {len(rows)} worms | cyan = outline, yellow ring = head " + f"({neuron_channel} neurons, magenta)", (20, 40), + cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2, cv2.LINE_AA) + cv2.imwrite(str(out / "survey.png"), img) + + keys = [k for k in rows[0] if not k.startswith("_")] if rows else ["worm"] + with (out / "worms.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=keys, extrasaction="ignore") + wr.writeheader(); wr.writerows(rows) + + print(f"{path.name}: {um_px:.3f} um/px, neuron channel {neuron_channel} (threshold {thr:.0f})") + print(f"{len(rows)} worms") + for w in rows: + head = "head found" if w["head_x_px"] != "" else "head unclear" + print(f" #{w['worm']:>2} {w['stage_estimate']:<22} {w['length_mm']:.2f} mm, " + f"{w['width_um']:.0f} um wide, neuron px {w['neuron_px']:>4}, {head} " + f"(ratio {w['head_confidence']})") + print(f"wrote {out}") + return out + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("nd2", type=Path) + ap.add_argument("--neuron-channel", default="RFP") + ap.add_argument("--min-length-um", type=float, default=150.0) + ap.add_argument("--dark-ratio", type=float, default=0.85) + a = ap.parse_args() + analyse(a.nd2, a.neuron_channel, a.min_length_um, a.dark_ratio) + + +if __name__ == "__main__": + main() From a6ecc06571a45544e00712a619d66bced3783564 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 16:13:16 -0400 Subject: [PATCH 36/42] aml18_survey: handle time-lapse and Z-stack .nd2 files Each time point is surveyed separately; a Z-stack is collapsed to a max projection for fluorescence and the sharpest plane for TD. Writes one annotated image per time point, worms.csv with a time column and counts.csv with worms and stages per time point. --- analysis/aml18_survey.py | 98 +++++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/analysis/aml18_survey.py b/analysis/aml18_survey.py index 8696798..3ef0499 100644 --- a/analysis/aml18_survey.py +++ b/analysis/aml18_survey.py @@ -45,15 +45,44 @@ STAGES = [(0.30, "L1"), (0.42, "L2"), (0.56, "L3"), (0.90, "L4"), (99.0, "adult")] -def load(path: Path) -> tuple[dict[str, np.ndarray], float]: +def load(path: Path) -> tuple[list[tuple[float, dict[str, np.ndarray]]], float]: + """Every time point as (seconds, {channel: 2-D image}). + + A Z-stack is collapsed per time point: fluorescence by maximum + projection (a neuron is counted wherever it is in focus), TD by + taking the single sharpest plane (a projection of brightfield smears + every plane's blur together). XY multipoint files are not handled. + """ with nd2.ND2File(path) as f: - if set(f.sizes) - {"C", "Y", "X"}: - raise SystemExit(f"expected one frame (C, Y, X), got {f.sizes} - " - "time-lapse / Z / multi-position files are not handled yet") + extra = set(f.sizes) - {"T", "Z", "C", "Y", "X"} + if extra: + raise SystemExit(f"unsupported dimensions {extra} in {f.sizes}") arr = f.asarray().astype(np.float32) + dims = list(f.sizes) names = [c.channel.name for c in f.metadata.channels] um_px = float(f.voxel_size().x) - return dict(zip(names, arr)), um_px + try: + times = [e.get("Time [s]", i) for i, e in enumerate(f.events())][: f.sizes.get("T", 1)] + except Exception: + times = [] + # Put the array in a fixed T, Z, C, Y, X order, adding missing axes. + for d in ("T", "Z", "C"): + if d not in dims: + arr = arr[np.newaxis] + dims.insert(0, d) + arr = np.transpose(arr, [dims.index(d) for d in ("T", "Z", "C", "Y", "X")]) + frames = [] + for t in range(arr.shape[0]): + ch = {} + for i, name in enumerate(names): + stack = arr[t, :, i] # Z, Y, X + if name == "TD": + sharp = [float(cv2.Laplacian(z, cv2.CV_32F).var()) for z in stack] + ch[name] = np.ascontiguousarray(stack[int(np.argmax(sharp))]) + else: + ch[name] = np.ascontiguousarray(stack.max(axis=0)) + frames.append((float(times[t]) if t < len(times) else float(t), ch)) + return frames, um_px def segment_worms(td: np.ndarray, um_px: float, dark_ratio: float) -> np.ndarray: @@ -101,13 +130,12 @@ def neuron_mask(ch: np.ndarray, um_px: float) -> tuple[np.ndarray, float]: return (cv2.GaussianBlur(ch, (0, 0), 1.0) > thr), thr -def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float) -> Path: - ch, um_px = load(path) +def survey_frame(ch: dict[str, np.ndarray], um_px: float, neuron_channel: str, + min_length_um: float, dark_ratio: float, title: str): + """One frame: worm rows and the annotated overlay (BGR).""" if "TD" not in ch or neuron_channel not in ch: raise SystemExit(f"need TD and {neuron_channel}; file has {list(ch)}") td, neu = ch["TD"], ch[neuron_channel] - out = path.with_name(path.stem + "_survey") - out.mkdir(exist_ok=True) worms = segment_worms(td, um_px, dark_ratio) nmask, thr = neuron_mask(neu, um_px) @@ -186,23 +214,51 @@ def end_signal(pts): h, wd = img.shape[:2] cv2.rectangle(img, (wd - 40 - px, h - 50), (wd - 40, h - 38), (255, 255, 255), -1) cv2.putText(img, "500 um", (wd - 40 - px, h - 60), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2) - cv2.putText(img, f"{path.name}: {len(rows)} worms | cyan = outline, yellow ring = head " + cv2.putText(img, f"{title}: {len(rows)} worms | cyan = outline, yellow ring = head " f"({neuron_channel} neurons, magenta)", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2, cv2.LINE_AA) - cv2.imwrite(str(out / "survey.png"), img) + return rows, img, thr + + +def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float) -> Path: + frames, um_px = load(path) + out = path.with_name(path.stem + "_survey") + out.mkdir(exist_ok=True) + multi = len(frames) > 1 + all_rows, counts = [], [] + print(f"{path.name}: {um_px:.3f} um/px, {len(frames)} time point(s), neuron channel {neuron_channel}") + for ti, (t_s, ch) in enumerate(frames): + title = f"{path.name}" + (f" t={t_s / 60:.1f} min ({ti + 1}/{len(frames)})" if multi else "") + rows, img, thr = survey_frame(ch, um_px, neuron_channel, min_length_um, dark_ratio, title) + cv2.imwrite(str(out / (f"survey_t{ti:03d}.png" if multi else "survey.png")), img) + for w in rows: + w.update(t_index=ti, t_s=round(t_s, 1)) + all_rows += rows + stages = [w["stage_estimate"].split()[0] for w in rows] + counts.append({"t_index": ti, "t_s": round(t_s, 1), "worms": len(rows), + **{st: stages.count(st) for _, st in STAGES}, + "heads_found": sum(1 for w in rows if w["head_x_px"] != "")}) + if not multi: + print(f"threshold {thr:.0f}; {len(rows)} worms") + for w in rows: + head = "head found" if w["head_x_px"] != "" else "head unclear" + print(f" #{w['worm']:>2} {w['stage_estimate']:<22} {w['length_mm']:.2f} mm, " + f"{w['width_um']:.0f} um wide, neuron px {w['neuron_px']:>4}, {head} " + f"(ratio {w['head_confidence']})") + else: + c = counts[-1] + print(f" t={t_s / 60:5.1f} min: {len(rows)} worms " + + " ".join(f"{st}:{c[st]}" for _, st in STAGES) + f" heads {c['heads_found']}") - keys = [k for k in rows[0] if not k.startswith("_")] if rows else ["worm"] + keys = ["t_index", "t_s"] + [k for k in (all_rows[0] if all_rows else {"worm": 0}) + if not k.startswith("_") and k not in ("t_index", "t_s")] with (out / "worms.csv").open("w", newline="", encoding="utf-8") as fh: wr = csv.DictWriter(fh, fieldnames=keys, extrasaction="ignore") - wr.writeheader(); wr.writerows(rows) - - print(f"{path.name}: {um_px:.3f} um/px, neuron channel {neuron_channel} (threshold {thr:.0f})") - print(f"{len(rows)} worms") - for w in rows: - head = "head found" if w["head_x_px"] != "" else "head unclear" - print(f" #{w['worm']:>2} {w['stage_estimate']:<22} {w['length_mm']:.2f} mm, " - f"{w['width_um']:.0f} um wide, neuron px {w['neuron_px']:>4}, {head} " - f"(ratio {w['head_confidence']})") + wr.writeheader(); wr.writerows(all_rows) + if multi: + with (out / "counts.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(counts[0])) + wr.writeheader(); wr.writerows(counts) print(f"wrote {out}") return out From b32ee9cf137c625a153a50540e4e0f04548893a5 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 16:19:57 -0400 Subject: [PATCH 37/42] aml18_survey: find larvae by their neurons; drop dark objects without any Small larvae are too faint for 4x brightfield but show as a chain of RFP signal, so glowing chains not claimed by a brightfield worm become worms too (orange in the overlay). Brightfield objects with no neuron signal are dropped - every AML18 worm glows. Adds --out, and fails loudly when an overlay cannot be written. --- analysis/aml18_survey.py | 88 +++++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/analysis/aml18_survey.py b/analysis/aml18_survey.py index 3ef0499..f146153 100644 --- a/analysis/aml18_survey.py +++ b/analysis/aml18_survey.py @@ -9,7 +9,8 @@ # python -m analysis.aml18_survey D:\...\CelegansAML-18\Test1.nd2 # python -m analysis.aml18_survey file.nd2 --neuron-channel GFP # -# Writes _survey/: worms.csv, survey.png (annotated overlay). +# Writes _survey/ (or --out): worms.csv, survey.png (annotated +# overlay); for a time-lapse, survey_tNNN.png per time point and counts.csv. # # WHICH FLUORESCENCE CHANNEL. The neuron channel must be captured at the # same instant as TD, or it shows where a crawling worm used to be. On @@ -144,36 +145,43 @@ def survey_frame(ch: dict[str, np.ndarray], um_px: float, neuron_channel: str, near = ndi.distance_transform_edt(~worms) <= 25 / um_px # neurons may sit just outside the TD edge rows = [] - for i in range(1, n + 1): - obj = lab == i + shape = td.shape + + def add_worm(obj: np.ndarray, width_map: np.ndarray | None, source: str) -> bool: skel = skeletonize(obj) if skel.sum() < 3: - continue + return False path_px, length_px = longest_path(skel) length_um = length_px * um_px - width_um = 2 * float(np.median(dist[skel])) * um_px - if length_um < min_length_um or length_um < 4 * width_um: - continue # egg or debris + width_um = 2 * float(np.median(width_map[skel])) * um_px if width_map is not None else None + if length_um < min_length_um or (width_um and length_um < 4 * width_um): + return False # egg or debris ys, xs = np.nonzero(obj) - edge = bool(ys.min() == 0 or xs.min() == 0 or ys.max() == td.shape[0] - 1 - or xs.max() == td.shape[1] - 1) + edge = bool(ys.min() == 0 or xs.min() == 0 or ys.max() == shape[0] - 1 + or xs.max() == shape[1] - 1) # Neuron signal belonging to this worm: inside the body or within - # 25 um of it, and not closer to another worm. - zone = ndi.binary_dilation(obj, iterations=int(25 / um_px)) & near + # 25 um of it. + zone = ndi.binary_dilation(obj, iterations=int(25 / um_px)) + if source == "TD": + zone &= near sig = np.where(zone & nmask, neu, 0) total = float(sig.sum()) + if total == 0: + # Every AML18 worm has glowing neurons; a dark TD object with + # none is agar texture or debris (Loop 16:13 worm 4). + return False # Head = the end whose first 15% of body length holds more signal. seg = max(3, int(0.15 * len(path_px))) r = int(60 / um_px) def end_signal(pts): - m = np.zeros_like(obj) + m = np.zeros(shape, np.uint8) for y, x in pts: - cv2.circle(m.view(np.uint8), (int(x), int(y)), r, 1, -1) + cv2.circle(m, (int(x), int(y)), r, 1, -1) return float(sig[m.astype(bool)].sum()) s_a, s_b = end_signal(path_px[:seg]), end_signal(path_px[-seg:]) - if total == 0 or max(s_a, s_b) < 1.5 * min(s_a, s_b) + 1: + if max(s_a, s_b) < 1.5 * min(s_a, s_b) + 1: head = None # no clear winner else: head = tuple(path_px[0] if s_a > s_b else path_px[-1]) @@ -181,19 +189,44 @@ def end_signal(pts): # not a real end - its neurons may be off-frame, so the # comparison is meaningless (Test1 worm 6 was called at the cut). hy, hx = head - if min(hy, hx) <= 3 or hy >= td.shape[0] - 4 or hx >= td.shape[1] - 4: + if min(hy, hx) <= 3 or hy >= shape[0] - 4 or hx >= shape[1] - 4: head = None stage = next(name for lim, name in STAGES if length_um / 1000 < lim) cy, cx = ndi.center_of_mass(obj) rows.append({ - "worm": len(rows) + 1, "x_px": round(cx), "y_px": round(cy), - "length_mm": round(length_um / 1000, 3), "width_um": round(width_um, 1), + "worm": len(rows) + 1, "found_by": source, "x_px": round(cx), "y_px": round(cy), + "length_mm": round(length_um / 1000, 3), + "width_um": round(width_um, 1) if width_um else "", "stage_estimate": stage + (" (cut by edge)" if edge else ""), "neuron_signal": round(total), "neuron_px": int((sig > 0).sum()), "head_x_px": head[1] if head else "", "head_y_px": head[0] if head else "", - "head_confidence": round(max(s_a, s_b) / (min(s_a, s_b) + 1), 1) if total else 0, + "head_confidence": round(max(s_a, s_b) / (min(s_a, s_b) + 1), 1), "_path": path_px, "_obj": obj, }) + return True + + for i in range(1, n + 1): + add_worm(lab == i, dist, "TD") + + # Worms seen only by their neurons: small larvae are too faint and + # thin in 4x brightfield (Timelapse1: ~6 glowing larvae unoutlined), + # but their neuron chain is a clear line of signal. Join each chain + # into one object and keep it if it is worm-shaped and not already + # part of a brightfield worm. Their length follows the neurons, so it + # can read a little short of the body. + claimed = np.zeros(shape, bool) + for w in rows: + claimed |= w["_obj"] + claimed = ndi.binary_dilation(claimed, iterations=int(40 / um_px)) + glow = ndi.binary_dilation(nmask, iterations=max(1, int(8 / um_px))) + glow = cv2.morphologyEx(glow.astype(np.uint8), cv2.MORPH_CLOSE, + cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (7, 7))).astype(bool) + glab, gn = ndi.label(glow) + for i in range(1, gn + 1): + obj = glab == i + if (obj & claimed).any() or (obj & nmask).sum() < 8: + continue + add_worm(obj, None, "neurons") # Annotated overlay: TD grey, neuron channel magenta, worm outlines cyan. g = np.clip((td - np.percentile(td, 0.5)) / (np.percentile(td, 99.5) - np.percentile(td, 0.5)), 0, 1) @@ -203,7 +236,8 @@ def end_signal(pts): img = cv2.resize(img, None, fx=2, fy=2, interpolation=cv2.INTER_NEAREST) for w in rows: cnts, _ = cv2.findContours(w["_obj"].astype(np.uint8), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE) - cv2.drawContours(img, [c * 2 for c in cnts], -1, (255, 255, 0), 1) + colour = (255, 255, 0) if w["found_by"] == "TD" else (0, 165, 255) + cv2.drawContours(img, [c * 2 for c in cnts], -1, colour, 2 if w["found_by"] != "TD" else 1) if w["head_x_px"] != "": cv2.circle(img, (2 * w["head_x_px"], 2 * w["head_y_px"]), 14, (0, 255, 255), 3) lbl = f"{w['worm']}: {w['stage_estimate'].split()[0]} {w['length_mm']:.2f}mm" @@ -214,15 +248,16 @@ def end_signal(pts): h, wd = img.shape[:2] cv2.rectangle(img, (wd - 40 - px, h - 50), (wd - 40, h - 38), (255, 255, 255), -1) cv2.putText(img, "500 um", (wd - 40 - px, h - 60), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2) - cv2.putText(img, f"{title}: {len(rows)} worms | cyan = outline, yellow ring = head " + cv2.putText(img, f"{title}: {len(rows)} worms | cyan = brightfield outline, orange = found by neurons, yellow ring = head " f"({neuron_channel} neurons, magenta)", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 1.0, (255, 255, 255), 2, cv2.LINE_AA) return rows, img, thr -def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float) -> Path: +def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: float, + out: Path | None = None) -> Path: frames, um_px = load(path) - out = path.with_name(path.stem + "_survey") + out = out or path.with_name(path.stem + "_survey") out.mkdir(exist_ok=True) multi = len(frames) > 1 all_rows, counts = [], [] @@ -230,7 +265,11 @@ def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: f for ti, (t_s, ch) in enumerate(frames): title = f"{path.name}" + (f" t={t_s / 60:.1f} min ({ti + 1}/{len(frames)})" if multi else "") rows, img, thr = survey_frame(ch, um_px, neuron_channel, min_length_um, dark_ratio, title) - cv2.imwrite(str(out / (f"survey_t{ti:03d}.png" if multi else "survey.png")), img) + dest = out / (f"survey_t{ti:03d}.png" if multi else "survey.png") + # cv2.imwrite returns False instead of raising - a file held open + # elsewhere silently kept the previous run's image on 2026-09-29. + if not cv2.imwrite(str(dest), img): + raise SystemExit(f"could not write {dest} - is it open in another program?") for w in rows: w.update(t_index=ti, t_s=round(t_s, 1)) all_rows += rows @@ -269,8 +308,9 @@ def main() -> None: ap.add_argument("--neuron-channel", default="RFP") ap.add_argument("--min-length-um", type=float, default=150.0) ap.add_argument("--dark-ratio", type=float, default=0.85) + ap.add_argument("--out", type=Path, help="output folder (default: _survey)") a = ap.parse_args() - analyse(a.nd2, a.neuron_channel, a.min_length_um, a.dark_ratio) + analyse(a.nd2, a.neuron_channel, a.min_length_um, a.dark_ratio, a.out) if __name__ == "__main__": From f39af5cde8ebfefb460e413d0629ee1576af4a7a Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 16:27:02 -0400 Subject: [PATCH 38/42] Add aml18_track: link surveyed worms across time-lapse frames Joins each frame's worms to the nearest similar-length worm of the previous frame, reports distance and speed per track, checks whether the neuron signal fades over the run, and renders the outlined frames as an MP4. --- analysis/aml18_track.py | 177 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 analysis/aml18_track.py diff --git a/analysis/aml18_track.py b/analysis/aml18_track.py new file mode 100644 index 0000000..f43055f --- /dev/null +++ b/analysis/aml18_track.py @@ -0,0 +1,177 @@ +# aml18_track.py +# ------------------------------------------------------------ +# Second step after aml18_survey on a time-lapse: link the worms found in +# each frame into tracks, measure how far and how fast each one moved, +# check whether the neuron signal fades over the run, and render the +# outlined frames as a movie. +# +# python -m analysis.aml18_track D:\...\CelegansAML-18\Timelapse_1h.nd2 +# +# Reads _survey/worms.csv and survey_tNNN.png, writes into the same +# folder: tracks.csv (one row per worm per frame, with a track id), +# track_summary.csv (one row per track), tracks.png (all paths on the +# first frame), signal.csv (neuron-channel brightness per frame) and +# survey_movie.mp4. +# +# LINKING. Frames are a minute apart, so a roaming worm can move several +# mm between them while a feeding one barely moves. Each worm is joined +# to the nearest unclaimed worm of the previous frame within --max-step-mm +# and of similar length (0.5-2x), closest pairs first. Anything further +# starts a new track. So a fast worm crossing the field may be split into +# pieces, and a worm leaving the 3.9 mm field simply ends - tracks are a +# lower bound on how long each worm was followed, not identities. +# +# FADING. The 99.9th percentile of the neuron channel per frame tracks the +# brightest neurons in view. Worms entering and leaving change it too, so +# only a steady decline over the whole run points to photobleaching. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import cv2 +import numpy as np + + +def link(rows: list[dict], um_px: float, max_step_mm: float) -> list[dict]: + frames = sorted({int(r["t_index"]) for r in rows}) + by_t = {t: [r for r in rows if int(r["t_index"]) == t] for t in frames} + next_id = 1 + prev: list[dict] = [] + for t in frames: + cur = by_t[t] + pairs = [] + for i, c in enumerate(cur): + for j, p in enumerate(prev): + d_mm = np.hypot(float(c["x_px"]) - float(p["x_px"]), + float(c["y_px"]) - float(p["y_px"])) * um_px / 1000 + ratio = float(c["length_mm"]) / max(float(p["length_mm"]), 1e-6) + if d_mm <= max_step_mm and 0.5 <= ratio <= 2.0: + pairs.append((d_mm, i, j)) + used_c, used_p = set(), set() + for d_mm, i, j in sorted(pairs): + if i in used_c or j in used_p: + continue + cur[i]["track"] = prev[j]["track"] + cur[i]["step_mm"] = round(d_mm, 3) + used_c.add(i); used_p.add(j) + for i, c in enumerate(cur): + if i not in used_c: + c["track"] = next_id + c["step_mm"] = "" + next_id += 1 + prev = cur + return rows + + +def summarise(rows: list[dict]) -> list[dict]: + out = [] + for tid in sorted({r["track"] for r in rows}): + tr = sorted((r for r in rows if r["track"] == tid), key=lambda r: int(r["t_index"])) + t0, t1 = float(tr[0]["t_s"]), float(tr[-1]["t_s"]) + dist = sum(float(r["step_mm"]) for r in tr if r["step_mm"] != "") + lengths = [float(r["length_mm"]) for r in tr] + stages = [r["stage_estimate"].split()[0] for r in tr] + out.append({ + "track": tid, "frames": len(tr), "start_min": round(t0 / 60, 1), + "end_min": round(t1 / 60, 1), "distance_mm": round(dist, 2), + "mean_speed_mm_per_min": round(dist / ((t1 - t0) / 60), 3) if t1 > t0 else "", + "median_length_mm": round(float(np.median(lengths)), 3), + "stage": max(set(stages), key=stages.count), + }) + return out + + +def draw_tracks(first_png: Path, rows: list[dict], dest: Path) -> None: + img = cv2.imread(str(first_png)) + if img is None: + return + rng = np.random.default_rng(1) + for tid in sorted({r["track"] for r in rows}): + tr = sorted((r for r in rows if r["track"] == tid), key=lambda r: int(r["t_index"])) + pts = np.array([[2 * int(r["x_px"]), 2 * int(r["y_px"])] for r in tr], np.int32) + col = tuple(int(v) for v in rng.integers(80, 256, 3)) + if len(pts) > 1: + cv2.polylines(img, [pts], False, col, 3, cv2.LINE_AA) + cv2.circle(img, tuple(pts[-1]), 7, col, -1) + cv2.putText(img, str(tid), tuple(pts[-1] + 10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, col, 2) + cv2.imwrite(str(dest), img) + + +def movie(folder: Path, fps: int) -> Path | None: + import imageio_ffmpeg + frames = sorted(folder.glob("survey_t*.png")) + if len(frames) < 2: + return None + first = cv2.imread(str(frames[0])) + h, w = (first.shape[0] // 2) * 2, (first.shape[1] // 2) * 2 + dest = folder / "survey_movie.mp4" + wr = imageio_ffmpeg.write_frames( + str(dest), (w, h), fps=fps, codec="libx264", quality=None, macro_block_size=2, + output_params=["-crf", "22", "-movflags", "+faststart"]) # write_frames already outputs yuv420p + wr.send(None) + for f in frames: + img = cv2.imread(str(f))[:h, :w] + wr.send(np.ascontiguousarray(img[..., ::-1])) + wr.close() + return dest + + +def signal_per_frame(nd2_path: Path, channel: str) -> list[dict]: + from analysis.aml18_survey import load + frames, _ = load(nd2_path) + return [{"t_min": round(t / 60, 2), + "p99_9": round(float(np.percentile(ch[channel], 99.9)), 1), + "max": round(float(ch[channel].max()), 1)} for t, ch in frames] + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("nd2", type=Path) + ap.add_argument("--survey", type=Path, help="survey folder (default _survey)") + ap.add_argument("--max-step-mm", type=float, default=1.0) + ap.add_argument("--channel", default="RFP") + ap.add_argument("--fps", type=int, default=6) + a = ap.parse_args() + folder = a.survey or a.nd2.with_name(a.nd2.stem + "_survey") + + import nd2 + with nd2.ND2File(a.nd2) as f: + um_px = float(f.voxel_size().x) + with (folder / "worms.csv").open(encoding="utf-8") as fh: + rows = list(csv.DictReader(fh)) + rows = link(rows, um_px, a.max_step_mm) + with (folder / "tracks.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=["track", "step_mm"] + [k for k in rows[0] + if k not in ("track", "step_mm")]) + wr.writeheader(); wr.writerows(rows) + summ = summarise(rows) + with (folder / "track_summary.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(summ[0])) + wr.writeheader(); wr.writerows(summ) + draw_tracks(folder / "survey_t000.png", rows, folder / "tracks.png") + + sig = signal_per_frame(a.nd2, a.channel) + with (folder / "signal.csv").open("w", newline="", encoding="utf-8") as fh: + wr = csv.DictWriter(fh, fieldnames=list(sig[0])) + wr.writeheader(); wr.writerows(sig) + mv = movie(folder, a.fps) + + long_tracks = [s for s in summ if s["frames"] >= 3] + print(f"{len(summ)} tracks, {len(long_tracks)} followed for 3+ frames") + for s in sorted(summ, key=lambda s: -s["frames"])[:15]: + print(f" track {s['track']:>3}: {s['stage']:<6} {s['frames']:>3} frames " + f"({s['start_min']}-{s['end_min']} min), {s['distance_mm']} mm, " + f"{s['mean_speed_mm_per_min']} mm/min") + first, last = sig[0]["p99_9"], sig[-1]["p99_9"] + print(f"{a.channel} brightest-neuron level: {first} -> {last} " + f"({(last / first - 1) * 100 if first else 0:+.0f}%) over {sig[-1]['t_min']} min") + print(f"wrote tracks.csv, track_summary.csv, tracks.png, signal.csv" + + (f", {mv.name}" if mv else "") + f" in {folder}") + + +if __name__ == "__main__": + main() From b0677257e97baa8815a272d962795ff3c364d20f Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Tue, 29 Sep 2026 18:09:40 -0400 Subject: [PATCH 39/42] aml18_survey: print worms found by neurons, which have no width --- analysis/aml18_survey.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/analysis/aml18_survey.py b/analysis/aml18_survey.py index f146153..6d557f1 100644 --- a/analysis/aml18_survey.py +++ b/analysis/aml18_survey.py @@ -282,7 +282,8 @@ def analyse(path: Path, neuron_channel: str, min_length_um: float, dark_ratio: f for w in rows: head = "head found" if w["head_x_px"] != "" else "head unclear" print(f" #{w['worm']:>2} {w['stage_estimate']:<22} {w['length_mm']:.2f} mm, " - f"{w['width_um']:.0f} um wide, neuron px {w['neuron_px']:>4}, {head} " + + (f"{w['width_um']:.0f} um wide, " if w["width_um"] != "" else "found by neurons, ") + + f"neuron px {w['neuron_px']:>4}, {head} " f"(ratio {w['head_confidence']})") else: c = counts[-1] From 4553b5715640acbb70762c8b1ae1d1c6e94d5b6c Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 30 Sep 2026 14:31:50 -0400 Subject: [PATCH 40/42] estop: make the panel mutex name a raw string '\c' is an invalid escape; Python 3.12 warns about it and a later release will make it a SyntaxError. The name's value is unchanged. --- acquisition/estop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acquisition/estop.py b/acquisition/estop.py index 425d472..45ef17b 100644 --- a/acquisition/estop.py +++ b/acquisition/estop.py @@ -75,7 +75,7 @@ #: Held by the running STOP panel for its whole life, so every process can #: tell whether one is already on screen. Windows frees it if the panel dies. -PANEL_MUTEX = "Local\confocal-mcp-estop-panel" +PANEL_MUTEX = r"Local\confocal-mcp-estop-panel" class EStopEngaged(RuntimeError): From 02af5b4fe684465e32d024789c7cfd5326405f3f Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 30 Sep 2026 14:44:42 -0400 Subject: [PATCH 41/42] Add nis_port_probe: HTTP control of NIS-Elements from a JOBS Python task A JOBS Python task serves /status, /move (relative, capped at 1 mm) and /objective (4x/10x only) on 127.0.0.1:8765 for 10 minutes. NIS macro functions are called through ctypes on g5_regprocs.dll, the way NIS's own limpy calls WaitText; signatures are from the NIS 6.20 macro reference. Nothing moves while the e-stop file exists and Z is never moved. Verified so far: the task binds the port inside nis_ar.exe and answers from outside. The /status, /move and /objective calls have not yet been run on the scope. --- acquisition/calibration/nis_port_probe.py | 117 ++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 acquisition/calibration/nis_port_probe.py diff --git a/acquisition/calibration/nis_port_probe.py b/acquisition/calibration/nis_port_probe.py new file mode 100644 index 0000000..26a0b4b --- /dev/null +++ b/acquisition/calibration/nis_port_probe.py @@ -0,0 +1,117 @@ +# IMPORTANT: 'limjob' must be imported like this (not from nor as) +import limjob + +# A JOBS Python task that serves a tiny HTTP control API from inside NIS for +# 10 minutes (or until Abort). From a terminal on this PC: +# curl http://127.0.0.1:8765/status +# curl "http://127.0.0.1:8765/move?dx=200&dy=0" relative, um +# curl "http://127.0.0.1:8765/objective?name=" +# +# NIS macro functions are called through ctypes on g5_regprocs.dll, the way +# Nikon's own limpy.macro calls WaitText. Signatures are from the NIS 6.20 +# macro reference (Docs/nis/eng_ar); macro char* is wchar_t* in this build. +# +# NIS calls are made only on the job's own thread (the run() loop); the HTTP +# thread just queues requests, since NIS is not known to be thread-safe. +# +# SAFETY: moves are relative and capped at MAX_STEP_UM per call; nothing +# moves while the e-stop file exists (same file acquisition/estop.py uses); +# Z is never moved; objective changes are limited to 4x and 10x, whose long +# working distances cannot reach the dish. +import ctypes as ct, http.server, json, os, queue, threading, time, urllib.parse + +PORT, RUN_S, MAX_STEP_UM = 8765, 600, 1000.0 +ESTOP = r"C:\Users\AX Confocal\.confocal-mcp\ESTOP" +ALLOWED_OBJECTIVES = ("4x", "10x") + +nis = ct.cdll.g5_regprocs +for fn, args in {"StgGetPosXY": [ct.POINTER(ct.c_double)] * 2, + "StgGetPosZ": [ct.POINTER(ct.c_double), ct.c_int], + "StgMoveXY": [ct.c_double, ct.c_double, ct.c_int], + "Stg_GetNosepiecePosition": [], + "Stg_GetNosepiecePositions": [], + "Stg_GetNosepieceObjectiveName": [ct.c_int, ct.c_wchar_p, ct.c_int], + "GetCurrentObjName": [ct.c_wchar_p], + "ChangeObjective": [ct.c_wchar_p]}.items(): + getattr(nis, fn).argtypes, getattr(nis, fn).restype = args, ct.c_int + + +def status(): + x, y, z = ct.c_double(), ct.c_double(), ct.c_double() + rc_xy = nis.StgGetPosXY(ct.byref(x), ct.byref(y)) + rc_z = nis.StgGetPosZ(ct.byref(z), 0) + cur = ct.create_unicode_buffer(255); nis.GetCurrentObjName(cur) + names = {} + for i in range(0, nis.Stg_GetNosepiecePositions() + 1): # index base unknown: try 0..n + b = ct.create_unicode_buffer(255) + if nis.Stg_GetNosepieceObjectiveName(i, b, 255) == 1 and b.value: + names[i] = b.value + return {"xy_um": [x.value, y.value], "xy_rc": rc_xy, "z_um": z.value, "z_rc": rc_z, + "nosepiece_position": nis.Stg_GetNosepiecePosition(), + "current_objective": cur.value, "nosepiece_objectives": names} + + +def move(dx, dy): + if os.path.exists(ESTOP): + return {"error": "e-stop engaged - not moving"} + if max(abs(dx), abs(dy)) > MAX_STEP_UM: + return {"error": f"step over {MAX_STEP_UM} um refused"} + before = status()["xy_um"] + rc = nis.StgMoveXY(dx, dy, 1) # 1 = MOVE_RELATIVE + return {"rc": rc, "before_um": before, "after_um": status()["xy_um"]} + + +def objective(name): + if os.path.exists(ESTOP): + return {"error": "e-stop engaged - not changing objective"} + if not any(k in name for k in ALLOWED_OBJECTIVES): + return {"error": f"only {ALLOWED_OBJECTIVES} objectives allowed here"} + before = status() + rc = nis.ChangeObjective(name) + after = status() + return {"rc": rc, "before": before["current_objective"], "after": after["current_objective"], + "nosepiece_before": before["nosepiece_position"], + "nosepiece_after": after["nosepiece_position"]} + + +jobs = queue.Queue() + + +class H(http.server.BaseHTTPRequestHandler): + def do_GET(self): + u = urllib.parse.urlparse(self.path); q = dict(urllib.parse.parse_qsl(u.query)) + call = {"/status": lambda: status(), + "/move": lambda: move(float(q.get("dx", 0)), float(q.get("dy", 0))), + "/objective": lambda: objective(q.get("name", ""))}.get(u.path) + if call is None: + out = {"error": "use /status, /move?dx=&dy=, /objective?name="} + else: + box = queue.Queue(); jobs.put((call, box)) + try: + out = box.get(timeout=60) + except queue.Empty: + out = {"error": "timed out waiting for NIS"} + body = (json.dumps(out, indent=1) + "\n").encode() + self.send_response(200); self.send_header("Content-Type", "application/json") + self.end_headers(); self.wfile.write(body) + + +def run(imgs: tuple[limjob.Image], Job: limjob.JobParam, macro: limjob.MacroParam, ctx: limjob.RunContext): + s = http.server.ThreadingHTTPServer(("127.0.0.1", PORT), H) + threading.Thread(target=s.serve_forever, daemon=True).start() + print(f"listening on 127.0.0.1:{PORT}, pid {os.getpid()}") + abort = getattr(ctx, "shouldAbort", lambda: False) + t0 = time.time() + try: + while time.time() - t0 < RUN_S and not abort(): + try: + call, box = jobs.get(timeout=0.2) + except queue.Empty: + continue + try: + box.put(call()) + except Exception as e: + box.put({"error": f"{type(e).__name__}: {e}"}) + finally: + s.shutdown(); s.server_close() + print("server stopped") From 79867993eb1699efe297f4806d05248c83c45066 Mon Sep 17 00:00:00 2001 From: Qurratul Ain Quais Date: Wed, 30 Sep 2026 15:36:03 -0400 Subject: [PATCH 42/42] Add make_soundtrack: original music for time-lapse movies Promoted from the one-off script that scored the Physarum Short. The arrangement is unchanged and stretches to any length; --movie sizes the track to a movie and muxes it on (video copied, AAC audio) with the ffmpeg bundled by imageio-ffmpeg. At 39 s the output is byte-identical to the original track. --- CHANGELOG.md | 5 ++ analysis/make_soundtrack.py | 165 ++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 analysis/make_soundtrack.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b9edf5..2ac3cec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions follow ## [Unreleased] ### Added +- `analysis/make_soundtrack.py` - original ambient soundtrack synthesised + with numpy (no samples, so no licensing questions), stretched to any + length. `--movie run.mp4` sizes it to the movie and muxes it on as AAC + with the video stream copied, writing `run_music.mp4`. At 39 s it + reproduces the Physarum Short's track exactly. - `acquisition/calibration/ti2_inventory.py` - read-only inventory of every device the Ti2 reports, with each one's valid range, unit, and `Control` value. Field names come from the COM type library at runtime and rows are diff --git a/analysis/make_soundtrack.py b/analysis/make_soundtrack.py new file mode 100644 index 0000000..c297615 --- /dev/null +++ b/analysis/make_soundtrack.py @@ -0,0 +1,165 @@ +# make_soundtrack.py +# ------------------------------------------------------------ +# Original ambient soundtrack for a time-lapse movie, synthesised with +# numpy - no samples and no third-party audio, so there are no licensing +# questions when a movie is posted or presented. +# +# python -m analysis.make_soundtrack --duration 39 --out track.wav +# python -m analysis.make_soundtrack --movie run.mp4 # -> run_music.mp4 +# +# With --movie the track is made exactly as long as the movie and muxed +# onto it (video stream copied untouched, audio AAC), using the ffmpeg +# that imageio-ffmpeg bundles. +# +# THE ARRANGEMENT is the one written for the 2026-09-25 Physarum Short: +# a quiet opening, a chord progression that builds with a rising +# arpeggio, a sparse held middle, a lighter second rise and a warm major +# resolve at the end. It was composed on a 39 s timeline; other lengths +# stretch that timeline, so sections keep their proportions. Notes keep +# their own length, so very long movies (minutes) get sparser, not +# slower-sounding - it suits roughly 20-120 s. +# ------------------------------------------------------------ + +from __future__ import annotations + +import argparse +import subprocess +import wave +from pathlib import Path + +import numpy as np + +SR = 44100 +SCORE_S = 39.0 # the timeline the arrangement was written on + + +def midi(m: float) -> float: + return 440.0 * 2 ** ((m - 69) / 12) + + +def render(duration: float, seed: int = 7) -> np.ndarray: + """Stereo float mix in [-1, 1], `duration` seconds long.""" + n = int(SR * duration) + t = np.arange(n) / SR + k = duration / SCORE_S # score seconds -> real seconds + rng = np.random.default_rng(seed) + + def env_between(a, b, fade): + e = np.minimum(np.clip((t - a * k) / fade, 0, 1), np.clip((b * k - t) / fade, 0, 1)) + return 0.5 - 0.5 * np.cos(np.pi * e) + + def pad(notes, a, b, level, bright=0.3): + """Soft pad: each note = detuned sines + weak harmonics, slow swell.""" + out = np.zeros((n, 2)) + e = env_between(a, b, fade=2.0) + for note in notes: + f = midi(note) + for det, pan in ((-0.12, 0.3), (0.0, 0.5), (0.12, 0.7)): + ff = f * 2 ** (det / 12) + ph = rng.uniform(0, 2 * np.pi) + s = (np.sin(2 * np.pi * ff * t + ph) + + bright * 0.35 * np.sin(2 * np.pi * 2 * ff * t + ph) + + bright * 0.12 * np.sin(2 * np.pi * 3 * ff * t + ph)) + out[:, 0] += s * (1 - pan) + out[:, 1] += s * pan + return out * (e * level / (len(notes) * 3))[:, None] + + def pluck(note, at, level, pan=0.5, decay=1.2): + out = np.zeros((n, 2)) + i0 = int(at * k * SR) + if i0 >= n: + return out + L = min(n - i0, int(SR * decay * 4)) + tt = np.arange(L) / SR + f = midi(note) + s = (np.sin(2 * np.pi * f * tt) + 0.3 * np.sin(2 * np.pi * 2 * f * tt) + + 0.1 * np.sin(2 * np.pi * 3 * f * tt)) + s *= np.exp(-tt / decay) * np.clip(tt / 0.005, 0, 1) * level + out[i0:i0 + L, 0] += s * (1 - pan) + out[i0:i0 + L, 1] += s * pan + return out + + mix = np.zeros((n, 2)) + # A minor world resolving to C major: Am, F, C, G + AM, F, C, G = [57, 64, 69, 72], [53, 60, 65, 69], [48, 55, 64, 67], [55, 62, 67, 71] + CMAJ = [48, 55, 60, 64, 67, 72] + + mix += pad([45, 57, 64], 0.0, 7.0, 0.55, bright=0.15) # opening + for s0, ch in ((6.0, AM), (8.75, F), (11.5, C), (14.25, G)): # progression + mix += pad(ch, s0 - 0.5, s0 + 3.4, 0.7 if s0 < 11 else 0.85, + bright=0.3 if s0 < 11 else 0.55) + for i, at in enumerate(np.arange(6.0, 11.0, 0.6875)): # root pulse + mix += pluck(45 if i % 2 == 0 else 52, at, 0.10, pan=0.4, decay=0.6) + arp = [69, 72, 76, 79, 81, 84, 81, 79] + for i, at in enumerate(np.arange(11.0, 17.2, 0.34375)): # rising arpeggio + note = arp[i % len(arp)] + (12 if at > 14.5 else 0) + mix += pluck(note - 12, at, 0.10 + 0.05 * (at - 11) / 6, pan=0.3 + 0.4 * (i % 2), decay=0.9) + mix += pad([45, 52, 60, 64], 16.5, 27.5, 0.5, bright=0.12) # held middle + for at, note in ((18.5, 76), (21.0, 72), (23.5, 71), (25.5, 69)): + mix += pluck(note, at, 0.07, pan=0.6, decay=2.0) + mix += pad(F, 26.8, 31.0, 0.6, bright=0.35) # second rise + mix += pad(G, 30.5, 34.5, 0.6, bright=0.35) + for i, at in enumerate(np.arange(27.2, 34.0, 0.4125)): + mix += pluck([72, 76, 79, 76][i % 4], at, 0.08, pan=0.35 + 0.3 * (i % 2), decay=0.8) + mix += pad(CMAJ, 33.8, 40.5, 0.9, bright=0.4) # resolve + for dt, note, lv in ((0.0, 60, 0.12), (0.15, 64, 0.10), (0.3, 67, 0.10)): + mix += pluck(note, 34.0 + dt, lv, decay=3.0) + + # Stereo reverb: convolve with decaying noise (FFT). + ir_len = int(SR * 2.2) + ir = rng.standard_normal((ir_len, 2)) * np.exp(-np.arange(ir_len) / (SR * 0.55))[:, None] + ir[0] = 0 + nfft = 1 << int(np.ceil(np.log2(n + ir_len))) + wet = np.stack([np.fft.irfft(np.fft.rfft(mix[:, c], nfft) * np.fft.rfft(ir[:, c], nfft), nfft)[:n] + for c in range(2)], axis=1) + wet *= np.abs(mix).max() / (np.abs(wet).max() + 1e-9) + out = 0.72 * mix + 0.38 * wet + + # Master: short fade-in, 2.5 s fade-out, normalise to -1 dBFS. + out *= (np.clip(t / 0.3, 0, 1) * np.clip((duration - t) / 2.5, 0, 1))[:, None] + return out * 10 ** (-1 / 20) / np.abs(out).max() + + +def write_wav(mix: np.ndarray, path: Path) -> None: + with wave.open(str(path), "wb") as w: + w.setnchannels(2); w.setsampwidth(2); w.setframerate(SR) + w.writeframes((mix * 32767).astype(np.int16).tobytes()) + + +def movie_seconds(movie: Path) -> float: + import imageio_ffmpeg + frames, secs = imageio_ffmpeg.count_frames_and_secs(str(movie)) + return float(secs) + + +def mux(movie: Path, wav: Path, out: Path) -> None: + import imageio_ffmpeg + cmd = [imageio_ffmpeg.get_ffmpeg_exe(), "-y", "-loglevel", "error", + "-i", str(movie), "-i", str(wav), "-map", "0:v:0", "-map", "1:a:0", + "-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-shortest", + "-movflags", "+faststart", str(out)] + subprocess.run(cmd, check=True) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--duration", type=float, help="seconds (default: the movie's length, else 39)") + ap.add_argument("--movie", type=Path, help="add the track to this movie -> _music.mp4") + ap.add_argument("--out", type=Path, help="output .wav (default beside the movie, or soundtrack.wav)") + ap.add_argument("--seed", type=int, default=7, help="changes pad phases and reverb, not the notes") + a = ap.parse_args() + + dur = a.duration or (movie_seconds(a.movie) if a.movie else SCORE_S) + wav = a.out or (a.movie.with_name(a.movie.stem + "_soundtrack.wav") if a.movie + else Path("soundtrack.wav")) + mix = render(dur, a.seed) + write_wav(mix, wav) + print(f"wrote {wav} ({dur:.1f} s)") + if a.movie: + dest = a.movie.with_name(a.movie.stem + "_music.mp4") + mux(a.movie, wav, dest) + print(f"wrote {dest}") + + +if __name__ == "__main__": + main()