From f42fbe1a274d5f54346e1e6f6ecf914aea4fbdd9 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:21:18 +0400 Subject: [PATCH 1/9] Test that the Modal runner records the image emitter's status The receipt's runner.telemetry.undelivered_events records what the image emitter's `microcosm-emitter status` answers after the tool: its runs, an empty list for exit 8, or one problem line. Builds and the image take the oldest emitter, 0.13.0, from one packaged file. The fake runner moves to test_support so that a new installed-emitter test can run a stage with the real emitter (#1522). Co-Authored-By: Claude Opus 5.5 --- .../shared/test_emitter_adapter.py | 26 +- .../us/test_us_modal_stage_plan_tool.py | 517 ++++++++++++++---- .../shared/test_installed_emitter_status.py | 193 +++++++ test_support/microcosm_build/emitter.py | 2 +- .../us_modal_stage_plan_tool.py | 152 ++++- 5 files changed, 763 insertions(+), 127 deletions(-) create mode 100644 packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py diff --git a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py index 40d84e187f..928f78a5d4 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py @@ -273,8 +273,10 @@ def _start_build_emitter(monkeypatch, tmp_path): @pytest.mark.parametrize( ("version_answer", "reported"), [ - # The release just before the minimum. - (_version_answer("0.11.8"), "reports version 0.11.8"), + # The release just before the minimum, 0.13.0. + (_version_answer("0.12.5"), "reports version 0.12.5"), + # The minimum until 2026-11-05, without the `status` command. + (_version_answer("0.11.9"), "reports version 0.11.9"), # Releases before 0.10.0 send telemetry to the collector's Cloud Run # address, which changes when the collector moves to another project. (_version_answer("0.9.0"), "reports version 0.9.0"), @@ -287,7 +289,8 @@ def _start_build_emitter(monkeypatch, tmp_path): ("not JSON", "reports no version"), ], ids=[ - "0.11.8", + "0.12.5", + "0.11.9", "0.9.0", "0.6.0", "0.5.1", @@ -297,7 +300,7 @@ def _start_build_emitter(monkeypatch, tmp_path): "not-json", ], ) -def test_an_emitter_older_than_0_11_9_is_never_started_and_the_build_continues( +def test_an_emitter_older_than_the_minimum_is_never_started_and_the_build_continues( monkeypatch, capsys, tmp_path, version_answer, reported ): # The emitter's telemetry module records the run's start as soon as the @@ -314,7 +317,7 @@ def test_an_emitter_older_than_0_11_9_is_never_started_and_the_build_continues( assert not emitter.available warning = capsys.readouterr().err assert f"The installed microcosm-emitter {reported};" in warning - assert "this build needs 0.11.9 or later" in warning + assert "this build needs 0.13.0 or later" in warning # The warning says what the build loses without the emitter, including # the one option that then fails the build. assert ( @@ -322,7 +325,7 @@ def test_an_emitter_older_than_0_11_9_is_never_started_and_the_build_continues( "and queues no Orrery graph publication; a UK build given " "--staging-read-back will fail at the read-back." in warning ) - assert "uv tool install --upgrade 'microcosm-emitter>=0.11.9'" in warning + assert "uv tool install --upgrade 'microcosm-emitter>=0.13.0'" in warning # It also says that a remote stage of the dataset bundle is refused, so # the pre-flight's later refusal does not contradict it. assert "will continue without it" not in warning @@ -346,10 +349,11 @@ def test_the_fake_client_reports_a_release_builds_start(): @pytest.mark.parametrize( "installed_version", - # 0.11.10 is compared by number, not as text, where it sorts before 0.11.9. - ["0.11.9", "0.11.10", "0.12.0", "1.0.0"], + # Compared by number, not as text: as text, 0.9.0, which is refused, + # sorts after 0.13.0. + ["0.13.0", "0.13.10", "0.14.0", "1.0.0"], ) -def test_an_emitter_from_0_11_9_on_starts( +def test_an_emitter_from_the_minimum_on_starts( monkeypatch, capsys, tmp_path, installed_version ): client = FakeEmitterClient() @@ -367,7 +371,7 @@ def test_an_emitter_from_0_11_9_on_starts( @pytest.mark.parametrize("operation", ["inventory", "publish"]) -def test_a_short_lived_graph_emitter_older_than_0_11_9_is_never_started( +def test_a_short_lived_graph_emitter_older_than_the_minimum_is_never_started( monkeypatch, capsys, tmp_path, operation ): client = FakeEmitterClient(modules={"orrery"}) @@ -386,7 +390,7 @@ def test_a_short_lived_graph_emitter_older_than_0_11_9_is_never_started( assert starts == [] assert not started.exists() assert client.calls == [] - assert "this build needs 0.11.9 or later" in capsys.readouterr().err + assert "this build needs 0.13.0 or later" in capsys.readouterr().err @pytest.mark.parametrize("failure", ["default_spool_path", "runtime_identity"]) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index 6076590a63..960967588b 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -39,6 +39,7 @@ DUMMY_COLLECTOR_TOKEN, FEED_SHA, LADDER_SHA, + RECORDED_EMITTER_STATUS, RECORDED_EMITTER_VERSION, ROOT, RUNNER_TOKEN_SECRET, @@ -46,9 +47,19 @@ TEST_COLLECTOR_ORIGIN, TEST_TOKEN_EXPIRES_AT, base_plan_data, + install_fake_runner, load_app, plan_lib, ) +from test_support.microcosm_build.us_modal_stage_plan_tool import ( + FAKE_PAYLOAD as _FAKE_PAYLOAD, +) +from test_support.microcosm_build.us_modal_stage_plan_tool import ( + fake_plan as _fake_plan, +) +from test_support.microcosm_build.us_modal_stage_plan_tool import ( + fake_tool as _fake_tool, +) from test_support.microcosm_build.us_modal_stage_plan_tool import ( plan_data as _plan_data, ) @@ -405,9 +416,11 @@ def test_image_installs_the_emitter_command_on_its_own(app) -> None: plan = plan_lib.parse_plan(_plan_data()) commands = plan_lib.image_build_commands(plan) sync = next(index for index, command in enumerate(commands) if "uv sync" in command) - install = commands.index( - "UV_TOOL_BIN_DIR=/usr/local/bin uv tool install 'microcosm-emitter>=0.11.9'" + install_command = ( + "UV_TOOL_BIN_DIR=/usr/local/bin uv tool install 'microcosm-emitter>=0.13.0'" ) + assert install_command in commands + install = commands.index(install_command) # Outside the build's environment, after it is synced and before the # clean-tree check. The image does not build unless the command runs. assert sync < install < len(commands) - 1 @@ -428,24 +441,94 @@ def test_image_installs_the_emitter_command_on_its_own(app) -> None: assert "MICROCOSM_EMITTER_COMMAND" not in removed -def test_the_image_installs_no_emitter_older_than_a_build_starts() -> None: - """The image's emitter floor is the build adapter's minimum release. +#: The one file that holds the oldest emitter release builds start and the +#: image installs: microcosm-build's package data. +MINIMUM_EMITTER_VERSION_FILE = ( + ROOT + / "packages" + / "microcosm-build" + / "src" + / "microcosm" + / "build" + / "minimum_emitter_version.json" +) + + +def _image_emitter_install(plan: plan_lib.Plan) -> str: + (install,) = [ + command + for command in plan_lib.image_build_commands(plan) + if "uv tool install" in command + ] + return install + - ``tools/modal_us_stage_plan.py`` imports nothing from the workspace, so it - repeats the version. This compares the two values of one checkout only. - At launch they can come from two trees: the install step from the - checkout that launches the app, and the minimum from the commit the plan - pins. So a launch can still combine an older checkout's floor with a - newer commit's minimum; that floor has no upper bound, so it resolves to - the newest release when the image is built. +def test_builds_and_the_image_take_the_oldest_emitter_from_one_file() -> None: + """The build adapter and the image's install step read one packaged file. + + The adapter reads it as package data of microcosm-build. The runner, + which imports nothing from the workspace, reads it by its path in the + checkout that launches the app. At launch the two can still come from + two trees: the install step from the launching checkout, the minimum + from the commit the plan pins. """ from microcosm.build.telemetry_emitter import MINIMUM_EMITTER_VERSION - assert MINIMUM_EMITTER_VERSION == "0.11.9" - assert plan_lib.IMAGE_EMITTER_REQUIREMENT == ( - f"microcosm-emitter>={MINIMUM_EMITTER_VERSION}" + assert MINIMUM_EMITTER_VERSION_FILE.is_file() + assert json.loads(MINIMUM_EMITTER_VERSION_FILE.read_text(encoding="utf-8")) == { + "minimum_emitter_version": MINIMUM_EMITTER_VERSION + } + plan = plan_lib.parse_plan(_plan_data()) + assert _image_emitter_install(plan).endswith( + f"uv tool install 'microcosm-emitter>={MINIMUM_EMITTER_VERSION}'" ) + # The runner runs the image emitter's `status`, which 0.13.0 added. + release = tuple(int(part) for part in MINIMUM_EMITTER_VERSION.split(".")) + assert release >= (0, 13, 0) + + +def test_the_image_installs_the_release_the_file_names_when_the_app_is_defined( + monkeypatch, tmp_path: Path +) -> None: + version_file = tmp_path / "minimum_emitter_version.json" + version_file.write_text('{"minimum_emitter_version": "9.8.7"}\n') + monkeypatch.setattr( + plan_lib, "MINIMUM_EMITTER_VERSION_FILE", version_file, raising=False + ) + + plan = plan_lib.parse_plan(_plan_data()) + + assert _image_emitter_install(plan) == ( + "UV_TOOL_BIN_DIR=/usr/local/bin uv tool install 'microcosm-emitter>=9.8.7'" + ) + + +@pytest.mark.parametrize( + "content", + [ + None, + "0.13.0\n", + '{"minimum_emitter_version": "0.13"}\n', + '{"minimum_emitter_version": 13}\n', + '{"minimum": "0.13.0"}\n', + '["0.13.0"]\n', + ], + ids=["missing", "not JSON", "not X.Y.Z", "a number", "another key", "a list"], +) +def test_the_image_is_not_defined_without_a_usable_emitter_floor( + monkeypatch, tmp_path: Path, content: str | None +) -> None: + version_file = tmp_path / "minimum_emitter_version.json" + if content is not None: + version_file.write_text(content) + monkeypatch.setattr( + plan_lib, "MINIMUM_EMITTER_VERSION_FILE", version_file, raising=False + ) + plan = plan_lib.parse_plan(_plan_data()) + + with pytest.raises(plan_lib.PlanError, match="minimum_emitter_version.json"): + plan_lib.image_build_commands(plan) def test_the_app_never_runs_an_emitter_installed_where_the_image_puts_it( @@ -464,12 +547,13 @@ def test_the_app_never_runs_an_emitter_installed_where_the_image_puts_it( monkeypatch.setattr(plan_lib, "IMAGE_REPO_ROOT", str(tmp_path / "repo")) recorded = { - "emitter_version": "0.11.9", + "emitter_version": "0.13.0", "protocol_versions": [1], "collector_origin": "https://collector.invalid", } assert app._emitter_version() == recorded assert app._runner_identity()["microcosm_emitter"] == recorded + assert app._emitter_status(dict(os.environ)) == RECORDED_EMITTER_STATUS assert not ran.exists() @@ -1893,102 +1977,12 @@ def test_push_copy_only_applies_to_a_clean_exit_of_a_resume_state_stage() -> Non # The run path end to end, against local directories # # --------------------------------------------------------------------------- # -# A stand-in for a resumable tool: its first run writes a checkpoint, a -# later run of the same state finds it and only rewrites the output. -_FAKE_TOOL = """\ -import os, pathlib, sys, time -state, data = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) -ckpt = state / "ckpt" / "step1.bin" -resumed = ckpt.exists() -if not resumed: - ckpt.parent.mkdir(parents=True, exist_ok=True) - ckpt.write_bytes(data.read_bytes() * 3) -(state / "out").mkdir(parents=True, exist_ok=True) -(state / "out" / "result.txt").write_text(f"resumed={resumed}\\n") -print("fake tool", "resumed" if resumed else "fresh", flush=True) -time.sleep(float(os.environ.get("MICROCOSM_FAKE_SLEEP", "0"))) -""" -_FAKE_PAYLOAD = b"fake input" -_FAKE_SHA = hashlib.sha256(_FAKE_PAYLOAD).hexdigest() - - -def _fake_tool( - timeout_s: int = 3600, overhead: int = 60, **stage: object -) -> plan_lib.ToolSpec: - resources = plan_lib.Resources("check", 2.0, 8 * 1024, timeout_s=timeout_s) - return plan_lib.ToolSpec( - name="fake-stage", - script=None, - inputs=("data",), - stages={ - "run": plan_lib.StageSpec( - "run", - resources, - ("data",), - runner_overhead_seconds=overhead, - **stage, # type: ignore[arg-type] - ) - }, - options={}, - owned_flags=frozenset(), - argv_builder=lambda plan, paths, state: [ - "-c", - _FAKE_TOOL, - state, - paths["data"], - ], - ) - - -def _fake_plan(**overrides) -> dict: - data = { - "schema": plan_lib.PLAN_SCHEMA, - "tool": "fake-stage", - "stage": "run", - "run_id": "fake-run", - "source": {"commit": "a" * 40, "branch": "main"}, - "inputs": { - "data": { - "uri": f"volume://cas/sha256/{_FAKE_SHA}/data.bin", - "sha256": _FAKE_SHA, - } - }, - } - data.update(overrides) - return data - @pytest.fixture def fake_runner(app, tmp_path: Path, monkeypatch): """The app with its mounts, clone and venv pointed at local directories.""" - def install(tool: plan_lib.ToolSpec): - monkeypatch.setitem(plan_lib.TOOLS, tool.name, tool) - return app - - for name in ("inputs", "runs", "work", "repo", "venv/bin"): - (tmp_path / name).mkdir(parents=True) - (tmp_path / "venv" / "bin" / "python").symlink_to(sys.executable) - cas = tmp_path / "inputs" / "cas" / "sha256" / _FAKE_SHA - cas.mkdir(parents=True) - (cas / "data.bin").write_bytes(_FAKE_PAYLOAD) - monkeypatch.setattr(plan_lib, "INPUTS_MOUNT", str(tmp_path / "inputs")) - monkeypatch.setattr(plan_lib, "RUNS_MOUNT", str(tmp_path / "runs")) - monkeypatch.setattr(plan_lib, "WORK_ROOT", str(tmp_path / "work")) - monkeypatch.setattr(plan_lib, "IMAGE_REPO_ROOT", str(tmp_path / "repo")) - monkeypatch.setattr(plan_lib, "IMAGE_VENV", str(tmp_path / "venv")) - monkeypatch.setattr( - app, - "_git_state", - lambda plan: { - "head_matches_plan": True, - "tree_clean": True, - "branch_verified": True, - "tool_present": True, - "tree_files_verified": True, - }, - ) - return install + return install_fake_runner(app, tmp_path, monkeypatch) def _new_container(tmp_path: Path) -> None: @@ -3002,7 +2996,8 @@ def test_a_stage_passes_its_token_to_the_tool_and_records_only_its_expiry_and_or "origin": TEST_COLLECTOR_ORIGIN, } assert telemetry["image_emitter_collector_origin"] == TEST_COLLECTOR_ORIGIN - assert telemetry["undelivered_events"] is None + # What the image emitter's `status` reported after the tool. + assert telemetry["undelivered_events"] == RECORDED_EMITTER_STATUS run_dir = tmp_path / "runs" / "runs" / "fake-run" for path in [*run_dir.rglob("*.json"), *run_dir.rglob("*.log")]: assert DUMMY_COLLECTOR_TOKEN not in path.read_text(), path @@ -3105,6 +3100,297 @@ def test_a_stage_whose_plan_opts_out_runs_without_a_token( assert receipt["plan"]["env"] == {"MICROCOSM_TELEMETRY_REQUIRED": "0"} +# The image emitter's `status`, after the tool ----------------------------- # + +# A stand-in tool that finishes at once and leaves, in its own session, a +# process that holds its output for 2 s and then writes a marker file: the +# place of the emitter a build starts, which delivers its last events after +# the build exits. +_EMITTER_HOLDER_TOOL = """\ +import pathlib, subprocess, sys +state, marker = pathlib.Path(sys.argv[1]), sys.argv[2] +(state / "out").mkdir(parents=True, exist_ok=True) +(state / "out" / "result.txt").write_text("done\\n") +hold = ( + "import pathlib, sys, time; time.sleep(2); " + "pathlib.Path(sys.argv[1]).write_text('finished')" +) +subprocess.Popen([sys.executable, "-c", hold, marker], start_new_session=True) +print("tool done", flush=True) +""" + + +def test_the_runner_asks_the_image_emitter_for_its_status_once_the_tool_and_its_emitter_finish( + fake_runner, tmp_path: Path, monkeypatch +) -> None: + marker = tmp_path / "emitter-finished" + app = fake_runner( + dataclasses.replace( + _fake_tool(), + argv_builder=lambda plan, paths, state: [ + "-c", + _EMITTER_HOLDER_TOOL, + state, + str(marker), + ], + ) + ) + answer = { + "runs": [ + { + "run_id": "fake-run.run", + "producer_id": "the-build", + "upload_state": "local_only", + "reason": "missing_huggingface_credential", + "undelivered_events": 4, + } + ], + "problem": None, + } + asked = [] + + def status(env: dict[str, str]) -> dict: + asked.append((marker.exists(), dict(env))) + return copy.deepcopy(answer) + + monkeypatch.setattr(app, "_emitter_status", status, raising=False) + receipt = app._run_stage(_fake_plan(max_wall_seconds=600)) + + assert receipt["status"] == "COMPLETED" + # Asked once, after the process holding the tool's output had finished. + assert [finished for finished, _ in asked] == [True] + # In the tool's environment, from which the build's emitter took the + # path of its event queue file. + ((_, env),) = asked + assert env["MICROCOSM_TELEMETRY_RUN_ID"] == "fake-run.run" + assert env["MICROCOSM_TELEMETRY_REQUIRED"] == "1" + assert env["XDG_CACHE_HOME"] == os.environ["XDG_CACHE_HOME"] + assert receipt["runner"]["telemetry"]["undelivered_events"] == answer + + +def _install_status_emitter( + monkeypatch, + directory: Path, + *, + stdout: str = "", + stderr: str = "", + exit_code: int = 0, + sleep_seconds: float = 0, +) -> Path: + """A stand-in for the image's emitter, at ``plan_lib.IMAGE_EMITTER_COMMAND``. + + It writes its arguments and environment as JSON to the returned file, + waits ``sleep_seconds``, prints ``stdout`` and ``stderr`` and exits with + ``exit_code``. + """ + + log = directory / "status-call.json" + command = directory / "microcosm-emitter" + command.write_text( + f"#!{sys.executable}\n" + "import json, os, sys, time\n" + f"with open({str(log)!r}, 'w') as log:\n" + " json.dump({'argv': sys.argv[1:], 'env': dict(os.environ)}, log)\n" + f"time.sleep({sleep_seconds!r})\n" + f"sys.stdout.write({stdout!r})\n" + f"sys.stderr.write({stderr!r})\n" + f"sys.exit({exit_code!r})\n" + ) + command.chmod(0o755) + monkeypatch.setattr(plan_lib, "IMAGE_EMITTER_COMMAND", str(command)) + return log + + +@pytest.fixture +def status_runner(tmp_path: Path, monkeypatch): + """The fake runner with the app's own ``_emitter_status``.""" + + app = load_app(monkeypatch, real_emitter_status=True) + return install_fake_runner(app, tmp_path, monkeypatch)(_fake_tool()) + + +_PRINTED_RUNS = [ + { + "run_id": "fake-run.run", + "producer_id": "the-build", + "upload_state": "local_only", + "reason": "run_registration_rejected", + "undelivered_events": 2, + }, + { + "run_id": "fake-run.run", + "producer_id": "the-child-stage", + "upload_state": "pending", + "reason": None, + "undelivered_events": 0, + }, +] +_NO_STATUS_COMMAND = ( + "usage: microcosm-emitter [-h] {host,version,telemetry-token} ...\n" + "microcosm-emitter: error: argument command: invalid choice: 'status' " + "(choose from host, version, telemetry-token)\n" +) + + +@pytest.mark.parametrize( + ("answer", "recorded"), + [ + ( + {"stdout": json.dumps({"runs": _PRINTED_RUNS}) + "\n"}, + {"runs": _PRINTED_RUNS, "problem": None}, + ), + ({"stdout": '{"runs": []}\n'}, {"runs": [], "problem": None}), + ( + { + "stderr": "No event queue file exists at /root/.cache/microcosm/" + "telemetry/events.sqlite3; nothing was read.\n", + "exit_code": 8, + }, + {"runs": [], "problem": None}, + ), + ( + { + "stderr": "The file at /root/.cache/microcosm/telemetry/" + "events.sqlite3 is not an event queue file.\n", + "exit_code": 9, + }, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 9: The file at " + "/root/.cache/microcosm/telemetry/events.sqlite3 is not an event " + "queue file.", + }, + ), + ( + {"stderr": _NO_STATUS_COMMAND, "exit_code": 2}, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 2: microcosm-emitter: " + "error: argument command: invalid choice: 'status' (choose from " + "host, version, telemetry-token)", + }, + ), + ( + {"exit_code": 1}, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 1 without a message", + }, + ), + ( + {"stdout": "runs: 2\n"}, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but did not print " + "its JSON list of runs", + }, + ), + ( + { + "stdout": json.dumps( + {"runs": [{**_PRINTED_RUNS[0], "undelivered_events": "2"}]} + ) + }, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but did not print " + "its JSON list of runs", + }, + ), + ( + {"stdout": json.dumps({"runs": [{"run_id": "fake-run.run"}]})}, + { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but did not print " + "its JSON list of runs", + }, + ), + ], + ids=[ + "runs", + "no runs", + "no event queue file", + "an unusable file", + "an emitter without status", + "a failure without a message", + "not JSON", + "a count that is not a number", + "a run without its fields", + ], +) +def test_the_receipt_records_what_the_image_emitter_s_status_answered( + status_runner, tmp_path: Path, monkeypatch, answer: dict, recorded: dict +) -> None: + _install_status_emitter(monkeypatch, tmp_path, **answer) + + receipt = status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + + assert receipt["runner"]["telemetry"]["undelivered_events"] == recorded + # Whatever `status` answered, the stage's result is the tool's. + assert receipt["status"] == "COMPLETED" + assert [item["outcome"] for item in _records(tmp_path)] == ["receipt"] + + +def test_a_problem_line_is_one_bounded_line( + status_runner, tmp_path: Path, monkeypatch +) -> None: + message = "The emitter\tsays " + "x" * 2000 + "\r\n" + _install_status_emitter(monkeypatch, tmp_path, stderr=message, exit_code=9) + + receipt = status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + + undelivered = receipt["runner"]["telemetry"]["undelivered_events"] + assert isinstance(undelivered, dict) + problem = undelivered["problem"] + assert problem.startswith("`microcosm-emitter status` exited 9: The emitter says x") + assert len(problem) == 300 + assert problem.endswith("…") + assert not {"\n", "\r", "\t"} & set(problem) + + +def test_the_receipt_records_a_status_that_hangs_or_cannot_start( + status_runner, tmp_path: Path, monkeypatch +) -> None: + monkeypatch.setattr(plan_lib, "EMITTER_STATUS_TIMEOUT_SECONDS", 1, raising=False) + _install_status_emitter( + monkeypatch, tmp_path, stdout='{"runs": []}', sleep_seconds=5 + ) + hung = status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + monkeypatch.setattr(plan_lib, "IMAGE_EMITTER_COMMAND", str(tmp_path / "absent")) + _new_container(tmp_path) + absent = status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + + assert hung["runner"]["telemetry"]["undelivered_events"] == { + "runs": None, + "problem": "`microcosm-emitter status` did not answer within 1 s", + } + assert absent["runner"]["telemetry"]["undelivered_events"] == { + "runs": None, + "problem": "`microcosm-emitter status` could not run (FileNotFoundError)", + } + assert (hung["status"], absent["status"]) == ("COMPLETED", "COMPLETED") + + +def test_the_image_emitter_s_status_gets_the_tool_s_environment_without_credentials( + status_runner, tmp_path: Path, monkeypatch +) -> None: + monkeypatch.setenv("HF_TOKEN", "hf_secret_not_a_credential") + log = _install_status_emitter(monkeypatch, tmp_path, stdout='{"runs": []}') + + status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + + assert log.is_file(), "the image emitter's status did not run" + call = json.loads(log.read_text()) + # The emitter's own default path, from the tool's environment. + assert call["argv"] == ["status"] + env = call["env"] + assert env["MICROCOSM_TELEMETRY_RUN_ID"] == "fake-run.run" + assert env["XDG_CACHE_HOME"] == os.environ["XDG_CACHE_HOME"] + assert env["PATH"] == os.environ["PATH"] + assert not {"HF_TOKEN", *RUNNER_TOKEN_SECRET, *COLLECTOR_TOKEN_NAMES} & set(env) + assert not any(DUMMY_COLLECTOR_TOKEN in value for value in env.values()) + + def test_check_stage_reports_whether_the_image_s_emitter_can_take_a_token( fake_runner, monkeypatch ) -> None: @@ -3189,18 +3475,29 @@ def test_a_plan_may_not_choose_the_emitter_the_build_starts() -> None: ) -def test_every_telemetry_record_has_the_same_fields() -> None: +def test_every_telemetry_record_has_the_same_fields(fake_runner) -> None: plan = plan_lib.parse_plan(_smoke_plan_data()) fields = set(plan_lib.TelemetryRecord.__annotations__) summary = plan_lib.summarize(plan)["telemetry"] check, _ = plan_lib.telemetry_check_report(plan, RECORDED_EMITTER_VERSION) - receipt, _ = plan_lib.telemetry_check( + before_the_tool, _ = plan_lib.telemetry_check( plan, _telemetry_environment(), RECORDED_EMITTER_VERSION, container_started=time.time(), ) - assert set(summary) == set(check) == set(receipt) == fields + receipt = fake_runner(_fake_tool())._run_stage(_fake_plan(max_wall_seconds=600)) + after_the_tool = receipt["runner"]["telemetry"] + assert set(summary) == set(check) == set(before_the_tool) == set(after_the_tool) + assert set(after_the_tool) == fields + # No emitter has been asked before the tool runs; the receipt holds what + # the image emitter's `status` answered after it. + assert summary["undelivered_events"] is None + assert check["undelivered_events"] is None + assert before_the_tool["undelivered_events"] is None + undelivered = after_the_tool["undelivered_events"] + assert isinstance(undelivered, dict) + assert set(undelivered) == {"runs", "problem"} def test_the_tool_gets_the_runner_s_token_values_stripped() -> None: diff --git a/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py new file mode 100644 index 0000000000..0c0ce38b08 --- /dev/null +++ b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py @@ -0,0 +1,193 @@ +"""The Modal stage runner records what the installed emitter left undelivered. + +The runner's other tests stand in for the image emitter's ``status``. These run +a stage through the runner's own ``_run_stage`` against local directories, with +the separately installed ``microcosm-emitter`` at the image's emitter path: the +stage's tool starts it through the interface with its telemetry module, as a +build does, and the runner then runs its ``status`` and records the answer in +the receipt. + +The tool's plan opts out of telemetry (``MICROCOSM_TELEMETRY_REQUIRED=0``), so +the emitter is given no collector token, and the test environment holds no +Hugging Face credential. The emitter therefore keeps the run's events in its +event queue file, as ``local_only``, and sends nothing. Its collector origin is +a closed loopback port all the same, so that nothing could reach the collector. + +``tools/run_installed_emitter_tests.sh`` installs the minimum release builds +accept and the newest pinned release, and runs this file once for each, with +``MICROCOSM_EMITTER_COMMAND`` naming the installed command. +""" + +from __future__ import annotations + +import dataclasses +import os +import shutil +import socket +import sysconfig +from pathlib import Path + +import pytest + +from test_support.microcosm_build.us_modal_stage_plan_tool import ( + fake_plan, + fake_tool, + install_fake_runner, + load_app, + plan_lib, +) + +#: The telemetry run a stage of the fake plan reports under. +RUN_ID = "fake-run.run" +PRODUCER_ID = "installed-emitter-status-test" +#: The fake runner's venv Python is a link to this interpreter, which then +#: does not find this environment's packages; a tool that imports the +#: interface puts them on its path, the first argument after its state. +_SITE_PACKAGES = sysconfig.get_paths()["purelib"] +#: A tool that starts the emitter with its telemetry module, records two +#: stages and the build's completion, and closes it, as a build program does. +#: The emitter, in its own session, holds the tool's output until it has +#: finished, so the runner asks for its status only then. +_EMITTING_TOOL = """\ +import pathlib, sys + +state, site_packages, collector, run_id, producer_id = sys.argv[1:6] +sys.path.insert(0, site_packages) +from microcosm_emitter_interface import EmitterClient, TelemetryRun, TelemetrySettings + +run = TelemetryRun( + run_id=run_id, + country_code="US", + pipeline="installed_emitter_status", + producer_id=producer_id, +) +client = EmitterClient.start( + telemetry=TelemetrySettings( + run=run, development_collector_url=collector, heartbeat_seconds=3600 + ) +) +if not client.available: + sys.exit("the installed emitter did not start") +client.telemetry.stage("first_stage") +client.telemetry.stage("second_stage") +client.telemetry.complete() +client.close() +(pathlib.Path(state) / "out").mkdir(parents=True, exist_ok=True) +(pathlib.Path(state) / "out" / "result.txt").write_text("done\\n") +""" +#: A tool that leaves a file that is not an event queue file where the +#: emitter would keep one, and starts no emitter. +_UNUSABLE_FILE_TOOL = """\ +import pathlib, sys + +sys.path.insert(0, sys.argv[2]) +from microcosm_emitter_interface import default_spool_path + +path = default_spool_path() +path.parent.mkdir(parents=True, exist_ok=True) +path.write_bytes(b"not an event queue file") +(pathlib.Path(sys.argv[1]) / "out").mkdir(parents=True, exist_ok=True) +(pathlib.Path(sys.argv[1]) / "out" / "result.txt").write_text("done\\n") +""" + + +@pytest.fixture +def installed_command() -> str: + command = os.environ.get("MICROCOSM_EMITTER_COMMAND", "").strip() + if not command or shutil.which(command) is None: + pytest.fail( + "Run these tests with tools/run_installed_emitter_tests.sh, which sets " + "MICROCOSM_EMITTER_COMMAND." + ) + return command + + +@pytest.fixture +def runner(installed_command: str, tmp_path: Path, monkeypatch): + """The runner's app, with the installed emitter at the image's emitter path. + + Only ``status`` runs at that path; ``version`` keeps its recorded answer. + """ + + app = load_app(monkeypatch, real_emitter_status=True) + monkeypatch.setattr(plan_lib, "IMAGE_EMITTER_COMMAND", installed_command) + return install_fake_runner(app, tmp_path, monkeypatch) + + +def _closed_loopback_origin() -> str: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + return f"http://127.0.0.1:{port}" + + +def _tool(script: str, *arguments: str) -> plan_lib.ToolSpec: + return dataclasses.replace( + fake_tool(), + argv_builder=lambda plan, paths, state: ["-c", script, state, *arguments], + ) + + +def _opted_out_plan() -> dict: + return fake_plan(max_wall_seconds=600, env={"MICROCOSM_TELEMETRY_REQUIRED": "0"}) + + +def test_the_runner_records_the_events_the_installed_emitter_still_holds( + runner, +) -> None: + app = runner( + _tool( + _EMITTING_TOOL, + _SITE_PACKAGES, + _closed_loopback_origin(), + RUN_ID, + PRODUCER_ID, + ) + ) + + receipt = app._run_stage(_opted_out_plan()) + + assert receipt["status"] == "COMPLETED" + undelivered = receipt["runner"]["telemetry"]["undelivered_events"] + assert undelivered == { + "runs": [ + { + "run_id": RUN_ID, + "producer_id": PRODUCER_ID, + "upload_state": "local_only", + "reason": "missing_huggingface_credential", + "undelivered_events": 4, + } + ], + "problem": None, + } + + +def test_the_runner_records_no_runs_when_no_emitter_kept_an_event_queue_file( + runner, +) -> None: + app = runner(fake_tool()) + + receipt = app._run_stage(_opted_out_plan()) + + assert receipt["status"] == "COMPLETED" + assert receipt["runner"]["telemetry"]["undelivered_events"] == { + "runs": [], + "problem": None, + } + + +def test_the_runner_records_the_installed_emitter_s_refusal_of_an_unusable_file( + runner, +) -> None: + app = runner(_tool(_UNUSABLE_FILE_TOOL, _SITE_PACKAGES)) + + receipt = app._run_stage(_opted_out_plan()) + + assert receipt["status"] == "COMPLETED" + path = Path(os.environ["XDG_CACHE_HOME"]) / "microcosm/telemetry/events.sqlite3" + assert receipt["runner"]["telemetry"]["undelivered_events"] == { + "runs": None, + "problem": f"`microcosm-emitter status` exited 9: The file at {path} is " + "not an event queue file.", + } diff --git a/test_support/microcosm_build/emitter.py b/test_support/microcosm_build/emitter.py index 0912e5f908..9c0e7d7b6b 100644 --- a/test_support/microcosm_build/emitter.py +++ b/test_support/microcosm_build/emitter.py @@ -150,7 +150,7 @@ class FakeEmitterClient: as the interface does after a timeout. """ - emitter_version = "0.11.9" + emitter_version = "0.13.0" def __init__( self, diff --git a/test_support/microcosm_build/us_modal_stage_plan_tool.py b/test_support/microcosm_build/us_modal_stage_plan_tool.py index d0fa68c3a4..f194ce6d39 100644 --- a/test_support/microcosm_build/us_modal_stage_plan_tool.py +++ b/test_support/microcosm_build/us_modal_stage_plan_tool.py @@ -3,9 +3,12 @@ from __future__ import annotations import copy +import hashlib import importlib.util import json import sys +from collections.abc import Callable +from pathlib import Path from test_support.paths import paths_for @@ -39,10 +42,25 @@ def _load(): #: What the image's emitter answers to ``microcosm-emitter version``. Tests #: get this answer instead of running a command at the image's emitter path. RECORDED_EMITTER_VERSION = { - "emitter_version": "0.11.9", + "emitter_version": "0.13.0", "protocol_versions": [1], "collector_origin": TEST_COLLECTOR_ORIGIN, } +#: What the runner records of the image emitter's ``microcosm-emitter status`` +#: after the tool. Tests get this record instead of running a command at the +#: image's emitter path. +RECORDED_EMITTER_STATUS = { + "runs": [ + { + "run_id": "fake-run.run", + "producer_id": "recorded-producer", + "upload_state": "pending", + "reason": "collector_token_expired", + "undelivered_events": 3, + } + ], + "problem": None, +} #: A made-up collector token. Tests never use a real credential. DUMMY_COLLECTOR_TOKEN = "dummy-collector-token-not-a-credential" #: Long after any test runs. @@ -56,7 +74,12 @@ def _load(): } -def load_app(monkeypatch, *, real_emitter_version: bool = False): +def load_app( + monkeypatch, + *, + real_emitter_version: bool = False, + real_emitter_status: bool = False, +): """``tools/modal_us_stage.py`` imported against a stub ``modal`` module. The stub answers ``is_local()`` with False, so the module defines its @@ -64,10 +87,12 @@ def load_app(monkeypatch, *, real_emitter_version: bool = False): calls (commit, reload) are recorded, not performed. Each ``@app.function`` keeps its function and records its Modal options as ``modal_options``. ``_emitter_version`` answers - ``RECORDED_EMITTER_VERSION``, so no test runs an emitter installed at the + ``RECORDED_EMITTER_VERSION`` and ``_emitter_status`` answers + ``RECORDED_EMITTER_STATUS``, so no test runs an emitter installed at the image's path (``/usr/local/bin/microcosm-emitter``) on the test machine; - ``real_emitter_version`` keeps the module's own, for a test that replaces - what it runs. The environment holds ``RUNNER_TOKEN_SECRET``, as a + ``real_emitter_version`` and ``real_emitter_status`` keep the module's + own, for a test that replaces what they run. The environment holds + ``RUNNER_TOKEN_SECRET``, as a container launched with ``--run`` does; a test of a container without the token removes it. """ @@ -97,6 +122,13 @@ def load_app(monkeypatch, *, real_emitter_version: bool = False): "_emitter_version", lambda: copy.deepcopy(RECORDED_EMITTER_VERSION), ) + if not real_emitter_status: + monkeypatch.setattr( + module, + "_emitter_status", + lambda env: copy.deepcopy(RECORDED_EMITTER_STATUS), + raising=False, + ) for name, value in RUNNER_TOKEN_SECRET.items(): monkeypatch.setenv(name, value) return module @@ -169,3 +201,113 @@ def smoke_plan_data() -> dict: }, }, } + + +# --------------------------------------------------------------------------- # +# The run path end to end, against local directories # +# --------------------------------------------------------------------------- # + +#: A stand-in for a resumable tool: its first run writes a checkpoint, a +#: later run of the same state finds it and only rewrites the output. +FAKE_TOOL_SCRIPT = """\ +import os, pathlib, sys, time +state, data = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +ckpt = state / "ckpt" / "step1.bin" +resumed = ckpt.exists() +if not resumed: + ckpt.parent.mkdir(parents=True, exist_ok=True) + ckpt.write_bytes(data.read_bytes() * 3) +(state / "out").mkdir(parents=True, exist_ok=True) +(state / "out" / "result.txt").write_text(f"resumed={resumed}\\n") +print("fake tool", "resumed" if resumed else "fresh", flush=True) +time.sleep(float(os.environ.get("MICROCOSM_FAKE_SLEEP", "0"))) +""" +FAKE_PAYLOAD = b"fake input" +FAKE_SHA = hashlib.sha256(FAKE_PAYLOAD).hexdigest() + + +def fake_tool( + timeout_s: int = 3600, overhead: int = 60, **stage: object +) -> plan_lib.ToolSpec: + """A registered tool whose one stage, ``run``, runs ``FAKE_TOOL_SCRIPT``.""" + + resources = plan_lib.Resources("check", 2.0, 8 * 1024, timeout_s=timeout_s) + return plan_lib.ToolSpec( + name="fake-stage", + script=None, + inputs=("data",), + stages={ + "run": plan_lib.StageSpec( + "run", + resources, + ("data",), + runner_overhead_seconds=overhead, + **stage, # type: ignore[arg-type] + ) + }, + options={}, + owned_flags=frozenset(), + argv_builder=lambda plan, paths, state: [ + "-c", + FAKE_TOOL_SCRIPT, + state, + paths["data"], + ], + ) + + +def fake_plan(**overrides) -> dict: + """A plan for ``fake_tool``'s stage, run ``fake-run``, with top-level overrides.""" + + data = { + "schema": plan_lib.PLAN_SCHEMA, + "tool": "fake-stage", + "stage": "run", + "run_id": "fake-run", + "source": {"commit": "a" * 40, "branch": "main"}, + "inputs": { + "data": { + "uri": f"volume://cas/sha256/{FAKE_SHA}/data.bin", + "sha256": FAKE_SHA, + } + }, + } + data.update(overrides) + return data + + +def install_fake_runner(app, tmp_path: Path, monkeypatch) -> Callable: + """Point ``app``'s mounts, clone and venv at local directories under ``tmp_path``. + + Returns ``install(tool)``, which registers ``tool`` and returns ``app``. + The venv's Python is this interpreter, and the clone's git state is a + stand-in that passes every check. + """ + + def install(tool: plan_lib.ToolSpec): + monkeypatch.setitem(plan_lib.TOOLS, tool.name, tool) + return app + + for name in ("inputs", "runs", "work", "repo", "venv/bin"): + (tmp_path / name).mkdir(parents=True) + (tmp_path / "venv" / "bin" / "python").symlink_to(sys.executable) + cas = tmp_path / "inputs" / "cas" / "sha256" / FAKE_SHA + cas.mkdir(parents=True) + (cas / "data.bin").write_bytes(FAKE_PAYLOAD) + monkeypatch.setattr(plan_lib, "INPUTS_MOUNT", str(tmp_path / "inputs")) + monkeypatch.setattr(plan_lib, "RUNS_MOUNT", str(tmp_path / "runs")) + monkeypatch.setattr(plan_lib, "WORK_ROOT", str(tmp_path / "work")) + monkeypatch.setattr(plan_lib, "IMAGE_REPO_ROOT", str(tmp_path / "repo")) + monkeypatch.setattr(plan_lib, "IMAGE_VENV", str(tmp_path / "venv")) + monkeypatch.setattr( + app, + "_git_state", + lambda plan: { + "head_matches_plan": True, + "tree_clean": True, + "branch_verified": True, + "tool_present": True, + "tree_files_verified": True, + }, + ) + return install From 7271775fb6b69c3f9310c70bf7d0a4d2f03ad4e4 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:27:38 +0400 Subject: [PATCH 2/9] Test that the write probe keeps 60 s for the image emitter's status The runner's reserve after the tool now also covers the image emitter's `status`, so the mirror's share of the base's 30 minutes is 1,560 s (#1522). Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_modal_stage_puf_support_base.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py index 23690d4d4e..b0e6be0946 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py @@ -2291,22 +2291,24 @@ def test_a_finished_base_pushes_only_its_output_and_evidence(tmp_path: Path) -> def test_write_probe_verdict_sizes_the_mirror_against_the_reserve() -> None: stage = BASE.stages["all"] gib = 1024**3 - # The reserve less the stop's grace and the time kept after the mirror + # The reserve less the stop's grace, the 60 s the image emitter's + # `status` may take after the tool, and the time kept after the mirror # for the runner identity, the receipt and the final commit. reserve = ( plan_lib.BASE_RUNNER_OVERHEAD_SECONDS - plan_lib.STOP_GRACE_SECONDS + - 60 - plan_lib.RECEIPT_RESERVE_SECONDS ) - # The slowest rate that still mirrors 50 GiB inside that (about 33 MB/s). + # The slowest rate that still mirrors 50 GiB inside that (about 34 MB/s). floor = 50 * gib / reserve - assert floor / 1e6 == pytest.approx(33.1, abs=0.1) + assert floor / 1e6 == pytest.approx(34.4, abs=0.1) fast = plan_lib.write_probe_verdict(stage, gib, gib / (floor * 1.01)) assert "problem" not in fast - assert fast["mirror_reserve_seconds"] == reserve == 1620 + assert fast["mirror_reserve_seconds"] == reserve == 1560 assert fast["receipt_reserve_seconds"] == 120 slow = plan_lib.write_probe_verdict(stage, gib, gib / (floor * 0.99)) - assert "more than the 1620s the runner keeps for the mirror" in slow["problem"] + assert "more than the 1560s the runner keeps for the mirror" in slow["problem"] # The runbook's one measured volume rate (reads, at least 58 MB/s) fits. assert "problem" not in plan_lib.write_probe_verdict(stage, 58_000_000, 1.0) acs = plan_lib.US_ACS_LOCAL_RELEASE.stages["materialize"] From b89e8a4abfb30dd3a7ef147b94d339763727bb65 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:33:49 +0400 Subject: [PATCH 3/9] Record the image emitter's status after each Modal stage After the tool exits and its output ends, the runner runs the image emitter's read-only `microcosm-emitter status` in the tool's environment without credentials, with a 60 s timeout that the reserve now keeps, and records its runs, an empty list for exit 8, or one problem line in runner.telemetry.undelivered_events; a failure never changes the stage's result. Builds and the image take microcosm-emitter 0.13.0 or later from one packaged file, minimum_emitter_version.json, and the installed-emitter pin rises to 0.13.0 (#1522). Co-Authored-By: Claude Opus 5.5 --- README.md | 19 +- .../1522-modal-undelivered-events.added.md | 1 + docs/agent-guide.md | 8 +- docs/orrery-publication.md | 45 ++-- docs/us-modal-stage-runbook.md | 90 +++++--- .../build/minimum_emitter_version.json | 3 + .../src/microcosm/build/telemetry_emitter.py | 56 +++-- .../us_modal_stage_plan_tool.py | 1 - tools/installed-emitter/requirements.txt | 2 +- tools/modal_us_stage.py | 45 +++- tools/modal_us_stage_plan.py | 207 ++++++++++++++++-- 11 files changed, 384 insertions(+), 93 deletions(-) create mode 100644 changelog.d/1522-modal-undelivered-events.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json diff --git a/README.md b/README.md index 885e7c1f00..58061201a9 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ change: ```bash uv tool install microcosm-emitter # once per machine uv tool upgrade microcosm-emitter # to take a new release -microcosm-emitter version # the installed release; builds need 0.11.9 or later +microcosm-emitter version # the installed release; builds need 0.13.0 or later ``` The build finds the `microcosm-emitter` command through @@ -76,15 +76,20 @@ no remote staging, and queues no Orrery graph publication; a UK build given its dataset bundle remotely, as it is by default, is refused before it starts, since the emitter uploads the bundle. Before it starts the emitter, the build reads its version from `microcosm-emitter version` and does not -start one older than 0.11.9. Releases before 0.10.0 send telemetry to the +start one older than 0.13.0, the release that added `microcosm-emitter +status`, which the US Modal stage runner runs after each stage to record how +many telemetry events the stage's emitter left undelivered. The release is +written once, in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`, +which builds read as package data and the Modal image's install step reads +from the launching checkout. Releases before 0.10.0 send telemetry to the collector's Cloud Run address, which changes when the collector moves to another Google Cloud project; from 0.10.0 on, releases send to the -collector's hostname, which stays the same. 0.10.0 would be enough for that; -0.11.9 was the newest release when practically no machine ran the emitter -yet, so requiring it cost nothing (see +collector's hostname, which stays the same. 0.13.0 was the newest release +when practically no machine ran the emitter yet, so requiring it cost +nothing (see [graph publication](docs/orrery-publication.md#emitter-dependency)). The build prints the installed and required versions, what it loses and the -upgrade command, `uv tool install --upgrade 'microcosm-emitter>=0.11.9'`, +upgrade command, `uv tool install --upgrade 'microcosm-emitter>=0.13.0'`, and continues as it does without the command. `uv tool upgrade` keeps any version constraint given at install time, such as `==0.7.0`; that `uv tool install --upgrade` command replaces it. Microcosm supplies build @@ -107,7 +112,7 @@ if the emitter cannot be prepared or is missing, its `microcosm-emitter version` times out or fails, it is older than the minimum, it lists no protocol version 1, its `hello` fails, it exits or times out before it is ready, or it starts without its telemetry module. -Every emitter a build starts (0.11.9 and later, the minimum) reads a given +Every emitter a build starts (0.13.0 and later, the minimum) reads a given collector token, and exits at that point when the collector-token variables it inherits (`MICROCOSM_COLLECTOR_TOKEN`, `MICROCOSM_COLLECTOR_TOKEN_EXPIRES_AT`, `MICROCOSM_COLLECTOR_ORIGIN`) are set diff --git a/changelog.d/1522-modal-undelivered-events.added.md b/changelog.d/1522-modal-undelivered-events.added.md new file mode 100644 index 0000000000..3bc2d98586 --- /dev/null +++ b/changelog.d/1522-modal-undelivered-events.added.md @@ -0,0 +1 @@ +The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run or output that is not its list of runs. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index c21f982c5b..c1091bce07 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -135,7 +135,13 @@ without `hf_xet`, or with it turned off, for files sent in one part. A request it has no route for, or that its handler fails on, fails the test. The tests check what builds rely on in the hand-over (one commit per bundle on the head it names, `copied_files`, `already_staged`, the recorded error codes and the -version check), not the emitter's own queue or delivery. The runner warns when +version check), not the emitter's own queue or delivery. +`test_installed_emitter_status.py` runs a stage through the US Modal runner's +`_run_stage` against local directories, with the installed command at the +image's emitter path: its tool starts the emitter's telemetry module through +the interface, with no credential, no collector token and a closed loopback +collector address, so the emitter keeps its events and sends nothing, and the +test checks what the runner records from the emitter's `status`. The runner warns when the package index has a newer release than the pin; raise the pin in a pull request then. `tools/installed-emitter/constraints.txt` pins the `huggingface_hub` release whose requests the stand-in answers; the runner also diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index 148a8c9c55..659921e9a8 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -92,7 +92,7 @@ can be rebuilt from it later. This does not hold for: | `MICROCOSM_EMITTER_COMMAND` | Optional path of the `microcosm-emitter` command, read by the interface in the build; without it the interface searches `PATH` | Operator's shell; the Modal image sets it | | Collector origin | Built into the installed emitter; the build sends no address unless a caller passes a loopback development address in the module settings | No producer variable or destination override | | Runs origin | Tracked `DEFAULT_GRAPH_PUBLICATION_ORIGIN` | No new producer variable | -| `XDG_CACHE_HOME` | Optional existing cache-location setting, read by the interface in the build; the build passes the spool path (`microcosm/telemetry/events.sqlite3`) and the hand-over database beside it (`events.handovers.sqlite3`) to the emitter on its standard input | Defaults to the user's cache directory | +| `XDG_CACHE_HOME` | Optional existing cache-location setting, read by the interface in the build; the build passes the path of the emitter's event queue file (`microcosm/telemetry/events.sqlite3`) and the hand-over database beside it (`events.handovers.sqlite3`) to the emitter on its standard input. On Modal, the stage runner's `microcosm-emitter status` reads it, with `HOME`, in the tool's environment after the tool, so it reads the file the build's emitter used | Defaults to the user's cache directory; on Modal, the container's defaults (the runner sets neither variable, and a plan may not) | | `MICROCOSM_TELEMETRY_REQUIRED` | Optional, read by the build adapter (`microcosm.build.telemetry_emitter`) each time a build starts its emitter; only builds that start the adapter read it. `1`: the build stops with `TelemetryRequiredError` if the emitter cannot be prepared or is missing, its `version` times out or fails, it is older than `MINIMUM_EMITTER_VERSION`, it lists no protocol version 1, its `hello` fails, it exits or times out before it is ready, or it starts without its telemetry module (for a failure the interface reports, its line saying the build will continue does not apply); unset or `0`: it continues without the emitter; any other value: `ValueError` | Whoever starts the build. The Modal stage runner sets `1` unless the plan sets `0`; workstations: the operator's shell. Leaving it off (unset or `0`) exists only for people outside PolicyEngine who run Microcosm without access to the collector; PolicyEngine's own builds keep telemetry required. Not a credential | | `MICROCOSM_TELEMETRY_RUN_ID` | Optional, read by the US build programs that open a run through `microcosm.build.program_telemetry`: the telemetry run ID they report under; the part before its last `.` becomes the run's candidate ID. Unset, each attempt makes its own ID (`--<8 hex digits>`); a value the collector would refuse stops the program with `ValueError` before any work | The Modal stage runner sets `.`, and a plan may not set it; workstations: the operator's shell, usually unset. Not a credential | | `MICROCOSM_TELEMETRY_PARENT_PID` | Read by the same programs: when it equals the process's parent's ID, the process is a child stage whose parent records its telemetry, and it starts no emitter | Set by `build_us_puf_support_base.py --stage all` for the child interpreters it starts; a plan may not set it. Not a credential | @@ -162,12 +162,16 @@ versions interfaces still send, and an emitter that lacks a module the build asks for starts without it. A build does not start an emitter older than `MINIMUM_EMITTER_VERSION` in -`microcosm.build.telemetry_emitter`, now 0.11.9, and the Modal image installs -at least the same release (`IMAGE_EMITTER_REQUIREMENT` in -`tools/modal_us_stage_plan.py`, `microcosm-emitter>=0.11.9`). A test checks -that the two agree in one checkout; at a Modal launch the install step comes -from the launching checkout and the minimum from the commit the plan pins -(see the [US Modal stage runbook](us-modal-stage-runbook.md)). Releases +`microcosm.build.telemetry_emitter`, now 0.13.0, and the Modal image installs +at least the same release (`image_emitter_requirement()` in +`tools/modal_us_stage_plan.py`, `microcosm-emitter>=0.13.0`). The release is +written once, in microcosm-build's package data, +`packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: +the adapter reads it as package data, and the runner, which imports nothing +from the workspace, reads it by its path in the checkout that launches the +app. At a Modal launch the install step therefore comes from the launching +checkout and the minimum from the commit the plan pins (see the +[US Modal stage runbook](us-modal-stage-runbook.md)). Releases before 0.10.0 send telemetry to the collector's Cloud Run address, which changes when the collector moves to another Google Cloud project. From 0.10.0 on, the emitter's default collector address is the collector's hostname, @@ -175,10 +179,14 @@ which stays the same. Microcosm passes the emitter no collector address: the adapter's `development_collector_url` parameter, which no build command sets, accepts only a loopback address. -0.10.0 would be enough for the hostname. The minimum is 0.11.9, the newest -release on 2026-11-02, because practically no machine ran the separately -installed emitter then, so refusing 0.10.0 to 0.11.8 cost nothing. A refused -emitter does cost the build: +The minimum is 0.13.0, the newest release on 2026-11-05. 0.13.0 added +`microcosm-emitter status`, which the US Modal stage runner runs after each +stage to record how many telemetry events the stage's emitter left +undelivered; builds require the same release so that it is written once. +Practically no machine ran the separately installed emitter then, so +refusing 0.11.9 to 0.12.5 cost nothing; the minimum before it, 0.11.9, came +from the collector's hostname, for which 0.10.0 would have been enough. A +refused emitter does cost the build: - no hosted telemetry, and no staging hand-overs (the staging run files are still written locally); @@ -190,9 +198,12 @@ emitter does cost the build: do by default, is refused before it starts, and the refusal gives the upgrade command built from the minimum. -The build's warning says so. Raise the minimum and the image's floor -together, and before raising them while machines still run older emitters, -weigh these losses against having each such machine upgrade first. +The build's warning says so. Raising the minimum in that one file raises +the image's floor too; before raising it while machines still run older +emitters, weigh these losses against having each such machine upgrade +first. Raise the installed-emitter job's pin +(`tools/installed-emitter/requirements.txt`) with it, so that the job never +tests a release older than the minimum. To adopt a new interface release, regenerate `uv.lock`, update `APPROVED_UV_LOCK_SHA256` in `microcosm.build.us_runtime.worker_identity` to @@ -200,7 +211,7 @@ the new lockfile digest and rerun the tests and wheel checks. ## Migrating an existing workstation -Install the emitter once with `uv tool install microcosm-emitter` (0.11.9 or +Install the emitter once with `uv tool install microcosm-emitter` (0.13.0 or later; a build does not start an older emitter, says how to upgrade it and continues without it). It keeps using the existing spool database, `$XDG_CACHE_HOME/microcosm/telemetry/events.sqlite3`, at the same Alembic @@ -213,7 +224,7 @@ modification. The emitter's README describes the repair for a machine that ran an earlier, unreleased build of the hand-over queue. No new credential, hosted service or infrastructure is required by this migration. -To upgrade a machine whose emitter is older than 0.11.9, run -`uv tool install --upgrade 'microcosm-emitter>=0.11.9'`; it also replaces a +To upgrade a machine whose emitter is older than 0.13.0, run +`uv tool install --upgrade 'microcosm-emitter>=0.13.0'`; it also replaces a version constraint given at install time, which `uv tool upgrade` keeps. `microcosm-emitter version` prints the installed release. diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index ae7bcb64cb..eb866a97c6 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -85,7 +85,7 @@ release directory onto the runs volume. Publication stays the human step in and asserts `HEAD` equals the commit. It then runs `uv sync --all-packages --extra us --frozen` against that tree's own `uv.lock` into `/opt/venv`, installs the newest `microcosm-emitter` - release (0.11.9 or later, the oldest a build starts) on its own with + release (0.13.0 or later, the oldest a build starts) on its own with `uv tool install` into `/usr/local/bin`, checks that `microcosm-emitter version` runs, and asserts `git status --porcelain` is empty. Emitter releases from 0.10.0 on send telemetry to the collector's @@ -94,14 +94,17 @@ release directory onto the runs volume. Publication stays the human step in `MICROCOSM_EMITTER_COMMAND` to that command, so a build that starts its emitter finds it; the check reports the version and the receipt's runner identity records it. An image built for a commit keeps the emitter it was - built with. The install step and its `>=0.11.9` floor come from the - checkout that launches the app (`image_build_commands`), while the - minimum the build enforces comes from the commit the plan pins - (`MINIMUM_EMITTER_VERSION`), so a launch can combine an older checkout's - floor with a newer commit's minimum, or the reverse. Neither floor has an - upper bound, so an image built now installs the newest release, which - every commit's minimum so far accepts. The test that requires the floor to - equal the minimum compares the two values of one checkout only. The + built with. The release is written once, in microcosm-build's package + data (`packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`): + the build adapter reads it as `MINIMUM_EMITTER_VERSION`, and the runner + reads the same file by its path in the checkout. The install step and its + `>=0.13.0` floor come from the checkout that launches the app + (`image_build_commands`, `image_emitter_requirement`), while the minimum + the build enforces comes from the commit the plan pins, so a launch can + combine an older checkout's floor with a newer commit's minimum, or the + reverse. Neither floor has an upper bound, so an image built now installs + the newest release, which every commit's minimum so far accepts. A test + checks that both read the one file in one checkout. The tool's environment withholds credentials and sets `HF_HUB_OFFLINE=1`, so an emitter started there keeps its Hub hand-overs local; it sends telemetry with the collector token the launch @@ -188,13 +191,34 @@ release directory onto the runs volume. Publication stays the human step in - `runner.telemetry`: whether the build had to send telemetry, the collector token's expiry and origin (never its value), the collector origin the image's emitter reported, and the time the token had to - last until (`token_needed_until`). Its `undelivered_events` is always - `null` for now, a gap left on purpose: the count of events the emitter - left undelivered comes later. A read-only `status` command in - microcosm-emitter will report it from the emitter's event queue file, - and the runner will run that command in the container after the tool - and record the counts here. Until then the runner does not read the - event queue file itself. + last until (`token_needed_until`), and `undelivered_events`: what the + image emitter's read-only `microcosm-emitter status` (0.13.0 and + later) reported after the tool. The runner runs it once the tool has + exited and its output has ended, which happens only after the + emitter the build started has delivered what it could and stopped. It + runs without `--event-queue-file`, in the tool's environment without + credentials or the token, so it reads the file the build's emitter + kept, and gets at most 60 s (`EMITTER_STATUS_TIMEOUT_SECONDS`). The + runner never reads the emitter's event queue file itself. + - `{"runs": [...], "problem": null}`: `runs` is `status`'s list as it + printed it, one entry per run and producer, with `upload_state`, + `reason` (the run's `local_only_reason`, or `null`) and + `undelivered_events`, the events the file still holds. The emitter + also prunes undelivered events older than 7 days or beyond 100 MiB, + so 0 means none are held, not that every event was delivered. + - `{"runs": [], "problem": null}`: no event queue file (exit 8), so no + emitter kept one in the container, for example a tool that failed + before it started one. + - `{"runs": null, "problem": ""}`: any other exit code, with + the last line `status` wrote on standard error (exit 2 is an emitter + without `status`, older than 0.13.0); a timeout; a command that + could not run; or output that is not its list of runs. The line has + at most 300 characters. + + A `status` failure is recorded only: it never changes the stage's + status or outcome. The plan's summary and the check have + `undelivered_events: null`, since no tool has run. A receipt written + before this change has `null` too. A failed stage still mirrors its state, so `calibrate` can resume from `weights_latest.npz`, and its receipt says `FAILED`. @@ -305,9 +329,10 @@ release directory onto the runs volume. Publication stays the human step in - **An image emitter without a collector origin.** The image installs the newest emitter release when Modal builds its install step, and Modal reuses that step for a commit while the step's command is the - same. Since #1455 (merged 2026-11-03), the command asks for - `microcosm-emitter>=0.11.9`, so every image this runner builds has an - emitter that reports a collector origin. An image whose install step an + same. Since #1455 (merged 2026-11-03), the command asks for at least + 0.11.9, and since #1522 for `microcosm-emitter>=0.13.0`, so every image + this runner builds has an emitter that reports a collector origin and + has `status`. An image whose install step an older checkout built (with `>=0.7.0`, possibly before 0.11.0) is not reused: the changed command rebuilds that step, for each commit, at its first launch from a checkout with this runner. The refusal stays for an @@ -331,9 +356,9 @@ release directory onto the runs volume. Publication stays the human step in run Microcosm without access to the collector. PolicyEngine's own builds keep telemetry required, so PolicyEngine's plans do not set it. - **Events left undelivered.** Events the emitter has not sent when the - build ends are lost with the container. The receipt does not record - how many there were until the emitter's `status` command and the - runner's use of it land (see `runner.telemetry` above). + build ends are lost with the container. The receipt records how many + each run left, from the image emitter's `status` (see + `runner.telemetry` above); nothing delivers them later. - **On Modal:** a paid smoke run on 3 November 2026 verified the launch, the per-launch secret and the check before staging (see "Verified on Modal, 3 November 2026: the telemetry token"); it did not cover a @@ -602,8 +627,9 @@ Hub were not touched. - **Not covered: a container reaching the collector.** `runner-smoke` does not start the emitter, so nothing sent telemetry. That waits until the registered US programs start the emitter (the design's PR 6). The receipt - does not record the undelivered-event count, which comes later with the - emitter's `status` command (see `runner.telemetry`). + did not record the undelivered-event count then; since #1522 (on + 2026-11-05) a receipt records it from the emitter's `status` command (see + `runner.telemetry`). ## Acceptance attempt: 23 September materialize on the state surface @@ -975,9 +1001,11 @@ The plan is `docs/us-modal-stage-route-a-base-plan.json` (run the tool at whichever comes first: the plan's budget, or 14,700 seconds after the function started (the timeout less the reserve). A slow pull or staging therefore shortens the tool's time, never the reserve. The - reserve covers stopping the tool (up to 60 s), then hashing and mirroring - the state, then the runner's identity, the receipt and the attempt's - final commit (`RECEIPT_RESERVE_SECONDS`, 120 s, an allowance). After a + reserve covers stopping the tool (up to 60 s), the image emitter's + `status` (up to 60 s, `EMITTER_STATUS_TIMEOUT_SECONDS`), then hashing and + mirroring the state, then the runner's identity, the receipt and the + attempt's final commit (`RECEIPT_RESERVE_SECONDS`, 120 s, an allowance). + After a stop or a failure the push copies the whole state, about 50 GiB (`BASE_MIRRORED_STATE_GIB`): the local run's 44 GB of checkpoints and 2.35 GB H5, plus the final checkpoint's hard-linked alias @@ -986,9 +1014,9 @@ The plan is `docs/us-modal-stage-route-a-base-plan.json` (run - *The write probe.* No container-to-volume write rate has been measured, so the check measures one. It copies 1 GiB to the runs volume, commits it, deletes it, and reports `runs_volume_write_probe`. The check fails - when 50 GiB at that rate would not fit in the 1,620 s the reserve leaves - for the mirror (1,800 s less the stop's 60 s and the receipt's 120 s), - which means below about 33 MB/s. The only measured volume rate here is + when 50 GiB at that rate would not fit in the 1,560 s the reserve leaves + for the mirror (1,800 s less the stop's 60 s, `status`'s 60 s and the + receipt's 120 s), which means below about 34 MB/s. The only measured volume rate here is reads of at least 58 MB/s (the acceptance attempt above). Modal describes volume bandwidth as up to 2.5 GB/s and not guaranteed (modal.com/docs/guide/volumes, read 29 September 2026). One small probe @@ -1429,7 +1457,7 @@ archives. H5 (10 GiB), summary and logs, with about 2 GiB of headroom. These are planning allowances, not output limits. The check measures the runs volume's write rate and refuses if 12 GiB would not mirror in the - reserve's 1,620 s after the stop and receipt allowances. Both output + reserve's 1,560 s after the stop, `status` and receipt allowances. Both output files and logs are mirrored after clean exits and failures; the staging builder has no registered checkpoint resume. diff --git a/packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json b/packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json new file mode 100644 index 0000000000..032511f1c6 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json @@ -0,0 +1,3 @@ +{ + "minimum_emitter_version": "0.13.0" +} diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 708c282959..f8e4bd20a7 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -39,6 +39,7 @@ import subprocess import sys from collections.abc import Mapping, Sequence +from importlib import resources as importlib_resources from pathlib import Path from typing import Any, Literal, Protocol @@ -98,15 +99,46 @@ #: The emitter took about four seconds to become ready on an M1 Max; this is #: only a ceiling on the wait. STARTUP_TIMEOUT_SECONDS = 15.0 -#: The oldest emitter release a build starts, and the newest release on -#: 2026-11-02, when it was chosen. Since 0.10.0 the emitter sends to the -#: collector's hostname by default, which stays the same when the collector -#: moves to another Google Cloud project; earlier releases send to its Cloud -#: Run address, which changes with the move. 0.11.1 fixed two cases in which -#: the telemetry module stopped; 0.11.2 to 0.11.9 change no emitter behaviour. -#: 0.10.0 would have been enough for the hostname. The newest release was -#: required instead because practically no machine ran the separately -#: installed emitter yet, so refusing 0.10.0 to 0.11.8 cost nothing then. A +#: The package data file that holds the oldest emitter release a build +#: starts, ``MINIMUM_EMITTER_VERSION``. It is the one place the release is +#: written: the Modal stage runner (``tools/modal_us_stage_plan.py``), which +#: imports nothing from the workspace, reads the same file by its path in the +#: checkout that launches it, and installs at least that release in its image. +MINIMUM_EMITTER_VERSION_RESOURCE = "minimum_emitter_version.json" +_RELEASE_NUMBER = re.compile(r"(\d+)\.(\d+)\.(\d+)") + + +def _packaged_minimum_emitter_version() -> str: + """The release ``MINIMUM_EMITTER_VERSION_RESOURCE`` names, as ``X.Y.Z``. + + A file without exactly that is a packaging defect, so importing this + module fails rather than start an emitter of any release. + """ + resource = importlib_resources.files("microcosm.build").joinpath( + MINIMUM_EMITTER_VERSION_RESOURCE + ) + data = json.loads(resource.read_text(encoding="utf-8")) + version = data.get("minimum_emitter_version") if isinstance(data, dict) else None + if not isinstance(version, str) or _RELEASE_NUMBER.fullmatch(version) is None: + raise ValueError( + f"microcosm/build/{MINIMUM_EMITTER_VERSION_RESOURCE} must hold " + '{"minimum_emitter_version": "X.Y.Z"}' + ) + return version + + +#: The oldest emitter release a build starts: 0.13.0 since 2026-11-05, the +#: newest release then. 0.13.0 added ``microcosm-emitter status``, which the +#: Modal stage runner runs in the container after a stage's tool to record how +#: many telemetry events each run left undelivered, so the image needs it; a +#: build requires the same release, so that the release is written once +#: (``MINIMUM_EMITTER_VERSION_RESOURCE``). Practically no machine ran the +#: separately installed emitter yet, so refusing 0.11.9 to 0.12.5 cost +#: nothing then. Earlier floors: 0.11.9 from 2026-11-02, because since 0.10.0 +#: the emitter sends to the collector's hostname by default, which stays the +#: same when the collector moves to another Google Cloud project, while +#: earlier releases send to its Cloud Run address, which changes with the +#: move; 0.11.1 fixed two cases in which the telemetry module stopped. A #: refused emitter costs a build its hosted telemetry, its staging hand-overs #: and its Orrery graph publication, fails a UK build given #: ``--staging-read-back``, and refuses a UK build that asks for remote @@ -117,9 +149,8 @@ #: ``stage_dataset_bundle``, ``dataset_bundle_receipt`` and #: ``check_write_access`` for staged dataset bundles) and its queue in #: ``events.handovers.sqlite3``; releases 0.4.0 to 0.5.1 accept the same -#: protocol version but start without that module. ``IMAGE_EMITTER_REQUIREMENT`` -#: in ``tools/modal_us_stage_plan.py`` repeats this release. -MINIMUM_EMITTER_VERSION = "0.11.9" +#: protocol version but start without that module. +MINIMUM_EMITTER_VERSION = _packaged_minimum_emitter_version() UPGRADE_EMITTER_COMMAND = ( f"uv tool install --upgrade 'microcosm-emitter>={MINIMUM_EMITTER_VERSION}'" ) @@ -142,7 +173,6 @@ "Upgrade it with `{upgrade}`, or point MICROCOSM_EMITTER_COMMAND at a " "newer one." ) -_RELEASE_NUMBER = re.compile(r"(\d+)\.(\d+)\.(\d+)") _MINIMUM_RELEASE = tuple(int(part) for part in MINIMUM_EMITTER_VERSION.split(".")) #: ``1``: this build must send telemetry, so it stops when its emitter cannot diff --git a/test_support/microcosm_build/us_modal_stage_plan_tool.py b/test_support/microcosm_build/us_modal_stage_plan_tool.py index f194ce6d39..fb19bff8a0 100644 --- a/test_support/microcosm_build/us_modal_stage_plan_tool.py +++ b/test_support/microcosm_build/us_modal_stage_plan_tool.py @@ -127,7 +127,6 @@ def load_app( module, "_emitter_status", lambda env: copy.deepcopy(RECORDED_EMITTER_STATUS), - raising=False, ) for name, value in RUNNER_TOKEN_SECRET.items(): monkeypatch.setenv(name, value) diff --git a/tools/installed-emitter/requirements.txt b/tools/installed-emitter/requirements.txt index d3e34da42d..68e5165e1e 100644 --- a/tools/installed-emitter/requirements.txt +++ b/tools/installed-emitter/requirements.txt @@ -4,4 +4,4 @@ # it, and tools/run_installed_emitter_tests.sh installs exactly this release and # warns when the package index has a newer one. Raise it in a pull request when # the emitter publishes a release. -microcosm-emitter==0.12.0 +microcosm-emitter==0.13.0 diff --git a/tools/modal_us_stage.py b/tools/modal_us_stage.py index e76c3b854e..4d260d0b9f 100644 --- a/tools/modal_us_stage.py +++ b/tools/modal_us_stage.py @@ -264,6 +264,43 @@ def _emitter_version() -> plan_lib.EmitterVersionAnswer | None: return version if isinstance(version, dict) else None +def _emitter_status(env: dict[str, str]) -> plan_lib.UndeliveredEventsRecord: + """What the image emitter's ``status`` reports, as the receipt records it. + + The runner calls this once the tool has exited and its output has ended: + the emitter the build started holds that output until it has delivered + what it could and stopped, so by then its event queue file is final. + ``status`` reads that file read-only and prints each run in it; the + runner never reads the file itself. It runs without + ``--event-queue-file``, in the tool's environment ``env`` without + credentials or the collector token, so the emitter's default path is the + one the build computed. A failure is recorded as a problem line and + never raises. + """ + + try: + answer = subprocess.run( + [plan_lib.IMAGE_EMITTER_COMMAND, "status"], + capture_output=True, + text=True, + errors="replace", + timeout=plan_lib.EMITTER_STATUS_TIMEOUT_SECONDS, + env=plan_lib.emitter_query_environment(env), + ) + except subprocess.TimeoutExpired: + return plan_lib.undelivered_events_problem( + "`microcosm-emitter status` did not answer within " + f"{plan_lib.EMITTER_STATUS_TIMEOUT_SECONDS} s" + ) + except (OSError, subprocess.SubprocessError) as error: + return plan_lib.undelivered_events_problem( + f"`microcosm-emitter status` could not run ({type(error).__name__})" + ) + return plan_lib.undelivered_events_from_status( + answer.returncode, answer.stdout, answer.stderr + ) + + def _runner_identity() -> dict[str, object]: identity: dict[str, object] = { "python": sys.version.split()[0], @@ -1037,6 +1074,10 @@ def _attempt_stage( wall = time.time() - started peak_kib = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss finished_at = _now() + # The output has ended, so the emitter the build started has stopped: + # what its event queue file still holds is final. Its own `status` + # reports each run's undelivered events; a failure is only recorded. + telemetry = {**telemetry, "undelivered_events": _emitter_status(env)} # 7. Outputs: mirror the state tree to the volume, hashing each file as it # is copied (a file pulled and left untouched keeps its verified @@ -1080,8 +1121,8 @@ def _attempt_stage( "home_seeds": home_seeds, "input_links": input_links, "tool_budget": tool_budget, - # The token's expiry and origin, never its value; the - # undelivered-event count is not recorded yet (telemetry_check). + # The token's expiry and origin, never its value, and what the + # image emitter's `status` reported after the tool. "telemetry": telemetry, }, prior_receipts=prior, diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index 844bdaafe4..1d476ab181 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -103,16 +103,26 @@ # child process (microcosm-emitter-interface); uv.lock pins only the # interface. The image installs the newest emitter release when it is built # for a plan commit, outside the synced environment; an image built for one -# commit keeps the emitter it was built with. A build does not start an -# emitter older than microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION, -# 0.11.9; releases before 0.10.0 send telemetry to the collector's Cloud Run -# address, which changes when the collector moves to another Google Cloud -# project. This module imports nothing from the workspace, so the floor -# repeats that release, and a test checks that the two agree in one -# checkout. At launch the two can differ: the image's install step comes from -# this module in the checkout that launches the app, and the minimum from the -# commit the plan pins. The stage's runner records the version. -IMAGE_EMITTER_REQUIREMENT = "microcosm-emitter>=0.11.9" +# commit keeps the emitter it was built with. The oldest release it accepts +# is the oldest a build starts (MINIMUM_EMITTER_VERSION in +# microcosm.build.telemetry_emitter), written once, in microcosm-build's +# package data. This module imports nothing from the workspace, so it reads +# that file by its path in the checkout it belongs to: the checkout that +# launches the app, when Modal defines the image (image_emitter_requirement). +# In the container this module is a copy without a checkout, and it never +# reads the file. So at launch the two can still differ: the image's install +# step comes from the launching checkout, and the minimum from the commit the +# plan pins. The stage's runner records the version. The runner needs 0.13.0 +# or later, which added the `status` command it runs after the tool. +MINIMUM_EMITTER_VERSION_FILE = ( + Path(__file__).resolve().parents[1] + / "packages" + / "microcosm-build" + / "src" + / "microcosm" + / "build" + / "minimum_emitter_version.json" +) IMAGE_EMITTER_BIN_DIR = "/usr/local/bin" IMAGE_EMITTER_COMMAND = f"{IMAGE_EMITTER_BIN_DIR}/microcosm-emitter" #: The image's environment variables, which the stage tool inherits. @@ -418,7 +428,8 @@ def estimated_usd(self, wall_seconds: float, multiplier: float = 1.0) -> float: BASE_MIRRORED_STATE_GIB = 50 # Staging has no frame checkpoints: its 10.7 GB H5 (~10 GiB), summary and # logs fit in a 12 GiB mirror allowance. Thirty minutes reserves 60 s for -# stopping, 120 s for the receipt, and 1,620 s for hashing/copying the state. +# stopping, 60 s for the image emitter's status, 120 s for the receipt, and +# 1,560 s for hashing/copying the state. # The check's write probe refuses a slower volume (about 8 MB/s required). ACS_STAGING_RUNNER_OVERHEAD_SECONDS = 30 * 60 ACS_STAGING_MIRRORED_STATE_GIB = 12 @@ -1540,6 +1551,38 @@ def image_pip_bootstrap_command( ) +_EMITTER_RELEASE = re.compile(r"\d+\.\d+\.\d+") + + +def minimum_emitter_version() -> str: + """The oldest emitter release a build starts, from ``MINIMUM_EMITTER_VERSION_FILE``. + + That file is microcosm-build's package data, which the build adapter + reads as ``MINIMUM_EMITTER_VERSION``; this reads it by its path in this + module's checkout. Raises ``PlanError`` when the file is missing or does + not hold ``{"minimum_emitter_version": "X.Y.Z"}``. + """ + + path = MINIMUM_EMITTER_VERSION_FILE + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise PlanError( + f"cannot read the emitter's minimum release from {path} " + f"({type(error).__name__})" + ) from None + version = data.get("minimum_emitter_version") if isinstance(data, dict) else None + if not isinstance(version, str) or _EMITTER_RELEASE.fullmatch(version) is None: + raise PlanError(f'{path} must hold {{"minimum_emitter_version": "X.Y.Z"}}') + return version + + +def image_emitter_requirement() -> str: + """What the image installs: at least the oldest emitter release a build starts.""" + + return f"microcosm-emitter>={minimum_emitter_version()}" + + def image_build_commands(plan: Plan, repo_root: str = IMAGE_REPO_ROOT) -> list[str]: """Shell steps that pin the image's tree and environment to the plan. @@ -1547,7 +1590,7 @@ def image_build_commands(plan: Plan, repo_root: str = IMAGE_REPO_ROOT) -> list[s and ``git status`` inside the tools report the real code vintage), and the environment is synced from that tree's own uv.lock with --frozen. The emitter command is installed on its own, outside that environment - (``IMAGE_EMITTER_REQUIREMENT``), and must run. + (``image_emitter_requirement``), and must run. """ root = shlex.quote(repo_root) @@ -1560,7 +1603,7 @@ def image_build_commands(plan: Plan, repo_root: str = IMAGE_REPO_ROOT) -> list[s f'test "$(git -C {root} rev-parse HEAD)" = {commit}', f"cd {root} && uv sync --all-packages --extra us --frozen", f"UV_TOOL_BIN_DIR={IMAGE_EMITTER_BIN_DIR} uv tool install " - f"{shlex.quote(IMAGE_EMITTER_REQUIREMENT)}", + f"{shlex.quote(image_emitter_requirement())}", f"{IMAGE_EMITTER_COMMAND} version", f'test -z "$(git -C {root} status --porcelain)"', ] @@ -1831,6 +1874,17 @@ def emitter_query_environment(base: Mapping[str, str]) -> dict[str, str]: #: How long the launch waits for ``telemetry-token``: one exchange with the #: collector. TELEMETRY_TOKEN_TIMEOUT_SECONDS = 120 +#: How long the runner waits for the image emitter's ``status`` after the +#: tool. It reads the event queue file and prints, so it normally answers in +#: well under a second; only a file another connection holds for a commit +#: makes it wait (the emitter's 5 s busy timeout). The runner's reserve after +#: the tool keeps this much for it (``write_probe_verdict``). +EMITTER_STATUS_TIMEOUT_SECONDS = 60 +#: ``status``'s exit code when no event queue file exists at the path it +#: reads: no emitter kept one in the container, so none holds an event. +EMITTER_STATUS_NO_FILE_EXIT_CODE = 8 +#: The longest problem line a receipt keeps of a ``status`` failure. +EMITTER_STATUS_PROBLEM_MAX_CHARS = 300 #: What each failing exit code of ``microcosm-emitter telemetry-token`` means #: (its ``cli`` module). 2 is argparse's usage error, which a release before #: 0.11.0, without the command, also gives. @@ -1875,6 +1929,38 @@ class EmitterVersionAnswer(TypedDict, total=False): collector_origin: str +class EmitterStatusRun(TypedDict): + """One run as ``microcosm-emitter status`` prints it (emitter 0.13.0). + + ``reason`` is the run's ``local_only_reason``: why a ``local_only`` run + is excluded from upload, or why a ``pending`` one waits. + ``undelivered_events`` counts the run's events the emitter's event queue + file still holds. The emitter also prunes events it never delivered + (older than 7 days, or beyond 100 MiB), so 0 means none are held, not + that every event was delivered. + """ + + run_id: str + producer_id: str + upload_state: str + reason: str | None + undelivered_events: int + + +class UndeliveredEventsRecord(TypedDict): + """What a receipt records of the image emitter's ``status`` after the tool. + + ``runs`` is ``status``'s list as printed; an empty list when it found no + event queue file (exit 8). On any other failure (another exit code, a + timeout, a command that could not run, output that is not its list of + runs) ``runs`` is None and ``problem`` one line saying why. A failure is + recorded only: it never changes the stage's status or outcome. + """ + + runs: list[EmitterStatusRun] | None + problem: str | None + + class TelemetryRecord(TypedDict): """What the plan's summary, the check and the receipt say about telemetry. @@ -1882,7 +1968,10 @@ class TelemetryRecord(TypedDict): is what a launch asks for; ``image_emitter_collector_origin`` what the image's emitter reported (None in the summary, which runs no emitter); ``collector_token`` and ``token_needed_until`` are known only in the - container that has the token. + container that has the token. ``undelivered_events`` is None until the + tool has run, so in the summary and the check; the receipt holds what the + image emitter's ``status`` answered after the tool + (``undelivered_events_from_status``). """ required: bool @@ -1890,10 +1979,7 @@ class TelemetryRecord(TypedDict): image_emitter_collector_origin: str | None collector_token: CollectorTokenRecord | None token_needed_until: str | None - # Not recorded yet: how many events the emitter left undelivered when the - # tool ended. The count comes later from the emitter's read-only `status` - # command, which the runner will run in the container after the tool. - undelivered_events: None + undelivered_events: UndeliveredEventsRecord | None def telemetry_required(env: Mapping[str, str]) -> bool: @@ -2141,6 +2227,83 @@ def telemetry_record( } +_STATUS_COMMAND = "`microcosm-emitter status`" +_STATUS_RUN_FIELDS = { + "run_id": str, + "producer_id": str, + "upload_state": str, + "reason": (str, type(None)), +} + + +def undelivered_events_problem(problem: str) -> UndeliveredEventsRecord: + """A record of a ``status`` that gave no runs, with ``problem`` as one line. + + Whitespace, line breaks included, becomes single spaces, and a line + longer than EMITTER_STATUS_PROBLEM_MAX_CHARS is cut to that length, + ending in an ellipsis. + """ + + line = " ".join(problem.split()) + if len(line) > EMITTER_STATUS_PROBLEM_MAX_CHARS: + line = line[: EMITTER_STATUS_PROBLEM_MAX_CHARS - 1] + "…" + return {"runs": None, "problem": line} + + +def _status_runs(stdout: str) -> list[EmitterStatusRun] | None: + """The runs ``status`` printed, or None if its output is not its JSON.""" + + try: + printed = json.loads(stdout) + except ValueError: + return None + runs = printed.get("runs") if isinstance(printed, dict) else None + if not isinstance(runs, list): + return None + for run in runs: + if not isinstance(run, dict): + return None + for name, kind in _STATUS_RUN_FIELDS.items(): + if name not in run or not isinstance(run[name], kind): + return None + count = run.get("undelivered_events") + if type(count) is not int or count < 0: + return None + return runs + + +def undelivered_events_from_status( + returncode: int, stdout: str, stderr: str +) -> UndeliveredEventsRecord: + """What a receipt records of the image emitter's ``status`` answer. + + Exit 0 gives the runs it printed, as printed; exit 8 (no event queue + file) an empty list. Any other exit code gives a problem naming the code + and the last line ``status`` wrote on standard error, which is its own + message, argparse's for an emitter without the command (exit 2), or a + traceback's last line. Output that is not its list of runs is a problem + too. + """ + + if returncode == EMITTER_STATUS_NO_FILE_EXIT_CODE: + return {"runs": [], "problem": None} + if returncode != 0: + lines = [line for line in stderr.splitlines() if line.strip()] + if not lines: + return undelivered_events_problem( + f"{_STATUS_COMMAND} exited {returncode} without a message" + ) + return undelivered_events_problem( + f"{_STATUS_COMMAND} exited {returncode}: {lines[-1]}" + ) + runs = _status_runs(stdout) + if runs is None: + return undelivered_events_problem( + f"{_STATUS_COMMAND} exited 0 but did not print its JSON list of runs" + ) + return {"runs": runs, "problem": None} + + def token_needed_until(plan: Plan, container_started: float) -> int: """When the collector token must still be usable, in seconds since the epoch. @@ -3389,7 +3552,8 @@ def write_probe_verdict( ``mirrored_state_gib``, that is the time mirroring its whole state would take after the tool (the push after a stop, which copies everything). The runner keeps ``runner_overhead_seconds`` after the tool's deadline, of - which the stop may take STOP_GRACE_SECONDS and the runner's identity, the + which the stop may take STOP_GRACE_SECONDS, the image emitter's + ``status`` EMITTER_STATUS_TIMEOUT_SECONDS, and the runner's identity, the receipt and the attempt's final commit RECEIPT_RESERVE_SECONDS; if the extrapolated mirror does not fit in the rest, a stop at the budget would run into the class timeout mid-mirror and leave no receipt, so the check @@ -3402,7 +3566,10 @@ def write_probe_verdict( rate = probe_bytes / max(probe_seconds, 1e-6) needed = stage.mirrored_state_gib * 1024**3 / rate reserve = ( - stage.runner_overhead_seconds - STOP_GRACE_SECONDS - RECEIPT_RESERVE_SECONDS + stage.runner_overhead_seconds + - STOP_GRACE_SECONDS + - EMITTER_STATUS_TIMEOUT_SECONDS + - RECEIPT_RESERVE_SECONDS ) verdict: dict[str, object] = { "bytes": probe_bytes, From b3f041917bf96bfc5d10d67d52b60004f827b54b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:54:34 +0400 Subject: [PATCH 4/9] Test the check's emitter floor in the container, and the ordering The check runs in the container with the plan module's copy and what the image adds beside it, so its summary must name the floor the image was built with. Deeply nested output from status is recorded as a problem, and the write probe reports the 60 s kept for status. The ordering tests now hold the runner's pipe only through standard error, as the emitter host does, and an installed-emitter test checks the real host has exited when status runs (#1523 review round 1). Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_modal_stage_plan_tool.py | 95 ++++++++++++++++++- .../test_us_modal_stage_puf_support_base.py | 2 + .../shared/test_installed_emitter_status.py | 94 ++++++++++++++++++ 3 files changed, 186 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index 960967588b..8f8cc84048 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -12,6 +12,7 @@ import copy import dataclasses import hashlib +import importlib.util import inspect import json import math @@ -22,7 +23,7 @@ import time import uuid from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath from types import SimpleNamespace from unittest.mock import MagicMock @@ -531,6 +532,63 @@ def test_the_image_is_not_defined_without_a_usable_emitter_floor( plan_lib.image_build_commands(plan) +def _import_plan_module_copy(path: Path, monkeypatch): + """Import the plan module at ``path`` as the container imports its copy.""" + + name = "modal_us_stage_plan_image_copy" + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # dataclasses resolve string annotations through sys.modules. + monkeypatch.setitem(sys.modules, name, module) + spec.loader.exec_module(module) + return module + + +def test_the_check_in_the_container_reports_the_emitter_floor_the_image_was_built_with( + app, monkeypatch, tmp_path: Path +) -> None: + """The container's plan module is the copy the image adds, without a checkout. + + The check (``check_stage``) runs there and calls ``summarize``, which + builds the image's install step. That must name the release the image + was built with: the launching checkout's, as the app defined the image. + """ + + version_file = tmp_path / "launching-checkout" / "minimum_emitter_version.json" + version_file.parent.mkdir() + version_file.write_text('{"minimum_emitter_version": "9.8.7"}\n') + monkeypatch.setattr(plan_lib, "MINIMUM_EMITTER_VERSION_FILE", version_file) + data = _plan_data() + monkeypatch.setattr(app, "_LOADED", (data, plan_lib.parse_plan(data))) + modal = sys.modules["modal"] + modal.reset_mock() + app._image() + + # What the image adds, all of it beside the plan module's copy, copied + # into a directory with no checkout around it. + container_root = tmp_path / "container-root" + container_root.mkdir() + beside = PurePosixPath(app.PLAN_MODULE_REMOTE).parent + added = [ + entry.args for entry in modal.mock_calls if entry[0].endswith("add_local_file") + ] + for local, remote in added: + assert PurePosixPath(remote).parent == beside + shutil.copyfile(local, container_root / PurePosixPath(remote).name) + copy_of_the_module = _import_plan_module_copy( + container_root / PurePosixPath(app.PLAN_MODULE_REMOTE).name, monkeypatch + ) + + summary = copy_of_the_module.summarize(copy_of_the_module.parse_plan(data)) + + assert [ + command + for command in summary["image_build_commands"] + if "uv tool install" in command + ] == ["UV_TOOL_BIN_DIR=/usr/local/bin uv tool install 'microcosm-emitter>=9.8.7'"] + + def test_the_app_never_runs_an_emitter_installed_where_the_image_puts_it( app, monkeypatch, tmp_path: Path ) -> None: @@ -3103,9 +3161,9 @@ def test_a_stage_whose_plan_opts_out_runs_without_a_token( # The image emitter's `status`, after the tool ----------------------------- # # A stand-in tool that finishes at once and leaves, in its own session, a -# process that holds its output for 2 s and then writes a marker file: the -# place of the emitter a build starts, which delivers its last events after -# the build exits. +# process that holds its standard error for 2 s and then writes a marker +# file: the place of the emitter a build starts, which delivers its last +# events after the build exits. _EMITTER_HOLDER_TOOL = """\ import pathlib, subprocess, sys state, marker = pathlib.Path(sys.argv[1]), sys.argv[2] @@ -3115,7 +3173,14 @@ def test_a_stage_whose_plan_opts_out_runs_without_a_token( "import pathlib, sys, time; time.sleep(2); " "pathlib.Path(sys.argv[1]).write_text('finished')" ) -subprocess.Popen([sys.executable, "-c", hold, marker], start_new_session=True) +# Like the emitter host the interface starts: in its own session, with its +# standard output sent nowhere, so it holds the runner's pipe only through +# the standard error it inherits. +subprocess.Popen( + [sys.executable, "-c", hold, marker], + start_new_session=True, + stdout=subprocess.DEVNULL, +) print("tool done", flush=True) """ @@ -3348,6 +3413,26 @@ def test_a_problem_line_is_one_bounded_line( assert not {"\n", "\r", "\t"} & set(problem) +def test_deeply_nested_output_is_recorded_as_a_problem( + status_runner, tmp_path: Path, monkeypatch +) -> None: + # Nested deeper than Python's JSON parser recurses (RecursionError). + depth = 1_000_000 + _install_status_emitter(monkeypatch, tmp_path, stdout="[" * depth + "]" * depth) + + try: + receipt = status_runner._run_stage(_fake_plan(max_wall_seconds=600)) + except RecursionError: + pytest.fail("deeply nested output from status ended the attempt") + + assert receipt["runner"]["telemetry"]["undelivered_events"] == { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but did not print its " + "JSON list of runs", + } + assert receipt["status"] == "COMPLETED" + + def test_the_receipt_records_a_status_that_hangs_or_cannot_start( status_runner, tmp_path: Path, monkeypatch ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py index b0e6be0946..e30e1c300e 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py @@ -2307,6 +2307,8 @@ def test_write_probe_verdict_sizes_the_mirror_against_the_reserve() -> None: assert "problem" not in fast assert fast["mirror_reserve_seconds"] == reserve == 1560 assert fast["receipt_reserve_seconds"] == 120 + # The 60 s kept for the image emitter's `status` is reported too. + assert fast.get("emitter_status_reserve_seconds") == 60 slow = plan_lib.write_probe_verdict(stage, gib, gib / (floor * 0.99)) assert "more than the 1560s the runner keeps for the mirror" in slow["problem"] # The runbook's one measured volume rate (reads, at least 58 MB/s) fits. diff --git a/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py index 0c0ce38b08..5474634fbb 100644 --- a/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py +++ b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py @@ -75,6 +75,39 @@ (pathlib.Path(state) / "out").mkdir(parents=True, exist_ok=True) (pathlib.Path(state) / "out" / "result.txt").write_text("done\\n") """ +#: A tool that starts the emitter the same way, writes the emitter host's +#: process ID to a file, records two stages and exits at once, without +#: finishing its run or closing the emitter, as a build that is killed does. +#: The emitter notices its parent's exit only afterwards, records the run's +#: failure and stops. +_UNFINISHED_TOOL = """\ +import os, pathlib, sys + +state, site_packages, collector, run_id, producer_id, pid_file = sys.argv[1:7] +sys.path.insert(0, site_packages) +from microcosm_emitter_interface import EmitterClient, TelemetryRun, TelemetrySettings + +run = TelemetryRun( + run_id=run_id, + country_code="US", + pipeline="installed_emitter_status", + producer_id=producer_id, +) +client = EmitterClient.start( + telemetry=TelemetrySettings( + run=run, development_collector_url=collector, heartbeat_seconds=3600 + ) +) +if not client.available: + sys.exit("the installed emitter did not start") +pathlib.Path(pid_file).write_text(str(client._process.process.pid)) +client.telemetry.stage("first_stage") +client.telemetry.stage("second_stage") +(pathlib.Path(state) / "out").mkdir(parents=True, exist_ok=True) +(pathlib.Path(state) / "out" / "result.txt").write_text("done\\n") +sys.stdout.flush() +os._exit(0) +""" #: A tool that leaves a file that is not an event queue file where the #: emitter would keep one, and starts no emitter. _UNUSABLE_FILE_TOOL = """\ @@ -163,6 +196,67 @@ def test_the_runner_records_the_events_the_installed_emitter_still_holds( } +def _process_exists(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def test_the_runner_asks_for_status_only_once_the_build_s_emitter_has_exited( + runner, monkeypatch, tmp_path: Path +) -> None: + """The emitter host holds the tool's output, which the runner reads to its end. + + The interface starts the host in its own session with its standard + output sent nowhere, so only the standard error it inherits holds the + runner's pipe; this test fails if a change there let the runner ask for + ``status`` while the host was still running. + """ + + pid_file = tmp_path / "emitter-host.pid" + app = runner( + _tool( + _UNFINISHED_TOOL, + _SITE_PACKAGES, + _closed_loopback_origin(), + RUN_ID, + PRODUCER_ID, + str(pid_file), + ) + ) + runner_s_own_status = app._emitter_status + host_running_when_asked = [] + + def status(env: dict[str, str]) -> dict: + host_running_when_asked.append(_process_exists(int(pid_file.read_text()))) + return runner_s_own_status(env) + + monkeypatch.setattr(app, "_emitter_status", status) + + receipt = app._run_stage(_opted_out_plan()) + + assert receipt["status"] == "COMPLETED" + assert host_running_when_asked == [False] + # The run's start, its two stages and the failure the emitter records + # once it notices the build has gone. + assert receipt["runner"]["telemetry"]["undelivered_events"] == { + "runs": [ + { + "run_id": RUN_ID, + "producer_id": PRODUCER_ID, + "upload_state": "local_only", + "reason": "missing_huggingface_credential", + "undelivered_events": 4, + } + ], + "problem": None, + } + + def test_the_runner_records_no_runs_when_no_emitter_kept_an_event_queue_file( runner, ) -> None: From 961ad8694d272a352ee2a173b63b7319d27aa94c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:56:44 +0400 Subject: [PATCH 5/9] Give the container the image's emitter floor beside the plan module The check runs in the container, where the plan module is the image's copy with no checkout around it, so reading the floor there failed for every plan. The image now adds the launching checkout's minimum_emitter_version.json beside that copy, which reads it first, so the check reports the install step the image was built with. Deeply nested status output is recorded as a problem, and the write probe's verdict reports the stop's and status's allowances (#1523 review round 1). Co-Authored-By: Claude Opus 5.5 --- .../1522-modal-undelivered-events.added.md | 2 +- docs/agent-guide.md | 3 +- docs/orrery-publication.md | 4 +- docs/us-modal-stage-runbook.md | 13 ++++- tools/modal_us_stage.py | 16 +++++- tools/modal_us_stage_plan.py | 50 +++++++++++++------ 6 files changed, 67 insertions(+), 21 deletions(-) diff --git a/changelog.d/1522-modal-undelivered-events.added.md b/changelog.d/1522-modal-undelivered-events.added.md index 3bc2d98586..36b7efc719 100644 --- a/changelog.d/1522-modal-undelivered-events.added.md +++ b/changelog.d/1522-modal-undelivered-events.added.md @@ -1 +1 @@ -The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run or output that is not its list of runs. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. +The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run or output that is not its list of runs. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s, and its verdict reports the stop's and `status`'s allowances beside the receipt's. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The image adds that checkout's file beside the plan module's copy, `/root/minimum_emitter_version.json`, which the copy reads first, so the check in the container reports the install step the image was built with. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index c1091bce07..a19365866e 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -141,7 +141,8 @@ version check), not the emitter's own queue or delivery. image's emitter path: its tool starts the emitter's telemetry module through the interface, with no credential, no collector token and a closed loopback collector address, so the emitter keeps its events and sends nothing, and the -test checks what the runner records from the emitter's `status`. The runner warns when +test checks what the runner records from the emitter's `status`, and that the +emitter host has exited before the runner runs it. The runner warns when the package index has a newer release than the pin; raise the pin in a pull request then. `tools/installed-emitter/constraints.txt` pins the `huggingface_hub` release whose requests the stand-in answers; the runner also diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index 659921e9a8..f6270791c8 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -169,7 +169,9 @@ written once, in microcosm-build's package data, `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: the adapter reads it as package data, and the runner, which imports nothing from the workspace, reads it by its path in the checkout that launches the -app. At a Modal launch the install step therefore comes from the launching +app. The image adds that checkout's file beside the runner's copy of the +plan module, so the check, which runs in the container, reads the same +file. At a Modal launch the install step therefore comes from the launching checkout and the minimum from the commit the plan pins (see the [US Modal stage runbook](us-modal-stage-runbook.md)). Releases before 0.10.0 send telemetry to the collector's Cloud Run address, which diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index eb866a97c6..8cc0be6cdf 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -97,7 +97,12 @@ release directory onto the runs volume. Publication stays the human step in built with. The release is written once, in microcosm-build's package data (`packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`): the build adapter reads it as `MINIMUM_EMITTER_VERSION`, and the runner - reads the same file by its path in the checkout. The install step and its + reads the same file by its path in the checkout. The container has no + checkout around the plan module's copy (`/root/modal_us_stage_plan.py`), + so the image adds the launching checkout's file beside it + (`/root/minimum_emitter_version.json`), and the copy reads that one: the + check, which runs there, reports the install step the image was built + with. The install step and its `>=0.13.0` floor come from the checkout that launches the app (`image_build_commands`, `image_emitter_requirement`), while the minimum the build enforces comes from the commit the plan pins, so a launch can @@ -1005,7 +1010,11 @@ The plan is `docs/us-modal-stage-route-a-base-plan.json` (run `status` (up to 60 s, `EMITTER_STATUS_TIMEOUT_SECONDS`), then hashing and mirroring the state, then the runner's identity, the receipt and the attempt's final commit (`RECEIPT_RESERVE_SECONDS`, 120 s, an allowance). - After a + The check's write probe reports each of these figures. Not counted: the + emitter the build started keeps the tool's output open while it finishes, + up to about 15.5 s after the tool exits (`EMITTER_EXIT_NOTICE_SECONDS` and + `EMITTER_DRAIN_SECONDS`), also after a stop; the runner waits for it before + `status`. After a stop or a failure the push copies the whole state, about 50 GiB (`BASE_MIRRORED_STATE_GIB`): the local run's 44 GB of checkpoints and 2.35 GB H5, plus the final checkpoint's hard-linked alias diff --git a/tools/modal_us_stage.py b/tools/modal_us_stage.py index 4d260d0b9f..b85bb6b8dd 100644 --- a/tools/modal_us_stage.py +++ b/tools/modal_us_stage.py @@ -60,7 +60,7 @@ import threading import time from datetime import UTC, datetime -from pathlib import Path +from pathlib import Path, PurePosixPath import modal @@ -74,6 +74,13 @@ HF_SECRET_ENV = "MICROCOSM_MODAL_HF_SECRET" APP_NAME = "microcosm-us-stage" PLAN_MODULE_REMOTE = "/root/modal_us_stage_plan.py" +#: Beside the plan module's copy, where that copy looks for it first +#: (``plan_lib.minimum_emitter_version_file``). +MINIMUM_EMITTER_VERSION_REMOTE = str( + PurePosixPath(PLAN_MODULE_REMOTE).with_name( + plan_lib.MINIMUM_EMITTER_VERSION_FILENAME + ) +) def _local_plan() -> tuple[dict, plan_lib.Plan] | None: @@ -124,6 +131,13 @@ def _image() -> modal.Image: image = image.add_local_file( str(_HERE / "modal_us_stage_plan.py"), PLAN_MODULE_REMOTE ) + # The launching checkout's emitter floor, beside the plan module's + # copy, which has no checkout around it: the check, which runs in the + # container, reports the install step from it. + image = image.add_local_file( + str(plan_lib.minimum_emitter_version_file()), + MINIMUM_EMITTER_VERSION_REMOTE, + ) return image diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index 1d476ab181..cb52696dd7 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -106,14 +106,20 @@ # commit keeps the emitter it was built with. The oldest release it accepts # is the oldest a build starts (MINIMUM_EMITTER_VERSION in # microcosm.build.telemetry_emitter), written once, in microcosm-build's -# package data. This module imports nothing from the workspace, so it reads -# that file by its path in the checkout it belongs to: the checkout that -# launches the app, when Modal defines the image (image_emitter_requirement). -# In the container this module is a copy without a checkout, and it never -# reads the file. So at launch the two can still differ: the image's install -# step comes from the launching checkout, and the minimum from the commit the -# plan pins. The stage's runner records the version. The runner needs 0.13.0 -# or later, which added the `status` command it runs after the tool. +# package data. This module imports nothing from the workspace, so on the +# launching machine it reads that file by its path in the checkout it belongs +# to (MINIMUM_EMITTER_VERSION_FILE), when Modal defines the image +# (image_emitter_requirement). In the container this module is the copy the +# image adds at /root, with no checkout around it, so the image also adds the +# launching checkout's file beside it (MINIMUM_EMITTER_VERSION_FILENAME), and +# a copy with that file beside it reads it first +# (minimum_emitter_version_file): the check, which runs there, then reports +# the requirement the image was built with. So at launch the image's install +# step comes from the launching checkout, and the minimum the build enforces +# from the commit the plan pins. The stage's runner records the version. The +# runner needs 0.13.0 or later, which added the `status` command it runs +# after the tool. +MINIMUM_EMITTER_VERSION_FILENAME = "minimum_emitter_version.json" MINIMUM_EMITTER_VERSION_FILE = ( Path(__file__).resolve().parents[1] / "packages" @@ -121,7 +127,7 @@ / "src" / "microcosm" / "build" - / "minimum_emitter_version.json" + / MINIMUM_EMITTER_VERSION_FILENAME ) IMAGE_EMITTER_BIN_DIR = "/usr/local/bin" IMAGE_EMITTER_COMMAND = f"{IMAGE_EMITTER_BIN_DIR}/microcosm-emitter" @@ -1554,16 +1560,27 @@ def image_pip_bootstrap_command( _EMITTER_RELEASE = re.compile(r"\d+\.\d+\.\d+") +def minimum_emitter_version_file() -> Path: + """The file that names the oldest emitter release a build starts. + + The copy beside this module when there is one: the image adds the + launching checkout's file beside its copy of this module. Otherwise the + file in this module's checkout, ``MINIMUM_EMITTER_VERSION_FILE``. + """ + + beside = Path(__file__).resolve().with_name(MINIMUM_EMITTER_VERSION_FILENAME) + return beside if beside.is_file() else MINIMUM_EMITTER_VERSION_FILE + + def minimum_emitter_version() -> str: - """The oldest emitter release a build starts, from ``MINIMUM_EMITTER_VERSION_FILE``. + """The oldest emitter release a build starts, from ``minimum_emitter_version_file``. That file is microcosm-build's package data, which the build adapter - reads as ``MINIMUM_EMITTER_VERSION``; this reads it by its path in this - module's checkout. Raises ``PlanError`` when the file is missing or does - not hold ``{"minimum_emitter_version": "X.Y.Z"}``. + reads as ``MINIMUM_EMITTER_VERSION``. Raises ``PlanError`` when the file + is missing or does not hold ``{"minimum_emitter_version": "X.Y.Z"}``. """ - path = MINIMUM_EMITTER_VERSION_FILE + path = minimum_emitter_version_file() try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError) as error: @@ -2255,7 +2272,8 @@ def _status_runs(stdout: str) -> list[EmitterStatusRun] | None: try: printed = json.loads(stdout) - except ValueError: + # Output nested deeper than the parser recurses raises RecursionError. + except (ValueError, RecursionError): return None runs = printed.get("runs") if isinstance(printed, dict) else None if not isinstance(runs, list): @@ -3578,6 +3596,8 @@ def write_probe_verdict( "mirrored_state_gib": stage.mirrored_state_gib, "implied_mirror_seconds": round(needed), "mirror_reserve_seconds": reserve, + "stop_grace_seconds": STOP_GRACE_SECONDS, + "emitter_status_reserve_seconds": EMITTER_STATUS_TIMEOUT_SECONDS, "receipt_reserve_seconds": RECEIPT_RESERVE_SECONDS, } if needed > reserve: From 72596ab7fb55e98aa5ca885165fc9a570f79bb9d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:17:40 +0400 Subject: [PATCH 6/9] Test runs too deep for the receipt and a stray version file Runs nested just under the JSON parser's limit pass the parse but break the receipt's write, so they must be recorded as a problem. In a checkout the package's version file must win over one left beside the plan module. The real-emitter ordering test treats an exited process awaiting collection as gone, and the probe test checks that its four figures add up to the reserve (#1523 review round 2). Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_modal_stage_plan_tool.py | 78 +++++++++++++++++++ .../test_us_modal_stage_puf_support_base.py | 11 ++- .../shared/test_installed_emitter_status.py | 30 ++++--- 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index 8f8cc84048..999b973a5d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -589,6 +589,36 @@ def test_the_check_in_the_container_reports_the_emitter_floor_the_image_was_buil ] == ["UV_TOOL_BIN_DIR=/usr/local/bin uv tool install 'microcosm-emitter>=9.8.7'"] +def test_the_package_s_version_file_wins_over_a_stray_one_beside_the_plan_module( + monkeypatch, tmp_path: Path +) -> None: + """In a checkout, a file left beside the plan module changes nothing. + + Only the container, which has no checkout around the plan module's copy, + reads the copy of the file the image adds beside it. + """ + + checkout = tmp_path / "checkout" + (checkout / "tools").mkdir(parents=True) + shutil.copyfile( + ROOT / "tools" / "modal_us_stage_plan.py", + checkout / "tools" / "modal_us_stage_plan.py", + ) + (checkout / "tools" / "minimum_emitter_version.json").write_text( + '{"minimum_emitter_version": "9.9.9"}\n' + ) + package = checkout / "packages" / "microcosm-build" / "src" / "microcosm" / "build" + package.mkdir(parents=True) + (package / "minimum_emitter_version.json").write_text( + '{"minimum_emitter_version": "1.2.3"}\n' + ) + copy_in_a_checkout = _import_plan_module_copy( + checkout / "tools" / "modal_us_stage_plan.py", monkeypatch + ) + + assert copy_in_a_checkout.image_emitter_requirement() == "microcosm-emitter>=1.2.3" + + def test_the_app_never_runs_an_emitter_installed_where_the_image_puts_it( app, monkeypatch, tmp_path: Path ) -> None: @@ -3413,6 +3443,54 @@ def test_a_problem_line_is_one_bounded_line( assert not {"\n", "\r", "\t"} & set(problem) +def _status_output_with_an_extra_field(depth: int) -> str: + """``status``'s output for one run with an extra field, objects ``depth`` deep.""" + + printed = json.dumps({"runs": [_PRINTED_RUNS[0]]}) + assert printed.endswith("}]}") + nested = '{"a": ' * depth + "0" + "}" * depth + return f'{printed[:-3]}, "extra": {nested}}}]}}' + + +def _deepest_extra_field_the_parser_accepts() -> int: + low, high = 1, 1_000_000 + while low < high: + middle = (low + high + 1) // 2 + try: + json.loads(_status_output_with_an_extra_field(middle)) + except RecursionError: + high = middle - 1 + else: + low = middle + return low + + +def test_runs_nested_deeper_than_a_receipt_can_hold_are_recorded_as_a_problem() -> None: + """Runs the parser accepts but the receipt cannot be written with. + + ``status``'s runs are recorded as printed, extra fields included. A field + nested just under the parser's limit is accepted, but the receipt holds + the runs four objects deeper, and writing it (after the mirror) then fails + on Python 3.13 and 3.14: the stage would get no receipt. + """ + + stdout = _status_output_with_an_extra_field( + _deepest_extra_field_the_parser_accepts() - 1 + ) + runs = json.loads(stdout)["runs"] + with pytest.raises(RecursionError): + json.dumps( + {"runner": {"telemetry": {"undelivered_events": {"runs": runs}}}}, + indent=2, + ) + + assert plan_lib.undelivered_events_from_status(0, stdout, "") == { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but printed runs nested " + "too deeply for the receipt", + } + + def test_deeply_nested_output_is_recorded_as_a_problem( status_runner, tmp_path: Path, monkeypatch ) -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py index e30e1c300e..c2f8de30c6 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_puf_support_base.py @@ -2307,8 +2307,17 @@ def test_write_probe_verdict_sizes_the_mirror_against_the_reserve() -> None: assert "problem" not in fast assert fast["mirror_reserve_seconds"] == reserve == 1560 assert fast["receipt_reserve_seconds"] == 120 - # The 60 s kept for the image emitter's `status` is reported too. + # The 60 s kept for the image emitter's `status` is reported too, with + # the stop's grace, and the four figures add up to the reserve. assert fast.get("emitter_status_reserve_seconds") == 60 + assert fast.get("stop_grace_seconds") == 60 + assert ( + fast["mirror_reserve_seconds"] + + fast["stop_grace_seconds"] + + fast["emitter_status_reserve_seconds"] + + fast["receipt_reserve_seconds"] + == plan_lib.BASE_RUNNER_OVERHEAD_SECONDS + ) slow = plan_lib.write_probe_verdict(stage, gib, gib / (floor * 0.99)) assert "more than the 1560s the runner keeps for the mirror" in slow["problem"] # The runbook's one measured volume rate (reads, at least 58 MB/s) fits. diff --git a/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py index 5474634fbb..aa6d020ee3 100644 --- a/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py +++ b/packages/microcosm-build/tests/installed_emitter/shared/test_installed_emitter_status.py @@ -24,6 +24,7 @@ import os import shutil import socket +import subprocess import sysconfig from pathlib import Path @@ -100,6 +101,8 @@ ) if not client.available: sys.exit("the installed emitter did not start") +# The interface (0.7.0) has no public way to name the host's process, so +# this reads the client's private handle on it. pathlib.Path(pid_file).write_text(str(client._process.process.pid)) client.telemetry.stage("first_stage") client.telemetry.stage("second_stage") @@ -196,14 +199,23 @@ def test_the_runner_records_the_events_the_installed_emitter_still_holds( } -def _process_exists(pid: int) -> bool: - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - return True +def _process_running(pid: int) -> bool: + """Whether ``pid`` is a process that has not exited. + + A process that has exited but that its parent, here the system's process + reaper since the tool exited first, has not yet collected still answers + a signal 0, so ``ps`` reads its state instead: none for a process that + is gone, ``Z`` for one that has exited and awaits collection. + """ + + answer = subprocess.run( + ["ps", "-o", "stat=", "-p", str(pid)], + capture_output=True, + text=True, + check=False, + ) + state = answer.stdout.strip() + return answer.returncode == 0 and bool(state) and not state.startswith("Z") def test_the_runner_asks_for_status_only_once_the_build_s_emitter_has_exited( @@ -232,7 +244,7 @@ def test_the_runner_asks_for_status_only_once_the_build_s_emitter_has_exited( host_running_when_asked = [] def status(env: dict[str, str]) -> dict: - host_running_when_asked.append(_process_exists(int(pid_file.read_text()))) + host_running_when_asked.append(_process_running(int(pid_file.read_text()))) return runner_s_own_status(env) monkeypatch.setattr(app, "_emitter_status", status) From 43cd7dc5048374d9331c3fff060a09d48a111b7b Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 22:19:37 +0400 Subject: [PATCH 7/9] Refuse runs too deep for the receipt, and prefer the package's floor The runs status printed are serialized once as deep as the receipt holds them, so output the parser accepts but the receipt's write would fail on is recorded as a problem instead of ending the attempt. In a checkout the plan module reads the package's version file, and only a copy without a checkout reads the one beside it. write_probe_verdict's docstring names the emitter's uncounted wait after the tool (#1523 review round 2). Co-Authored-By: Claude Opus 5.5 --- .../1522-modal-undelivered-events.added.md | 2 +- docs/us-modal-stage-runbook.md | 5 +- tools/modal_us_stage_plan.py | 52 ++++++++++++++++--- 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/changelog.d/1522-modal-undelivered-events.added.md b/changelog.d/1522-modal-undelivered-events.added.md index 36b7efc719..eae8307de0 100644 --- a/changelog.d/1522-modal-undelivered-events.added.md +++ b/changelog.d/1522-modal-undelivered-events.added.md @@ -1 +1 @@ -The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run or output that is not its list of runs. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s, and its verdict reports the stop's and `status`'s allowances beside the receipt's. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The image adds that checkout's file beside the plan module's copy, `/root/minimum_emitter_version.json`, which the copy reads first, so the check in the container reports the install step the image was built with. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. +The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run, output that is not its list of runs, or runs nested too deeply for the receipt to be written. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s, and its verdict reports the stop's and `status`'s allowances beside the receipt's. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The image adds that checkout's file beside the plan module's copy, `/root/minimum_emitter_version.json`, which the copy reads there, having no checkout around it, so the check in the container reports the install step the image was built with; in a checkout the package's file always wins over one left beside the plan module. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index 8cc0be6cdf..bf29ae70ab 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -217,8 +217,9 @@ release directory onto the runs volume. Publication stays the human step in - `{"runs": null, "problem": ""}`: any other exit code, with the last line `status` wrote on standard error (exit 2 is an emitter without `status`, older than 0.13.0); a timeout; a command that - could not run; or output that is not its list of runs. The line has - at most 300 characters. + could not run; output that is not its list of runs; or runs nested + so deeply that the receipt could not be written with them. The line + has at most 300 characters. A `status` failure is recorded only: it never changes the stage's status or outcome. The plan's summary and the check have diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index cb52696dd7..a7b9b32133 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -112,7 +112,7 @@ # (image_emitter_requirement). In the container this module is the copy the # image adds at /root, with no checkout around it, so the image also adds the # launching checkout's file beside it (MINIMUM_EMITTER_VERSION_FILENAME), and -# a copy with that file beside it reads it first +# a copy without a checkout around it reads that one # (minimum_emitter_version_file): the check, which runs there, then reports # the requirement the image was built with. So at launch the image's install # step comes from the launching checkout, and the minimum the build enforces @@ -1563,13 +1563,15 @@ def image_pip_bootstrap_command( def minimum_emitter_version_file() -> Path: """The file that names the oldest emitter release a build starts. - The copy beside this module when there is one: the image adds the - launching checkout's file beside its copy of this module. Otherwise the - file in this module's checkout, ``MINIMUM_EMITTER_VERSION_FILE``. + In a checkout, the package's file, ``MINIMUM_EMITTER_VERSION_FILE``, so a + file left beside this module there changes nothing. Only without a + checkout around this module, as for the copy in the container, the file + beside it, which the image adds there from the launching checkout. """ - beside = Path(__file__).resolve().with_name(MINIMUM_EMITTER_VERSION_FILENAME) - return beside if beside.is_file() else MINIMUM_EMITTER_VERSION_FILE + if MINIMUM_EMITTER_VERSION_FILE.parent.is_dir(): + return MINIMUM_EMITTER_VERSION_FILE + return Path(__file__).resolve().with_name(MINIMUM_EMITTER_VERSION_FILENAME) def minimum_emitter_version() -> str: @@ -2245,6 +2247,10 @@ def telemetry_record( _STATUS_COMMAND = "`microcosm-emitter status`" +#: How many objects deep the receipt holds the runs (the receipt, ``runner``, +#: ``telemetry``, ``undelivered_events``), and a margin for what passes the +#: receipt on, such as Modal's return of it to the launch. +_RECEIPT_NESTING_OF_RUNS = 4 + 4 _STATUS_RUN_FIELDS = { "run_id": str, "producer_id": str, @@ -2319,9 +2325,34 @@ def undelivered_events_from_status( return undelivered_events_problem( f"{_STATUS_COMMAND} exited 0 but did not print its JSON list of runs" ) + if not _receipt_can_hold(runs): + return undelivered_events_problem( + f"{_STATUS_COMMAND} exited 0 but printed runs nested too deeply " + "for the receipt" + ) return {"runs": runs, "problem": None} +def _receipt_can_hold(runs: list[EmitterStatusRun]) -> bool: + """Whether the receipt can be written with ``runs`` in it. + + The runs are kept as printed, extra fields included, and a field nested + just under the JSON parser's limit passes the parse; held deeper in the + receipt, it would make the receipt's write fail after the mirror and + leave no receipt. So they are serialized once here, as deep as the + receipt holds them and a little deeper, as the receipt is written. + """ + + held: object = {"runs": runs} + for _ in range(_RECEIPT_NESTING_OF_RUNS): + held = {"held": held} + try: + json.dumps(held, indent=2) + except (ValueError, RecursionError): + return False + return True + + def token_needed_until(plan: Plan, container_started: float) -> int: """When the collector token must still be usable, in seconds since the epoch. @@ -3572,8 +3603,13 @@ def write_probe_verdict( runner keeps ``runner_overhead_seconds`` after the tool's deadline, of which the stop may take STOP_GRACE_SECONDS, the image emitter's ``status`` EMITTER_STATUS_TIMEOUT_SECONDS, and the runner's identity, the - receipt and the attempt's final commit RECEIPT_RESERVE_SECONDS; if the - extrapolated mirror does not fit in the rest, a stop at the budget would + receipt and the attempt's final commit RECEIPT_RESERVE_SECONDS; the + verdict reports each figure. Not counted: after the tool exits, also + after a stop, the emitter the build started keeps the tool's output open + while it finishes, up to EMITTER_EXIT_NOTICE_SECONDS plus + EMITTER_DRAIN_SECONDS, and the runner waits for it before ``status``. If + the extrapolated mirror does not fit in the mirror's share, a stop at the + budget would run into the class timeout mid-mirror and leave no receipt, so the check reports a problem. One small probe from the check container is a hint, not a guarantee. From d913a13b04b62fc97c10d74afa043d20b8cef02a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:05:57 +0400 Subject: [PATCH 8/9] Test a 32-level bound on the runs status prints, without deep dumps The test of runs nested just under the parser's limit wrote indented JSON tens of thousands of levels deep, which takes memory growing with the square of the depth: about 3 GB more on Python 3.14 here, the likely cause of the runner shutdown in engine-free (3.14) of run 42083209447. It is replaced by a test of a fixed bound: a field nested 32 levels deep is recorded as printed, one nested 33 levels deep as a problem (#1523). Co-Authored-By: Claude Opus 5.5 --- .../us/test_us_modal_stage_plan_tool.py | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py index 999b973a5d..41a663d184 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_modal_stage_plan_tool.py @@ -3452,43 +3452,31 @@ def _status_output_with_an_extra_field(depth: int) -> str: return f'{printed[:-3]}, "extra": {nested}}}]}}' -def _deepest_extra_field_the_parser_accepts() -> int: - low, high = 1, 1_000_000 - while low < high: - middle = (low + high + 1) // 2 - try: - json.loads(_status_output_with_an_extra_field(middle)) - except RecursionError: - high = middle - 1 - else: - low = middle - return low - - -def test_runs_nested_deeper_than_a_receipt_can_hold_are_recorded_as_a_problem() -> None: - """Runs the parser accepts but the receipt cannot be written with. - - ``status``'s runs are recorded as printed, extra fields included. A field - nested just under the parser's limit is accepted, but the receipt holds - the runs four objects deeper, and writing it (after the mirror) then fails - on Python 3.13 and 3.14: the stage would get no receipt. +@pytest.mark.parametrize(("depth", "recorded"), [(32, True), (33, False)]) +def test_runs_with_a_field_nested_deeper_than_32_levels_are_recorded_as_a_problem( + depth: int, recorded: bool +) -> None: + """The runs are kept as printed, but none nested more than 32 levels deep. + + The receipt holds the runs six levels deeper and is written with indented + JSON, whose memory grows with the square of the depth: a field nested + tens of thousands of levels deep, which the parser accepts, would take + gigabytes to write, or fail to and leave no receipt. ``status`` prints + flat fields, so a bound of 32 refuses nothing it prints. """ - stdout = _status_output_with_an_extra_field( - _deepest_extra_field_the_parser_accepts() - 1 - ) - runs = json.loads(stdout)["runs"] - with pytest.raises(RecursionError): - json.dumps( - {"runner": {"telemetry": {"undelivered_events": {"runs": runs}}}}, - indent=2, - ) + stdout = _status_output_with_an_extra_field(depth) - assert plan_lib.undelivered_events_from_status(0, stdout, "") == { - "runs": None, - "problem": "`microcosm-emitter status` exited 0 but printed runs nested " - "too deeply for the receipt", - } + record = plan_lib.undelivered_events_from_status(0, stdout, "") + + if recorded: + assert record == {"runs": json.loads(stdout)["runs"], "problem": None} + else: + assert record == { + "runs": None, + "problem": "`microcosm-emitter status` exited 0 but printed runs " + "nested more than 32 levels deep", + } def test_deeply_nested_output_is_recorded_as_a_problem( From 84b4c7ad91031c29ba65383b038f86933f078dfb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:07:57 +0400 Subject: [PATCH 9/9] Bound the runs' field nesting at 32 instead of dumping them Serializing the runs as indented JSON to see whether the receipt could hold them costs memory growing with the square of the nesting depth, up to several gigabytes for output the parser accepts. A field nested more than 32 levels deep, found with an iterative walk, is now recorded as a problem; status prints flat fields, so no answer it gives is refused (#1523). Co-Authored-By: Claude Opus 5.5 --- .../1522-modal-undelivered-events.added.md | 2 +- docs/us-modal-stage-runbook.md | 8 ++- tools/modal_us_stage_plan.py | 55 +++++++++++-------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/changelog.d/1522-modal-undelivered-events.added.md b/changelog.d/1522-modal-undelivered-events.added.md index eae8307de0..c00aa30f3f 100644 --- a/changelog.d/1522-modal-undelivered-events.added.md +++ b/changelog.d/1522-modal-undelivered-events.added.md @@ -1 +1 @@ -The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run, output that is not its list of runs, or runs nested too deeply for the receipt to be written. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s, and its verdict reports the stop's and `status`'s allowances beside the receipt's. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The image adds that checkout's file beside the plan module's copy, `/root/minimum_emitter_version.json`, which the copy reads there, having no checkout around it, so the check in the container reports the install step the image was built with; in a checkout the package's file always wins over one left beside the plan module. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. +The US Modal stage runner records how many telemetry events each run's emitter left undelivered (microcosm#1522). After the tool exits and its output ends, which is after the emitter the build started has stopped, `tools/modal_us_stage.py` runs the image emitter's read-only `microcosm-emitter status` (microcosm-emitter 0.13.0) without `--event-queue-file`, in the tool's environment without credentials or the collector token, with a 60 s timeout, and never reads the emitter's event queue file itself. The receipt's `runner.telemetry.undelivered_events`, `null` until now, becomes `{"runs": [...], "problem": null}` with `status`'s runs as printed (`run_id`, `producer_id`, `upload_state`, `reason`, `undelivered_events`); `{"runs": [], "problem": null}` when no event queue file exists (exit 8); and `{"runs": null, "problem": ""}` for any other exit code, a timeout, a command that cannot run, output that is not its list of runs, or runs with a field nested more than 32 levels deep. A failure is recorded only and never changes the stage's status or outcome. The runner's reserve after the tool keeps those 60 s, so the check's write probe leaves the mirror 1,560 s of a 30-minute reserve instead of 1,620 s, and its verdict reports the stop's and `status`'s allowances beside the receipt's. Builds and the Modal image now require microcosm-emitter 0.13.0 or later instead of 0.11.9, written once in `packages/microcosm-build/src/microcosm/build/minimum_emitter_version.json`: `microcosm.build.telemetry_emitter.MINIMUM_EMITTER_VERSION` reads it as package data and the runner's image install step reads it by its path in the launching checkout, replacing the repeated value and the test that kept the two equal. The image adds that checkout's file beside the plan module's copy, `/root/minimum_emitter_version.json`, which the copy reads there, having no checkout around it, so the check in the container reports the install step the image was built with; in a checkout the package's file always wins over one left beside the plan module. The installed-emitter job's pin rises to 0.13.0, and a new installed-emitter test runs a stage through the runner with the real emitter. diff --git a/docs/us-modal-stage-runbook.md b/docs/us-modal-stage-runbook.md index bf29ae70ab..0739351828 100644 --- a/docs/us-modal-stage-runbook.md +++ b/docs/us-modal-stage-runbook.md @@ -217,9 +217,11 @@ release directory onto the runs volume. Publication stays the human step in - `{"runs": null, "problem": ""}`: any other exit code, with the last line `status` wrote on standard error (exit 2 is an emitter without `status`, older than 0.13.0); a timeout; a command that - could not run; output that is not its list of runs; or runs nested - so deeply that the receipt could not be written with them. The line - has at most 300 characters. + could not run; output that is not its list of runs; or runs with a + field nested more than 32 levels deep + (`EMITTER_STATUS_MAX_FIELD_NESTING`; `status` prints flat fields, + and the receipt's indented JSON needs memory that grows with the + square of the depth). The line has at most 300 characters. A `status` failure is recorded only: it never changes the stage's status or outcome. The plan's summary and the check have diff --git a/tools/modal_us_stage_plan.py b/tools/modal_us_stage_plan.py index a7b9b32133..0c61f9a161 100644 --- a/tools/modal_us_stage_plan.py +++ b/tools/modal_us_stage_plan.py @@ -2247,10 +2247,12 @@ def telemetry_record( _STATUS_COMMAND = "`microcosm-emitter status`" -#: How many objects deep the receipt holds the runs (the receipt, ``runner``, -#: ``telemetry``, ``undelivered_events``), and a margin for what passes the -#: receipt on, such as Modal's return of it to the launch. -_RECEIPT_NESTING_OF_RUNS = 4 + 4 +#: How deep a run's fields may nest (a scalar is 0, an object or list one +#: more than its deepest member). ``status`` prints scalars; a deeper field +#: would reach the receipt, which is written as indented JSON, whose memory +#: grows with the square of the depth, and which fails past the +#: interpreter's recursion limits. +EMITTER_STATUS_MAX_FIELD_NESTING = 32 _STATUS_RUN_FIELDS = { "run_id": str, "producer_id": str, @@ -2306,7 +2308,8 @@ def undelivered_events_from_status( and the last line ``status`` wrote on standard error, which is its own message, argparse's for an emitter without the command (exit 2), or a traceback's last line. Output that is not its list of runs is a problem - too. + too, and so are runs with a field nested more than + EMITTER_STATUS_MAX_FIELD_NESTING levels deep. """ if returncode == EMITTER_STATUS_NO_FILE_EXIT_CODE: @@ -2325,32 +2328,38 @@ def undelivered_events_from_status( return undelivered_events_problem( f"{_STATUS_COMMAND} exited 0 but did not print its JSON list of runs" ) - if not _receipt_can_hold(runs): + if any( + _nesting(value) > EMITTER_STATUS_MAX_FIELD_NESTING + for run in runs + for value in run.values() + ): return undelivered_events_problem( - f"{_STATUS_COMMAND} exited 0 but printed runs nested too deeply " - "for the receipt" + f"{_STATUS_COMMAND} exited 0 but printed runs nested more than " + f"{EMITTER_STATUS_MAX_FIELD_NESTING} levels deep" ) return {"runs": runs, "problem": None} -def _receipt_can_hold(runs: list[EmitterStatusRun]) -> bool: - """Whether the receipt can be written with ``runs`` in it. +def _nesting(value: object) -> int: + """How deep ``value`` nests: 0 for a scalar, one more than its deepest member. - The runs are kept as printed, extra fields included, and a field nested - just under the JSON parser's limit passes the parse; held deeper in the - receipt, it would make the receipt's write fail after the mirror and - leave no receipt. So they are serialized once here, as deep as the - receipt holds them and a little deeper, as the receipt is written. + Walked with a list rather than by recursion, so any depth the parser + accepted costs memory in proportion to its size. """ - held: object = {"runs": runs} - for _ in range(_RECEIPT_NESTING_OF_RUNS): - held = {"held": held} - try: - json.dumps(held, indent=2) - except (ValueError, RecursionError): - return False - return True + deepest = 0 + pending = [(value, 0)] + while pending: + item, depth = pending.pop() + if isinstance(item, dict): + members = list(item.values()) + elif isinstance(item, list): + members = item + else: + continue + deepest = max(deepest, depth + 1) + pending.extend((member, depth + 1) for member in members) + return deepest def token_needed_until(plan: Plan, container_started: float) -> int: