From 7a2981a477d5371e347737f7d5e66a69bc204683 Mon Sep 17 00:00:00 2001 From: Matt Galligan Date: Thu, 11 Jun 2026 13:32:21 -0400 Subject: [PATCH] test: add dispatch fixture corpus --- AGENTS.md | 2 + docs/development/design.md | 2 + docs/usage/README.md | 15 ++ justfile | 4 + pyproject.toml | 1 + scripts/check_pypi_smoke.py | 163 ++++++++++++++++++ tests/client/test_client.py | 36 +--- tests/client/test_models.py | 71 ++++---- tests/fixtures/README.md | 21 +++ tests/fixtures/__init__.py | 45 +++++ .../app_server/config_read/current.json | 9 + .../turn_failure_unsupported_model.jsonl | 4 + .../app_server/model_list/current.json | 69 ++++++++ .../legacy_additional_speed_tiers.json | 18 ++ .../app_server/thread_list/basic.json | 26 +++ .../app_server/thread_read/with_turns.json | 31 ++++ tests/fixtures/cli_smoke/README.md | 10 ++ tests/fixtures/registry/builders.py | 74 ++++++++ tests/fixtures/test_corpus.py | 115 ++++++++++++ .../long_history_top_and_tail.jsonl | 6 + .../transcripts/malformed_lines.jsonl | 6 + tests/fixtures/transcripts/minimal.jsonl | 3 + tests/registry/test_store.py | 41 +---- 23 files changed, 664 insertions(+), 108 deletions(-) create mode 100644 scripts/check_pypi_smoke.py create mode 100644 tests/fixtures/README.md create mode 100644 tests/fixtures/__init__.py create mode 100644 tests/fixtures/app_server/config_read/current.json create mode 100644 tests/fixtures/app_server/events/turn_failure_unsupported_model.jsonl create mode 100644 tests/fixtures/app_server/model_list/current.json create mode 100644 tests/fixtures/app_server/model_list/legacy_additional_speed_tiers.json create mode 100644 tests/fixtures/app_server/thread_list/basic.json create mode 100644 tests/fixtures/app_server/thread_read/with_turns.json create mode 100644 tests/fixtures/cli_smoke/README.md create mode 100644 tests/fixtures/registry/builders.py create mode 100644 tests/fixtures/test_corpus.py create mode 100644 tests/fixtures/transcripts/long_history_top_and_tail.jsonl create mode 100644 tests/fixtures/transcripts/malformed_lines.jsonl create mode 100644 tests/fixtures/transcripts/minimal.jsonl diff --git a/AGENTS.md b/AGENTS.md index 411dde7..064fe2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,7 @@ dispatch owns one `codex app-server` subprocess (stdio JSONL, shared `~/.codex`) - `docs/usage/` — operator docs for the CLI, MCP, triggers, and plugin setup. - `.agents/plans/v0/` — phased plan (`PLAN.md`) + references (`REFS.md`); tracked. - `spikes/` — App Server probe scripts; seed of the integration suite. +- `tests/fixtures/` — small named App Server, JSONL, CLI-smoke, and registry fixtures. - `.agents/notes/` — working notes, session recaps, learnings; **gitignored, local only**. - `skills/` — first-party Codex skills for operating dispatch (`dispatch`) and dispatch-backed direct messages (`dm`). - `plugins/dispatch/` — workspace-local Codex plugin bundle exposing the skills and MCP server. @@ -64,6 +65,7 @@ Use the project language consistently: - **App Server access only via `client/`.** Never spawn or speak to `codex app-server` outside the client layer. See [client rules](.claude/rules/client.md). - **Async core, sync CLI.** The daemon is asyncio end-to-end; the CLI is a thin sync client over the control socket. No blocking calls in the loop (use `aiosqlite`, asyncio subprocess, `run_in_executor`). See [python-conventions](.claude/rules/python-conventions.md). - **Never touch the user's live state in tests.** Integration tests use a real ephemeral app-server with an isolated `CODEX_HOME` and `ephemeral:true` lanes. +- **Fixtures should be exercised.** Add checked-in cases under `tests/fixtures/` only when a test loads them; prefer Python builders over binary SQLite fixtures. ## Source control diff --git a/docs/development/design.md b/docs/development/design.md index 301b428..e3d7ce3 100644 --- a/docs/development/design.md +++ b/docs/development/design.md @@ -160,6 +160,7 @@ The client supports the full responder loop. v1 surfaces `waiting_on_approval` a - Async: stdlib **asyncio** (subprocess + streams + unix socket server). DB: **aiosqlite** (hand-written SQL; no ORM). Logging: **structlog** (also feeds the audit log). - MCP: the official Python **`mcp`** SDK (stdio transport first). Scheduling: small custom asyncio scheduler + `croniter` for cron (interval needs no lib). No `dateutil`/RRULE in v1. - Tests: **pytest** + **pytest-asyncio**. Hooks: **lefthook** (polyglot; runs ruff/mypy/pytest). Task runner: **just** (justfile) for `test`/`lint`/`typecheck`/`run`. Daemon keep-alive: **launchd** LaunchAgent plist. CI: GitHub Actions + `astral-sh/setup-uv`. +- Fixture corpus: `tests/fixtures/` stores small named App Server payloads, Codex JSONL sync sources, CLI-smoke notes, and registry builders. Every checked-in fixture should be loaded by a test. Prefer builders over binary SQLite files. ## Data model (registry, SQLite) @@ -185,6 +186,7 @@ The client supports the full responder loop. v1 surfaces `waiting_on_approval` a - Promote the existing probe scripts (`/tmp/codex_{stdio,dm,lab4,fanout}.py`) into the integration suite, run against a **real ephemeral app-server with an isolated `CODEX_HOME`** (zero pollution; `ephemeral:true` lanes). - `test_examples(registry)` runs op examples as assertions. - Unit: message router (canned JSONL), trigger/guard evaluation, registry, error projections. +- Release smoke: `just pypi-smoke -- --package-spec outfitter-dispatch==` installs the published package with `uvx`, uses a temporary `DISPATCH_HOME`, verifies daemon/model/list paths, and shuts down cleanly. ## Rough build slices (detailed by the implementation plan) diff --git a/docs/usage/README.md b/docs/usage/README.md index 8a285a7..a64c409 100644 --- a/docs/usage/README.md +++ b/docs/usage/README.md @@ -41,6 +41,18 @@ dispatch daemon status dispatch down --json ``` +Maintainers can run the same release smoke from the repository against the +published package: + +```bash +just pypi-smoke -- --package-spec outfitter-dispatch==0.5.0 +``` + +The smoke installs with `uvx`, uses a temporary `DISPATCH_HOME`, verifies the +derived `models` schema, starts the daemon, reads the live App Server model +catalog, verifies the cached registry read, checks the empty first-run lane list, +and shuts the daemon down. + If `dispatch doctor` fails before the app-server smoke because the Codex CLI is not installed or authenticated, fix that first and rerun the doctor. Use `dispatch doctor --no-app-server` when you only need to inspect package, PATH, @@ -141,6 +153,9 @@ release, bump `project.version` in `pyproject.toml`, run: just check ``` +After the GitHub Release publishes to PyPI, run `just pypi-smoke -- --package-spec +outfitter-dispatch==` to verify the public install path. + Then create and publish a GitHub Release for the same tag, for example `v0.1.0`. Do not upload with a long-lived PyPI token unless the trusted publisher path is unavailable and the maintainer explicitly chooses that diff --git a/justfile b/justfile index 1788c7d..5744895 100644 --- a/justfile +++ b/justfile @@ -18,6 +18,10 @@ test *args: test-int *args: uv run pytest -m integration {{args}} +# Smoke-test the published PyPI package from a clean temporary DISPATCH_HOME. +pypi-smoke *args: + uv run python scripts/check_pypi_smoke.py {{args}} + # Lint with ruff. lint: uv run ruff check . diff --git a/pyproject.toml b/pyproject.toml index 83b944e..81992e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ exclude = ["/src/**/AGENTS.md"] include = [ "/src", "/tests", + "/scripts", "/skills", "/plugins/dispatch", "/README.md", diff --git a/scripts/check_pypi_smoke.py b/scripts/check_pypi_smoke.py new file mode 100644 index 0000000..997dddd --- /dev/null +++ b/scripts/check_pypi_smoke.py @@ -0,0 +1,163 @@ +"""Smoke-test the published PyPI package from a clean Dispatch home. + +This is intentionally not part of ``just check``: it installs from PyPI with +``uvx`` and starts a real daemon/app-server. Run it after publishing or when +validating the clean-install path tracked by GitHub issue #27. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import tomllib +from pathlib import Path +from typing import Any + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(argv) + package_spec = args.package_spec or f"outfitter-dispatch=={_project_version()}" + home = Path(tempfile.mkdtemp(prefix="dispatch-pypi-smoke.")) + env = os.environ.copy() + env["DISPATCH_HOME"] = str(home) + print(f"DISPATCH_HOME={home}") + print(f"package={package_spec}") + try: + version = _dispatch(package_spec, ["--version"], env) + _expect(version.stdout.strip().startswith("dispatch "), version.stdout) + + schema = _dispatch_json(package_spec, ["schema", "models"], env) + _expect(schema.get("op") == "models", "schema op is not models") + _expect(_path(schema, "input", "properties", "refresh", "type") == "boolean", schema) + _expect(_path(schema, "output", "properties", "models", "type") == "array", schema) + + up = _dispatch_json(package_spec, ["up", "--json"], env, timeout=args.timeout) + _expect(up.get("status") in {"started", "running"}, up) + + models = _dispatch_json(package_spec, ["models", "--json"], env, timeout=args.timeout) + _expect(models.get("source") == "app-server", models) + _expect(_nonempty_list(models.get("models")), models) + configured = models.get("configured_default") + _expect(isinstance(configured, dict), models) + _expect(isinstance(configured.get("model"), str), models) + + cached = _dispatch_json( + package_spec, ["models", "--no-refresh", "--json"], env, timeout=args.timeout + ) + _expect(cached.get("source") == "registry", cached) + _expect(_nonempty_list(cached.get("models")), cached) + + lanes = _dispatch_json(package_spec, ["list", "--json"], env) + _expect(isinstance(lanes.get("lanes"), list), lanes) + + down = _dispatch_json(package_spec, ["down", "--json"], env) + _expect(down.get("status") == "stopped", down) + print("PyPI clean-install smoke passed") + return 0 + finally: + if not args.keep_home: + _dispatch(package_spec, ["down", "--json"], env, check=False) + shutil.rmtree(home, ignore_errors=True) + + +def _parse_args(argv: list[str] | None) -> argparse.Namespace: + raw_args = sys.argv[1:] if argv is None else argv + if raw_args and raw_args[0] == "--": + raw_args = raw_args[1:] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--package-spec", + help="uvx package spec to install, e.g. outfitter-dispatch==0.5.0", + ) + parser.add_argument( + "--timeout", + type=float, + default=90.0, + help="seconds to allow each daemon/app-server command", + ) + parser.add_argument( + "--keep-home", + action="store_true", + help="keep the temporary DISPATCH_HOME for debugging", + ) + return parser.parse_args(raw_args) + + +def _project_version() -> str: + with Path("pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + version = project["version"] + if not isinstance(version, str): + raise SystemExit("pyproject.toml project.version is not a string") + return version + + +def _dispatch( + package_spec: str, + args: list[str], + env: dict[str, str], + *, + timeout: float = 90.0, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + ["uvx", "--from", package_spec, "dispatch", *args], + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if check and result.returncode != 0: + raise SystemExit( + f"dispatch {' '.join(args)} failed with {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +def _dispatch_json( + package_spec: str, + args: list[str], + env: dict[str, str], + *, + timeout: float = 90.0, +) -> dict[str, Any]: + result = _dispatch(package_spec, args, env, timeout=timeout) + try: + parsed = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise SystemExit( + f"dispatch {' '.join(args)} did not produce JSON\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) from exc + if not isinstance(parsed, dict): + raise SystemExit(f"dispatch {' '.join(args)} produced non-object JSON") + return parsed + + +def _path(data: dict[str, Any], *parts: str) -> Any: + value: Any = data + for part in parts: + if not isinstance(value, dict): + return None + value = value.get(part) + return value + + +def _nonempty_list(value: object) -> bool: + return isinstance(value, list) and len(value) > 0 + + +def _expect(condition: bool, detail: object) -> None: + if not condition: + raise SystemExit(f"PyPI smoke assertion failed: {detail!r}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/client/test_client.py b/tests/client/test_client.py index 588acd1..c6b0f08 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -9,6 +9,7 @@ from outfitter.dispatch.client.client import AppServerClient from outfitter.dispatch.client.errors import AppServerError, ProtocolError, TransportError from outfitter.dispatch.client.events import TurnCompleted +from tests.fixtures import load_json from .conftest import FakeTransport, Responder @@ -220,45 +221,18 @@ async def test_config_read_and_model_list_parse_current_catalog_shape( client: tuple[AppServerClient, FakeTransport], ) -> None: c, fake = client - fake.auto = _result_for( - "config/read", - { - "config": { - "model": "gpt-5.5", - "modelProvider": "openai", - "serviceTier": "priority", - "modelReasoningEffort": "xhigh", - } - }, - ) + fake.auto = _result_for("config/read", load_json("app_server", "config_read", "current.json")) config = await c.config_read() assert config.model == "gpt-5.5" assert config.model_provider == "openai" - assert config.service_tier == "priority" + assert config.service_tier == "fast" assert config.model_reasoning_effort == "xhigh" assert fake.sent[-1] == {"id": 1, "method": "config/read", "params": {}} - fake.auto = _result_for( - "model/list", - { - "data": [ - { - "id": "gpt-5.5", - "defaultReasoningEffort": "xhigh", - "supportedReasoningEfforts": ["low", "xhigh"], - "serviceTiers": [ - { - "id": "priority", - "name": "Fast", - "description": "1.5x speed, increased usage", - } - ], - } - ] - }, - ) + fake.auto = _result_for("model/list", load_json("app_server", "model_list", "current.json")) models = await c.model_list() assert models[0].id == "gpt-5.5" + assert models[0].supported_reasoning_efforts == ["low", "medium", "high", "xhigh"] assert models[0].service_tiers[0].id == "priority" assert models[0].service_tiers[0].name == "Fast" assert fake.sent[-1] == {"id": 2, "method": "model/list", "params": {}} diff --git a/tests/client/test_models.py b/tests/client/test_models.py index b1190eb..972229f 100644 --- a/tests/client/test_models.py +++ b/tests/client/test_models.py @@ -24,6 +24,7 @@ TurnStartParams, TurnSteerParams, ) +from tests.fixtures import load_json def test_thread_start_sandbox_is_string_enum() -> None: @@ -185,52 +186,42 @@ def test_thread_info_keeps_observed_model_service_tier() -> None: def test_config_and_model_catalog_wire_models_accept_camel_case() -> None: - config = ConfigInfo.model_validate( - { - "model": "gpt-5.5", - "modelProvider": "openai", - "serviceTier": "priority", - "modelReasoningEffort": "xhigh", - } + config_payload = load_json("app_server", "config_read", "current.json")["config"] + config = ConfigInfo.model_validate(config_payload) + catalog = ModelListResult.model_validate(load_json("app_server", "model_list", "current.json")) + + assert config.model_provider == "openai" + assert config.service_tier == "fast" + assert catalog.data[0] == AppModel( + id="gpt-5.5", + model="gpt-5.5", + display_name="GPT-5.5", + description="Frontier model for complex coding, research, and real-world work.", + is_default=True, + hidden=False, + default_reasoning_effort="medium", + supported_reasoning_efforts=["low", "medium", "high", "xhigh"], + service_tiers=[ + ModelServiceTier( + id="priority", + name="Fast", + description="1.5x speed, increased usage", + ) + ], ) + + +def test_legacy_model_catalog_fixture_keeps_speed_tier_fallback() -> None: catalog = ModelListResult.model_validate( - { - "data": [ - { - "id": "gpt-5.5", - "displayName": "GPT-5.5", - "defaultReasoningEffort": "xhigh", - "supportedReasoningEfforts": [ - {"reasoningEffort": "low", "description": "faster"}, - {"reasoningEffort": "xhigh", "description": "deeper"}, - ], - "serviceTiers": [ - { - "id": "priority", - "name": "Fast", - "description": "1.5x speed, increased usage", - } - ], - "additionalSpeedTiers": ["fast"], - } - ] - } + load_json("app_server", "model_list", "legacy_additional_speed_tiers.json") ) - assert config.model_provider == "openai" assert catalog.data == [ AppModel( - id="gpt-5.5", - display_name="GPT-5.5", - default_reasoning_effort="xhigh", - supported_reasoning_efforts=["low", "xhigh"], - service_tiers=[ - ModelServiceTier( - id="priority", - name="Fast", - description="1.5x speed, increased usage", - ) - ], + id="legacy-fast-model", + display_name="Legacy Fast Model", + default_reasoning_effort="medium", + supported_reasoning_efforts=["low", "medium"], additional_speed_tiers=["fast"], ) ] diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..6bdf4a5 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1,21 @@ +# Dispatch Test Fixture Corpus + +This directory holds small, named fixtures for App Server protocol payloads, +Codex JSONL sources, CLI smoke recipes, and registry builders. Fixtures here are +for tests, not product docs or generated outputs. + +Rules: + +- Keep checked-in fixtures small and readable. +- Prefer JSON/JSONL text over binary artifacts. +- Prefer Python builders over committed SQLite database files. +- Add a test that loads every new fixture; unexercised fixtures rot. +- Use synthetic ids, paths, and prompts. Do not copy private thread content. + +Layout: + +- `app_server/` — raw App Server result payloads and notifications. +- `transcripts/` — Codex persisted JSONL source files for sync parsing. +- `registry/` — builders for registry rows and migration test setup. +- `cli_smoke/` — notes and recipes for install/first-run smoke checks. + diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py new file mode 100644 index 0000000..6816073 --- /dev/null +++ b/tests/fixtures/__init__.py @@ -0,0 +1,45 @@ +"""Shared fixture loading helpers. + +Fixtures are intentionally data-first. Tests import these helpers so fixture +paths stay stable while individual test modules remain focused on behavior. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import cast + +JsonObject = dict[str, object] + +FIXTURE_ROOT = Path(__file__).resolve().parent + + +def fixture_path(*parts: str) -> Path: + return FIXTURE_ROOT.joinpath(*parts) + + +def load_json(*parts: str) -> JsonObject: + data = json.loads(fixture_path(*parts).read_text()) + if not isinstance(data, dict): + raise AssertionError(f"expected JSON object fixture: {'/'.join(parts)}") + return cast(JsonObject, data) + + +def load_jsonl(*parts: str) -> list[JsonObject]: + rows: list[JsonObject] = [] + for line_number, line in enumerate(fixture_path(*parts).read_text().splitlines(), start=1): + if not line: + continue + data = json.loads(line) + if not isinstance(data, dict): + raise AssertionError(f"expected JSON object at {'/'.join(parts)}:{line_number}") + rows.append(cast(JsonObject, data)) + return rows + + +def copy_fixture(*parts: str, to: Path) -> Path: + to.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(fixture_path(*parts), to) + return to diff --git a/tests/fixtures/app_server/config_read/current.json b/tests/fixtures/app_server/config_read/current.json new file mode 100644 index 0000000..86e4fc8 --- /dev/null +++ b/tests/fixtures/app_server/config_read/current.json @@ -0,0 +1,9 @@ +{ + "config": { + "model": "gpt-5.5", + "modelProvider": "openai", + "serviceTier": "fast", + "modelReasoningEffort": "xhigh" + } +} + diff --git a/tests/fixtures/app_server/events/turn_failure_unsupported_model.jsonl b/tests/fixtures/app_server/events/turn_failure_unsupported_model.jsonl new file mode 100644 index 0000000..4893b52 --- /dev/null +++ b/tests/fixtures/app_server/events/turn_failure_unsupported_model.jsonl @@ -0,0 +1,4 @@ +{"method":"turn/started","params":{"threadId":"019f0000-0000-7000-9000-000000000001","turnId":"turn-1"}} +{"method":"turn/failed","params":{"threadId":"019f0000-0000-7000-9000-000000000001","turnId":"turn-1","message":"unsupported model: gpt-5.5-codex"}} +{"method":"thread/status/changed","params":{"threadId":"019f0000-0000-7000-9000-000000000001","status":{"activeFlags":[]}}} + diff --git a/tests/fixtures/app_server/model_list/current.json b/tests/fixtures/app_server/model_list/current.json new file mode 100644 index 0000000..8dcfd3a --- /dev/null +++ b/tests/fixtures/app_server/model_list/current.json @@ -0,0 +1,69 @@ +{ + "data": [ + { + "id": "gpt-5.5", + "model": "gpt-5.5", + "displayName": "GPT-5.5", + "description": "Frontier model for complex coding, research, and real-world work.", + "isDefault": true, + "hidden": false, + "defaultReasoningEffort": "medium", + "supportedReasoningEfforts": [ + { + "reasoningEffort": "low", + "description": "Fastest responses." + }, + { + "reasoningEffort": "medium", + "description": "Balanced responses." + }, + { + "reasoningEffort": "high", + "description": "Deeper reasoning." + }, + { + "reasoningEffort": "xhigh", + "description": "Maximum reasoning." + } + ], + "defaultServiceTier": null, + "serviceTiers": [ + { + "id": "priority", + "name": "Fast", + "description": "1.5x speed, increased usage" + } + ] + }, + { + "id": "gpt-5.3-codex-spark", + "model": "gpt-5.3-codex-spark", + "displayName": "GPT-5.3-Codex-Spark", + "description": "Ultra-fast coding model.", + "isDefault": false, + "hidden": false, + "defaultReasoningEffort": "high", + "supportedReasoningEfforts": [ + { + "reasoningEffort": "low", + "description": "Fastest responses." + }, + { + "reasoningEffort": "medium", + "description": "Balanced responses." + }, + { + "reasoningEffort": "high", + "description": "Deeper reasoning." + }, + { + "reasoningEffort": "xhigh", + "description": "Maximum reasoning." + } + ], + "defaultServiceTier": null, + "serviceTiers": [] + } + ] +} + diff --git a/tests/fixtures/app_server/model_list/legacy_additional_speed_tiers.json b/tests/fixtures/app_server/model_list/legacy_additional_speed_tiers.json new file mode 100644 index 0000000..9706b23 --- /dev/null +++ b/tests/fixtures/app_server/model_list/legacy_additional_speed_tiers.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "id": "legacy-fast-model", + "displayName": "Legacy Fast Model", + "defaultReasoningEffort": "medium", + "supportedReasoningEfforts": [ + "low", + "medium" + ], + "serviceTiers": [], + "additionalSpeedTiers": [ + "fast" + ] + } + ] +} + diff --git a/tests/fixtures/app_server/thread_list/basic.json b/tests/fixtures/app_server/thread_list/basic.json new file mode 100644 index 0000000..affca55 --- /dev/null +++ b/tests/fixtures/app_server/thread_list/basic.json @@ -0,0 +1,26 @@ +{ + "data": [ + { + "id": "019f0000-0000-7000-9000-000000000001", + "sessionId": "019f0000-0000-7000-9000-000000000001", + "name": "[dispatch] fixture", + "status": { + "type": "idle", + "activeFlags": [] + }, + "cwd": "/fixture/repo", + "path": "/tmp/dispatch-fixture.jsonl", + "preview": "Fixture thread preview", + "source": "appServer", + "threadSource": "user", + "modelProvider": "openai", + "model": "gpt-5.5", + "reasoningEffort": "xhigh", + "serviceTier": "priority", + "createdAt": 1781190000, + "updatedAt": 1781190300 + } + ], + "nextCursor": "cursor-1" +} + diff --git a/tests/fixtures/app_server/thread_read/with_turns.json b/tests/fixtures/app_server/thread_read/with_turns.json new file mode 100644 index 0000000..c1e9c25 --- /dev/null +++ b/tests/fixtures/app_server/thread_read/with_turns.json @@ -0,0 +1,31 @@ +{ + "thread": { + "id": "019f0000-0000-7000-9000-000000000001", + "sessionId": "019f0000-0000-7000-9000-000000000001", + "name": "[dispatch] fixture", + "cwd": "/fixture/repo", + "modelProvider": "openai", + "model": "gpt-5.5", + "reasoningEffort": "xhigh", + "serviceTier": "priority", + "turns": [ + { + "id": "turn-1", + "status": "completed", + "items": [ + { + "id": "item-user-1", + "type": "userMessage", + "text": "Summarize the fixture." + }, + { + "id": "item-agent-1", + "type": "agentMessage", + "text": "Fixture summary." + } + ] + } + ] + } +} + diff --git a/tests/fixtures/cli_smoke/README.md b/tests/fixtures/cli_smoke/README.md new file mode 100644 index 0000000..e701d94 --- /dev/null +++ b/tests/fixtures/cli_smoke/README.md @@ -0,0 +1,10 @@ +# CLI Smoke Fixtures + +The reusable clean-install smoke lives in `scripts/check_pypi_smoke.py`. It uses +a temporary `DISPATCH_HOME`, installs the published package with `uvx`, starts a +daemon, verifies the model catalog path, verifies cached registry reads, and +shuts the daemon down. + +This directory is reserved for future input/output fixtures if the smoke needs +stable golden payloads. Keep networked or machine-specific outputs out of git. + diff --git a/tests/fixtures/registry/builders.py b/tests/fixtures/registry/builders.py new file mode 100644 index 0000000..2d3ad78 --- /dev/null +++ b/tests/fixtures/registry/builders.py @@ -0,0 +1,74 @@ +"""Registry fixture builders. + +Prefer these builders over checked-in SQLite files. They keep storage tests +reviewable while still giving future tests stable, reusable shapes. +""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from outfitter.dispatch.registry.models import ( + LaneModelSettings, + ModelCatalogEntry, + ServiceTierEntry, +) + + +def fixed_now() -> datetime: + return datetime(2026, 6, 11, 12, 0, 0, tzinfo=UTC) + + +def fixed_now_iso() -> str: + return fixed_now().isoformat() + + +def service_tier_entry( + *, + id: str = "priority", + name: str = "Fast", + description: str = "1.5x speed, increased usage", +) -> ServiceTierEntry: + return ServiceTierEntry(id=id, name=name, description=description) + + +def model_catalog_entry( + *, + id: str = "gpt-5.5", + provider: str = "openai", + now: str | None = None, + service_tiers: list[ServiceTierEntry] | None = None, +) -> ModelCatalogEntry: + seen_at = now or fixed_now_iso() + return ModelCatalogEntry( + id=id, + provider=provider, + display_name="GPT-5.5", + is_default=True, + hidden=False, + default_reasoning_effort="xhigh", + supported_reasoning_efforts=["low", "xhigh"], + default_service_tier="priority", + service_tiers=service_tiers if service_tiers is not None else [service_tier_entry()], + additional_speed_tiers=["fast"], + first_seen_at=seen_at, + last_seen_at=seen_at, + ) + + +def lane_model_settings( + *, + lane: str = "L1", + updated_at: str | None = None, +) -> LaneModelSettings: + return LaneModelSettings( + lane=lane, + model_provider="openai", + model="gpt-5.5", + reasoning_effort="xhigh", + requested_service_tier="fast", + resolved_service_tier="priority", + service_tier_name="Fast", + service_tier_source="dispatch", + updated_at=updated_at or fixed_now_iso(), + ) diff --git a/tests/fixtures/test_corpus.py b/tests/fixtures/test_corpus.py new file mode 100644 index 0000000..8ebb4ac --- /dev/null +++ b/tests/fixtures/test_corpus.py @@ -0,0 +1,115 @@ +"""Tests that keep the fixture corpus executable and honest.""" + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +from outfitter.dispatch.client.events import ( + LaneIdle, + TurnFailed, + TurnStarted, + project_notification, +) +from outfitter.dispatch.client.models import ( + ConfigInfo, + ModelListResult, + ThreadInfo, + ThreadListResult, +) +from outfitter.dispatch.core.sync import SyncLimits, scan_codex_jsonl + +from . import copy_fixture, load_json, load_jsonl + + +def test_app_server_protocol_fixtures_validate_against_wire_models() -> None: + config_payload = load_json("app_server", "config_read", "current.json")["config"] + config = ConfigInfo.model_validate(config_payload) + catalog = ModelListResult.model_validate(load_json("app_server", "model_list", "current.json")) + legacy_catalog = ModelListResult.model_validate( + load_json("app_server", "model_list", "legacy_additional_speed_tiers.json") + ) + thread_list = ThreadListResult.model_validate( + load_json("app_server", "thread_list", "basic.json") + ) + thread_payload = load_json("app_server", "thread_read", "with_turns.json")["thread"] + thread = ThreadInfo.model_validate(thread_payload) + + assert config.model == "gpt-5.5" + assert config.service_tier == "fast" + assert catalog.data[0].supported_reasoning_efforts == ["low", "medium", "high", "xhigh"] + assert catalog.data[0].service_tiers[0].id == "priority" + assert legacy_catalog.data[0].additional_speed_tiers == ["fast"] + assert thread_list.data[0].model == "gpt-5.5" + assert thread_list.next_cursor == "cursor-1" + assert thread.turns[0]["id"] == "turn-1" + + +def test_app_server_event_fixture_projects_to_normalized_events() -> None: + projected = [] + for message in load_jsonl("app_server", "events", "turn_failure_unsupported_model.jsonl"): + method = message.get("method") + params = message.get("params") + assert isinstance(method, str) + assert isinstance(params, dict) + projected.extend(project_notification(method, cast(dict[str, object], params))) + + assert projected[0] == TurnStarted("019f0000-0000-7000-9000-000000000001", "turn-1") + assert projected[1] == TurnFailed( + "019f0000-0000-7000-9000-000000000001", + "turn-1", + "unsupported model: gpt-5.5-codex", + ) + assert isinstance(projected[-1], LaneIdle) + + +def test_transcript_fixtures_scan_minimal_complete_case(tmp_path: Path) -> None: + path = copy_fixture("transcripts", "minimal.jsonl", to=tmp_path / "minimal.jsonl") + + facts = scan_codex_jsonl(str(path), full=True) + + assert facts.state == "complete" + assert facts.line_count == 3 + assert facts.session_id == "019f0000-0000-7000-9000-000000000001" + assert facts.cwd == "/fixture/repo" + assert facts.model == "gpt-5.5" + assert facts.reasoning_effort == "xhigh" + assert facts.latest_turn_id == "turn-1" + + +def test_transcript_fixture_preserves_top_identity_and_tail_recency(tmp_path: Path) -> None: + path = copy_fixture( + "transcripts", + "long_history_top_and_tail.jsonl", + to=tmp_path / "long_history_top_and_tail.jsonl", + ) + + facts = scan_codex_jsonl( + str(path), + limits=SyncLimits(top_bytes=512, tail_bytes=256, tail_lines=1), + ) + + assert facts.state == "partial" + assert facts.session_id == "019f0000-0000-7000-9000-000000000010" + assert facts.cwd == "/fixture/long" + assert facts.source_kind == "cli" + assert facts.model_provider == "openai" + assert facts.latest_turn_id == "turn-9" + assert facts.latest_event_at == "2026-06-11T12:00:05.000Z" + + +def test_transcript_fixture_ignores_malformed_jsonl_records(tmp_path: Path) -> None: + path = copy_fixture( + "transcripts", + "malformed_lines.jsonl", + to=tmp_path / "malformed_lines.jsonl", + ) + + facts = scan_codex_jsonl(str(path), full=True) + + assert facts.state == "complete" + assert facts.line_count == 6 + assert facts.session_id == "019f0000-0000-7000-9000-000000000099" + assert facts.model == "gpt-5.3-codex-spark" + assert facts.reasoning_effort == "low" + assert facts.latest_turn_id == "turn-malformed" diff --git a/tests/fixtures/transcripts/long_history_top_and_tail.jsonl b/tests/fixtures/transcripts/long_history_top_and_tail.jsonl new file mode 100644 index 0000000..89bbad8 --- /dev/null +++ b/tests/fixtures/transcripts/long_history_top_and_tail.jsonl @@ -0,0 +1,6 @@ +{"type":"session_meta","timestamp":"2026-06-11T12:00:00.000Z","payload":{"id":"019f0000-0000-7000-9000-000000000010","cwd":"/fixture/long","source":"cli","thread_source":"user","model_provider":"openai"}} +{"type":"turn_context","timestamp":"2026-06-11T12:00:01.000Z","payload":{"cwd":"/fixture/long","model":"gpt-5.4","effort":"medium"}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:02.000Z","payload":{"type":"token_count","total":100}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:03.000Z","payload":{"type":"token_count","total":200}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:04.000Z","payload":{"type":"token_count","total":300}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:05.000Z","payload":{"type":"task_complete","turn_id":"turn-9"}} diff --git a/tests/fixtures/transcripts/malformed_lines.jsonl b/tests/fixtures/transcripts/malformed_lines.jsonl new file mode 100644 index 0000000..7b7652a --- /dev/null +++ b/tests/fixtures/transcripts/malformed_lines.jsonl @@ -0,0 +1,6 @@ +not json +{"type":"session_meta","timestamp":"2026-06-11T12:00:00.000Z","payload":{"id":"019f0000-0000-7000-9000-000000000099","cwd":"/fixture/malformed","source":"cli","thread_source":"user","model_provider":"openai"}} +[] +{"type":"turn_context","timestamp":"2026-06-11T12:00:01.000Z","payload":{"cwd":"/fixture/malformed","model":"gpt-5.3-codex-spark","effort":"low"}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:02.000Z","payload":{"type":"task_complete","turn_id":"turn-malformed"}} +{"type": "event_msg" diff --git a/tests/fixtures/transcripts/minimal.jsonl b/tests/fixtures/transcripts/minimal.jsonl new file mode 100644 index 0000000..f6b7cf8 --- /dev/null +++ b/tests/fixtures/transcripts/minimal.jsonl @@ -0,0 +1,3 @@ +{"type":"session_meta","timestamp":"2026-06-11T12:00:00.000Z","payload":{"id":"019f0000-0000-7000-9000-000000000001","cwd":"/fixture/repo","source":"cli","thread_source":"user","model_provider":"openai"}} +{"type":"turn_context","timestamp":"2026-06-11T12:00:01.000Z","payload":{"cwd":"/fixture/repo","model":"gpt-5.5","effort":"xhigh"}} +{"type":"event_msg","timestamp":"2026-06-11T12:00:02.000Z","payload":{"type":"task_complete","turn_id":"turn-1"}} diff --git a/tests/registry/test_store.py b/tests/registry/test_store.py index 4167818..d509c6b 100644 --- a/tests/registry/test_store.py +++ b/tests/registry/test_store.py @@ -11,14 +11,10 @@ import pytest_asyncio from outfitter.dispatch.contracts.errors import NotFoundError -from outfitter.dispatch.registry.models import ( - LaneModelSettings, - LaneSync, - ModelCatalogEntry, - ServiceTierEntry, -) +from outfitter.dispatch.registry.models import LaneSync from outfitter.dispatch.registry.refs import BASE58BTC_ALPHABET, codex_ref_payload from outfitter.dispatch.registry.store import SCHEMA_VERSION, Registry +from tests.fixtures.registry.builders import lane_model_settings, model_catalog_entry def _clock() -> datetime: @@ -236,26 +232,7 @@ async def test_migrates_v3_registry_with_runtime_columns(tmp_path: Path) -> None async def test_model_catalog_and_lane_model_settings_roundtrip(store: Registry) -> None: now = store.now_iso() - entry = ModelCatalogEntry( - id="gpt-5.5", - provider="openai", - display_name="GPT-5.5", - is_default=True, - hidden=False, - default_reasoning_effort="xhigh", - supported_reasoning_efforts=["low", "xhigh"], - default_service_tier="priority", - service_tiers=[ - ServiceTierEntry( - id="priority", - name="Fast", - description="1.5x speed, increased usage", - ) - ], - additional_speed_tiers=["fast"], - first_seen_at=now, - last_seen_at=now, - ) + entry = model_catalog_entry(now=now) await store.upsert_model_catalog([entry]) refreshed = entry.model_copy(update={"last_seen_at": "2026-06-03T12:05:00+00:00"}) await store.upsert_model_catalog([refreshed]) @@ -265,17 +242,7 @@ async def test_model_catalog_and_lane_model_settings_roundtrip(store: Registry) assert await store.list_model_catalog() == [got] lane = await store.add_lane(id="L1", handle="@alpha", source="own") - settings = LaneModelSettings( - lane=lane.id, - model_provider="openai", - model="gpt-5.5", - reasoning_effort="xhigh", - requested_service_tier="fast", - resolved_service_tier="priority", - service_tier_name="Fast", - service_tier_source="dispatch", - updated_at=now, - ) + settings = lane_model_settings(lane=lane.id, updated_at=now) await store.upsert_lane_model_settings(settings) assert await store.get_lane_model_settings(lane.id) == settings