From 6fb6e7f9fb4ef6b25d15e771dee8d24eb806e6a6 Mon Sep 17 00:00:00 2001 From: Andy Xu Date: Wed, 16 Sep 2026 20:05:21 +0000 Subject: [PATCH] Add unmanaged model discovery integration matrix --- scripts/run_integration.py | 21 ++ tests/README.md | 37 +- tests/integration/README.md | 54 +-- tests/integration/conftest.py | 15 + .../test_ug_claude_model_discovery.py | 322 ++++++++++++++++++ .../test_ug_codex_model_discovery.py | 278 +++++++++++++++ tests/integration/utils/harness.py | 22 ++ tests/test_integration_contract.py | 33 ++ 8 files changed, 746 insertions(+), 36 deletions(-) create mode 100644 tests/integration/test_ug_claude_model_discovery.py create mode 100644 tests/integration/test_ug_codex_model_discovery.py diff --git a/scripts/run_integration.py b/scripts/run_integration.py index b7b418e3..916c890d 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -105,6 +105,11 @@ def arguments(): default="main.ucode.ci_e2e_anthropic_relay_mps", help="Existing relayed (subscription-relay) Anthropic MPS for the hybrid-routing CUJ.", ) + parser.add_argument( + "--claude-provider-model", + default="claude-haiku-4-5-20251001", + help="Only model exposed by the Anthropic MPS discovery fixture.", + ) parser.add_argument( "--codex-provider", default="main.ucode.ci_openai_mps", @@ -120,6 +125,16 @@ def arguments(): default="main.ucode", help="Schema containing the dedicated model-discovery Model Services.", ) + parser.add_argument( + "--claude-parent-model", + default="main.ucode.ci_e2e_claude", + help="Claude-compatible Model Service in --parent-schema.", + ) + parser.add_argument( + "--codex-parent-model", + default="main.ucode.ci_e2e_codex", + help="Codex-compatible Model Service in --parent-schema.", + ) parser.add_argument("--python", default=sys.executable, help="Python 3.12+ path or uv version.") parser.add_argument("--dependency", action="append", default=[], metavar="PACKAGE==VERSION") parser.add_argument("--constraints", type=Path, help="Replay a previous dependencies.txt.") @@ -293,9 +308,12 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "codex_model": args.codex_model, "claude_provider": args.claude_provider, "claude_relayed_provider": args.claude_relayed_provider, + "claude_provider_model": args.claude_provider_model, "codex_provider": args.codex_provider, "codex_provider_model": args.codex_provider_model, "parent_schema": args.parent_schema, + "claude_parent_model": args.claude_parent_model, + "codex_parent_model": args.codex_parent_model, "dependencies": args.dependency, "workspace": args.workspace, }, @@ -534,9 +552,12 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_CLAUDE_PROVIDER": args.claude_provider, "UG_INTEGRATION_CLAUDE_RELAYED_PROVIDER": args.claude_relayed_provider, "UG_INTEGRATION_CLAUDE_OAUTH_TOKEN": oauth_token, + "UG_INTEGRATION_CLAUDE_PROVIDER_MODEL": args.claude_provider_model, "UG_INTEGRATION_CODEX_PROVIDER": args.codex_provider, "UG_INTEGRATION_CODEX_PROVIDER_MODEL": args.codex_provider_model, "UG_INTEGRATION_PARENT_SCHEMA": args.parent_schema, + "UG_INTEGRATION_CLAUDE_PARENT_MODEL": args.claude_parent_model, + "UG_INTEGRATION_CODEX_PARENT_MODEL": args.codex_parent_model, "UCODE_TEST_WORKSPACE": args.workspace or "", "DATABRICKS_BEARER": bearer, } diff --git a/tests/README.md b/tests/README.md index 957bbfec..fcee9451 100644 --- a/tests/README.md +++ b/tests/README.md @@ -38,6 +38,18 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_configure_codex_databricks` | Configure Databricks Hosted; execute the generated auth helper; open Codex TUI and read a file | Generated helper invokes `ug` with clean token stdout; completed assistant answer contains the file value; normal exit and reopen | | `test_ug_configure_codex_openai_mps` | Select OpenAI MPS in the real configure picker; launch Codex | Saved provider in status; completed TUI file task; normal exit | | `test_ug_claude_custom_oauth_cli_boots`, `test_ug_codex_custom_oauth_cli_boots` | Launch with `ENABLE_CUSTOM_OAUTH_FROM_CLI=1`, `--workspace`, and `--client-id databricks-cli` | Real TUI reaches a usable prompt, accepts keyboard input, exits normally, and saves `client_id = databricks-cli` in its generated CLI profile; Claude also reads the OS-managed settings and requires a profile-only `apiKeyHelper` | +| `test_case_13_configured_claude_reuses_saved_model_location` | Configure Claude with a model location, then launch without options | The saved parent supplies the discovered catalog | +| `test_case_14_configured_codex_reuses_saved_model_location` | Configure Codex with a model location, then launch without options | The saved parent supplies the discovered catalog | +| `test_case_15_fresh_claude_uses_system_models_when_discovery_disabled` | Launch fresh Claude with discovery disabled | Workspace `system.ai` models appear; Claude Code creates no cache after picker launch | +| `test_case_16_fresh_codex_uses_system_models_when_discovery_disabled` | Launch fresh Codex with discovery disabled | Workspace `system.ai` models appear without a scoped catalog | +| `test_case_17_*` | Launch configured and fresh Claude with a provider | Automatic discovery supplies exactly the provider catalog | +| `test_case_18_*` | Launch configured and fresh Codex with a provider | The provider supplies exactly its model catalog | +| `test_case_19_*` | Launch configured and fresh Claude with a model location | The explicit parent supplies exactly its picker catalog | +| `test_case_20_*` | Launch configured and fresh Codex with a model location | The explicit parent supplies exactly its model catalog | +| `test_case_21_*` | Launch configured and fresh Claude with a provider and discovery disabled | Claude uses native families; Claude Code creates no cache after picker launch | +| `test_case_22_*` | Launch configured and fresh Codex with a provider and discovery disabled | Codex uses its native catalog and ug writes no scoped catalog | +| `test_case_23_*` | Launch configured and fresh Claude with a parent and discovery disabled | Claude uses native families; Claude Code creates no cache after picker launch | +| `test_case_24_*` | Launch configured and fresh Codex with a parent and discovery disabled | Codex uses its native catalog and ug writes no scoped catalog | | `test_ug_claude_headless_prompt_argument`, `test_ug_claude_headless_prompt_stdin`, `test_ug_claude_headless_prompt_after_separator` | Run Claude from a script using each prompt form | Structured final answer contains the file value; exit zero; no routing | | `test_ug_codex_headless_prompt_argument`, `test_ug_codex_headless_prompt_stdin`, `test_ug_codex_headless_prompt_after_separator` | Run Codex from a script using each prompt form | Completed turn and final answer contain the file value; exit zero; no routing | | `test_ug_claude_headless_explicit_model_bypasses_routing` | Pass `--model VALUE` / `--model=VALUE` with routing enabled | Real file task completes; no routing wrapper | @@ -64,16 +76,16 @@ All tests live directly in `integration/`; shared mechanics live in `utils/`. | `test_ug_and_ucode_auth_helpers_emit_only_the_supplied_bearer` | Run both auth helper commands with the public bearer override, with and without forced refresh | Exact token-only stdout, no warnings or ANSI escapes; no workspace authentication or saved state | | `test_ug_and_ucode_web_search_helpers_preserve_mcp_stdio` | Initialize and list tools through both web-search helper commands | Exactly the MCP JSON-RPC responses; no text/ANSI contamination; existing server/tool identities preserved; no model request | -With both agents selected there are **42 live cases** (6 interactive TUI cases), -**4 managed-workspace cases** (marker `managed`, run against a separate workspace that -publishes a CodingAgentConfig), **28 managed-fixture cases** (marker `managed_fixture`), and -**5 installation checks**. The 12 numbered scenarios fetch the published config once per agent, -replace that agent's static model source with its dedicated test MPS, drop its incompatible static -defaults, and reuse the result across 24 explicit configured/fresh journeys. The other four -collected cases, from three test functions, inject focused model and MCP shapes. Parametrization -varies argument spelling or routing mode, never hides the -agent/provider in the test name. Duplicate boot-only cases are incorporated into the Databricks -configuration TUI journeys. +With both agents selected there are **62 live cases** (16 interactive TUI cases), +**4 un-stubbed managed-workspace cases** (marker `managed`), **28 managed-config fixture cases** +(marker `managed_fixture`), and **5 installation checks**. The 24 numbered scenarios comprise +**44 explicit journeys**, and all groups total **99 executions**. Cases 1–12 fetch the published +config once per agent, replace that agent's static model source with its dedicated test MPS, drop +its incompatible static defaults, and reuse the result across 24 configured/fresh journeys. The +other four managed-fixture executions inject focused model and MCP shapes. The un-stubbed cases +retain coverage of the config fetch/wire contract. Parametrization varies argument spelling or +routing mode, never hides the agent/provider in the test name. Duplicate boot-only cases are +incorporated into the Databricks configuration TUI journeys. Generated-file cleanup and strict app-server stdout assertions remain enforced. ug no longer runs a post-configure agent probe; the deprecated `--skip-validate` @@ -94,7 +106,7 @@ dependency graph to reproduce a user's combination. Every relevant same-reposito PR and push to `main` runs both smoke and the full CUJ suite. Smoke covers the Databricks Hosted configure/TUI, custom OAuth CLI TUI, and headless argument journeys for both agents, in two parallel jobs. After smoke finishes, the full -suite runs all 41 cases across two parallel agent jobs: one Claude VM and one +suite runs all 62 live cases across two parallel agent jobs: one Claude VM and one Codex VM, each running its configure, headless, and commands/lifecycle cases serially. Each agent is installed once for the full suite, and no two full jobs for the same agent overlap within a run. @@ -126,7 +138,8 @@ pending. The descriptive jobs provide the actual coverage and diagnostics. | --- | --- | | Live MCP and skills functionality | Deferred; installation tests cover the local web-search MCP handshake and tool listing, not upstream proxying or a real search request | | Broad configure flags, tracing, multiple workspaces, and PAT flows | Deferred while focusing on basic CUJs | -| Provider switching, relayed/subscription MPS | Not covered by the four provider journeys | +| Relayed/subscription MPS discovery | Not covered by the scoped discovery journeys | +| Fresh provider/parent validation and mixed Bedrock filtering | Not covered after removing the duplicate model-discovery suites | | TUI initial prompt supplied on the launch command line | Not yet covered; headless prompt arguments are covered | | Follow-up turns and conversation resume | Not covered; reopen proves startup, not conversation resume | | Claude/Codex interactive smart routing | Deferred at the user's request; routing jobs and live journeys removed. Unit/component routing tests remain, but do not establish live routing behavior. | diff --git a/tests/integration/README.md b/tests/integration/README.md index c772bd4c..b0fe11e9 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -100,6 +100,8 @@ test_ug_configure_claude_lifecycle.py # repeat setup, revert, rejected credent test_ug_configure_codex_lifecycle.py # repeat setup, revert, rejected credentials test_ug_claude_managed_model_discovery.py # fetched/reused Claude MPS policy cases test_ug_codex_managed_model_discovery.py # fetched/reused Codex MPS policy cases +test_ug_claude_model_discovery.py # Tests-tab cases 13, 15, 17, 19, 21, 23 +test_ug_codex_model_discovery.py # Tests-tab cases 14, 16, 18, 20, 22, 24 test_ug_configure_managed.py # managed workspace: static model list, no agent selector test_ug_configure_managed_models.py # injected model lists: pickers and Codex fallback metadata test_ug_configure_managed_mcp.py # injected managed MCP list @@ -151,27 +153,31 @@ MPS CUJs select the existing services already used by e2e: - Codex: `main.ucode.ci_openai_mps`, using its allowed `gpt-5-nano` model. Use `--claude-provider` / `--claude-relayed-provider` / `--codex-provider` to -reproduce another existing service. Use `--codex-provider-model` when that OpenAI -service allows a different model. Those choices are recorded in `versions.json`. +reproduce another existing service. Use `--claude-provider-model` / +`--codex-provider-model` when it allows a different model. Those choices are recorded in `versions.json`. No service is created or modified. A missing service, permission, or OAuth token fails the selected CUJ, rather than skipping it. -There are **42 live cases** (including 6 TUI journeys) and **5 installation -checks** with both agents. A separate **4 managed-workspace cases** (one per agent, -an idempotent re-configure, and cache reuse within the TTL; marker `managed`) run against a -workspace that publishes a CodingAgentConfig; see "Managed-workspace journeys" below. A further -**28 `managed_fixture` cases** use `UCODE_MANAGED_CONFIG_STUB`: 24 explicit configured/fresh -managed-discovery journeys fetch the published config once per agent module, replace that agent's -static model source with its dedicated MPS, drop its incompatible static defaults, and reuse the -result; four collected cases from three test functions inject focused model/MCP shapes. -See the named coverage and gaps matrix in +Scoped discovery additionally requires Model Services +`main.ucode.ci_e2e_claude` and `main.ucode.ci_e2e_codex`. Override them with +`--parent-schema`, `--claude-parent-model`, or `--codex-parent-model`. The tests +consume but never create or modify them. + +There are **62 live cases** (including 16 TUI journeys), **4 un-stubbed managed-workspace cases** +(marker `managed`), **28 managed-config fixture cases** (marker `managed_fixture`), and +**5 installation checks** with both agents. The 24 numbered scenarios comprise 44 explicit +journeys, and all groups total 99 executions. Cases 1–12 fetch the published config once per agent +module, replace that agent's static model source with its dedicated MPS, drop its incompatible +static defaults, and reuse it across 24 configured/fresh journeys. Four additional fixture +executions inject focused model/MCP shapes. The un-stubbed cases retain the config fetch/wire +contract; see "Managed-workspace journeys" below and the named coverage and gaps matrix in [../README.md](../README.md). ```bash # Append one of these selections to the runner command: -- -m live # default: all live user journeys -- -m smoke # six Hosted, custom OAuth CLI TUI, and headless journeys --- -m tui # six interactive TUI journeys +-- -m 'live and tui' # sixteen interactive live configuration/model-discovery journeys -- -k test_ug_codex_app_server_client_initializes # one named journey and its variants # Use --installation-only before -- for package checks without credentials. ``` @@ -235,20 +241,20 @@ cannot receive those secrets. The workspace check requires the secret to match `https://eng-ml-inference-team-us-east-1.cloud.databricks.com` (a trailing slash -is accepted). It never changes the secret or switches workspaces. There is no CI -model-discovery or model-selection job. Real `ug configure` performs its normal -workspace discovery inside each test; only explicit-model scenarios choose and -record a discovered `system.ai` model as a test argument. +is accepted). It never changes the secret or switches workspaces. There is no +separate CI model-selection job; the full agent lanes include scoped model +discovery. Real `ug configure` performs its normal workspace discovery inside +each test; only explicit-model scenarios choose and record a discovered +`system.ai` model as a test argument. Every same-repository PR and push to `main` runs **Smoke journeys**, followed by -**Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, -headless argument, and custom OAuth CLI TUI journeys for each agent (six cases, -two agent jobs). Full runs all 42 live cases, including those smoke cases, in two -disjoint agent lanes: +**Full journeys** even if smoke fails. Smoke runs the Hosted configure/TUI, headless argument, +and custom OAuth CLI TUI journey for each agent (six cases, two agent jobs). Full runs all 62 +live cases, including those smoke cases, in two disjoint agent lanes: | Agent lane | Marker | Cases | | --- | --- | --- | -| Claude | `live and claude` | 17 | -| Codex | `live and codex` | 25 | +| Claude | `live and claude` | 27 | +| Codex | `live and codex` | 35 | Each lane installs only its agent CLI, once, and runs all its configure, headless, commands, lifecycle, and applicable app-server journeys. Cases remain serial @@ -329,7 +335,7 @@ The workflow consumes the stored bearer; it does not mint or refresh credentials For a manual run, use **Actions → Integration → Run workflow**, select the branch, and choose `full` (default), `smoke`, `tui`, or `installation`. `live` remains an alias for `full`. Manual subsets are explicit: `smoke` runs just the six smoke -cases; `tui` adds `and tui` to each agent lane's marker and runs all six TUI cases. Installation +cases; `tui` adds `and tui` to each agent lane's marker and runs all 16 live TUI cases. Installation checks always run. Set the ug/agent versions. From the CLI: ```bash @@ -500,7 +506,7 @@ uv run --no-project --python 3.12 python scripts/run_integration.py \ unset DATABRICKS_BEARER ``` -This runs all 42 live cases. For the five installation checks, run the same +This runs all 62 live cases. For the five installation checks, run the same runner/version/index arguments with `--installation-only` and omit `-- -m live`; no bearer or workspace is needed. Results remain under `.integration-runs/`. Each invocation needs a new output directory; an existing one is rejected. diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7f14fd53..56cd6ef9 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -106,6 +106,11 @@ def claude_oauth_token(): return token +@pytest.fixture(scope="session") +def claude_provider_model(): + return os.environ["UG_INTEGRATION_CLAUDE_PROVIDER_MODEL"] + + @pytest.fixture(scope="session") def codex_provider(): return os.environ["UG_INTEGRATION_CODEX_PROVIDER"] @@ -119,3 +124,13 @@ def codex_provider_model(): @pytest.fixture(scope="session") def parent_schema(): return os.environ["UG_INTEGRATION_PARENT_SCHEMA"] + + +@pytest.fixture(scope="session") +def claude_parent_model(): + return os.environ["UG_INTEGRATION_CLAUDE_PARENT_MODEL"] + + +@pytest.fixture(scope="session") +def codex_parent_model(): + return os.environ["UG_INTEGRATION_CODEX_PARENT_MODEL"] diff --git a/tests/integration/test_ug_claude_model_discovery.py b/tests/integration/test_ug_claude_model_discovery.py new file mode 100644 index 00000000..5c5596ca --- /dev/null +++ b/tests/integration/test_ug_claude_model_discovery.py @@ -0,0 +1,322 @@ +"""Claude CUJs for Tests-table cases 13, 15, 17, 19, 21, and 23.""" + +import pytest +from utils.terminal import AgentTerminal + +pytestmark = pytest.mark.claude + +CLAUDE_NATIVE_MODEL_FAMILIES = ("Opus", "Sonnet", "Haiku") + + +def _assert_scoped_models_in_picker(session, screen, expected_ids): + models = session.claude_gateway_models() + assert [model.get("id") for model in models] == expected_ids, models + display_names = [model.get("display_name") for model in models] + assert all(isinstance(name, str) and name for name in display_names), models + for display_name in display_names: + assert display_name in screen, screen + + +def _assert_native_models_in_picker(screen): + for family in CLAUDE_NATIVE_MODEL_FAMILIES: + assert family in screen, screen + + +def _assert_no_claude_owned_gateway_cache_after_launch(session): + """Check Claude Code's own cache only after its picker process has exited.""" + assert not (session.home / ".claude/cache/gateway-models.json").exists() + + +@pytest.mark.live +@pytest.mark.tui +def test_case_13_configured_claude_reuses_saved_model_location( + live_session, workspace, parent_schema, claude_parent_model +): + """Scenario: configure Claude with --model-location, then launch without options. + + Expected: the saved parent supplies Claude's discovered model catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "claude", + "--workspace", + workspace, + "--model-location", + parent_schema, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + command = [str(session.binary), "claude"] + with AgentTerminal(session, "claude", command, "case-13-saved-location") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_scoped_models_in_picker(session, screen, [claude_parent_model]) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_15_fresh_claude_uses_system_models_when_discovery_disabled(live_session, workspace): + """Scenario: launch fresh Claude with UG_ENABLE_MODEL_DISCOVERY=0. + + Expected: ug uses its discovered system.ai family models without a gateway catalog. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + command = [str(session.binary), "claude", "--workspace", workspace] + with AgentTerminal(session, "claude", command, "case-15-system-models") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + models = session.workspace_state()["claude_models"] + assert models + assert all(model.startswith("system.ai.") for model in models.values()) + for model_id in models.values(): + assert model_id in screen, screen + _assert_no_claude_owned_gateway_cache_after_launch(session) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_17_configured_claude_provider_discovers_models_by_default( + live_session, workspace, claude_provider, claude_provider_model +): + """Scenario: configure Claude, then launch with --provider and no opt-in flag. + + Expected: automatic provider discovery supplies its exact picker catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "claude", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + command = [str(session.binary), "claude", "--provider", claude_provider] + with AgentTerminal(session, "claude", command, "case-17-provider-default") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_scoped_models_in_picker(session, screen, [claude_provider_model]) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_17_fresh_claude_provider_discovers_models_by_default( + live_session, workspace, claude_provider, claude_provider_model +): + """Scenario: launch fresh Claude with --provider and no opt-in flag. + + Expected: automatic provider discovery supplies its exact picker catalog. + """ + session = live_session + command = [ + str(session.binary), + "claude", + "--workspace", + workspace, + "--provider", + claude_provider, + ] + with AgentTerminal(session, "claude", command, "case-17-provider-default") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_scoped_models_in_picker(session, screen, [claude_provider_model]) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_19_configured_claude_model_location_overrides_saved_setup( + live_session, workspace, parent_schema, claude_parent_model +): + """Scenario: configure Claude, then launch with --model-location. + + Expected: the explicit parent overrides saved setup with its exact picker catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "claude", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + command = [str(session.binary), "claude", "--model-location", parent_schema] + with AgentTerminal(session, "claude", command, "case-19-location-default") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_scoped_models_in_picker(session, screen, [claude_parent_model]) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_19_fresh_claude_model_location_discovers_parent_models( + live_session, workspace, parent_schema, claude_parent_model +): + """Scenario: launch fresh Claude with --model-location. + + Expected: the explicit parent supplies its exact picker catalog. + """ + session = live_session + command = [ + str(session.binary), + "claude", + "--workspace", + workspace, + "--model-location", + parent_schema, + ] + with AgentTerminal(session, "claude", command, "case-19-location-default") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_scoped_models_in_picker(session, screen, [claude_parent_model]) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_21_configured_claude_provider_uses_native_models_when_discovery_disabled( + live_session, workspace, claude_provider +): + """Scenario: configure Claude, disable discovery, and launch with --provider. + + Expected: Claude uses native aliases and creates no cache after picker launch. + """ + session = live_session + session.run( + "configure", + "--agents", + "claude", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + + command = [str(session.binary), "claude", "--provider", claude_provider] + with AgentTerminal(session, "claude", command, "case-21-provider-disabled") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_no_claude_owned_gateway_cache_after_launch(session) + _assert_native_models_in_picker(screen) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_21_fresh_claude_provider_uses_native_models_when_discovery_disabled( + live_session, workspace, claude_provider +): + """Scenario: disable discovery and launch fresh Claude with --provider. + + Expected: Claude uses native aliases and creates no cache after picker launch. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + + command = [ + str(session.binary), + "claude", + "--workspace", + workspace, + "--provider", + claude_provider, + ] + with AgentTerminal(session, "claude", command, "case-21-provider-disabled") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + _assert_no_claude_owned_gateway_cache_after_launch(session) + _assert_native_models_in_picker(screen) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_23_configured_claude_location_uses_native_models_when_discovery_disabled( + live_session, workspace, parent_schema +): + """Scenario: configure Claude, disable discovery, and launch with a parent. + + Expected: Claude uses native aliases and creates no cache after picker launch. + """ + session = live_session + session.run( + "configure", + "--agents", + "claude", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + + command = [str(session.binary), "claude", "--model-location", parent_schema] + with AgentTerminal(session, "claude", command, "case-23-location-disabled") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + models = session.workspace_state()["claude_models"] + assert models + assert all(model.startswith("system.ai.") for model in models.values()) + _assert_no_claude_owned_gateway_cache_after_launch(session) + _assert_native_models_in_picker(screen) + + +@pytest.mark.live +@pytest.mark.tui +def test_case_23_fresh_claude_location_uses_native_models_when_discovery_disabled( + live_session, workspace, parent_schema +): + """Scenario: disable discovery and launch fresh Claude with a parent. + + Expected: Claude uses native aliases and creates no cache after picker launch. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + + command = [ + str(session.binary), + "claude", + "--workspace", + workspace, + "--model-location", + parent_schema, + ] + with AgentTerminal(session, "claude", command, "case-23-location-disabled") as tui: + tui.boot() + screen = tui.open_model_picker() + tui.exit_normally() + + models = session.workspace_state()["claude_models"] + assert models + assert all(model.startswith("system.ai.") for model in models.values()) + _assert_no_claude_owned_gateway_cache_after_launch(session) + _assert_native_models_in_picker(screen) diff --git a/tests/integration/test_ug_codex_model_discovery.py b/tests/integration/test_ug_codex_model_discovery.py new file mode 100644 index 00000000..fa71d4b7 --- /dev/null +++ b/tests/integration/test_ug_codex_model_discovery.py @@ -0,0 +1,278 @@ +"""Codex CUJs for Tests-table cases 14, 16, 18, 20, 22, and 24.""" + +import pytest + +pytestmark = pytest.mark.codex + + +@pytest.mark.live +def test_case_14_configured_codex_reuses_saved_model_location( + live_session, workspace, parent_schema, codex_parent_model +): + """Scenario: configure Codex with --model-location, then launch without options. + + Expected: the saved parent supplies Codex's discovered model catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "codex", + "--workspace", + workspace, + "--model-location", + parent_schema, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + models = session.codex_model_ids(["app-server", "--listen", "stdio://"]) + + assert models == [codex_parent_model] + + +@pytest.mark.live +def test_case_16_fresh_codex_uses_system_models_when_discovery_disabled(live_session, workspace): + """Scenario: launch fresh Codex with UG_ENABLE_MODEL_DISCOVERY=0. + + Expected: ug uses its discovered system.ai models without a scoped catalog. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + models = session.codex_model_ids( + ["--workspace", workspace, "--", "app-server", "--listen", "stdio://"] + ) + + discovered = session.workspace_state()["codex_models"] + assert models + assert discovered + assert all(model.startswith("system.ai.") for model in discovered) + assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) + + +@pytest.mark.live +def test_case_18_configured_codex_provider_discovers_models_by_default( + live_session, workspace, codex_provider, codex_provider_model +): + """Scenario: configure Codex, then launch with --provider. + + Expected: the explicit provider supplies its exact catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "codex", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + + models = session.codex_model_ids( + ["--provider", codex_provider, "--", "app-server", "--listen", "stdio://"] + ) + + assert models == [codex_provider_model] + + +@pytest.mark.live +def test_case_18_fresh_codex_provider_discovers_models_by_default( + live_session, workspace, codex_provider, codex_provider_model +): + """Scenario: launch fresh Codex with --workspace and --provider. + + Expected: the explicit provider supplies its exact catalog. + """ + session = live_session + models = session.codex_model_ids( + [ + "--workspace", + workspace, + "--provider", + codex_provider, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + + assert models == [codex_provider_model] + + +@pytest.mark.live +def test_case_20_configured_codex_model_location_overrides_saved_setup( + live_session, workspace, parent_schema, codex_parent_model +): + """Scenario: configure Codex, then launch with --model-location. + + Expected: the explicit parent supplies its exact catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "codex", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + models = session.codex_model_ids( + [ + "--model-location", + parent_schema, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + + assert models == [codex_parent_model] + + +@pytest.mark.live +def test_case_20_fresh_codex_model_location_overrides_saved_setup( + live_session, workspace, parent_schema, codex_parent_model +): + """Scenario: launch fresh Codex with --workspace and --model-location. + + Expected: the explicit parent supplies its exact catalog. + """ + session = live_session + models = session.codex_model_ids( + [ + "--workspace", + workspace, + "--model-location", + parent_schema, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + + assert models == [codex_parent_model] + + +@pytest.mark.live +def test_case_22_configured_codex_provider_uses_native_models_when_discovery_disabled( + live_session, workspace, codex_provider +): + """Scenario: configure Codex, disable discovery, then launch with --provider. + + Expected: Codex uses the native catalog and ug writes no scoped catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "codex", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + models = session.codex_model_ids( + ["--provider", codex_provider, "--", "app-server", "--listen", "stdio://"] + ) + + assert models + assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) + + +@pytest.mark.live +def test_case_22_fresh_codex_provider_uses_native_models_when_discovery_disabled( + live_session, workspace, codex_provider +): + """Scenario: disable discovery, then launch fresh Codex with a provider. + + Expected: Codex uses the native catalog and ug writes no scoped catalog. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + models = session.codex_model_ids( + [ + "--workspace", + workspace, + "--provider", + codex_provider, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + + assert models + assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) + + +@pytest.mark.live +def test_case_24_configured_codex_location_uses_native_models_when_discovery_disabled( + live_session, workspace, parent_schema +): + """Scenario: configure Codex, disable discovery, then launch with a parent. + + Expected: Codex uses the native catalog and ug writes no scoped catalog. + """ + session = live_session + session.run( + "configure", + "--agents", + "codex", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + timeout=240, + ) + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + models = session.codex_model_ids( + ["--model-location", parent_schema, "--", "app-server", "--listen", "stdio://"] + ) + + discovered = session.workspace_state()["codex_models"] + assert models + assert discovered + assert all(model.startswith("system.ai.") for model in discovered) + assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) + + +@pytest.mark.live +def test_case_24_fresh_codex_location_uses_native_models_when_discovery_disabled( + live_session, workspace, parent_schema +): + """Scenario: disable discovery, then launch fresh Codex with a parent. + + Expected: Codex uses the native catalog and ug writes no scoped catalog. + """ + session = live_session + session.env["UG_ENABLE_MODEL_DISCOVERY"] = "0" + + models = session.codex_model_ids( + [ + "--workspace", + workspace, + "--model-location", + parent_schema, + "--", + "app-server", + "--listen", + "stdio://", + ] + ) + + discovered = session.workspace_state()["codex_models"] + assert models + assert discovered + assert all(model.startswith("system.ai.") for model in discovered) + assert not list((session.home / ".ucode").glob("codex-model-catalog-*.json")) diff --git a/tests/integration/utils/harness.py b/tests/integration/utils/harness.py index 9644bf4c..e9564418 100644 --- a/tests/integration/utils/harness.py +++ b/tests/integration/utils/harness.py @@ -192,6 +192,28 @@ def assert_not_routed(self) -> None: for name in ("codex-v2-interposer.log", "claude-v2-pty.log"): assert not (self.home / ".ucode" / name).exists(), f"Unexpected routing: {name}" + def claude_gateway_models(self, name: str = "claude-gateway-models") -> list[dict]: + """Inspect Claude Code's own cache after its model picker launched and exited.""" + path = Path(self.env["CLAUDE_CONFIG_DIR"]) / "cache/gateway-models.json" + assert path.is_file(), f"Claude Code did not create its gateway model cache: {path}" + payload = json.loads(path.read_text()) + models = payload.get("models") if isinstance(payload, dict) else None + assert isinstance(models, list), f"Invalid Claude gateway model cache: {payload!r}" + assert all(isinstance(model, dict) for model in models), ( + f"Invalid Claude gateway model entries: {models!r}" + ) + self.record(f"{name}.json", payload) + return models + + def claude_gateway_model_ids(self, name: str = "claude-gateway-models") -> list[str]: + """Read IDs from Claude Code's own post-launch gateway catalog cache.""" + models = self.claude_gateway_models(name) + ids = [model.get("id") for model in models if isinstance(model, dict)] + assert len(ids) == len(models) and all(isinstance(model_id, str) for model_id in ids), ( + f"Invalid Claude gateway model entries: {models!r}" + ) + return ids + def app_server_handshake( self, args: list[str], diff --git a/tests/test_integration_contract.py b/tests/test_integration_contract.py index f0499ec1..29c5589a 100644 --- a/tests/test_integration_contract.py +++ b/tests/test_integration_contract.py @@ -1,6 +1,7 @@ """Keep the black-box suite independent of application internals and test doubles.""" import ast +import re from pathlib import Path @@ -71,6 +72,38 @@ def test_live_integration_cases_belong_to_exactly_one_ci_agent(): assert len(marks & {"claude", "codex"}) == 1, node.name +def test_model_discovery_cases_match_the_tests_table(): + root = Path(__file__).parent / "integration" + seen = [] + for path in root.glob("test_ug_*_model_discovery.py"): + tree = ast.parse(path.read_text()) + module_marks = _markers( + node + for node in tree.body + if isinstance(node, ast.Assign) + and any( + isinstance(target, ast.Name) and target.id == "pytestmark" + for target in node.targets + ) + ) + for node in tree.body: + if not isinstance(node, ast.FunctionDef): + continue + match = re.match(r"test_case_(\d{2})_", node.name) + if match is None: + continue + case = int(match.group(1)) + seen.append(case) + marks = module_marks | _markers(node.decorator_list) + expected = {"managed_fixture"} if case <= 12 else {"live"} + assert marks & {"managed_fixture", "managed", "live"} == expected, node.name + assert not any(arg.arg == "configured" for arg in node.args.args), node.name + assert set(seen) == set(range(1, 25)) + assert len(seen) == 44 + for case in range(1, 25): + assert seen.count(case) == (1 if 13 <= case <= 16 else 2), case + + def test_smoke_covers_hosted_custom_oauth_and_headless_for_both_agents(): smoke = set() for path in (Path(__file__).parent / "integration").glob("test_*.py"):