From f21fdc4145b049e21fa9897e404635db3c728e9f Mon Sep 17 00:00:00 2001 From: Mason Cao Date: Thu, 17 Sep 2026 15:42:38 +0000 Subject: [PATCH 1/3] Add Codex Azure OpenAI MPS test coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover routing Codex through an azure_openai Model Provider Service, backed by main.ucode.ci_azure_openai_mps (allowlist: gpt-5-nano, gpt-5.6-luna reasoning; gpt-4.1-mini non-reasoning). - e2e TestAzureOpenAiMps: direct Responses-API probes through the gateway MPS (no agent binary; gateway shard, independent of #685) — reasoning-capable models route and are switchable, a non-reasoning allow-listed model rejects reasoning.effort (parity: what fails natively fails here), the same model serves without reasoning, and an off-allowlist model is refused. - integration test_ug_configure_codex_azure_openai_mps: the real ug CLI picks the Azure MPS in the configure picker and completes a file task on gpt-5-nano. Requires #685 (adds azure_openai to Codex's provider-type allowlist). - run_integration.py / conftest.py: --codex-azure-provider(-model) plumbing. Co-authored-by: Isaac --- scripts/run_integration.py | 14 ++++ tests/integration/conftest.py | 10 +++ tests/integration/test_ug_configure_codex.py | 40 ++++++++++ tests/test_e2e.py | 84 ++++++++++++++++++++ 4 files changed, 148 insertions(+) diff --git a/scripts/run_integration.py b/scripts/run_integration.py index b7b418e3..4c452cd8 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -115,6 +115,16 @@ def arguments(): default="gpt-5-nano", help="Model allowed by the OpenAI MPS selected in the configure CUJ.", ) + parser.add_argument( + "--codex-azure-provider", + default="main.ucode.ci_azure_openai_mps", + help="Existing Azure OpenAI MPS (reasoning-capable model) for the Codex azure_openai CUJ.", + ) + parser.add_argument( + "--codex-azure-provider-model", + default="gpt-5-nano", + help="Reasoning-capable model allowed by the Azure OpenAI MPS.", + ) parser.add_argument( "--parent-schema", default="main.ucode", @@ -295,6 +305,8 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "claude_relayed_provider": args.claude_relayed_provider, "codex_provider": args.codex_provider, "codex_provider_model": args.codex_provider_model, + "codex_azure_provider": args.codex_azure_provider, + "codex_azure_provider_model": args.codex_azure_provider_model, "parent_schema": args.parent_schema, "dependencies": args.dependency, "workspace": args.workspace, @@ -536,6 +548,8 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_CLAUDE_OAUTH_TOKEN": oauth_token, "UG_INTEGRATION_CODEX_PROVIDER": args.codex_provider, "UG_INTEGRATION_CODEX_PROVIDER_MODEL": args.codex_provider_model, + "UG_INTEGRATION_CODEX_AZURE_PROVIDER": args.codex_azure_provider, + "UG_INTEGRATION_CODEX_AZURE_PROVIDER_MODEL": args.codex_azure_provider_model, "UG_INTEGRATION_PARENT_SCHEMA": args.parent_schema, "UCODE_TEST_WORKSPACE": args.workspace or "", "DATABRICKS_BEARER": bearer, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7f14fd53..b70bea90 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -116,6 +116,16 @@ def codex_provider_model(): return os.environ["UG_INTEGRATION_CODEX_PROVIDER_MODEL"] +@pytest.fixture(scope="session") +def codex_azure_provider(): + return os.environ["UG_INTEGRATION_CODEX_AZURE_PROVIDER"] + + +@pytest.fixture(scope="session") +def codex_azure_provider_model(): + return os.environ["UG_INTEGRATION_CODEX_AZURE_PROVIDER_MODEL"] + + @pytest.fixture(scope="session") def parent_schema(): return os.environ["UG_INTEGRATION_PARENT_SCHEMA"] diff --git a/tests/integration/test_ug_configure_codex.py b/tests/integration/test_ug_configure_codex.py index bc5d6822..86e7f392 100644 --- a/tests/integration/test_ug_configure_codex.py +++ b/tests/integration/test_ug_configure_codex.py @@ -97,3 +97,43 @@ def test_ug_configure_codex_openai_mps( tui.exit_normally() task.assert_completed(session, "codex") session.assert_not_routed() + + +def test_ug_configure_codex_azure_openai_mps( + live_session, workspace, codex_azure_provider, codex_azure_provider_model +): + """Scenario: route Codex through an Azure OpenAI MPS backed by a reasoning model. + + Expected: ug saves the Azure provider, and launching Codex with one of its allowed + models completes a file-reading task. Codex always sends `reasoning.effort`, so a + completed task also proves two things a plain routing check would miss: the provider-type + allowlist accepts `azure_openai` for Codex, and the backing deployment is reasoning-capable + (a non-reasoning Azure model 400s the request `reasoning.effort`). + """ + session = live_session + task = FileTask(session) + + command = [ + str(session.binary), + "configure", + "--workspace", + workspace, + "--skip-upgrade", + "--disable-databricks-ai-tools", + ] + with ConfigureTerminal(session, "codex", command, "configure-azure-provider") as configure: + configure.select_agent("Codex") + configure.choose("How should Codex get its models?", "External Models") + configure.choose("Select a model provider service:", codex_azure_provider) + configure.finish(timeout=240) + assert session.workspace_state()["provider_services"]["codex"] == codex_azure_provider + assert codex_azure_provider in session.run("status").stdout + + command = [str(session.binary), "codex", "--", "--model", codex_azure_provider_model] + with AgentTerminal(session, "codex", command, "azure-provider-session") as tui: + tui.boot() + tui.submit(task.prompt) + tui.wait_for_task(task, timeout=300) + tui.exit_normally() + task.assert_completed(session, "codex") + session.assert_not_routed() diff --git a/tests/test_e2e.py b/tests/test_e2e.py index a6ae5da1..e9748479 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -57,6 +57,12 @@ ) CI_OPENAI_MPS = "main.ucode.ci_openai_mps" # api-key OpenAI (for codex) CI_OPENAI_MODEL = "gpt-5-nano" +# Azure OpenAI MPS for codex, with a mixed allowlist: two reasoning-capable models plus one +# non-reasoning model. Codex always sends `reasoning.effort`, so this single MPS exercises the +# reasoning gate, model switching, and allowlist rejection (see TestAzureOpenAiMps). +CI_AZURE_OPENAI_MPS = "main.ucode.ci_azure_openai_mps" +CI_AZURE_REASONING_MODELS = ("gpt-5-nano", "gpt-5.6-luna") +CI_AZURE_NON_REASONING_MODEL = "gpt-4.1-mini" # Claude Code's client-side model pre-flight is flaky for MPS-routed ids and emits this; the # launch retries on it before giving up (see TestModelProviderLaunch). @@ -848,6 +854,84 @@ def test_mps_is_listed_usable_and_nonrelayed(self, e2e_workspace, e2e_token): assert not entry.get("relayed") +class TestAzureOpenAiMps: + """Azure OpenAI MPS for codex, pinned to CI_AZURE_OPENAI_MPS. + + Exercised without the agent binary — direct Responses API calls through the gateway MPS, + the same surface codex uses. The allowlist mixes reasoning-capable models (gpt-5 family) + with a non-reasoning one (gpt-4.1-mini); codex always sends reasoning.effort, so these probes + pin the behavior a real codex launch depends on: any reasoning model routes (so a session can + switch between them), a non-reasoning model rejects reasoning.effort, and an off-allowlist + model is refused up front. + """ + + @staticmethod + def _responses_request( + workspace: str, token: str, model: str, *, reasoning: bool = True + ) -> httpx.Response: + body: dict = { + "model": model, + "input": "say hi in 5 words or less", + "max_output_tokens": 16, + } + if reasoning: + body["reasoning"] = {"effort": "low"} + return httpx.post( + f"{build_tool_base_url('codex', workspace)}/responses", + headers={ + "Authorization": f"Bearer {token}", + "Databricks-Model-Provider-Service": CI_AZURE_OPENAI_MPS, + }, + json=body, + timeout=60, + ) + + @staticmethod + def _skip_if_unusable(resp: httpx.Response) -> None: + # Environmental account conditions, not ucode bugs (mirrors TestAnthropicNonRelayMps). The + # allowlist 403 ("not in the allowed models list") is a real assertion, not a skip — only a + # connection-permission 401/403 or an exhausted account falls outside the test's scope. + text = resp.text + if resp.status_code in (401, 403) and ("USE CONNECTION" in text or "EXECUTE" in text): + pytest.skip(f"no permission on {CI_AZURE_OPENAI_MPS}: {text[:200]}") + if "quota" in text.lower(): + pytest.skip(f"{CI_AZURE_OPENAI_MPS} account is out of quota: {text[:200]}") + + @pytest.mark.parametrize("model", CI_AZURE_REASONING_MODELS) + def test_reasoning_model_serves_codex_reasoning_effort(self, model, e2e_workspace, e2e_token): + # codex's request shape (reasoning.effort) routes on every reasoning-capable allowed model, + # so an in-session /model switch between them stays within the allowlist. + resp = self._responses_request(e2e_workspace, e2e_token, model) + self._skip_if_unusable(resp) + assert resp.status_code == 200, f"model={model} HTTP {resp.status_code}: {resp.text[:300]}" + + def test_non_reasoning_model_rejects_reasoning_effort(self, e2e_workspace, e2e_token): + # Parity invariant: a model that would not work with codex natively must not work through the + # gateway either. gpt-4.1-mini is allow-listed but non-reasoning, and codex always sends + # reasoning.effort, so the gateway must surface the provider's 400 — not paper over it by + # stripping the param. + resp = self._responses_request(e2e_workspace, e2e_token, CI_AZURE_NON_REASONING_MODEL) + self._skip_if_unusable(resp) + assert resp.status_code == 400, f"HTTP {resp.status_code}: {resp.text[:300]}" + assert "reasoning.effort" in resp.text + + def test_non_reasoning_model_serves_without_reasoning(self, e2e_workspace, e2e_token): + # The rejection above is the reasoning parameter, not the model or the allowlist: the same + # model serves a plain request. + resp = self._responses_request( + e2e_workspace, e2e_token, CI_AZURE_NON_REASONING_MODEL, reasoning=False + ) + self._skip_if_unusable(resp) + assert resp.status_code == 200, f"HTTP {resp.status_code}: {resp.text[:300]}" + + def test_rejects_model_outside_allowlist(self, e2e_workspace, e2e_token): + # The MPS enforces its allowlist up front: a model it doesn't declare is refused. + resp = self._responses_request(e2e_workspace, e2e_token, "gpt-4o") + self._skip_if_unusable(resp) + assert resp.status_code == 403, f"HTTP {resp.status_code}: {resp.text[:300]}" + assert "not in the allowed models list" in resp.text + + class TestGeminiLaunch: """Run the real Gemini CLI against every available gemini model.""" From f6b284b16f6dbeadc20f9f9ece9fcad97ef41776 Mon Sep 17 00:00:00 2001 From: Mason Cao Date: Fri, 18 Sep 2026 14:53:37 +0000 Subject: [PATCH 2/3] Also cover Microsoft Foundry in the Codex OpenAI-dialect MPS test Foundry speaks the same OpenAI dialect as Azure OpenAI, so extract the configure-picker + launch CUJ into a shared helper and add a thin test_ug_configure_codex_foundry_mps wrapper reusing it. Mirror the Azure flags/fixtures as --codex-foundry-provider / --codex-foundry-provider-model. Requires #685 (adds microsoft_foundry to Codex's allowlist) and a CI Foundry MPS. Co-authored-by: Isaac --- scripts/run_integration.py | 14 ++++++ tests/integration/conftest.py | 10 ++++ tests/integration/test_ug_configure_codex.py | 53 +++++++++++++------- 3 files changed, 60 insertions(+), 17 deletions(-) diff --git a/scripts/run_integration.py b/scripts/run_integration.py index 4c452cd8..04d2e0c7 100644 --- a/scripts/run_integration.py +++ b/scripts/run_integration.py @@ -125,6 +125,16 @@ def arguments(): default="gpt-5-nano", help="Reasoning-capable model allowed by the Azure OpenAI MPS.", ) + parser.add_argument( + "--codex-foundry-provider", + default="main.ucode.ci_foundry_mps", + help="Existing Microsoft Foundry MPS (reasoning-capable model) for the Codex foundry CUJ.", + ) + parser.add_argument( + "--codex-foundry-provider-model", + default="gpt-5-nano", + help="Reasoning-capable model allowed by the Microsoft Foundry MPS.", + ) parser.add_argument( "--parent-schema", default="main.ucode", @@ -307,6 +317,8 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "codex_provider_model": args.codex_provider_model, "codex_azure_provider": args.codex_azure_provider, "codex_azure_provider_model": args.codex_azure_provider_model, + "codex_foundry_provider": args.codex_foundry_provider, + "codex_foundry_provider_model": args.codex_foundry_provider_model, "parent_schema": args.parent_schema, "dependencies": args.dependency, "workspace": args.workspace, @@ -550,6 +562,8 @@ def run(command, *, cwd=output, env=base_env, timeout=600) -> str: "UG_INTEGRATION_CODEX_PROVIDER_MODEL": args.codex_provider_model, "UG_INTEGRATION_CODEX_AZURE_PROVIDER": args.codex_azure_provider, "UG_INTEGRATION_CODEX_AZURE_PROVIDER_MODEL": args.codex_azure_provider_model, + "UG_INTEGRATION_CODEX_FOUNDRY_PROVIDER": args.codex_foundry_provider, + "UG_INTEGRATION_CODEX_FOUNDRY_PROVIDER_MODEL": args.codex_foundry_provider_model, "UG_INTEGRATION_PARENT_SCHEMA": args.parent_schema, "UCODE_TEST_WORKSPACE": args.workspace or "", "DATABRICKS_BEARER": bearer, diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b70bea90..7399b074 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -126,6 +126,16 @@ def codex_azure_provider_model(): return os.environ["UG_INTEGRATION_CODEX_AZURE_PROVIDER_MODEL"] +@pytest.fixture(scope="session") +def codex_foundry_provider(): + return os.environ["UG_INTEGRATION_CODEX_FOUNDRY_PROVIDER"] + + +@pytest.fixture(scope="session") +def codex_foundry_provider_model(): + return os.environ["UG_INTEGRATION_CODEX_FOUNDRY_PROVIDER_MODEL"] + + @pytest.fixture(scope="session") def parent_schema(): return os.environ["UG_INTEGRATION_PARENT_SCHEMA"] diff --git a/tests/integration/test_ug_configure_codex.py b/tests/integration/test_ug_configure_codex.py index 86e7f392..93da270f 100644 --- a/tests/integration/test_ug_configure_codex.py +++ b/tests/integration/test_ug_configure_codex.py @@ -99,18 +99,15 @@ def test_ug_configure_codex_openai_mps( session.assert_not_routed() -def test_ug_configure_codex_azure_openai_mps( - live_session, workspace, codex_azure_provider, codex_azure_provider_model -): - """Scenario: route Codex through an Azure OpenAI MPS backed by a reasoning model. - - Expected: ug saves the Azure provider, and launching Codex with one of its allowed - models completes a file-reading task. Codex always sends `reasoning.effort`, so a - completed task also proves two things a plain routing check would miss: the provider-type - allowlist accepts `azure_openai` for Codex, and the backing deployment is reasoning-capable - (a non-reasoning Azure model 400s the request `reasoning.effort`). +def _run_codex_openai_dialect_mps_cuj(session, workspace, provider, model, label): + """Configure Codex through an OpenAI-dialect MPS, then complete a task on one of its models. + + Azure OpenAI and Microsoft Foundry both speak the OpenAI dialect the gateway fronts for Codex, + so the CUJ is identical apart from which MPS the picker selects. Codex always sends + `reasoning.effort`, so a completed task also proves two things a plain routing check would miss: + the provider-type allowlist accepts the MPS's type for Codex, and the backing deployment is + reasoning-capable (a non-reasoning model 400s the request's `reasoning.effort`). """ - session = live_session task = FileTask(session) command = [ @@ -121,19 +118,41 @@ def test_ug_configure_codex_azure_openai_mps( "--skip-upgrade", "--disable-databricks-ai-tools", ] - with ConfigureTerminal(session, "codex", command, "configure-azure-provider") as configure: + with ConfigureTerminal(session, "codex", command, f"configure-{label}-provider") as configure: configure.select_agent("Codex") configure.choose("How should Codex get its models?", "External Models") - configure.choose("Select a model provider service:", codex_azure_provider) + configure.choose("Select a model provider service:", provider) configure.finish(timeout=240) - assert session.workspace_state()["provider_services"]["codex"] == codex_azure_provider - assert codex_azure_provider in session.run("status").stdout + assert session.workspace_state()["provider_services"]["codex"] == provider + assert provider in session.run("status").stdout - command = [str(session.binary), "codex", "--", "--model", codex_azure_provider_model] - with AgentTerminal(session, "codex", command, "azure-provider-session") as tui: + command = [str(session.binary), "codex", "--", "--model", model] + with AgentTerminal(session, "codex", command, f"{label}-provider-session") as tui: tui.boot() tui.submit(task.prompt) tui.wait_for_task(task, timeout=300) tui.exit_normally() task.assert_completed(session, "codex") session.assert_not_routed() + + +def test_ug_configure_codex_azure_openai_mps( + live_session, workspace, codex_azure_provider, codex_azure_provider_model +): + """Route Codex through an Azure OpenAI (`azure_openai`) MPS backed by a reasoning model.""" + _run_codex_openai_dialect_mps_cuj( + live_session, workspace, codex_azure_provider, codex_azure_provider_model, "azure" + ) + + +def test_ug_configure_codex_foundry_mps( + live_session, workspace, codex_foundry_provider, codex_foundry_provider_model +): + """Route Codex through a Microsoft Foundry (`microsoft_foundry`) MPS backed by a reasoning model. + + Foundry speaks the same OpenAI dialect as Azure OpenAI, so this reuses the Azure CUJ; the + separate test exercises the `microsoft_foundry` provider-type allowlist entry for Codex. + """ + _run_codex_openai_dialect_mps_cuj( + live_session, workspace, codex_foundry_provider, codex_foundry_provider_model, "foundry" + ) From ba50bb40429840ca6b172df9802d5b9301edcdfc Mon Sep 17 00:00:00 2001 From: Mason Cao Date: Fri, 18 Sep 2026 15:55:42 +0000 Subject: [PATCH 3/3] Give the Codex MPS test wrappers Scenario/Expected docstrings The shared-helper refactor left the azure/foundry test wrappers with one-line docstrings, which test_integration_contract flags (every integration test must describe Scenario + Expected). Restore both. Co-authored-by: Isaac --- tests/integration/test_ug_configure_codex.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_ug_configure_codex.py b/tests/integration/test_ug_configure_codex.py index 93da270f..871edb60 100644 --- a/tests/integration/test_ug_configure_codex.py +++ b/tests/integration/test_ug_configure_codex.py @@ -139,7 +139,11 @@ def _run_codex_openai_dialect_mps_cuj(session, workspace, provider, model, label def test_ug_configure_codex_azure_openai_mps( live_session, workspace, codex_azure_provider, codex_azure_provider_model ): - """Route Codex through an Azure OpenAI (`azure_openai`) MPS backed by a reasoning model.""" + """Scenario: select an Azure OpenAI (`azure_openai`) MPS in ug configure's picker for Codex. + + Expected: ug saves the provider and a real Codex session completes a file task on one of its + reasoning-capable models (see _run_codex_openai_dialect_mps_cuj). + """ _run_codex_openai_dialect_mps_cuj( live_session, workspace, codex_azure_provider, codex_azure_provider_model, "azure" ) @@ -148,10 +152,11 @@ def test_ug_configure_codex_azure_openai_mps( def test_ug_configure_codex_foundry_mps( live_session, workspace, codex_foundry_provider, codex_foundry_provider_model ): - """Route Codex through a Microsoft Foundry (`microsoft_foundry`) MPS backed by a reasoning model. + """Scenario: select a Microsoft Foundry (`microsoft_foundry`) MPS in ug configure's picker. - Foundry speaks the same OpenAI dialect as Azure OpenAI, so this reuses the Azure CUJ; the - separate test exercises the `microsoft_foundry` provider-type allowlist entry for Codex. + Expected: ug saves the provider and a real Codex session completes a file task, exercising the + `microsoft_foundry` allowlist entry for Codex. Foundry speaks the same OpenAI dialect as Azure + OpenAI, so this reuses the Azure CUJ (see _run_codex_openai_dialect_mps_cuj). """ _run_codex_openai_dialect_mps_cuj( live_session, workspace, codex_foundry_provider, codex_foundry_provider_model, "foundry"