Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions scripts/repro_routing_race.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Reproduce the smart-routing / custom-catalog model-id race.

The race
--------
At launch, smart routing builds ``available_models`` from one of two sources
(`v2.launch_codex` / `codex._launch_smart_routing`)::

available_models = custom_catalog_models() or _cached_routing_models(state)

* ``custom_catalog_models()`` -> bare hyphenated slugs from the per-workspace
MPS catalog (``codex/v1/models``), e.g. ``gpt-5-6-luna``. This is also the
router's own arm vocabulary.
* ``_cached_routing_models()`` -> ``system.ai.``-prefixed discovery ids cached
in state, e.g. ``system.ai.gpt-5-6-luna``. This is the fallback when the
catalog file is absent or didn't load in time.

Both lists normalize to the same router arms (via ``_normalize_route_model``),
so the router picks correctly either way. The bug is one step later: the
routed model is passed through ``codex_model_id`` to produce the dotted alias
the gateway actually resolves (``gpt-5.6-luna``). Before the fix,
``codex_model_id`` only translated *prefixed* ids; a bare slug like
``gpt-5-6-luna`` was returned unchanged, which the gateway cannot resolve.

So whether the catalog loaded in time decided which dialect
``available_models`` was in, and only the prefixed dialect got dotted-ized.
Users on the catalog path got the broken ``gpt-5-6-luna``; users on the
fallback path got the working ``gpt-5.6-luna`` alias.

Run this script to see both dialects converge to ``gpt-5.6-luna`` after the
fix. To see the pre-fix divergence, revert the one-line change in
``codex_routing.codex_model_id`` (``else: bare = tail`` -> ``else: return model``).
"""
from __future__ import annotations

from ucode.smart_routing import codex_routing

ROUTER_ARM = "gpt-5-6-luna"

SCENARIOS = [
("catalog present (bare slugs)", ["gpt-5-6-luna", "gpt-5-6-sol"]),
("catalog absent / fallback (system.ai. prefixed)", [
"system.ai.gpt-5-6-luna",
"system.ai.gpt-5-6-sol",
]),
]


def simulate(codex_routing_mod, available_models: list[str], router_arm: str) -> dict:
"""Simulate the full model-translation path for one launch scenario.

Mirrors the three places ``codex_model_id`` is applied:
- start_model : ``codex._launch_smart_routing`` -> ``codex_model_id(models[0])``
- interposer rewrite: ``codex_interposer._Session`` -> ``codex_model_id(decision.model)``
- subagent route : ``codex_routing.route_pre_tool_use`` -> ``model_id_mapper``
All three reduce to ``codex_model_id`` on the available_models entry the
router resolved to, so we compute that once.
"""
cr = codex_routing_mod
available = {cr._normalize_route_model(m): m for m in available_models}
resolved = available.get(cr._normalize_route_model(router_arm))
if resolved is None:
return {"resolved": None, "start_model": None, "routed_model": None}
return {
"resolved": resolved,
"start_model": cr.codex_model_id(available_models[0]),
"routed_model": cr.codex_model_id(resolved),
}


def main() -> int:
print("=" * 72)
print("RACE REPRO: codex_model_id translation across available_models dialects")
print("=" * 72)

print("\ncodex_model_id behavior:")
print(f" codex_model_id('gpt-5-6-luna') = {codex_routing.codex_model_id('gpt-5-6-luna')!r}")
print(f" codex_model_id('system.ai.gpt-5-6-luna') = {codex_routing.codex_model_id('system.ai.gpt-5-6-luna')!r}")

print("\nScenario -> model sent to the gateway (via codex_model_id):")
results: dict[str, dict] = {}
for label, models in SCENARIOS:
r = simulate(codex_routing, models, ROUTER_ARM)
results[label] = r
print(f" [{label}]")
print(f" available_models = {models}")
print(f" router arm = {ROUTER_ARM!r} -> resolved {r['resolved']!r}")
print(f" start_model = {r['start_model']!r}")
print(f" routed model = {r['routed_model']!r}")

routed_models = {r["routed_model"] for r in results.values()}
if len(routed_models) == 1:
model = next(iter(routed_models))
print(f"\n => Both dialects converge to {model!r}")
assert model == "gpt-5.6-luna", f"unexpected model: {model}"
print(" FIX VERIFIED: catalog-present and catalog-absent paths produce")
print(" the same gateway-resolvable dotted alias.")
return 0
else:
print("\n => DIVERGENCE (the race):")
for label, r in results.items():
print(f" {label:55s} -> {r['routed_model']}")
print(" Users on the catalog path get a model the gateway can't resolve;")
print(" users on the fallback path get the working dotted alias.")
return 1


if __name__ == "__main__":
raise SystemExit(main())
8 changes: 7 additions & 1 deletion src/ucode/smart_routing/codex_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ def codex_model_id(model: str) -> str:
Codex's bundled GPT catalog owns the model metadata for these aliases,
while the AI Gateway resolves them back to the matching ``system.ai`` service.
Leave non-GPT models unchanged because their metadata comes from the gateway catalog.

The router's arm vocabulary and the per-workspace MPS catalog both use the
bare hyphenated form (gpt-5-6-luna). Translate those too — not just the
system.ai./databricks- prefixed ids — so the routed and starting
model is always the dotted alias the gateway resolves, regardless of
whether the model list came from the custom catalog or cached discovery.
"""
tail = model.rsplit("/", 1)[-1]
if tail in {"databricks-gpt-5-2-codex", "databricks-gpt-5-4-nano"}:
Expand All @@ -176,7 +182,7 @@ def codex_model_id(model: str) -> str:
elif tail.startswith("databricks-"):
bare = tail.removeprefix("databricks-")
else:
return model
bare = tail
match = _GPT_RE.fullmatch(bare)
if not match:
return model
Expand Down
54 changes: 54 additions & 0 deletions tests/test_codex_routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,60 @@ def test_codex_model_id_maps_uc_gpt_models_to_codex_slugs():
assert codex_routing.codex_model_id("system.ai.glm-5-2") == "system.ai.glm-5-2"


def test_codex_model_id_translates_bare_hyphenated_gpt_slugs():
"""Bare hyphenated slugs (from the MPS catalog / router arms) get dotted too.

This is the race condition: when the custom catalog is present, available_models
contains bare slugs like gpt-5-6-luna; when absent, it falls back to
system.ai.gpt-5-6-luna. Both must produce the same dotted alias.
"""
assert codex_routing.codex_model_id("gpt-5-6-luna") == "gpt-5.6-luna"
assert codex_routing.codex_model_id("gpt-5-6-sol") == "gpt-5.6-sol"
assert codex_routing.codex_model_id("gpt-5-6-terra") == "gpt-5.6-terra"
assert codex_routing.codex_model_id("gpt-5-2") == "gpt-5.2"
# Already-dotted form is idempotent.
assert codex_routing.codex_model_id("gpt-5.6-luna") == "gpt-5.6-luna"
# Major-only slugs are unchanged (no minor to dot-ize).
assert codex_routing.codex_model_id("gpt-6-astra") == "gpt-6-astra"
# Non-GPT bare slugs are still passed through.
assert codex_routing.codex_model_id("glm-5-2") == "glm-5-2"


def test_routing_with_bare_catalog_slugs_produces_dotted_alias(monkeypatch):
"""Reproduce the race: bare catalog slugs must route to the dotted alias.

When custom_catalog_models() returns bare hyphenated slugs (the MPS catalog
format), the router resolves the arm back to the bare slug, and codex_model_id
must translate it to the dotted alias the gateway resolves. Before the fix,
codex_model_id returned bare slugs unchanged, so users on the catalog path
got gpt-5-6-luna (broken) while users on the fallback path got
gpt-5.6-luna (working).
"""
monkeypatch.setattr(
codex_routing,
"request_routing_decision",
lambda *args, **kwargs: (
codex_routing.RoutingDecision(
model="gpt-5-6-luna",
raw_model="gpt-5-6-luna",
),
None,
),
)

output = codex_routing.route_pre_tool_use(
{
"tool_name": "collaborationspawn_agent",
"tool_input": {"task_name": "routing-smoke-test", "message": "encrypted"},
},
workspace=WS,
token="token",
available_models=["gpt-5-6-luna", "gpt-5-6-sol"],
)

assert output["hookSpecificOutput"]["updatedInput"]["model"] == "gpt-5.6-luna"


def test_spawn_glm_decision_applies_glm_model(monkeypatch):
# GLM is no longer skipped for Codex subagents: a GLM routing decision is
# applied like any other arm.
Expand Down
Loading