diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a0a1bf810..9a76dbd57 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -693,7 +693,10 @@ jobs: echo "LLM_API_BASE_FILE=$llm_api_base_file" >> "$GITHUB_ENV" - name: Prepare GitHub Models fallback credentials - if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' + # github_models is included because its STRIX_FALLBACK_MODELS chain + # ends in openai-direct/gpt-5.4, which needs the direct-OpenAI key and + # API base to authenticate and route after the primary is exhausted. + if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' || steps.gate.outputs.provider_mode == 'github_models' env: GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} OPENAI_FALLBACK_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} @@ -725,6 +728,9 @@ jobs: openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" printf '%s' "$openai_trimmed" > "$openai_fallback_key_file" echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + openai_fallback_api_base_file="$RUNNER_TEMP/openai_fallback_api_base.txt" + printf '%s' 'https://api.openai.com/v1' > "$openai_fallback_api_base_file" + echo "STRIX_OPENAI_FALLBACK_API_BASE_FILE=$openai_fallback_api_base_file" >> "$GITHUB_ENV" fi - name: Prepare Vertex AI credentials @@ -857,6 +863,7 @@ jobs: STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} + STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" diff --git a/CHANGELOG.md b/CHANGELOG.md index 312d6d33b..cef0acda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,18 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] - +- Route Strix cross-provider fallbacks to explicit direct-OpenAI models + (`openai-direct/...`) through the OpenAI inference endpoint instead of + inheriting a provider-specific primary base: the workflow now provisions + `STRIX_OPENAI_FALLBACK_API_BASE_FILE` (`https://api.openai.com/v1`), while + standalone caller-supplied `LLM_API_BASE_FILE` values remain honored for + OpenAI-compatible endpoints. Known GitHub Models, NVIDIA NIM, and OpenRouter + bases are never inherited, and LiteLLM uses native OpenAI defaults only when + no base is supplied. A non-https override fails configuration. This removes the NVIDIA-NIM-edge + `404 page not found` that made the contracted final fallback unreachable + after NIM exhaustion. +- Align stale `gpt-5.6-luna` test expectations with the valid `gpt-5.4` + contract left behind by the earlier model rename. - Honor each trusted base project's exact, integrity-bearing pnpm `packageManager` specification in OpenCode coverage images through the pinned Node distribution's Corepack runtime, instead of admitting the specification diff --git a/docs/doctoring/strix-openai-fallback-api-base-routing.md b/docs/doctoring/strix-openai-fallback-api-base-routing.md new file mode 100644 index 000000000..d38d222d4 --- /dev/null +++ b/docs/doctoring/strix-openai-fallback-api-base-routing.md @@ -0,0 +1,83 @@ +# Strix direct-OpenAI fallback API-base routing: evidence and design record + +## Decision + +A Strix cross-provider fallback to an explicit direct-OpenAI model +(`openai-direct/...` or `openai_direct/...`) must route through the OpenAI +inference endpoint, never through the primary provider's `LLM_API_BASE`. The +gate now prefers an explicit `STRIX_OPENAI_FALLBACK_API_BASE_FILE` for such +models. In standalone runs, a caller-supplied `LLM_API_BASE_FILE` remains in +force for an OpenAI-compatible endpoint; known GitHub Models, NVIDIA NIM, and +OpenRouter primary endpoints are not inherited. Litellm uses its default +`https://api.openai.com/v1` endpoint only when no base file is supplied (or +when that provider-specific base is rejected). + +The central workflow writes `https://api.openai.com/v1` into +`$RUNNER_TEMP/openai_fallback_api_base.txt` and exports +`STRIX_OPENAI_FALLBACK_API_BASE_FILE` whenever it publishes the OpenAI +fallback key file, so every provider chain that ends in +`openai-direct/gpt-5.4` (NVIDIA NIM primary, OpenRouter primary, +GitHub Models primary) inherits correct routing automatically. + +## Failure this fixes + +Required-CI evidence (BandScope PR #1021 strix run 32800796577, 2026-08-25) +showed the NVIDIA NIM primary and first fallback exhausting provider +availability, then the contracted final fallback `openai-direct/gpt-5.4` +failing with a plain-text gateway error: + +```text +LLM CONNECTION FAILED +Could not establish connection to the language model. +Error: 404 page not found +``` + +Root cause: with `provider_mode=nvidia_nim`, the workflow sets +`LLM_API_BASE_FILE=https://integrate.api.nvidia.com/v1`. The gate reused that +base for the openai-direct fallback child, so litellm sent OpenAI requests to +the NVIDIA NIM edge, whose Go gateway answered `404 page not found`. The +fallback key was already routed correctly (`STRIX_OPENAI_FALLBACK_KEY_FILE`); +only the base URL leaked from the primary provider. Because no vulnerability +report artifact was produced, the gate failed closed — correct policy on an +incomplete scan, but caused by routing rather than by any repository finding. + +## Trust boundary + +The override is a runner-provisioned regular file under `$RUNNER_TEMP`, +resolved through the same `resolve_trusted_input_file` boundary as the other +API-base files: it must be a regular non-symlink file inside the trusted input +root, must trim to a single `https://` URL, and must not contain whitespace or +control characters. Absent or empty overrides preserve a caller-supplied +`LLM_API_BASE_FILE` for standalone local gate runs; when both files are absent, +litellm selects its default endpoint. Known GitHub Models, NVIDIA NIM, and +OpenRouter bases are explicitly rejected for a direct OpenAI model so a +missing OpenAI key remains a provider-unavailable outcome instead of a +configuration error. + +## Verification contract + +Regression evidence proves that: + +1. with a NVIDIA NIM primary base configured, `openai-direct/gpt-5.4` + resolves through the explicit OpenAI fallback base when provided; +2. without either base file, the resolver returns no base so litellm defaults + to `https://api.openai.com/v1`; +3. a standalone caller-supplied custom `LLM_API_BASE_FILE` remains effective; +4. known GitHub Models, NVIDIA NIM, and OpenRouter primary bases are not + inherited by a direct-OpenAI fallback; +5. NVIDIA NIM primary attempts keep resolving through the NIM edge; +6. `github_models/*` fallbacks keep their dedicated GitHub Models endpoint; +7. a non-https override fails configuration (exit 2) instead of scanning; +8. the workflow provisions the override file and passes it into the gate env; +9. the required-workflow smoke contract pins both sides of the wiring; and +10. the stale `gpt-5.6-luna` expectations left behind by the model rename are + aligned with the valid `gpt-5.4` contract in queue-contract tests. + +## Limitations + +This change restores reachability of the final fallback; it does not create +OpenAI quota. If the OpenAI key is absent or exhausted after NIM exhaustion, +the gate still fails closed as provider-unavailable — by design, because no +complete authoritative scan exists. Hosted model catalogs may also change +independently of this repository; model-name updates remain manual contract +changes reviewed through CI. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index cfc97a63c..c9aa41545 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -827,6 +827,17 @@ is_github_models_api_base() { esac } +is_known_foreign_provider_api_base() { + case "$1" in + https://models.github.ai/* | https://integrate.api.nvidia.com/* | https://openrouter.ai/*) + return 0 + ;; + *) + return 1 + ;; + esac +} + PRIMARY_MODEL="$(normalize_model "$STRIX_LLM")" if [ "$PRIMARY_MODEL" != "$STRIX_LLM" ]; then echo "Normalized STRIX_LLM to provider-qualified model '$PRIMARY_MODEL'." @@ -2426,15 +2437,20 @@ resolved_llm_api_base_for_model() { if is_vertex_model "$model"; then return 0 fi - if is_explicit_openai_model "$model" && ! is_explicit_openai_model "$PRIMARY_MODEL"; then - # A direct-OpenAI fallback must not inherit a foreign primary provider's - # endpoint (for example NVIDIA NIM or OpenRouter). - return 0 - fi - - local api_base_file="$LLM_API_BASE_FILE" + local api_base_file="${LLM_API_BASE_FILE:-}" local api_base_file_name="LLM_API_BASE_FILE" - if is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then + if is_explicit_openai_model "$model" && [ -n "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ]; then + # Cross-provider fallback: openai-direct/* candidates must reach the + # direct OpenAI API even when the primary provider selected a + # different LLM_API_BASE_FILE endpoint (e.g. NVIDIA NIM). Without + # this the fallback hits the primary gateway and 404s. + api_base_file="$STRIX_OPENAI_FALLBACK_API_BASE_FILE" + api_base_file_name="STRIX_OPENAI_FALLBACK_API_BASE_FILE" + # The workflow always provisions this file for cross-provider fallbacks. + # In standalone runs, an explicitly supplied LLM_API_BASE_FILE remains + # a caller-owned custom OpenAI-compatible endpoint rather than being + # silently discarded. + elif is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then # Cross-provider fallback: when the active primary provider uses a # different API base (for example OpenRouter), github_models/* fallback # attempts must still route through the GitHub Models inference endpoint. @@ -2470,6 +2486,15 @@ resolved_llm_api_base_for_model() { echo "ERROR: LLM_API_BASE must be an https URL when configured." >&2 return 2 fi + # Never let a known provider-specific base leak into an explicit + # direct-OpenAI fallback when no separate OpenAI override was provisioned. + # Other caller-supplied OpenAI-compatible endpoints remain valid standalone + # configuration and are intentionally preserved. + if is_explicit_openai_model "$model" \ + && [ -z "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ] \ + && is_known_foreign_provider_api_base "$llm_api_base_value"; then + return 0 + fi if is_github_models_api_base "$llm_api_base_value" && ! is_github_models_api_compatible_model "$model"; then echo "ERROR: LLM_API_BASE may route through GitHub Models only when STRIX_LLM uses a GitHub Models-compatible model." >&2 return 2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index abcb5ed07..3d3449dae 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -374,6 +374,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" + assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" + assert_file_contains "$workflow_file" "https://api.openai.com/v1" "strix workflow uses the OpenAI platform endpoint for direct fallbacks" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py new file mode 100644 index 000000000..f078dd2a0 --- /dev/null +++ b/tests/test_strix_openai_fallback_api_base.py @@ -0,0 +1,240 @@ +"""Regression contract for direct-OpenAI fallback API-base routing. + +When the Strix primary provider is NVIDIA NIM (or OpenRouter / GitHub Models), +the workflow's ``LLM_API_BASE_FILE`` points at that provider's endpoint. A +cross-provider fallback to ``openai-direct/gpt-5.4`` must never inherit that +base: routing an OpenAI model through the NVIDIA NIM edge yields a plain-text +gateway 404 ("404 page not found") instead of OpenAI responses, so the final +contracted fallback could never complete a scan. + +The gate must therefore prefer an explicit +``STRIX_OPENAI_FALLBACK_API_BASE_FILE`` for explicit direct-OpenAI models, and + fall back to a caller-supplied ``LLM_API_BASE_FILE`` for standalone custom + endpoints, or to litellm's default OpenAI endpoint when no base is supplied. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +OPENAI_FALLBACK_BASE = "https://api.openai.com/v1" + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) \{{\n.*?^\}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _resolver_helpers() -> list[str]: + """Collect every helper the production API-base resolver depends on.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + utils_source = STRIX_MODEL_UTILS.read_text(encoding="utf-8") + helper_names = ( + ("is_vertex_model", gate_source), + ("is_github_models_model", gate_source), + ("is_github_models_api_base", gate_source), + ("is_known_foreign_provider_api_base", gate_source), + ("is_github_models_api_compatible_model", gate_source), + ("is_explicit_openai_model", gate_source), + ("resolve_trusted_input_file", gate_source), + ("trim_whitespace", utils_source), + ) + helpers: list[str] = [] + for name, source in helper_names: + try: + helpers.append(_function_block(source, name)) + except AssertionError as exc: # pragma: no cover - shape drift guard + raise AssertionError(f"resolver helper missing: {name}") from exc + return helpers + + +def _resolve_api_base(env: dict[str, str], model: str) -> tuple[int, str]: + """Execute the production API-base resolver for one model and env.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + resolver_source = _function_block(gate_source, "resolved_llm_api_base_for_model") + helper_sources = _resolver_helpers() + with tempfile.TemporaryDirectory(prefix="strix-openai-fallback-base-") as temp_dir: + base_path = Path(temp_dir) / "primary_base.txt" + if "LLM_API_BASE_FILE" in env: + base_path.write_text(env["LLM_API_BASE_FILE"], encoding="utf-8") + env = {**env, "LLM_API_BASE_FILE": str(base_path)} + fallback_path = Path(temp_dir) / "openai_fallback_api_base.txt" + if "STRIX_OPENAI_FALLBACK_API_BASE_FILE" in env: + fallback_path.write_text( + env["STRIX_OPENAI_FALLBACK_API_BASE_FILE"], + encoding="utf-8", + ) + env = { + **env, + "STRIX_OPENAI_FALLBACK_API_BASE_FILE": str(fallback_path), + } + github_models_path = Path(temp_dir) / "github_models_api_base.txt" + if "STRIX_GITHUB_MODELS_API_BASE_FILE" in env: + github_models_path.write_text( + env["STRIX_GITHUB_MODELS_API_BASE_FILE"], + encoding="utf-8", + ) + env = { + **env, + "STRIX_GITHUB_MODELS_API_BASE_FILE": str(github_models_path), + } + script_lines = [ + "set -euo pipefail", + f'STRIX_INPUT_FILE_ROOT="{temp_dir}"', + *helper_sources, + resolver_source, + ] + command_env = { + key: value + for key, value in env.items() + if key in { + "LLM_API_BASE_FILE", + "STRIX_GITHUB_MODELS_API_BASE_FILE", + "STRIX_OPENAI_FALLBACK_API_BASE_FILE", + } + } + completed = subprocess.run( + [ + "bash", + "-c", + "\n".join([*script_lines, 'resolved_llm_api_base_for_model "$1"']), + "strix-resolver", + model, + ], + check=False, + capture_output=True, + text=True, + env={ + "PATH": "/usr/bin:/bin:/usr/local/bin", + "HOME": temp_dir, + **command_env, + }, + ) + return completed.returncode, completed.stdout.strip() + + +class ExplicitOpenAIFallbackRouting(unittest.TestCase): + """Direct-OpenAI fallbacks must not inherit the primary provider base.""" + + def test_nvidia_primary_with_override_routes_to_openai(self) -> None: + """openai-direct/gpt-5.4 uses the explicit OpenAI API base file.""" + + rc, api_base = _resolve_api_base( + { + "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", + "STRIX_OPENAI_FALLBACK_API_BASE_FILE": OPENAI_FALLBACK_BASE, + }, + "openai-direct/gpt-5.4", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, OPENAI_FALLBACK_BASE) + + def test_standalone_custom_base_is_honored_without_override(self) -> None: + """A standalone caller's explicit custom endpoint remains effective.""" + + rc, api_base = _resolve_api_base( + {"LLM_API_BASE_FILE": "https://api.example.com/v1"}, + "openai-direct/gpt-5.4", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, "https://api.example.com/v1") + + def test_nvidia_base_is_not_inherited_without_override(self) -> None: + """A cross-provider fallback never inherits the NVIDIA NIM base.""" + + rc, api_base = _resolve_api_base( + {"LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1"}, + "openai-direct/gpt-5.4", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, "") + + def test_direct_openai_without_any_base_uses_default_openai(self) -> None: + """With no base file, LiteLLM still selects the native OpenAI endpoint.""" + + rc, api_base = _resolve_api_base({}, "openai-direct/gpt-5.4") + self.assertEqual(rc, 0) + self.assertEqual(api_base, "") + + def test_github_models_base_is_not_inherited_without_override(self) -> None: + """A cross-provider fallback never inherits GitHub Models routing.""" + + rc, api_base = _resolve_api_base( + {"LLM_API_BASE_FILE": "https://models.github.ai/inference"}, + "openai-direct/gpt-5.4", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, "") + + def test_primary_provider_models_keep_their_base(self) -> None: + """NVIDIA NIM primary attempts still resolve through the NIM edge.""" + + rc, api_base = _resolve_api_base( + {"LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1"}, + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, "https://integrate.api.nvidia.com/v1") + + def test_github_models_fallback_keeps_github_models_base(self) -> None: + """github_models/* fallbacks keep their dedicated inference endpoint.""" + + rc, api_base = _resolve_api_base( + { + "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", + "STRIX_GITHUB_MODELS_API_BASE_FILE": "https://models.github.ai/inference", + }, + "github_models/openai/gpt-5.4", + ) + self.assertEqual(rc, 0) + self.assertEqual(api_base, "https://models.github.ai/inference") + + def test_invalid_https_override_is_configuration_failure(self) -> None: + """A non-https override fails configuration instead of scanning.""" + + rc, _ = _resolve_api_base( + { + "LLM_API_BASE_FILE": "https://integrate.api.nvidia.com/v1", + "STRIX_OPENAI_FALLBACK_API_BASE_FILE": "http://api.example.com/v1", + }, + "openai-direct/gpt-5.4", + ) + self.assertEqual(rc, 2) + + +class WorkflowProvisionsFallbackBase(unittest.TestCase): + """The workflow must publish and pass the explicit OpenAI fallback base.""" + + def test_workflow_writes_openai_fallback_api_base_file(self) -> None: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("STRIX_OPENAI_FALLBACK_API_BASE_FILE=", workflow) + self.assertIn(OPENAI_FALLBACK_BASE, workflow) + + def test_workflow_passes_override_into_gate_environment(self) -> None: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn( + "STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }}", + workflow, + ) + + +if __name__ == "__main__": + unittest.main()