diff --git a/.github/workflows/strix-changed-path-quality-ci.yml b/.github/workflows/strix-changed-path-quality-ci.yml index 31924910a..f92f47aba 100644 --- a/.github/workflows/strix-changed-path-quality-ci.yml +++ b/.github/workflows/strix-changed-path-quality-ci.yml @@ -9,7 +9,9 @@ on: - "CHANGELOG.md" - "docs/doctoring/strix-legal-git-paths.md" - "docs/doctoring/strix-model-behavior-error.md" + - "docs/doctoring/strix-openai-direct-fallback.md" - "docs/doctoring/strix-quality-timeout-fixtures.md" + - "scripts/ci/strix_model_utils.sh" - "scripts/ci/strix_quick_gate.sh" - "scripts/ci/test_strix_quick_gate.sh" - "tests/test_strix_changed_path_policy.py" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index b3248d943..aa469466e 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -696,6 +696,22 @@ jobs: printf '%s' 'https://models.github.ai/inference' > "$github_models_api_base_file" echo "STRIX_GITHUB_MODELS_API_BASE_FILE=$github_models_api_base_file" >> "$GITHUB_ENV" + - name: Prepare direct OpenAI fallback credentials + if: steps.gate.outputs.provider_mode == 'github_models' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' + env: + OPENAI_FALLBACK_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} + run: | + umask 077 + sanitized="$(printf '%s' "$OPENAI_FALLBACK_API_KEY" | tr -d '\r\n')" + trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [ -z "$trimmed" ]; then + echo '::notice::No direct OpenAI key available; cross-provider Strix scans run without that fallback.' + exit 0 + fi + openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" + printf '%s' "$trimmed" > "$openai_fallback_key_file" + echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + - name: Prepare Vertex AI credentials if: steps.gate.outputs.provider_mode == 'vertex_ai' env: @@ -825,6 +841,7 @@ jobs: STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} + STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" NPM_CONFIG_IGNORE_SCRIPTS: "true" diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b0ef8d44..56f5f70a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Rewrite the workflow-facing `openai-direct/` Strix fallback alias to LiteLLM + `openai/` before dispatch, clear a non-OpenAI primary API base, and load the + established direct OpenAI secret through a trusted runtime file so NVIDIA NIM + catalog 404s can complete a scan instead of failing closed on + `LLM Provider NOT provided`. Missing fallback credentials remain fail-closed. + Incomplete scans and reported vulnerabilities both still fail closed. + - Honor each trusted base project's exact, integrity-bearing pnpm `packageManager` specification in OpenCode coverage images through the pinned Node distribution's Corepack runtime, instead of admitting the specification diff --git a/docs/doctoring/strix-openai-direct-fallback.md b/docs/doctoring/strix-openai-direct-fallback.md new file mode 100644 index 000000000..5a813a9ef --- /dev/null +++ b/docs/doctoring/strix-openai-direct-fallback.md @@ -0,0 +1,37 @@ +# Strix openai-direct fallback: evidence and design record + +## Decision + +The central Strix workflow advertises `openai-direct/gpt-5.6-luna` as the +cross-provider fallback after NVIDIA NIM catalog misses. LiteLLM does not +recognize the hyphenated `openai-direct/` provider prefix. The gate rewrites +that alias to `openai_direct/` and dispatches it as LiteLLM `openai/`, clears a +non-OpenAI primary API base, and authenticates with the established +`STRIX_OPENAI_API_KEY` / `OPENAI_API_KEY` secret through a trusted runtime +file. + +## Trust boundary + +- `normalize_model` rewrites only `openai-direct/?*` to `openai_direct/`. +- `run_strix_once` normalizes the candidate before `child_model_for_api_base`. +- Cross-provider `openai_direct/*` attempts do not inherit NVIDIA, OpenRouter, + or GitHub Models API bases. +- A missing `STRIX_OPENAI_FALLBACK_KEY_FILE` during a cross-provider fallback + is configuration exit 2. The notice-only workflow step that skips writing + the file when the secret is empty does not weaken that gate. +- Incomplete scans, exhausted fallbacks, and reported vulnerabilities remain + fail-closed. This change does not ignore findings or skip the scanner. + +## Observed incident + +ScopeWeave #589 job 97204514255 (run 32643656525) on +`dce2424b45833fa6a942fae1edb5d16f0d687bdb` produced an empty SARIF and a +penetration report with no product findings. The last attempt log recorded +`model=openai-direct/gpt-5.6-luna` and `litellm.BadRequestError: LLM Provider +NOT provided`. That is infrastructure failure, not a Stripe HMAC finding. + +## Verification + +- `python3 -m unittest tests.test_strix_nvidia_nim_not_found_fallback` +- `bash scripts/ci/test_strix_quick_gate.sh` (normalize_model alias + workflow + contract for `STRIX_OPENAI_FALLBACK_KEY_FILE`) diff --git a/scripts/ci/strix_model_utils.sh b/scripts/ci/strix_model_utils.sh index 9f20eae67..8f255696b 100755 --- a/scripts/ci/strix_model_utils.sh +++ b/scripts/ci/strix_model_utils.sh @@ -105,6 +105,10 @@ normalize_model() { printf '%s\n' "$model" return 0 ;; + openai-direct/?*) + printf 'openai_direct/%s\n' "${model#openai-direct/}" + return 0 + ;; */*) printf '%s\n' "$model" return 0 diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 337373001..b1eae25ea 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -380,6 +380,23 @@ if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ]; then fi fi +STRIX_OPENAI_FALLBACK_KEY_FILE="${STRIX_OPENAI_FALLBACK_KEY_FILE:-}" +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && { [ ! -f "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] || [ -L "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; }; then + echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must reference a regular file containing the API key." >&2 + exit 2 +fi +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && ! STRIX_OPENAI_FALLBACK_KEY_FILE="$(resolve_trusted_input_file "STRIX_OPENAI_FALLBACK_KEY_FILE" "$STRIX_OPENAI_FALLBACK_KEY_FILE")"; then + exit 2 +fi +STRIX_OPENAI_FALLBACK_KEY="" +if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; then + STRIX_OPENAI_FALLBACK_KEY="$(trim_whitespace "$(cat -- "$STRIX_OPENAI_FALLBACK_KEY_FILE")")" + if [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must contain a non-empty API key." >&2 + exit 2 + fi +fi + require_non_negative_integer() { local value="$1" local label="$2" @@ -2380,6 +2397,14 @@ resolved_llm_api_base_for_model() { if is_vertex_model "$model"; then return 0 fi + case "$(normalize_model "$model"):$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + # Cross-provider fallback: do not inherit NVIDIA/OpenRouter/GitHub + # Models API bases. LiteLLM openai/* talks to api.openai.com. + return 0 + ;; + esac local api_base_file="$LLM_API_BASE_FILE" local api_base_file_name="LLM_API_BASE_FILE" @@ -2490,20 +2515,32 @@ run_strix_once() { if ! llm_api_base_value="$(resolved_llm_api_base_for_model "$model")"; then return 2 fi - child_model="$(child_model_for_api_base "$model" "$llm_api_base_value")" + local normalized_model + normalized_model="$(normalize_model "$model")" + child_model="$(child_model_for_api_base "$normalized_model" "$llm_api_base_value")" if ! resolved_target_path="$(resolve_current_target_path "$TARGET_PATH")"; then return 1 fi local start_epoch start_epoch="$(date +%s)" local child_llm_api_key="" - if ! is_vertex_model "$(normalize_model "$model")"; then + if ! is_vertex_model "$normalized_model"; then child_llm_api_key="$LLM_API_KEY" - if is_github_models_model "$(normalize_model "$model")" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then + if is_github_models_model "$normalized_model" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then # Cross-provider fallback: github_models/* models authenticate # with the GitHub Models token, not the direct-OpenAI key. child_llm_api_key="$STRIX_GITHUB_MODELS_KEY" fi + case "$normalized_model:$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + if [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "ERROR: direct OpenAI fallback requires STRIX_OPENAI_FALLBACK_KEY_FILE." >&2 + return 2 + fi + child_llm_api_key="$STRIX_OPENAI_FALLBACK_KEY" + ;; + esac fi set -o pipefail set +e diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index bf0a8693e..ebc849f93 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -367,6 +367,11 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" + assert_file_contains "$workflow_file" "Prepare direct OpenAI fallback credentials" "strix workflow provisions a direct OpenAI key for hyphenated openai-direct fallbacks" + assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix workflow passes the OpenAI fallback key through a trusted input file" + assert_file_contains "$GATE_SCRIPT" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix gate reads the optional direct OpenAI fallback key file" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_model_utils.sh" 'openai-direct/?*)' "strix gate rewrites the workflow-facing openai-direct alias before LiteLLM dispatch" + assert_file_contains "$GATE_SCRIPT" 'normalized_model="$(normalize_model "$model")"' "strix gate normalizes fallback models before LiteLLM child dispatch" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" assert_file_contains "$workflow_file" '${strix_model#github_models/}' "strix workflow strips manual github_models routing prefix for OpenAI GPT model names before passing model names to LiteLLM" assert_file_contains "$workflow_file" "openai_direct/%s" "strix workflow keeps manual direct OpenAI scans distinct from GitHub Models openai/gpt-* routing" @@ -12312,6 +12317,11 @@ assert_normalized_model \ "projects/my-proj/locations/us-central1/publishers/google/models/gemini-2.5-pro" \ "vertex_ai" \ "vertex_ai/gemini-2.5-pro" +assert_normalized_model \ + "openai-direct-hyphen-alias" \ + "openai-direct/gpt-5.6-luna" \ + "openai" \ + "openai_direct/gpt-5.6-luna" assert_model_requires_vertex_auth "explicit-vertex" "vertex_ai/gemini-2.5-pro" "gemini" "0" assert_model_requires_vertex_auth "explicit-vertex-beta" "vertex_ai_beta/gemini-2.5-pro" "gemini" "0" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 990269725..719720f44 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -17,6 +17,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" FREE_NVIDIA_FALLBACK = ( @@ -73,6 +74,64 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 +def _run_strix_once_resolves_child_model(model: str, primary_model: str) -> str: + """Execute run_strix_once's own model/API-base resolution lines verbatim. + + Extracts the exact statements `run_strix_once` uses to compute the model + it hands to LiteLLM, rather than reimplementing that composition, so a + future call-site edit that stops normalizing the workflow-facing alias + before dispatch fails this test instead of only failing in CI against + live NVIDIA NIM traffic. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + call_site_match = re.search( + r"(?m)^\tif ! llm_api_base_value=.*\n" + r"(?:.*\n)+?" + r'\tchild_model="\$\(child_model_for_api_base [^\n]+\)"\n', + gate_source, + ) + if call_site_match is None: + raise AssertionError( + "missing run_strix_once model/API-base resolution call site" + ) + function_source = "\n".join( + _function_block(gate_source, name) + for name in ( + "is_vertex_model", + "is_github_models_api_base", + "is_github_models_model", + "is_github_models_api_compatible_model", + "resolved_llm_api_base_for_model", + "child_model_for_api_base", + ) + ) + script = "\n".join( + ( + "set -euo pipefail", + STRIX_MODEL_UTILS.read_text(encoding="utf-8"), + function_source, + 'PRIMARY_MODEL="$2"', + 'LLM_API_BASE_FILE=""', + "run_strix_once_child_model() {", + 'local model="$1"', + "local llm_api_base_value", + "local child_model", + call_site_match.group(0), + 'printf %s "$child_model"', + "}", + 'run_strix_once_child_model "$1"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-run-once", model, primary_model], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" @@ -213,6 +272,34 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: )[0] self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate) + def test_run_strix_once_normalizes_hyphenated_fallback_before_dispatch( + self, + ) -> None: + """Dispatch the NIM-exhaustion fallback as a real LiteLLM provider. + + `normalize_model()` rewrites the workflow-facing `openai-direct/` alias + to LiteLLM's `openai_direct/` selector, and `child_model_for_api_base()` + rewrites `openai_direct/*` to `openai/*`. Observed live: NVIDIA NIM + returned 404, the run advanced to `openai-direct/gpt-5.6-luna`, and + LiteLLM recorded `model=openai-direct/gpt-5.6-luna` before + `BadRequestError: LLM Provider NOT provided` ended the scan. + """ + + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai-direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai_direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: """Reject provider-like target text in the outer neutralization gate."""