diff --git a/docs/adr/0002-product-technical-gap-baseline.md b/docs/adr/0002-product-technical-gap-baseline.md index 4caf92257..25371a09b 100644 --- a/docs/adr/0002-product-technical-gap-baseline.md +++ b/docs/adr/0002-product-technical-gap-baseline.md @@ -8,6 +8,17 @@ - Figma File ID: N/A. This repository has no customer UI. A UI-owning repository must replace N/A with its real Figma File ID before a UI PR is accepted and must provide Storybook and design-token evidence. - Consequence: The document is an operational snapshot, not a merge authorization or substitute for protected GitHub review. Hourly agents must re-collect exact head SHAs, reviews, threads, and required Checks before merge. Papers/standards live in `docs/doctoring/product-technical-gap-baseline.md` and must remain consistent with this ADR. +## Amendment: provider-specific direct-OpenAI fallback routing (2026-08-24) + +The Strix control plane must route an `openai-direct/` fallback to the +OpenAI platform API base, while retaining provider-specific credentials and +failing closed when no fallback credential or structured vulnerability report +exists. ContextualWisdomLab/.github#1295 exact head +`d376c33a3fdf013c588ab7fd30971f54f9dcee1b` records that routing and its +provider-404 regression contracts. This does not convert provider failure into +clean security evidence or authorize a merge; current-head Checks and +independent approvals remain required. + ## Amendment: central Strix fallback contract (2026-08-25) The current `main` workflow (`a724582`) intentionally replaced the unavailable diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 252249095..cdd9a92e0 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -202,6 +202,7 @@ flowchart LR - ContextualWisdomLab/.github#1265 stays GitHub CLEAN on `d4d4c2b0589065976e4bdcf5c5ae429bc21ed680` with hosted Checks green and no current-head OpenCode APPROVE after repeated `@opencode-agent` requests. CLEAN is not merge authorization. - ContextualWisdomLab/.github#1263 head `50a6ad9129b2da55d049600ecd2516ee0999d7f1` still has required Strix FAILURE on protected-main gate. - Open count is 98. No additional `.github` PR merged this pass. +- ContextualWisdomLab/.github#1295 exact head `d376c33a3fdf013c588ab7fd30971f54f9dcee1b` now binds `openai-direct/` fallbacks to the OpenAI platform endpoint and preserves fail-closed behavior for provider 404/no-report outcomes. The focused provider-routing tests pass locally; hosted Checks remain the authority. ## 2026-08-25 central Strix fallback contract recheck diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index cfc97a63c..b3f5bfe36 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -396,8 +396,9 @@ fi # models (openai-direct/... or openai_direct/...). When the primary model runs # against NVIDIA NIM, OpenRouter, or GitHub Models, its LLM_API_KEY cannot # authenticate a direct-OpenAI fallback; this file carries the OpenAI key. -# Optional: without it, explicit direct-OpenAI models keep using LLM_API_KEY, -# which is correct whenever the primary already runs against direct OpenAI. +# When the primary already runs against direct OpenAI, its LLM_API_KEY is safe +# to reuse for direct-OpenAI fallback models. A cross-provider fallback without +# this key fails closed instead of receiving the primary provider's credential. STRIX_OPENAI_FALLBACK_KEY_FILE="${STRIX_OPENAI_FALLBACK_KEY_FILE:-}" if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && { [ ! -f "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] || [ -L "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; }; then echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must reference a regular file containing the API key." >&2 @@ -2432,6 +2433,15 @@ resolved_llm_api_base_for_model() { return 0 fi + if is_explicit_openai_model "$model" && ! is_explicit_openai_model "$PRIMARY_MODEL"; then + # Cross-provider fallback: direct-OpenAI models must always use the + # OpenAI platform endpoint. Inheriting the primary provider's API + # base (NVIDIA NIM, OpenRouter, GitHub Models) sends an OpenAI + # model and key to a foreign host, which answers "404 page not + # found" and turns every rate-limit outage into a failed fallback. + return 0 + fi + local api_base_file="$LLM_API_BASE_FILE" local api_base_file_name="LLM_API_BASE_FILE" if is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then @@ -2562,11 +2572,21 @@ run_strix_once() { # with the GitHub Models token, not the direct-OpenAI key. child_llm_api_key="$STRIX_GITHUB_MODELS_KEY" fi - if is_explicit_openai_model "$model" && [ -n "$STRIX_OPENAI_FALLBACK_KEY" ]; then - # Cross-provider fallback: explicit direct-OpenAI models - # authenticate with the OpenAI key, not the primary provider's - # key (NVIDIA NIM, OpenRouter, or GitHub Models). - child_llm_api_key="$STRIX_OPENAI_FALLBACK_KEY" + if is_explicit_openai_model "$model"; then + if [ "$model" != "$PRIMARY_MODEL" ] && [ -n "$STRIX_OPENAI_FALLBACK_KEY" ]; then + # Cross-provider fallback: explicit direct-OpenAI models + # authenticate with the OpenAI key, not the primary provider's + # key (NVIDIA NIM, OpenRouter, or GitHub Models). The same + # dedicated key also takes precedence for same-provider fallback. + child_llm_api_key="$STRIX_OPENAI_FALLBACK_KEY" + elif is_explicit_openai_model "$PRIMARY_MODEL"; then + # Same-provider fallback: reuse the primary direct-OpenAI key + # only when no dedicated fallback key was configured. + : + else + echo "ERROR: direct-OpenAI fallback '$model' requires STRIX_OPENAI_FALLBACK_KEY_FILE when the primary model uses another provider." >&2 + return 2 + fi fi fi set -o pipefail @@ -4233,6 +4253,8 @@ run_current_target_scan() { read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" fallback_tried=0 + local fallback_attempts=0 + local fallback_config_failures=0 for candidate_raw in "${FALLBACK_MODELS[@]}"; do candidate="$(normalize_model "$candidate_raw")" if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then @@ -4246,6 +4268,7 @@ run_current_target_scan() { fi fallback_tried=1 + fallback_attempts=$((fallback_attempts + 1)) if is_vertex_model "$PRIMARY_MODEL"; then echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." else @@ -4264,7 +4287,9 @@ run_current_target_scan() { return 0 fi if [ "$fallback_scan_rc" -eq 2 ]; then - return 2 + fallback_config_failures=$((fallback_config_failures + 1)) + echo "Skipping fallback model '$candidate' because its provider configuration is invalid; trying the next configured fallback." >&2 + continue fi local strict_fallback_provider_signal=0 @@ -4332,6 +4357,10 @@ run_current_target_scan() { fi return 1 fi + if [ "$fallback_config_failures" -eq "$fallback_attempts" ]; then + echo "ERROR: All configured fallback models failed provider configuration." >&2 + return 2 + fi if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index abcb5ed07..e275272fe 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3455,6 +3455,114 @@ REPORT ;; esac ;; + same-provider-direct-openai-fallback-key) + case "${STRIX_LLM:-}" in + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected primary direct-OpenAI key (${LLM_API_KEY:-})" >&2 + exit 19 + fi + echo "Error: litellm.RateLimitError: Error code: 429" + exit 1 + ;; + openai/gpt-5.5) + if [ "${LLM_API_KEY:-}" != "same-provider-fallback-token" ]; then + echo "unexpected same-provider fallback key (${LLM_API_KEY:-})" >&2 + exit 20 + fi + echo "scan ok with same-provider fallback key" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; + nvidia-nim-quota-openai-direct-fallback-missing-key) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected primary key for NVIDIA NIM (${LLM_API_KEY:-})" >&2 + exit 17 + fi + echo "Error: litellm.RateLimitError: Error code: 429" + exit 1 + ;; + openai/gpt-5.6-luna) + echo "unexpected direct-OpenAI fallback invocation without a fallback key" >&2 + exit 18 + ;; + deepseek/deepseek-v3-0324) + echo "scan ok after invalid direct-OpenAI fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; + nvidia-nim-quota-openai-direct-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected primary key for NVIDIA NIM (${LLM_API_KEY:-})" >&2 + exit 25 + fi + echo "Error: litellm.RateLimitError: Error code: 429" + exit 1 + ;; + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "dedicated-openai-fallback-token" ]; then + echo "unexpected direct-OpenAI fallback key (${LLM_API_KEY:-})" >&2 + exit 26 + fi + if [ "${LLM_API_BASE:-}" != "" ]; then + echo "unexpected direct-OpenAI fallback API base (${LLM_API_BASE:-})" >&2 + exit 27 + fi + echo "scan ok with dedicated direct-OpenAI fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 28 + ;; + esac + ;; + all-fallbacks-invalid) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected primary key for NVIDIA NIM (${LLM_API_KEY:-})" >&2 + exit 21 + fi + echo "Error: litellm.RateLimitError: Error code: 429" + exit 1 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 22 + ;; + esac + ;; + mixed-fallback-config-and-retryable) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|deepseek/deepseek-v3-0324) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected NVIDIA NIM key for fallback probe (${LLM_API_KEY:-})" >&2 + exit 23 + fi + echo "Error: litellm.RateLimitError: Error code: 429" + exit 1 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 24 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -4399,7 +4507,7 @@ EOS echo "Error: deepseek model was rewritten (${STRIX_LLM:-})" >&2 exit 33 ;; - preserve-existing-api-base) + preserve-existing-api-base|preserve-existing-direct-openai-api-base) if [ "${LLM_API_BASE:-}" = "https://preexisting.invalid" ]; then echo "scan ok with preserved api base" exit 0 @@ -5706,6 +5814,27 @@ PY env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") env_cmd+=(STRIX_GITHUB_MODELS_KEY_FILE="$tmp_dir/github_models_key.txt") fi + if [ "$scenario" = "nvidia-nim-quota-openai-direct-fallback-missing-key" ]; then + # Exercise the cross-provider path with no OpenAI fallback credential, + # then prove a later valid fallback is still attempted. + env_cmd+=(STRIX_FALLBACK_MODELS="openai-direct/gpt-5.6-luna deepseek/deepseek-v3-0324") + fi + if [ "$scenario" = "nvidia-nim-quota-openai-direct-fallback-success" ]; then + printf '%s' 'dedicated-openai-fallback-token' >"$tmp_dir/openai_fallback_key.txt" + env_cmd+=(STRIX_FALLBACK_MODELS="openai-direct/gpt-5.6-luna") + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + fi + if [ "$scenario" = "all-fallbacks-invalid" ]; then + env_cmd+=(STRIX_FALLBACK_MODELS="openai-direct/gpt-5.6-luna openai_direct/gpt-5.5") + fi + if [ "$scenario" = "mixed-fallback-config-and-retryable" ]; then + env_cmd+=(STRIX_FALLBACK_MODELS="openai-direct/gpt-5.6-luna deepseek/deepseek-v3-0324") + fi + if [ "$scenario" = "same-provider-direct-openai-fallback-key" ]; then + printf '%s' 'same-provider-fallback-token' >"$tmp_dir/openai_fallback_key.txt" + env_cmd+=(STRIX_FALLBACK_MODELS="openai-direct/gpt-5.5") + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + fi if [ "$min_fail_severity" = "__UNSET__" ]; then local next_env_cmd=() local env_pair @@ -6434,6 +6563,18 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-fallback-requires-api-base) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/missing-primary" \ + "openai/openai/gpt-5.4" \ + "2" \ + "All configured fallback models failed provider configuration." \ + "1" \ + "vertex_ai/missing-primary" \ + "" \ + "vertex_ai" \ + "" + ;; endpoint-in-excluded-dir) run_gate_case "endpoint-in-excluded-dir" \ "vertex_ai/excluded-dir-primary" \ @@ -6477,6 +6618,74 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ "|" ;; + nvidia-nim-quota-openai-direct-fallback-missing-key) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "0" \ + "scan ok after invalid direct-OpenAI fallback" \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|deepseek/deepseek-v3-0324" \ + "" \ + "nvidia_nim" + ;; + nvidia-nim-quota-openai-direct-fallback-success) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "0" \ + "scan ok with dedicated direct-OpenAI fallback" \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|openai/gpt-5.6-luna" \ + "https://example.invalid|" \ + "nvidia_nim" + ;; + all-fallbacks-invalid) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "2" \ + "All configured fallback models failed provider configuration" \ + "1" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "nvidia_nim" + ;; + mixed-fallback-config-and-retryable) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "1" \ + "Configured model and fallback models were unavailable." \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|deepseek/deepseek-v3-0324" \ + "https://example.invalid|https://example.invalid" \ + "nvidia_nim" + ;; + same-provider-direct-openai-fallback-key) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "openai_direct/gpt-5.6-luna" \ + "" \ + "0" \ + "scan ok with same-provider fallback key" \ + "2" \ + "openai/gpt-5.6-luna|openai/gpt-5.5" \ + "|" \ + "openai" + ;; + preserve-existing-direct-openai-api-base) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "openai_direct/gpt-5.6-luna" \ + "" \ + "0" \ + "scan ok with preserved api base" \ + "1" \ + "openai/gpt-5.6-luna" \ + "https://preexisting.invalid" \ + "openai" \ + "" \ + "https://preexisting.invalid" + ;; total-timeout) run_total_timeout_case ;; @@ -11096,6 +11305,18 @@ run_gate_case "preserve-existing-api-base" \ "" \ "https://preexisting.invalid" +run_gate_case "preserve-existing-direct-openai-api-base" \ + "openai_direct/gpt-5.6-luna" \ + "" \ + "0" \ + "scan ok with preserved api base" \ + "1" \ + "openai/gpt-5.6-luna" \ + "https://preexisting.invalid" \ + "openai" \ + "" \ + "https://preexisting.invalid" + run_gate_case "default-fallback-order-fast-first" \ "vertex_ai/missing-primary" \ "" \ @@ -12589,7 +12810,7 @@ run_gate_case "github-models-fallback-requires-api-base" \ "vertex_ai/missing-primary" \ "openai/openai/gpt-5.4" \ "2" \ - "GitHub Models Strix scans require LLM_API_BASE_FILE" \ + "All configured fallback models failed provider configuration." \ "1" \ "vertex_ai/missing-primary" \ "" \ @@ -12691,6 +12912,38 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "" \ "github_models/openai/o3" +# A cross-provider direct-OpenAI fallback must fail closed when its dedicated +# credential is absent; it must never receive the primary NVIDIA key. +run_gate_case "nvidia-nim-quota-openai-direct-fallback-missing-key" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "0" \ + "scan ok after invalid direct-OpenAI fallback" \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|deepseek/deepseek-v3-0324" \ + "" \ + "nvidia_nim" + +run_gate_case "nvidia-nim-quota-openai-direct-fallback-success" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "0" \ + "scan ok with dedicated direct-OpenAI fallback" \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|openai/gpt-5.6-luna" \ + "https://example.invalid|" \ + "nvidia_nim" + +run_gate_case "mixed-fallback-config-and-retryable" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ + "1" \ + "Configured model and fallback models were unavailable." \ + "2" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5|deepseek/deepseek-v3-0324" \ + "https://example.invalid|https://example.invalid" \ + "nvidia_nim" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \