diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index b510b532d..3b653b900 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -37,13 +37,16 @@ on: # Same conservative doc/image-only skip for PR scans. GitHub evaluates these # path filters against the PR's full base..head diff, so a PR is skipped only # when EVERY changed file is a non-executable doc/image asset; any code, - # config, build, or workflow change still triggers the scan. Concurrency is - # PR-number based for status grouping, but Strix runs intentionally do not + # config, build, or workflow change still triggers the scan. The run-name + # includes the PR number and head SHA for status grouping, while the + # concurrency group is scoped per repository and event class to prevent + # shared-provider key rate-limit storms. Strix runs intentionally do not # cancel in progress because a pre-job cancellation leaves no scanner log to - # review. Queue pressure should be handled by stale-run cleanup outside this - # current-head evidence path. For PRs the merge scheduler manages, same-head - # Strix evidence is still forced at merge time via repository_dispatch (which - # paths-ignore does not affect), so merged code never loses evidence. + # review. GitHub keeps one active and one pending run per group; the merge + # scheduler re-dispatches exact-head evidence when a pending run is + # superseded. For PRs the merge scheduler manages, same-head Strix evidence + # is still forced at merge time via repository_dispatch (which paths-ignore + # does not affect), so merged code never loses evidence. paths-ignore: - '**/*.md' - '**/*.markdown' @@ -68,13 +71,32 @@ on: concurrency: # Include the event name so default-branch repository_dispatch evidence cannot cancel - # the required pull_request_target Strix context that branch protection reads. - # PR-number scope keeps the queue on the current HEAD within each event class. + # or interleave with the required pull_request_target Strix context that branch + # protection reads. Closed PR events use a separate group so their cancellation + # job can run immediately instead of waiting behind the scan it must cancel. + # + # Rate-limit root-cause fix (2026-08-24): the group is scoped per REPOSITORY + # (not per PR) so sibling pull requests in the same repository scan + # sequentially instead of concurrently. Concurrent per-PR scans each retry + # the shared NVIDIA NIM key up to three times, producing guaranteed + # litellm.RateLimitError storms and fail-closed gate failures across every + # open PR (observed 2026-08-23/24). Serializing per repository and event + # class keeps at most one provider-backed PR scan in flight per class. Push + # and scheduled scans retain the branch ref so one protected branch cannot + # supersede another branch's pending evidence. GitHub's native concurrency + # contract retains one active and one pending run; the scheduler re-dispatches + # the exact current head after pending-run supersession, and accuracy is + # prioritized over scan latency. group: >- - strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}-${{ - github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number) || - github.event_name == 'repository_dispatch' && github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number) || github.ref }} - cancel-in-progress: true + strix-${{ + github.event_name == 'pull_request_target' && + github.event.action == 'closed' && + format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number) || + (github.event_name == 'pull_request_target' || github.event_name == 'repository_dispatch') && + format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) || + format('{0}-{1}-{2}', github.event_name, github.repository, github.ref) + }} + cancel-in-progress: false # Scorecard Token-Permissions (alert #43): keep the workflow-level token # read-only and scope same-repo status publication to the Strix scan job. @@ -87,8 +109,60 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest + # Prefer the established scheduler credential, but let the close event use + # its job-scoped token so abandoned scans are cancelled even when that + # optional secret is unavailable. This job never checks out PR code. + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} + CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} + CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CURRENT_RUN_ID: ${{ github.run_id }} steps: - - run: echo "PR closed; this run only cancels older runs through workflow concurrency." + - name: Cancel queued and running scans for the closed pull request + shell: bash + run: | + set -euo pipefail + + cancel_runs() { + local status="$1" + local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100" + local runs_json + if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/strix-close-gh-error)"; then + echo "::warning::Strix close cleanup could not inspect ${TARGET_REPOSITORY}; leaving runs unchanged." + sed 's/^/ /' /tmp/strix-close-gh-error >&2 || true + return 0 + fi + local run_ids + if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ + --arg current "$CURRENT_RUN_ID" ' + .workflow_runs[] + | select((.id | tostring) != $current) + | select(.name == "Strix Security Scan") + | select(.event == "pull_request_target") + | select(.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr))) + | .id + ' <<<"$runs_json")"; then + echo "::warning::Strix close cleanup received invalid run data for ${TARGET_REPOSITORY}; leaving runs unchanged." + return 0 + fi + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then + echo "Cancelled Strix run ${run_id} in ${TARGET_REPOSITORY} for closed PR #${CLOSED_PR_NUMBER}." + else + echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${TARGET_REPOSITORY}; it may have finished or the credential lacks Actions write access." + sed 's/^/ /' /tmp/strix-close-cancel-error >&2 || true + fi + done <<<"$run_ids" + } + + for active_status in queued in_progress requested waiting pending; do + cancel_runs "$active_status" + done strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' @@ -110,6 +184,9 @@ jobs: statuses: write env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + STRIX_NVIDIA_ALLOWED_MODELS: >- + nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1 steps: - name: Harden runner uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 @@ -276,6 +353,9 @@ jobs: "") is_private="" for target_visibility_attempt in 1 2 3 4 5 6; do + # The single-quoted jq program intentionally expands jq's + # `$visibility`, not a shell variable (ShellCheck SC2016). + # shellcheck disable=SC2016 if is_private="$( gh api "repos/${TARGET_REPOSITORY}" --jq ' (.visibility // "" | ascii_downcase) as $visibility @@ -469,10 +549,49 @@ jobs: printf 'Materialized central Strix dependency lock from same-repository PR head.\n' fi + - name: Resolve live NVIDIA NIM Strix models + id: resolve_nvidia_models + env: + STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} + TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + run: | + set -euo pipefail + if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then + printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + resolver="$TRUSTED_STRIX_SOURCE/scripts/ci/select_nvidia_nim_model.py" + primary_rc=0 + primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_ALLOWED_MODELS")" || primary_rc=$? + if [ "$primary_rc" -eq 75 ]; then + echo '::warning::NVIDIA NIM model catalog is unavailable; using the contracted OpenAI fallback.' + printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + [ "$primary_rc" -eq 0 ] || exit "$primary_rc" + + fallback_rc=0 + fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_ALLOWED_MODELS" --exclude "$primary")" || fallback_rc=$? + if [ "$fallback_rc" -eq 75 ]; then + echo '::warning::NVIDIA NIM fallback resolution is unavailable; retaining the resolved primary and contracted OpenAI fallback.' + fallback="" + else + [ "$fallback_rc" -eq 0 ] || exit "$fallback_rc" + fi + { + printf 'primary=nvidia_nim/%s\n' "$primary" + if [ -n "$fallback" ]; then + printf 'fallback=nvidia_nim/%s\n' "$fallback" + else + printf 'fallback=\n' + fi + } >> "$GITHUB_OUTPUT" + - name: Gate Strix secrets id: gate env: - STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4') }} + STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4') }} STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} STRIX_OPENAI_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} STRIX_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} @@ -482,9 +601,6 @@ jobs: TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then - strix_model="gpt-5.4" - fi echo "strix_model=$strix_model" >> "$GITHUB_OUTPUT" case "$strix_model" in openai/gpt-5-mini* | openai/gpt-5-nano* | \ @@ -526,7 +642,17 @@ jobs: exit 1 fi ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + # Trusted-main bootstrap compatibility until this PR merges; this is + # the provider-qualified form of the canonical allowlist default: + # nvidia_nim/nvidia/nemotron-3-super-120b-a12b + nvidia_nim/*) + case " $STRIX_NVIDIA_ALLOWED_MODELS " in + *" ${strix_model#nvidia_nim/} "*) ;; + *) + echo '::error::STRIX_LLM selected an NVIDIA NIM model outside the reviewed allowlist.' + exit 1 + ;; + esac if [ "$TARGET_REPOSITORY_PRIVATE" != "false" ]; then echo '::error::NVIDIA NIM hosted trial scans are limited to public repositories.' exit 1 @@ -826,7 +952,7 @@ jobs: openrouter/free | openrouter/openrouter/free) printf '%s' 'openrouter/free' > "$strix_llm_file" ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + nvidia_nim/*) printf '%s' "$strix_model" > "$strix_llm_file" ;; vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash) @@ -870,9 +996,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - # Trusted-main smoke compatibility marker only; never executed: - # nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openrouter/free openai-direct/gpt-5.4' || '' }} + # `openrouter/free` is OpenRouter's authenticated dynamic router, not a + # pinned underlying model id; OpenRouter performs live model selection. + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} @@ -937,8 +1063,10 @@ jobs: strix_rc=0 strix_gate_attempt=1 strix_gate_deadline=$(( SECONDS + 6000 )) - strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS" - strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}" + # Reserve the scanner process budget, not the gate's total wrapper + # budget. The latter includes setup/cleanup overhead already spent + # by the current attempt and can make every retry impossible. + strix_gate_attempt_budget_seconds="$process_budget_seconds" set +e while : ; do strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_${strix_gate_attempt}.log" diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f023e4a..c217d6c10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ Semantic Versioning where the repository publishes a release. during materialization and then rejecting every version except pnpm 11.5.3; route generic coverage and docstring package scripts through the same Corepack boundary instead of invoking a removed bare `pnpm` binary. +- Review scans now run in a controlled order so each pull request receives a + complete result instead of a rate-limit interruption. Open the pull request + after the active scan finishes to review the latest result. +- Closed pull-request cleanup now preserves the review record and reports any + authorization or malformed-data issue for follow-up. Reopen the pull request + or update its credentials when the cleanup message asks you to act. - Keep `--trust-lockfile` only for pnpm 11.3 and newer (`trustLockfile` landed in pnpm 11.3). pnpm 9, 10, and 11.0–11.2 reject that flag and previously failed LineageWeave JavaScript coverage before diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index 213429e01..228274414 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -5,14 +5,23 @@ Strix treats an authenticated NVIDIA NIM model-catalog `404 Not Found` as provider availability evidence, not as a target-application vulnerability. The gate does not retry the same unavailable model. It proceeds to a distinct -reviewed NVIDIA hosted model and only then to the existing GitHub Models -candidates. - -Public-repository scans now default to -`nvidia/nemotron-3-super-120b-a12b`. The first fallback is -`nvidia/llama-3.3-nemotron-super-49b-v1.5`. Private repositories retain the -contracted provider because NVIDIA hosted trial inputs are restricted to public -repositories by the central workflow. +reviewed NVIDIA hosted model and only then to the direct OpenAI fallback. + +For public-repository scans, the trusted workflow queries NVIDIA's authenticated +`/v1/models` catalog and selects the first served entry from reviewed primary +and fallback pools. The default pool prefers +`nvidia/nemotron-3-super-120b-a12b`; the distinct fallback is +`nvidia/llama-3.1-nemotron-ultra-253b-v1`. The retired +`nvidia/llama-3.3-nemotron-super-49b-v1.5` is no longer executable workflow +configuration. Private repositories retain the contracted provider because +NVIDIA hosted trial inputs are restricted to public repositories. + +OpenRouter remains a supported transport and API-base capability. Required CI +run `33012371359` exposed a wrapped HTTP 502 from the authenticated +`openrouter/free` dynamic router. The same-model retry classifier did not match +because LiteLLM wrapped `APIError` and `OpenrouterException` onto separate +terminal lines. The classifier now recognizes that bounded signature, so the +NVIDIA exhaustion chain retains OpenRouter before direct OpenAI. ## Trust boundary @@ -48,21 +57,28 @@ Regression evidence proves that: 4. a provider-like source literal on one line without LiteLLM `NotFoundError` context is not recognized; 5. model-catalog 404s enter cross-model fallback but never same-model retry; -6. the primary and first fallback are current NVIDIA hosted models; -7. GitHub Models remain later cross-provider fallbacks; +6. the primary and first fallback are present in the live NVIDIA catalog; +7. OpenRouter's authenticated dynamic free router and direct OpenAI remain the + later cross-provider fallbacks; 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. the required-workflow smoke contract pins these properties. +10. wrapped OpenRouter 502 output enters a bounded same-model retry; and +11. the required-workflow smoke contract pins these properties. ## Limitations -Hosted model catalogs may change independently of this repository. A model-card -page or supported self-hosted NIM container does not guarantee indefinite hosted -trial availability. The ordered model plan must therefore be reviewed against -current NVIDIA documentation whenever a provider returns a catalog 404. This -change does not treat arbitrary provider errors as success and does not weaken -Strix severity, changed-file attribution, or independent approval requirements. +Hosted model catalogs and capacity may change independently of this repository. +Catalog membership prevents deterministic retired-model selection but does not +prove capacity, so HTTP 429 exhaustion remains fail-closed if every fallback is +unavailable. This change does not treat provider errors as success and does not +weaken Strix severity, changed-file attribution, or approval requirements. + +Operationally, at least one configured provider must have usable request +capacity or credit before a required scan can produce authoritative evidence. +The live catalog resolver verifies model availability, not quota, rate-limit +headroom, or account balance. Restoring those provider resources is a runtime +prerequisite; adding another fixed model identifier is not a substitute. ## Current fallback contract (2026-08-25) @@ -78,12 +94,12 @@ focused `test_strix_quick_gate.sh` case; provider failures remain non-passing. Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 -NVIDIA Corporation. (2025). *Llama-3.3-Nemotron-Super-49B-v1.5* [Model card]. -NVIDIA NIM. https://build.nvidia.com/nvidia/llama-3_3-nemotron-super-49b-v1_5/modelcard +NVIDIA Corporation. (2026a). *Models*. NVIDIA NIM. +https://build.nvidia.com/models -NVIDIA Corporation. (2026a). *NVIDIA-Nemotron-3-Super-120B-A12B* [Model +NVIDIA Corporation. (2026b). *NVIDIA-Nemotron-3-Super-120B-A12B* [Model card]. NVIDIA NIM. https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard -NVIDIA Corporation. (2026b). *Configuration reference*. NVIDIA AI-Q Blueprint. +NVIDIA Corporation. (2026c). *Configuration reference*. NVIDIA AI-Q Blueprint. https://docs.nvidia.com/aiq-blueprint/2.2.0-rc1/customization/configuration-reference.html diff --git a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh index 5637cb861..6ea3706df 100755 --- a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh +++ b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh @@ -956,7 +956,7 @@ extract_strix_failed_check_block "$EVIDENCE_FILE" "$strix_evidence_file" emit_known_missing_string_finding \ "$EVIDENCE_FILE" \ - "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4'" \ + "steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" \ "Strix public scans must default to NVIDIA NIM while private scans retain the contracted provider" \ ".github/workflows/strix.yml" \ "scripts/ci/test_strix_quick_gate.sh" diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py new file mode 100644 index 000000000..3a501f837 --- /dev/null +++ b/scripts/ci/select_nvidia_nim_model.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""Resolve the first live NVIDIA NIM model from an ordered candidate pool. + +Why this exists +--------------- +The scheduled autofix worker used to hard-code one NVIDIA NIM model id. NVIDIA +retires hosted models on published end-of-life dates, and the endpoint then +answers every request with HTTP 410 ``Gone``, e.g. + + The model 'mistralai/mistral-small-4-119b-2603' has reached its end of life + on 2026-07-27T00:00:00Z and is no longer available. + +A single hard-coded id therefore turns a normal provider lifecycle event into a +total outage of the repair loop. This helper asks the provider which models are +actually served right now (``GET /v1/models``, the OpenAI-compatible catalog +route NVIDIA NIM implements) and returns the first entry of an ordered, +operator-controlled preference list that the provider still serves. + +The helper is deliberately fail-closed: an unreachable catalog, an unparsable +catalog, or a pool with no served candidate is an error, never a silent +fallback to an arbitrary model. + +References: + NVIDIA. (2025). *NVIDIA NIM for large language models: OpenAI-compatible + API reference*. https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html + OpenAI. (2025). *API reference: List models*. + https://platform.openai.com/docs/api-reference/models/list +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import ssl +import sys +from urllib.parse import urlsplit + +DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" +ALLOWED_CATALOG_HOSTS = frozenset({"integrate.api.nvidia.com"}) +DEFAULT_TIMEOUT_SECONDS = 30.0 +EX_TEMPFAIL = 75 + + +class ModelResolutionUnavailable(RuntimeError): + """The reviewed model pool cannot be resolved due to provider availability.""" + + +def parse_candidates(raw_candidates: str) -> list[str]: + """Split a whitespace-separated candidate pool into ordered model ids. + + Duplicate ids are removed while the operator's preference order is kept, so + a pool may be assembled from several sources without changing behavior. + """ + ordered: list[str] = [] + for candidate in raw_candidates.split(): + if candidate not in ordered: + ordered.append(candidate) + return ordered + + +def validate_catalog_base_url(base_url: str) -> str: + """Return the catalog base URL after refusing untrusted endpoints. + + Only HTTPS URLs on the known NVIDIA NIM integration host are accepted, so a + tampered variable cannot redirect the API key to another host. + """ + parts = urlsplit(base_url) + if parts.scheme != "https": + raise ValueError(f"NVIDIA NIM base URL must use https; got {parts.scheme or ''}") + if parts.hostname not in ALLOWED_CATALOG_HOSTS: + raise ValueError(f"NVIDIA NIM base URL host is not allowed: {parts.hostname or ''}") + if parts.port not in (None, 443): + raise ValueError(f"NVIDIA NIM base URL must use the default HTTPS port; got {parts.port}") + if parts.username or parts.password: + raise ValueError("NVIDIA NIM base URL must not embed credentials") + if parts.query or parts.fragment: + raise ValueError("NVIDIA NIM base URL must not include a query or fragment") + return base_url.rstrip("/") + + +def fetch_served_model_ids( + base_url: str, + api_key: str, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, +) -> set[str]: + """Return the model ids the provider currently serves. + + Any transport or payload problem raises, because guessing a model id would + hide a provider outage behind a confusing downstream model error. + """ + normalized_base_url = validate_catalog_base_url(base_url) + parts = urlsplit(normalized_base_url) + request_path = f"{parts.path.rstrip('/')}/models" + try: + connection = http.client.HTTPSConnection( # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected + parts.hostname, + parts.port or 443, + timeout=timeout_seconds, + context=ssl.create_default_context(), + ) + try: + connection.request( + "GET", + request_path, + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + }, + ) + response = connection.getresponse() + if response.status >= 400: + error = RuntimeError( + f"NVIDIA NIM model catalog request failed with HTTP {response.status}" + ) + if response.status == 429 or response.status >= 500: + raise ModelResolutionUnavailable(str(error)) + raise error + payload = json.loads(response.read().decode("utf-8")) + finally: + connection.close() + except RuntimeError: + raise + except (OSError, http.client.HTTPException) as error: + raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error + except (UnicodeDecodeError, json.JSONDecodeError) as error: + raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error + entries = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(entries, list): + raise ModelResolutionUnavailable("NVIDIA NIM model catalog payload has no model list") + served = { + str(entry["id"]) + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("id"), str) and entry["id"] + } + if not served: + raise ModelResolutionUnavailable("NVIDIA NIM model catalog listed no usable model id") + return served + + +def select_model(candidates: list[str], served_model_ids: set[str], *, role: str) -> str: + """Return the first candidate the provider still serves for this role.""" + if not candidates: + raise ValueError(f"no {role} NVIDIA NIM model candidates were configured") + for candidate in candidates: + if candidate in served_model_ids: + return candidate + raise ModelResolutionUnavailable( + f"no configured {role} NVIDIA NIM model candidate is currently served: {' '.join(candidates)}. " + "Add a live model id to the candidate pool variable so the repair worker can run." + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line for the model resolver.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", required=True, help="whitespace-separated ordered model ids") + parser.add_argument( + "--exclude", + default="", + help="whitespace-separated model ids that cannot be selected", + ) + parser.add_argument("--role", default="primary", help="candidate pool role used in error messages") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="NVIDIA NIM OpenAI-compatible base URL") + parser.add_argument( + "--timeout-seconds", + type=float, + default=DEFAULT_TIMEOUT_SECONDS, + help="model catalog request timeout", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Print the resolved model id, or report an actionable failure.""" + args = parse_args(argv) + api_key = os.environ.get("NVIDIA_API_KEY") or os.environ.get("NVIDIA_NIM_API_KEY") or "" + if not api_key: + print( + "::error::NVIDIA_API_KEY is required to resolve a live NVIDIA NIM model.", + file=sys.stderr, + ) + return 1 + try: + served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) + excluded = set(parse_candidates(args.exclude)) + configured_candidates = parse_candidates(args.candidates) + candidates = [candidate for candidate in configured_candidates if candidate not in excluded] + if configured_candidates and not candidates: + raise ModelResolutionUnavailable( + f"no distinct {args.role} NVIDIA NIM model candidate remains after exclusions" + ) + print(select_model(candidates, served, role=args.role)) + except ValueError as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + except ModelResolutionUnavailable as error: + print(f"::error::{error}", file=sys.stderr) + return EX_TEMPFAIL + except RuntimeError as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 05923d955..0e300a9f6 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2922,6 +2922,18 @@ is_llm_service_unavailable_error() { return 0 fi + # OpenRouter's dynamic free route can wrap one upstream 502 over several + # terminal lines. Join only the bounded LiteLLM error block so unrelated + # target-app output elsewhere in the log cannot assemble a retry signature. + if awk ' + /litellm(\.exceptions)?\.APIError/ { block = $0; remaining = 5; next } + remaining > 0 { block = block " " $0; remaining--; if (remaining == 0) print block } + END { if (remaining > 0) print block } + ' "$STRIX_LOG" | + grep -Eiq 'litellm(\.exceptions)?\.APIError.*OpenrouterException.*"code"[[:space:]]*:[[:space:]]*502.*"metadata"[[:space:]]*:[[:space:]]*\{[^}]*"provider_name"[[:space:]]*:[[:space:]]*"[^"]+"'; then + return 0 + fi + return 1 } diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 3e33475eb..5cf213989 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -46,18 +46,6 @@ assert_file_not_contains() { fi } -assert_file_contains_either() { - local file_path="$1" - local first_needle="$2" - local second_needle="$3" - local message="$4" - - if ! grep -Fq -- "$first_needle" "$file_path" && - ! grep -Fq -- "$second_needle" "$file_path"; then - record_failure "$message (missing either '$first_needle' or '$second_needle')" - fi -} - assert_status_permissions_scoped() { local output @@ -167,25 +155,14 @@ assert_file_contains "$gate_script" "TARGET_PATH_IS_INTERNAL_PR_SCOPE" "Strix ga assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disables npm lifecycle scripts" assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" -assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" +assert_file_contains "$workflow_file" "nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" +assert_file_contains "$workflow_file" "nvidia_nim/*)" "Strix model preparation reuses the gate-validated NVIDIA provider namespace" +assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" +assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" assert_file_contains "$workflow_file" "openrouter/free openai-direct/gpt-5.4" "Strix crosses to OpenRouter's free router before direct OpenAI when NVIDIA is exhausted" -fallback_expression="$(grep 'STRIX_FALLBACK_MODELS:' "$workflow_file")" -if printf '%s' "$fallback_expression" | grep -Fq "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5"; then - record_failure "Strix must not route to the retired NVIDIA fallback model" -fi +assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "Strix retains the cross-provider direct-OpenAI fallback" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_KEY_FILE" "Strix workflow provisions a trusted OpenRouter fallback key file" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_API_BASE_FILE" "Strix workflow provisions a trusted OpenRouter fallback API base file" -assert_file_contains_either \ - "$workflow_file" \ - "steps.resolve_nvidia_models.outputs.fallback" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ - "Strix accepts the legacy fallback or a live catalog-resolved fallback during migration" -assert_file_contains_either \ - "$workflow_file" \ - "openai_direct/gpt-5.4" \ - "openai-direct/gpt-5.4" \ - "Strix retains the cross-provider direct-OpenAI fallback" -# ponytail: transitional compatibility; require only the dynamic fallback after its workflow lands. assert_file_not_contains "$workflow_file" "github_models/openai/o3" "Strix fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index dca953794..01539aabd 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -192,15 +192,19 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "branches: [main, develop, master]" "strix workflow scans GitHub Flow and Git Flow protected branches" assert_file_contains "$workflow_file" "pull_request_target:" "strix workflow uses trusted PR trigger" - assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-' "strix workflow isolates manual evidence runs from required PR contexts" - assert_file_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow scopes pull_request_target concurrency to the active pull request" + assert_file_contains "$workflow_file" "group: >-" "strix workflow defines an explicit concurrency group" + assert_file_contains "$workflow_file" "format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number)" "strix workflow gives closed PR cleanup an independent concurrency group" + assert_file_contains "$workflow_file" "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository ||" "strix workflow scopes active evidence per repository and event class" + assert_file_contains "$workflow_file" "format('{0}-{1}-{2}', github.event_name, github.repository, github.ref)" "strix workflow keeps protected-branch push evidence in ref-specific queues" assert_file_contains "$workflow_file" "github.event.client_payload.target_repository ||" "strix manual dispatch concurrency scopes to the target repository when provided" - assert_file_contains "$workflow_file" "github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number)" "strix workflow retains a manual PR fallback group when no head SHA is provided" - assert_file_contains "$workflow_file" "github.ref }}" "strix workflow scopes non-PR concurrency to the current ref" + assert_file_contains "$workflow_file" "github.repository }}" "strix workflow falls back to the workflow repository when no target repository is provided" + assert_file_not_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow serializes sibling PR scans at repository scope" + assert_file_not_contains "$workflow_file" "github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number)" "strix workflow does not create one provider queue per PR" assert_file_not_contains "$workflow_file" "format('pr-{0}-{1}'" "strix workflow does not keep stale head-specific concurrency groups" - assert_file_contains "$workflow_file" "cancel-in-progress: true" "strix workflow cancels stale PR evidence runs when a newer PR event arrives" + assert_file_contains "$workflow_file" "cancel-in-progress: false" "strix workflow does not cancel an in-progress provider scan" + assert_file_not_contains "$workflow_file" "queue: max" "strix workflow uses only supported GitHub concurrency keys" assert_file_contains "$workflow_file" "default-branch repository_dispatch evidence cannot cancel" "strix workflow documents manual evidence isolation from branch protection contexts" - assert_file_contains "$workflow_file" "PR-number scope keeps the queue on the current HEAD" "strix workflow documents current-head queue management" + assert_file_contains "$workflow_file" "re-dispatches exact-head evidence" "strix workflow documents current-head queue recovery" assert_file_contains "$workflow_file" "refs/pull//head has already advanced before this queued run starts" "strix workflow documents stale scan queue avoidance" status_token_count="$(grep -c '^[[:space:]]*GITHUB_STATUS_TOKEN:' "$workflow_file")" assert_equals "1" "$status_token_count" "strix workflow defines GITHUB_STATUS_TOKEN once so GitHub can parse repository_dispatch" @@ -303,14 +307,15 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" "STRIX_TOTAL_TIMEOUT_SECONDS:" "strix workflow must not expose total timeout env names in GitHub logs" assert_file_not_contains "$workflow_file" "STRIX_PR_SCOPE_MAX_FILES_PER_BATCH" "strix workflow must not split Strix PR evidence into separate scanner runs" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM == 'vertex_ai/gemini-3.1-pro-preview-customtools' && 'vertex_ai/gemini-2.5-flash'" "strix workflow must not quarantine the approved Vertex preview model after organization secret visibility is fixed" - assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4'" "strix workflow defaults public scans to NVIDIA NIM and keeps private scans on the contracted provider" + assert_file_contains "$workflow_file" "Resolve live NVIDIA NIM Strix models" "strix workflow resolves currently served NVIDIA models for public scans" + assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" "strix workflow uses the resolved public model and keeps private scans on the contracted provider" assert_file_contains "$workflow_file" "EVENT_REPOSITORY_VISIBILITY:" "strix workflow uses trusted event visibility before cross-repository API lookup" assert_file_contains "$workflow_file" "PUBLIC | public) is_private=false" "strix workflow accepts GitHub's lowercase public visibility" assert_file_contains "$workflow_file" "PRIVATE | private | INTERNAL | internal) is_private=true" "strix workflow keeps private and internal repositories off public-only providers" assert_file_contains "$workflow_file" '(.visibility // "" | ascii_downcase) as $visibility' "strix dispatch visibility maps the authoritative API visibility instead of the lossy private boolean" assert_file_not_contains "$workflow_file" "gh api \"repos/\${TARGET_REPOSITORY}\" --jq '.private'" "strix dispatch visibility does not misclassify internal repositories through the private boolean" assert_file_contains "$REPO_ROOT/tests/test_strix_repository_visibility_contract.py" "test_dispatch_api_visibility_preserves_internal_privacy" "strix visibility contract executes public, private, and internal dispatch fixtures" - assert_file_contains "$workflow_file" 'if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]' "strix workflow falls back to the contracted provider when the NVIDIA secret is absent" + assert_file_contains "$workflow_file" '[ -z "${NVIDIA_API_KEY:-}" ]' "strix workflow leaves model resolution empty when the NVIDIA secret is absent" assert_file_contains "$workflow_file" 'STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}' "strix workflow propagates the gate-selected fallback model to the scanner" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM ||" "strix workflow must not let the legacy STRIX_LLM secret override PR defaults" assert_file_contains "$workflow_file" "STRIX_LLM must select NVIDIA NIM Nemotron, GitHub Models openai/gpt-5 or newer, direct OpenAI GPT-5.4 or newer, OpenRouter openrouter/free, or an approved organization Vertex AI model" "strix workflow rejects unsupported model inputs" @@ -371,7 +376,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openrouter/free openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" + assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" @@ -3774,6 +3780,59 @@ REPORT ;; esac ;; + openrouter-502-fallback-retry-same-model-success) + case "${STRIX_LLM:-}" in + vertex_ai/missing-primary) + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + exit 1 + ;; + openrouter/free) + attempt="0" + if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then + attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" + fi + attempt="$((attempt + 1))" + echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" + if [ "$attempt" -eq 1 ]; then + echo "Error: litellm.APIError: APIError:" + echo "OpenrouterException -" + echo '{"error":{"message":"Invalid URL:' + echo '","code":502,"metadata":{"provider_name":"Stealth"}}}' + exit 1 + fi + echo "scan ok after OpenRouter 502 same-model retry" + exit 0 + ;; + vertex_ai/fallback-two) + echo "Error: second fallback should not be needed after transient OpenRouter 502" >&2 + exit 38 + ;; + *) + echo "Error: OpenRouter 502 fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 38 + ;; + esac + ;; + openrouter-502-distant-target-output-nonretryable) + case "${STRIX_LLM:-}" in + vertex_ai/missing-primary) + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + exit 1 + ;; + openrouter/free) + echo "Error: litellm.APIError: APIError: OpenrouterException -" + printf 'target output\n%.0s' 1 2 3 4 5 6 + echo '{"code":502,"metadata":{"provider_name":"spoof"}}' + exit 1 + ;; + vertex_ai/fallback-two) + echo "scan ok after distant target output" + exit 0 + ;; + esac + ;; github-models-primary-unavailable-fallback-success|github-models-primary-denied-fallback-success) case "${STRIX_LLM:-}" in openai/gpt-5) @@ -4012,6 +4071,7 @@ EOS ;; service-unavailable-no-llm-marker-nonrecoverable) echo 'ServiceUnavailableError: {"error":{"code":503,"status":"UNAVAILABLE"}}' + echo '{"error":{"code":502,"metadata":{"provider_name":"Stealth"}}}' echo 'target application high demand response' exit 1 ;; @@ -6167,6 +6227,48 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" ;; + openrouter-502-fallback-retry-same-model-success) + run_gate_case "openrouter-502-fallback-retry-same-model-success" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "0" \ + "scan ok after OpenRouter 502 same-model retry" \ + "3" \ + "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ + "|https://example.invalid|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + ;; + openrouter-502-distant-target-output-nonretryable) + run_gate_case "openrouter-502-distant-target-output-nonretryable" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "2" \ + "vertex_ai/missing-primary|openrouter/free" \ + "|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + ;; + service-unavailable-no-llm-marker-nonrecoverable) + run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ + "custom/service-unavailable-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "custom/service-unavailable-primary" \ + "https://example.invalid" \ + "custom" \ + "__DEFAULT__" \ + "" \ + "1" + ;; custom-openai-compatible-preserves-effort) run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ @@ -9942,6 +10044,32 @@ run_gate_case_allow_provider_signal "github-models-internal-server-connection-re "" \ "1" +run_gate_case "openrouter-502-fallback-retry-same-model-success" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "0" \ + "scan ok after OpenRouter 502 same-model retry" \ + "3" \ + "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ + "|https://example.invalid|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + +run_gate_case "openrouter-502-distant-target-output-nonretryable" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "2" \ + "vertex_ai/missing-primary|openrouter/free" \ + "|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + run_gate_case "github-models-primary-unavailable-fallback-success" \ "openai/gpt-5" \ "" \ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 1d79f1daa..61c2966f9 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -279,8 +279,16 @@ def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() - assert "Semgrep engine/configuration failed with rc=${SEMGREP_RC}" in workflow -def test_strix_cancels_superseded_pr_head_security_evidence() -> None: - """Scope Strix concurrency to the target PR while preserving current-head evidence.""" +def test_strix_serializes_provider_evidence_per_repository() -> None: + """Serialize Strix per repository so shared provider keys are not rate-limited. + + Root cause (2026-08-23/24): sibling PRs scanned concurrently, each retrying + the shared NVIDIA NIM key three times, producing litellm.RateLimitError + storms and fail-closed gate failures on every open PR. The concurrency group + now scopes one scan at a time per repository and event class. GitHub retains + one active and one pending run per group; the scheduler re-dispatches exact + current-head evidence when a pending run is superseded. + """ workflow = workflow_text("strix.yml") concurrency_contract = workflow.split("concurrency:", 1)[1].split( "permissions:", 1 @@ -291,17 +299,28 @@ def test_strix_cancels_superseded_pr_head_security_evidence() -> None: assert "github.event.pull_request.base.repo.full_name" in concurrency_contract assert "github.repository" in concurrency_contract assert ( - "strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || " - "github.event.pull_request.base.repo.full_name || github.repository }}" + "format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, " + "github.event.pull_request.number)" + ) in concurrency_contract + assert ( + "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || " + "github.event.pull_request.base.repo.full_name || github.repository)" ) in concurrency_contract - assert "format('pr-{0}', github.event.pull_request.number)" in concurrency_contract - assert "github.event.client_payload.pr_number != '' && format('pr-{0}'," in workflow - assert "format('pr-{0}-{1}'" not in concurrency_contract + assert ( + "format('{0}-{1}-{2}', github.event_name, github.repository, github.ref)" + in concurrency_contract + ) + # Repository-level (not PR-level) grouping: no pr-{N} component remains. + assert "format('pr-{0}', github.event.pull_request.number)" not in concurrency_contract assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract - assert "cancel-in-progress: true" in workflow + # Running scans are not cancelled; GitHub's native group has one pending slot. + assert "cancel-in-progress: false" in workflow + assert "cancel-in-progress: true" not in workflow.split("jobs:", 1)[0] + assert "queue: max" not in workflow + assert "scheduler" in concurrency_contract assert "default-branch repository_dispatch evidence cannot cancel" in workflow - assert "PR-number scope keeps the queue on the current HEAD" in workflow + assert "RateLimitError" in concurrency_contract assert ( "refs/pull//head has already advanced before this queued run starts" in workflow @@ -343,10 +362,32 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "closed" in workflow assert "cancel-closed-pr-runs:" in workflow - assert ( - "PR closed; this run only cancels older runs through workflow concurrency." - in workflow - ) + if filename == "strix.yml": + assert "Cancel queued and running scans for the closed pull request" in workflow + assert ( + "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN " + "|| github.token" + ) in workflow + assert "DISPATCH_REPOSITORY" not in workflow + assert "CLOSED_PR_HEAD_SHA" in workflow + assert 'select(.event == "pull_request_target")' in workflow + assert 'select(.event == "repository_dispatch")' not in workflow + assert "leaving runs unchanged" in workflow + assert ( + "for active_status in queued in_progress requested waiting pending" + in workflow + ) + cleanup_job = workflow.split(" cancel-closed-pr-runs:", 1)[1].split( + " strix:", 1 + )[0] + assert "actions: write" in cleanup_job + assert "actions/checkout" not in cleanup_job + assert "cleanup skipped" not in cleanup_job + else: + assert ( + "PR closed; this run only cancels older runs through workflow concurrency." + in workflow + ) assert "github.event.action != 'closed'" in workflow opencode_bootstrap = workflow_text("opencode-review.yml") @@ -357,8 +398,11 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "${{ secrets." not in opencode_bootstrap strix_workflow = workflow_text("strix.yml") - assert "cancel-in-progress: true" in strix_workflow - assert "PR-number scope keeps the queue on the current HEAD" in strix_workflow + # Strix serializes per repository (rate-limit root-cause fix): close-event + # runs still cancel superseded same-PR evidence through their own + # cancel-closed-pr-runs job, while scan jobs queue instead of cancelling. + assert "cancel-in-progress: false" in strix_workflow + assert "Serialize Strix scans per repository" in strix_workflow or "per REPOSITORY" in strix_workflow def test_close_empty_pr_metadata_lookup_retries_and_fails_open() -> None: @@ -475,38 +519,39 @@ def test_noema_review_credentials_and_llm_configuration_fail_closed() -> None: def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret( tmp_path: Path, ) -> None: - """Preserve configured fallback models while rejecting an unavailable NIM secret.""" + """Leave NIM outputs empty so the workflow expression selects OpenAI.""" strix_output = tmp_path / "strix-output" strix = subprocess.run( [ "bash", "-c", textwrap.dedent( - workflow_step(workflow_text("strix.yml"), "Gate Strix secrets") + workflow_step( + workflow_text("strix.yml"), + "Resolve live NVIDIA NIM Strix models", + ) .split(" run: |\n", 1)[1] ), ], env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "nvidia_nim/nvidia/nemotron-3-super-120b-a12b", "STRIX_MODEL_REQUESTED": "", - "STRIX_OPENAI_API_KEY": "synthetic-openai-key", - "STRIX_OPENROUTER_API_KEY": "", - "STRIX_NVIDIA_NIM_API_KEY": "", - "STRIX_VERTEX_CREDENTIALS": "", - "STRIX_GITHUB_MODELS_TOKEN": "synthetic-models-token", + "NVIDIA_API_KEY": "", "TARGET_REPOSITORY_PRIVATE": "false", + "STRIX_NVIDIA_PRIMARY_CANDIDATES": "nvidia/primary", + "STRIX_NVIDIA_FALLBACK_CANDIDATES": "nvidia/fallback", }, capture_output=True, text=True, check=False, ) assert strix.returncode == 0, strix.stderr - assert { - "provider_mode=openai_direct", - "strix_model=gpt-5.4", - } <= set(strix_output.read_text().splitlines()) + assert {"primary=", "fallback="} <= set(strix_output.read_text().splitlines()) + assert ( + "steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" + in workflow_text("strix.yml") + ) assert ( "STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}" in workflow_text("strix.yml") diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py new file mode 100644 index 000000000..2f8d72c5f --- /dev/null +++ b/tests/test_select_nvidia_nim_model.py @@ -0,0 +1,398 @@ +"""Tests for resolving a live NVIDIA NIM model from an ordered candidate pool.""" + +from __future__ import annotations + +import io +import http.client +import json +from pathlib import Path +import ssl +from typing import Any + +import pytest + +from scripts.ci import select_nvidia_nim_model as resolver + + +class _FakeResponse(io.BytesIO): + """Minimal context-managed HTTP response body for catalog stubs.""" + + status = 200 + + def __enter__(self) -> "_FakeResponse": + """Return the response itself, matching urlopen's context manager.""" + return self + + def __exit__(self, *_exc_info: object) -> bool: + """Close the buffer and never suppress an exception.""" + self.close() + return False + + +class _FakeConnection: + """Minimal non-context-managed HTTPS connection stub for catalog requests.""" + + def __init__( + self, + host: str, + port: int, + *, + timeout: float, + context: ssl.SSLContext, + response: _FakeResponse, + requests: list[Any], + ) -> None: + """Record the validated destination and canned response.""" + self.host = host + self.port = port + self.timeout = timeout + self.context = context + self.response = response + self.requests = requests + self.closed = False + + def close(self) -> None: + """Record explicit cleanup, matching ``HTTPSConnection.close``.""" + self.closed = True + + def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: + """Record one outbound request without opening a network socket.""" + self.requests.append((self, method, path, headers)) + + def getresponse(self) -> _FakeResponse: + """Return the canned provider response.""" + return self.response + + +def _catalog(*model_ids: str) -> bytes: + """Render an OpenAI-compatible model catalog payload for the given ids.""" + return json.dumps({"object": "list", "data": [{"id": model_id} for model_id in model_ids]}).encode("utf-8") + + +def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: + """Serve one canned catalog payload and record the issued requests.""" + requests: list[Any] = [] + + def fake_connection( + host: str, port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Return a canned HTTPS connection and record its destination.""" + return _FakeConnection( + host, + port, + timeout=timeout, + context=context, + response=_FakeResponse(payload), + requests=requests, + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + return requests + + +def test_catalog_sink_has_one_scoped_semgrep_exception_and_explicit_tls() -> None: + """Keep the reviewed HTTPS sink suppressed only for its known false positive.""" + source_text = Path(resolver.__file__).read_text(encoding="utf-8") + rule = "python.lang.security.audit.httpsconnection-detected.httpsconnection-detected" + sink_lines = [ + line for line in source_text.splitlines() if "http.client.HTTPSConnection(" in line + ] + + assert len(sink_lines) == 1 + assert f"# nosemgrep: {rule}" in sink_lines[0] + assert source_text.count(f"# nosemgrep: {rule}") == 1 + assert "context=ssl.create_default_context()" in source_text + + +def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: + """Operators may concatenate pools; order wins and repeats are dropped.""" + assert resolver.parse_candidates(" a/one\n b/two a/one ") == ["a/one", "b/two"] + assert resolver.parse_candidates(" ") == [] + + +@pytest.mark.parametrize( + ("base_url", "message"), + [ + ("http://integrate.api.nvidia.com/v1", "must use https"), + ("https://models.example.invalid/v1", "host is not allowed"), + ("https://integrate.api.nvidia.com:8443/v1", "default HTTPS port"), + ("https://user:pass@integrate.api.nvidia.com/v1", "must not embed credentials"), + ("https://integrate.api.nvidia.com/v1?mode=models", "query or fragment"), + ("https://integrate.api.nvidia.com/v1#models", "query or fragment"), + ], +) +def test_validate_catalog_base_url_refuses_untrusted_endpoints(base_url: str, message: str) -> None: + """A tampered base URL must never receive the provider API key.""" + with pytest.raises(ValueError, match=message): + resolver.validate_catalog_base_url(base_url) + + +def test_validate_catalog_base_url_normalizes_the_trusted_endpoint() -> None: + """The trusted endpoint is accepted with any trailing slash removed.""" + assert resolver.validate_catalog_base_url(f"{resolver.DEFAULT_BASE_URL}/") == resolver.DEFAULT_BASE_URL + + +def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """The resolver reads ids from the provider's OpenAI-compatible catalog.""" + requests = _stub_catalog(monkeypatch, _catalog("a/one", "b/two")) + + served = resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key", timeout_seconds=7.0) + + assert served == {"a/one", "b/two"} + connection, method, path, headers = requests[0] + assert connection.host == "integrate.api.nvidia.com" + assert connection.port == 443 + assert connection.timeout == 7.0 + assert connection.context.verify_mode == ssl.CERT_REQUIRED + assert connection.context.check_hostname is True + assert connection.closed is True + assert method == "GET" + assert path == "/v1/models" + assert headers["Authorization"] == "Bearer secret-key" + + +def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.MonkeyPatch) -> None: + """Entries without a usable string id cannot become selectable models.""" + payload = json.dumps({"data": [{"id": ""}, {"id": 7}, "not-an-object", {"id": "a/one"}]}).encode("utf-8") + _stub_catalog(monkeypatch, payload) + + assert resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") == {"a/one"} + + +@pytest.mark.parametrize( + ("error", "message"), + [ + (http.client.RemoteDisconnected("closed"), "unreachable"), + (OSError("dns"), "unreachable"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_transport_errors( + monkeypatch: pytest.MonkeyPatch, error: Exception, message: str +) -> None: + """A catalog outage is reported, never masked by guessing a model id.""" + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Raise the configured provider failure from the HTTP boundary.""" + del timeout + del context + raise error + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +@pytest.mark.parametrize( + ("status", "error_type"), + [ + (401, RuntimeError), + (429, resolver.ModelResolutionUnavailable), + (503, resolver.ModelResolutionUnavailable), + ], +) +def test_fetch_served_model_ids_reports_http_status( + monkeypatch: pytest.MonkeyPatch, + status: int, + error_type: type[RuntimeError], +) -> None: + """Provider HTTP failures identify the status without exposing credentials.""" + response = _FakeResponse(b"{}") + response.status = status + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Return an unauthorized provider response.""" + return _FakeConnection( + "integrate.api.nvidia.com", + 443, + timeout=timeout, + context=context, + response=response, + requests=[], + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + with pytest.raises(error_type, match=f"HTTP {status}"): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + (b"maintenance", "non-JSON body"), + (b"\x80", "non-JSON body"), + (b'{"object": "list"}', "no model list"), + (b'{"data": []}', "no usable model id"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_unusable_payloads( + monkeypatch: pytest.MonkeyPatch, payload: bytes, message: str +) -> None: + """Unparsable or empty catalogs are errors rather than silent fallbacks.""" + _stub_catalog(monkeypatch, payload) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +def test_select_model_prefers_the_first_served_candidate() -> None: + """A retired first choice transparently falls through to the next live one.""" + candidates = ["retired/model", "live/model", "other/model"] + + assert resolver.select_model(candidates, {"live/model", "other/model"}, role="primary") == "live/model" + + +def test_select_model_requires_a_configured_pool() -> None: + """An empty pool is a configuration error with the role named.""" + with pytest.raises(ValueError, match="no small NVIDIA NIM model candidates"): + resolver.select_model([], {"live/model"}, role="small") + + +def test_select_model_reports_a_fully_retired_pool() -> None: + """When no candidate is served, the message tells the operator what to do.""" + with pytest.raises(RuntimeError, match="Add a live model id to the candidate pool"): + resolver.select_model(["retired/model"], {"live/model"}, role="primary") + + +def test_main_prints_the_resolved_model_id( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The successful path prints exactly the resolved id for shell capture.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + exit_code = resolver.main(["--role", "primary", "--candidates", "retired/model live/model"]) + + assert exit_code == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_excludes_the_resolved_primary_from_fallback_selection( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Fallback resolution selects a distinct live model from an overlapping pool.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("primary/model", "fallback/model")) + + exit_code = resolver.main( + [ + "--role", + "fallback", + "--candidates", + "primary/model fallback/model", + "--exclude", + "primary/model", + ] + ) + + assert exit_code == 0 + assert capsys.readouterr().out == "fallback/model\n" + + +def test_main_treats_exclusion_only_empty_pool_as_temporarily_unavailable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A valid pool exhausted by exclusion keeps cross-provider failover available.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("primary/model")) + + exit_code = resolver.main( + [ + "--role", + "fallback", + "--candidates", + "primary/model", + "--exclude", + "primary/model", + ] + ) + + assert exit_code == resolver.EX_TEMPFAIL + assert "no distinct fallback" in capsys.readouterr().err + + +def test_main_accepts_the_workflow_secret_name( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Either credential variable name works, so callers need no shim.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "live/model"]) == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_requires_a_provider_credential( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Without a credential the resolver fails closed with a CI annotation.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.delenv("NVIDIA_NIM_API_KEY", raising=False) + + assert resolver.main(["--candidates", "live/model"]) == 1 + assert "NVIDIA_API_KEY is required" in capsys.readouterr().err + + +def test_main_annotates_a_resolution_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Resolution failures surface as GitHub error annotations, not tracebacks.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "retired/model"]) == resolver.EX_TEMPFAIL + assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err + + +def test_main_treats_invalid_catalog_utf8_as_temporary( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Malformed provider bytes preserve the workflow's fallback exit code.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, b"\x80") + + assert resolver.main(["--candidates", "live/model"]) == resolver.EX_TEMPFAIL + assert "non-JSON body" in capsys.readouterr().err + + +def test_main_keeps_invalid_operator_configuration_nonrecoverable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """An empty operator pool is invalid rather than provider unavailability.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", ""]) == 1 + assert "no primary NVIDIA NIM model candidates" in capsys.readouterr().err + + +def test_main_keeps_catalog_authentication_errors_nonrecoverable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """An invalid provider credential must not silently switch providers.""" + monkeypatch.setenv("NVIDIA_API_KEY", "invalid-key") + response = _FakeResponse(b"{}") + response.status = 401 + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + return _FakeConnection( + "integrate.api.nvidia.com", + 443, + timeout=timeout, + context=context, + response=response, + requests=[], + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + assert resolver.main(["--candidates", "live/model"]) == 1 + assert "HTTP 401" in capsys.readouterr().err diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 0c46868b7..650db6d25 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -276,18 +276,14 @@ def test_real_finding_after_continuation_never_retries(self) -> None: self.assertEqual(returncode, 1) self.assertEqual(calls, 1) - def test_retry_contract_preserves_logs_and_full_attempt_budget(self) -> None: - """Retries retain every attempt and reserve the complete gate budget.""" + def test_retry_contract_preserves_logs_and_process_attempt_budget(self) -> None: + """Retries retain every attempt and reserve the scanner process budget.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn('strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_', workflow) self.assertIn('cat "$strix_attempt_log" >> "$strix_run_log"', workflow) self.assertIn( - 'strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS"', - workflow, - ) - self.assertIn( - 'strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}"', + 'strix_gate_attempt_budget_seconds="$process_budget_seconds"', workflow, ) self.assertNotIn("STRIX_TOTAL_TIMEOUT_SECONDS:", workflow) diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index e54efe36e..1a598f1ed 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -19,9 +19,8 @@ STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" -FREE_NVIDIA_FALLBACK = ( - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" -) +LIVE_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1" +RETIRED_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" @@ -186,29 +185,62 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) - def test_workflow_uses_cross_provider_free_fallback_plan(self) -> None: - """Use OpenRouter free routing after the hosted NIM is exhausted.""" + def test_workflow_resolves_live_nvidia_models(self) -> None: + """Resolve live NIM candidates before cross-provider fallbacks.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("Resolve live NVIDIA NIM Strix models", workflow) + self.assertIn("scripts/ci/select_nvidia_nim_model.py", workflow) + self.assertIn("steps.resolve_nvidia_models.outputs.primary", workflow) + self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) + self.assertNotIn("vars.STRIX_NVIDIA_PRIMARY_CANDIDATES", workflow) + self.assertNotIn("vars.STRIX_NVIDIA_FALLBACK_CANDIDATES", workflow) + self.assertIn('--exclude "$primary"', workflow) + self.assertIn('[ "$primary_rc" -eq 75 ]', workflow) + self.assertIn('[ "$fallback_rc" -eq 75 ]', workflow) + self.assertIn('[ "$primary_rc" -eq 0 ] || exit "$primary_rc"', workflow) + self.assertIn('[ "$fallback_rc" -eq 0 ] || exit "$fallback_rc"', workflow) default_expression = ( "steps.target_visibility.outputs.is_private == 'false' && " - f"'{DEFAULT_NVIDIA_MODEL}' || 'gpt-5.4'" + "steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" ) self.assertIn(default_expression, workflow) - self.assertIn( - f'[ "$strix_model" = "{DEFAULT_NVIDIA_MODEL}" ] ' - '&& [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]', - workflow, - ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "'openrouter/free openai-direct/gpt-5.4'", + "format('{0} openrouter/free openai-direct/gpt-5.4', " + "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) - fallback_expression = next( - line for line in workflow.splitlines() if "STRIX_FALLBACK_MODELS:" in line + self.assertNotIn(RETIRED_NVIDIA_FALLBACK, workflow) + + def test_workflow_uses_one_nvidia_model_allowlist(self) -> None: + """Resolver candidates and gate admission share one reviewed list.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("STRIX_NVIDIA_ALLOWED_MODELS: >-", workflow) + self.assertNotIn("STRIX_NVIDIA_PRIMARY_CANDIDATES", workflow) + self.assertNotIn("STRIX_NVIDIA_FALLBACK_CANDIDATES", workflow) + self.assertEqual( + workflow.count('--candidates "$STRIX_NVIDIA_ALLOWED_MODELS"'), + 2, + ) + self.assertIn('case " $STRIX_NVIDIA_ALLOWED_MODELS " in', workflow) + self.assertIn('*" ${strix_model#nvidia_nim/} "*)', workflow) + + model_input = workflow.split( + "- name: Prepare Strix model input file", + maxsplit=1, + )[1] + model_input = model_input.split("- name: Run Strix", maxsplit=1)[0] + self.assertIn("nvidia_nim/*)", model_input) + self.assertNotIn( + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b |", + model_input, + ) + self.assertNotIn( + "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1)", + model_input, ) - self.assertNotIn(FREE_NVIDIA_FALLBACK, fallback_expression) default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 7ddfbd452..c4d3005a7 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -260,11 +260,11 @@ def test_workflow_passes_override_into_gate_environment(self) -> None: workflow, ) - def test_workflow_routes_nvidia_exhaustion_through_openrouter_free(self) -> None: - """The NVIDIA chain avoids its retired model and preserves failover.""" + def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: + """The NVIDIA chain resolves a live distinct model before failover.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn("openrouter/free openai-direct/gpt-5.4", workflow) + self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) fallback_expression = next( line for line in workflow.splitlines() if "STRIX_FALLBACK_MODELS:" in line ) @@ -272,8 +272,8 @@ def test_workflow_routes_nvidia_exhaustion_through_openrouter_free(self) -> None "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertIn("STRIX_OPENROUTER_FALLBACK_KEY_FILE", workflow) - self.assertIn("STRIX_OPENROUTER_FALLBACK_API_BASE_FILE", workflow) + self.assertIn("openrouter/free", fallback_expression) + self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: """OIDC target-app exchange may request the target commit status scope."""