diff --git a/.github/workflows/pr-review-autofix.yml b/.github/workflows/pr-review-autofix.yml index 786357722..b9702ce7a 100644 --- a/.github/workflows/pr-review-autofix.yml +++ b/.github/workflows/pr-review-autofix.yml @@ -247,6 +247,39 @@ jobs: python3 "$GITHUB_WORKSPACE/trusted-autofix-source/scripts/ci/pr_review_autofix_context.py" \ "${context_args[@]}" + - name: Resolve live NVIDIA NIM autofix models + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + # Ordered preference pools. NVIDIA retires hosted models on published + # end-of-life dates and then answers HTTP 410 Gone, so the worker + # resolves the first pool entry the provider still serves instead of + # hard-coding one id that a lifecycle event can retire. + AUTOFIX_MODEL_CANDIDATES: >- + ${{ vars.NVIDIA_NIM_AUTOFIX_MODEL_CANDIDATES || + 'nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + AUTOFIX_SMALL_MODEL_CANDIDATES: >- + ${{ vars.NVIDIA_NIM_AUTOFIX_SMALL_MODEL_CANDIDATES || 'nvidia/nemotron-3-nano-30b-a3b nvidia/llama-3.1-nemotron-nano-8b-v1' }} + run: | + set -euo pipefail + if [ -z "${NVIDIA_API_KEY:-}" ]; then + echo "::error::NVIDIA_NIM_API_KEY is required for scheduled OpenCode autofix." + exit 1 + fi + resolver="$GITHUB_WORKSPACE/trusted-autofix-source/scripts/ci/select_nvidia_nim_model.py" + autofix_model_id="$( + python3 "$resolver" --role primary --candidates "$AUTOFIX_MODEL_CANDIDATES" + )" + autofix_small_model_id="$( + python3 "$resolver" --role small --candidates "$AUTOFIX_SMALL_MODEL_CANDIDATES" + )" + echo "Resolved autofix model: ${autofix_model_id}" + echo "Resolved autofix small model: ${autofix_small_model_id}" + { + printf 'AUTOFIX_MODEL_ID=%s\n' "$autofix_model_id" + printf 'AUTOFIX_SMALL_MODEL_ID=%s\n' "$autofix_small_model_id" + } >>"$GITHUB_ENV" + - name: Prepare isolated OpenCode autofix workspace env: OPENCODE_AUTOFIX_WORKDIR: ${{ runner.temp }}/opencode-autofix-project @@ -277,10 +310,13 @@ jobs: Do not execute shell commands. Do not invent broad features or claim external approval/check latency is fixed. Queued reviews or checks remain merge blockers, but their latency is not a reason to invent a code change or stop the broader scheduler from processing other eligible work. EOF - jq -n --arg workspace "$TARGET_WORKSPACE" '{ + jq -n \ + --arg workspace "$TARGET_WORKSPACE" \ + --arg model_id "$AUTOFIX_MODEL_ID" \ + --arg small_model_id "$AUTOFIX_SMALL_MODEL_ID" '{ "$schema": "https://opencode.ai/config.json", - "model": "nvidia-nim/mistralai/mistral-small-4-119b-2603", - "small_model": "nvidia-nim/nvidia/nemotron-3-nano-30b-a3b", + "model": "nvidia-nim/\($model_id)", + "small_model": "nvidia-nim/\($small_model_id)", "enabled_providers": ["nvidia-nim"], "permission": { "edit": { @@ -306,8 +342,7 @@ jobs: "ci-autofix": { "description": "Conservative CI pull request review autofix agent", "mode": "primary", - "model": "nvidia-nim/mistralai/mistral-small-4-119b-2603", - "reasoningEffort": "high", + "model": "nvidia-nim/\($model_id)", "prompt": "{file:./autofix-prompt.md}", "steps": 12, "permission": { @@ -341,8 +376,8 @@ jobs: "apiKey": "{env:NVIDIA_API_KEY}" }, "models": { - "mistralai/mistral-small-4-119b-2603": { - "name": "Mistral Small 4 119B 2603", + ($model_id): { + "name": $model_id, "tool_call": true, "reasoning": true, "options": { @@ -353,8 +388,8 @@ jobs: "output": 4096 } }, - "nvidia/nemotron-3-nano-30b-a3b": { - "name": "Nemotron 3 Nano 30B A3B", + ($small_model_id): { + "name": $small_model_id, "tool_call": true, "reasoning": true, "limit": { @@ -371,7 +406,6 @@ jobs: if: env.RESOLVE_CONFLICT != 'true' env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} - MODEL: nvidia-nim/mistralai/mistral-small-4-119b-2603 SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" @@ -382,6 +416,11 @@ jobs: echo "::error::NVIDIA_NIM_API_KEY is required for scheduled OpenCode autofix." exit 1 fi + if [ -z "${AUTOFIX_MODEL_ID:-}" ]; then + echo "::error::Resolved NVIDIA NIM autofix model is missing." + exit 1 + fi + MODEL="nvidia-nim/${AUTOFIX_MODEL_ID}" prompt_file="${RUNNER_TEMP}/opencode-autofix-prompt.md" allowed_paths_zlist="${RUNNER_TEMP}/pr-review-autofix-allowed-paths.zlist" allowed_paths_context="$( @@ -546,7 +585,6 @@ jobs: GITHUB_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.target_app_token.outputs.token }} GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || steps.target_app_token.outputs.token }} MUTATION_CREDENTIAL_AVAILABLE: ${{ secrets.PR_REVIEW_MERGE_TOKEN != '' || secrets.OPENCODE_APPROVE_TOKEN != '' || steps.target_app_token.outputs.available == 'true' }} - MODEL: nvidia-nim/mistralai/mistral-small-4-119b-2603 SHARE: "false" NPM_CONFIG_IGNORE_SCRIPTS: "true" NO_COLOR: "1" @@ -561,6 +599,11 @@ jobs: echo "::error::NVIDIA_NIM_API_KEY is required for scheduled OpenCode autofix." exit 1 fi + if [ -z "${AUTOFIX_MODEL_ID:-}" ]; then + echo "::error::Resolved NVIDIA NIM autofix model is missing." + exit 1 + fi + MODEL="nvidia-nim/${AUTOFIX_MODEL_ID}" cd "$TARGET_WORKSPACE" # Merge the base branch into the detached head. A clean merge stays diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ab82ccea..edb7dec7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ Semantic Versioning where the repository publishes a release. - Run the bounded Clearfolio PR review-feedback repair caller at minute 23 of every hour while keeping the shared scheduler free of product-specific timers and repository names for modular reuse by naruon, contextual-orchestrator, Inkspan, and other CWL services. - Run the bounded DiskSage repair heartbeat at minute 37 of every hour, dispatch no more than one exact-head repair, and wait two hours before redispatching an unchanged head so legitimate OpenCode or NVIDIA NIM latency does not create duplicate writers. - Run the bounded fast-mlsirm repair heartbeat at minute 49 of every hour with one-dispatch scope and a two-hour same-head floor, without weakening true-parameter recovery, CPU/GPU parity, skipped-test, or Rust-ownership gates. -- Use NVIDIA NIM `mistralai/mistral-small-4-119b-2603` with explicit high reasoning for scheduled repair and `nvidia/nemotron-3-nano-30b-a3b` for bounded helper work instead of GitHub Models in the write-capable autofix worker. +- Resolve the write-capable autofix worker's NVIDIA NIM models at run time from ordered, operator-controlled reasoning-capable candidate pools instead of hard-coding one model id: the retired `mistralai/mistral-small-4-119b-2603` answered every repair request with HTTP 410 after its end-of-life date, so the whole hourly repair loop failed on a routine provider lifecycle event. Resolution queries the live provider catalog, keeps explicit high reasoning without sending it to instruct fallbacks, and fails closed with an actionable annotation when the catalog is unreachable or every candidate is retired. - Apply one NUL-delimited exact-path and complete pre/post-worktree verification contract to both ordinary review repair and merge-conflict repair rather than relying on a visible post-model diff for the ordinary path. ### Changed diff --git a/docs/automation/hourly-review-repair.md b/docs/automation/hourly-review-repair.md index 722724958..2ff8c0d6f 100644 --- a/docs/automation/hourly-review-repair.md +++ b/docs/automation/hourly-review-repair.md @@ -44,8 +44,9 @@ not overlap its successor. At most one repair dispatch is created per run. The caller passes only the established `PR_REVIEW_MERGE_TOKEN` and `OPENCODE_APPROVE_TOKEN` scheduler credentials. It does not receive or forward -`NVIDIA_NIM_API_KEY`; the model credential is scoped exclusively to the two -OpenCode execution steps in the separately reviewed autofix worker. +`NVIDIA_NIM_API_KEY`; the model credential is scoped exclusively to the live +model-resolution step and the two OpenCode execution steps in the separately +reviewed autofix worker. ## Orgmetra execution contract diff --git a/docs/doctoring/clearfolio-hourly-review-caller.md b/docs/doctoring/clearfolio-hourly-review-caller.md index 239fdbd3e..ae0578440 100644 --- a/docs/doctoring/clearfolio-hourly-review-caller.md +++ b/docs/doctoring/clearfolio-hourly-review-caller.md @@ -62,8 +62,9 @@ The caller passes exactly two established optional scheduler credentials: It does not use `secrets: inherit`. It does not receive `NVIDIA_NIM_API_KEY`, because queue inspection and dispatch are not model -execution. The NVIDIA credential is bound only inside the separately reviewed -`PR Review Autofix` workflow's two OpenCode execution steps. +execution. The NVIDIA credential is bound only inside three provider-bound +steps in the separately reviewed `PR Review Autofix` workflow: live model +resolution and its two OpenCode execution steps. Both the caller and reusable scheduler keep the workflow-generated `GITHUB_TOKEN` read-only with only `contents: read`; neither declares job-level diff --git a/docs/doctoring/hourly-nvidia-nim-autofix.md b/docs/doctoring/hourly-nvidia-nim-autofix.md index 6b05c6bd6..90634d5b8 100644 --- a/docs/doctoring/hourly-nvidia-nim-autofix.md +++ b/docs/doctoring/hourly-nvidia-nim-autofix.md @@ -67,14 +67,40 @@ OpenAI-compatible adapter and NVIDIA hosted endpoint: https://integrate.api.nvidia.com/v1 ``` -The primary repair model is `mistralai/mistral-small-4-119b-2603`. The -`ci-autofix` agent and its model configuration both request high reasoning -through OpenCode's provider-option contract (`reasoningEffort: "high"`). NVIDIA's -Mistral Small 4 NIM API documents the corresponding request behavior as +The primary repair model is resolved at run time, not hard-coded. NVIDIA retires +hosted models on published end-of-life dates and then answers every request with +HTTP 410 `Gone`, which turns a normal provider lifecycle event into a total +outage of the repair loop. The `Resolve live NVIDIA NIM autofix models` step +therefore runs `scripts/ci/select_nvidia_nim_model.py`, which reads the provider +catalog (`GET /v1/models`) and selects the first entry of an ordered preference +pool that the provider still serves: + +| Role | Pool variable | Default order | +| --- | --- | --- | +| primary | `NVIDIA_NIM_AUTOFIX_MODEL_CANDIDATES` | `nvidia/llama-3.3-nemotron-super-49b-v1.5`, `nvidia/nemotron-3-super-120b-a12b`, `nvidia/llama-3.1-nemotron-ultra-253b-v1` | +| small | `NVIDIA_NIM_AUTOFIX_SMALL_MODEL_CANDIDATES` | `nvidia/nemotron-3-nano-30b-a3b`, `nvidia/llama-3.1-nemotron-nano-8b-v1` | + +Both default pools contain only reasoning-capable NVIDIA NIM models, because +the generated provider entries deliberately request high reasoning. Operator +overrides must preserve that contract; a non-reasoning instruct model must not +be added to either pool. To change the preference order, set the matching +Actions variable on this repository; no workflow edit is required. Resolution +is fail-closed: an unreachable or unparsable catalog, and a pool whose every +entry is retired, both stop the run with an actionable annotation instead of +silently substituting an arbitrary model. The resolved ids are exported once +as `AUTOFIX_MODEL_ID` and `AUTOFIX_SMALL_MODEL_ID` and are the only model +identifiers the generated OpenCode configuration and both `opencode run` +invocations use, so the writer agent and its provider entry cannot drift apart. + +Each resolved model entry requests high reasoning through OpenCode's +provider-option contract (`reasoningEffort: "high"`). NVIDIA's +NIM LLM API documents the corresponding request behavior as `reasoning_effort: "high"`, which enables the model's reasoning mode. The small -model used for bounded helper work remains `nvidia/nemotron-3-nano-30b-a3b` and -is not a fallback provider. GitHub Models configuration, identifiers, base URLs, -and model-auth fallbacks are absent from the scheduled autofix execution path. +model is used for bounded helper work only and is not a fallback provider. +GitHub Models configuration, identifiers, base URLs, and model-auth fallbacks are +absent from the scheduled autofix execution path. Model resolution shares the +same `NVIDIA_NIM_API_KEY` credential as the two OpenCode runs, and no other step +receives it. The high-reasoning setting is deliberate for write-capable review repair. This workflow optimizes correctness, evidence quality, and controllability rather than @@ -82,6 +108,20 @@ latency. It does not imply that deeper reasoning is universally superior; the setting is an explicit operational choice for this bounded, security-sensitive writer role and remains subject to exact-head regression evidence. +### Reviewed static-analysis exception + +The catalog resolver keeps Python's `http.client.HTTPSConnection` because the +destination host is allowlisted, the request path is derived only from that +validated endpoint, and the call supplies `ssl.create_default_context()`. +Python documents that an explicit `SSLContext` controls the HTTPS options and +that certificate and hostname checks are enabled by default. Semgrep's generic +HTTPSConnection rule is therefore a reviewed false positive at this one sink. +The source uses one rule-specific `# nosemgrep` comment, and the resolver test +asserts that the exception occurs exactly once and that the default TLS context +remains present. This is a scoped exception, not a repository-wide suppression; +the central SARIF gate removes only explicitly suppressed results and continues +to fail on every other finding. + ## Credential boundary The organization secret is bound as: @@ -90,10 +130,10 @@ The organization secret is bound as: NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} ``` -It is present only on the two steps that execute OpenCode: ordinary -review-feedback repair and merge-conflict repair. Metadata collection, -checkout, context preparation, validation, commit, and push do not receive the -NVIDIA credential. A missing key is a fatal configuration error rather than a +It is present only on the three steps that need a provider credential: live +model resolution, ordinary review-feedback repair, and merge-conflict repair. +Metadata collection, checkout, context preparation, validation, commit, and push +do not receive the NVIDIA credential. A missing key is a fatal configuration error rather than a signal to choose another provider. The ordinary model execution step does not bind a GitHub write token. Its later @@ -264,20 +304,22 @@ That exact-head evidence is historical after any later documentation commit and must be re-established on the new current head. The later writer-model and mutation-authority hardening was likewise captured by -permanent RED contracts before the implementation changed. Those contracts pin -the exact NVIDIA Mistral Small 4 writer, high reasoning, absence of the obsolete -Mistral Nemotron identifier, explicit mutation credentials, and guards that run -before any Git write. Predecessor-head successes are historical TDD evidence, -not merge evidence. The final integrated head must establish every required -quality, security, review, and protection gate again. +permanent RED contracts before the implementation changed. Those predecessor +contracts asserted a fixed writer-model identity, high reasoning, absence of the +obsolete Mistral Nemotron identifier, explicit mutation credentials, and guards +that run before any Git write. The current implementation resolves the writer at +run time, so those predecessor assertions are historical TDD evidence rather +than a live model pin or merge evidence. The final integrated head must establish +every required quality, security, review, and protection gate again. ## Verification contract Automated tests prove: 1. the caller retains its approved one-hour cadence; -2. OpenCode enables only NVIDIA NIM, uses the exact Mistral Small 4 writer with - high reasoning, and receives the model key only in its two execution steps; +2. OpenCode enables only NVIDIA NIM, resolves the primary and small model ids + from the live ordered candidate pools, requests high reasoning, and receives + those ids only in its two execution steps; 3. missing model credentials fail closed and model children receive no GitHub or OIDC write credential; 4. mutation-capable ordinary and conflict paths accept only established explicit @@ -345,9 +387,20 @@ GitHub, Inc. (n.d.-b). *Secrets reference*. GitHub Docs. Retrieved August 7, NVIDIA Corporation. (n.d.-a). *LLM APIs*. NVIDIA API Catalog. Retrieved August 7, 2026, from https://docs.api.nvidia.com/nim/reference/llm-apis -NVIDIA Corporation. (2026). *Query the Mistral-Small-4-119B-2603 API*. NVIDIA -NIM for Vision Language Models. Retrieved August 8, 2026, from -https://docs.nvidia.com/nim/vision-language-models/1.7.0/examples/mistral-small-4-119b-2603/api.html +NVIDIA Corporation. (n.d.-d). *NVIDIA NIM for large language models: +OpenAI-compatible API reference*. Retrieved August 20, 2026, from +https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html + +Python Software Foundation. (2026). *http.client — HTTP protocol client*. +Python 3.14 documentation. Retrieved August 21, 2026, from +https://docs.python.org/3.14/library/http.client.html + +Semgrep, Inc. (2026). *Rule structure syntax examples: Rule ideas*. +Retrieved August 21, 2026, from +https://semgrep.dev/docs/writing-rules/rule-ideas + +OpenAI. (2025). *API reference: List models*. Retrieved August 20, 2026, from +https://platform.openai.com/docs/api-reference/models/list NVIDIA Corporation. (n.d.-c). *NVIDIA / nemotron-3-nano-30b-a3b*. NVIDIA API Catalog. Retrieved August 7, 2026, from diff --git a/organization_commercial_readiness_fixtures.py b/organization_commercial_readiness_fixtures.py index d86596196..2dc5d9916 100644 --- a/organization_commercial_readiness_fixtures.py +++ b/organization_commercial_readiness_fixtures.py @@ -90,6 +90,7 @@ def __init__( repositories: list[dict[str, Any]], snapshots: dict[str, list[RepositorySnapshot | Exception]], ) -> None: + """Record the repositories and per-repository snapshot scripts to replay.""" self.repositories = repositories self.snapshots = snapshots self.dispatched_repairs: list[tuple[str, str]] = [] diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index c00cfa1e0..54a6e2ab3 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -239,6 +239,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: + """Bind a required organization-scoped token and a per-call timeout.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py new file mode 100644 index 000000000..d68eb2d4e --- /dev/null +++ b/scripts/ci/select_nvidia_nim_model.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Resolve the first live NVIDIA NIM model from an ordered candidate pool. + +Why this exists +--------------- +The scheduled autofix worker used to hard-code one NVIDIA NIM model id. NVIDIA +retires hosted models on published end-of-life dates, and the endpoint then +answers every request with HTTP 410 ``Gone``, e.g. + + The model 'mistralai/mistral-small-4-119b-2603' has reached its end of life + on 2026-07-27T00:00:00Z and is no longer available. + +A single hard-coded id therefore turns a normal provider lifecycle event into a +total outage of the repair loop. This helper asks the provider which models are +actually served right now (``GET /v1/models``, the OpenAI-compatible catalog +route NVIDIA NIM implements) and returns the first entry of an ordered, +operator-controlled preference list that the provider still serves. + +The helper is deliberately fail-closed: an unreachable catalog, an unparsable +catalog, or a pool with no served candidate is an error, never a silent +fallback to an arbitrary model. + +References: + NVIDIA. (2025). *NVIDIA NIM for large language models: OpenAI-compatible + API reference*. https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html + OpenAI. (2025). *API reference: List models*. + https://platform.openai.com/docs/api-reference/models/list +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import ssl +import sys +from urllib.parse import urlsplit + +DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" +ALLOWED_CATALOG_HOSTS = frozenset({"integrate.api.nvidia.com"}) +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +def parse_candidates(raw_candidates: str) -> list[str]: + """Split a whitespace-separated candidate pool into ordered model ids. + + Duplicate ids are removed while the operator's preference order is kept, so + a pool may be assembled from several sources without changing behavior. + """ + ordered: list[str] = [] + for candidate in raw_candidates.split(): + if candidate not in ordered: + ordered.append(candidate) + return ordered + + +def validate_catalog_base_url(base_url: str) -> str: + """Return the catalog base URL after refusing untrusted endpoints. + + Only HTTPS URLs on the known NVIDIA NIM integration host are accepted, so a + tampered variable cannot redirect the API key to another host. + """ + parts = urlsplit(base_url) + if parts.scheme != "https": + raise ValueError(f"NVIDIA NIM base URL must use https; got {parts.scheme or ''}") + if parts.hostname not in ALLOWED_CATALOG_HOSTS: + raise ValueError(f"NVIDIA NIM base URL host is not allowed: {parts.hostname or ''}") + if parts.port not in (None, 443): + raise ValueError(f"NVIDIA NIM base URL must use the default HTTPS port; got {parts.port}") + if parts.username or parts.password: + raise ValueError("NVIDIA NIM base URL must not embed credentials") + if parts.query or parts.fragment: + raise ValueError("NVIDIA NIM base URL must not include a query or fragment") + return base_url.rstrip("/") + + +def fetch_served_model_ids( + base_url: str, + api_key: str, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, +) -> set[str]: + """Return the model ids the provider currently serves. + + Any transport or payload problem raises, because guessing a model id would + hide a provider outage behind a confusing downstream model error. + """ + normalized_base_url = validate_catalog_base_url(base_url) + parts = urlsplit(normalized_base_url) + request_path = f"{parts.path.rstrip('/')}/models" + try: + connection = http.client.HTTPSConnection( # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected + parts.hostname, + parts.port or 443, + timeout=timeout_seconds, + context=ssl.create_default_context(), + ) + try: + connection.request( + "GET", + request_path, + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + }, + ) + response = connection.getresponse() + if response.status >= 400: + raise RuntimeError( + f"NVIDIA NIM model catalog request failed with HTTP {response.status}" + ) + payload = json.loads(response.read().decode("utf-8")) + finally: + connection.close() + except RuntimeError: + raise + except (OSError, http.client.HTTPException) as error: + raise RuntimeError("NVIDIA NIM model catalog is unreachable") from error + except json.JSONDecodeError as error: + raise RuntimeError("NVIDIA NIM model catalog returned a non-JSON body") from error + entries = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(entries, list): + raise RuntimeError("NVIDIA NIM model catalog payload has no model list") + served = { + str(entry["id"]) + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("id"), str) and entry["id"] + } + if not served: + raise RuntimeError("NVIDIA NIM model catalog listed no usable model id") + return served + + +def select_model(candidates: list[str], served_model_ids: set[str], *, role: str) -> str: + """Return the first candidate the provider still serves for this role.""" + if not candidates: + raise ValueError(f"no {role} NVIDIA NIM model candidates were configured") + for candidate in candidates: + if candidate in served_model_ids: + return candidate + raise RuntimeError( + f"no configured {role} NVIDIA NIM model candidate is currently served: {' '.join(candidates)}. " + "Add a live model id to the candidate pool variable so the repair worker can run." + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line for the model resolver.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", required=True, help="whitespace-separated ordered model ids") + parser.add_argument("--role", default="primary", help="candidate pool role used in error messages") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="NVIDIA NIM OpenAI-compatible base URL") + parser.add_argument( + "--timeout-seconds", + type=float, + default=DEFAULT_TIMEOUT_SECONDS, + help="model catalog request timeout", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Print the resolved model id, or report an actionable failure.""" + args = parse_args(argv) + api_key = os.environ.get("NVIDIA_API_KEY") or os.environ.get("NVIDIA_NIM_API_KEY") or "" + if not api_key: + print( + "::error::NVIDIA_API_KEY is required to resolve a live NVIDIA NIM model.", + file=sys.stderr, + ) + return 1 + try: + served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) + print(select_model(parse_candidates(args.candidates), served, role=args.role)) + except (RuntimeError, ValueError) as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 1c05feb6f..31a5ff09b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1506,6 +1506,7 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$workflow_file" "github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number)" "scheduler scopes pull_request_target concurrency to the active PR" assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].number && format('pr-{0}', github.event.workflow_run.pull_requests[0].number)" "scheduler scopes workflow_run concurrency to the completed review PR" assert_file_contains "$workflow_file" "github.event_name == 'schedule' && format('schedule-{0}', github.event.schedule)" "scheduler isolates the 15-minute organization sweep from the separate 30-minute scheduled scan" + assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && format('repo-dispatch-{0}', github.repository)" "scheduler keeps manual queue scans isolated per repository dispatch target" assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && github.event.client_payload.target_repository != '' && github.event.client_payload.pr_number != ''" "scheduler scopes targeted manual queue scans to the requested PR" assert_file_contains "$workflow_file" "cancel-in-progress: \${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review' || github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_run' && !github.event.workflow_run.pull_requests[0].number) }}" "scheduler cancels stale PR/review/manual queue scans instead of accumulating merge/update attempts" assert_file_contains "$workflow_file" "timeout-minutes: 60" "organization sweep has enough headroom to finish the complete repository walk" diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 1bbd98750..ccf56b404 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -40,22 +40,29 @@ def test_scheduled_autofix_uses_only_nvidia_nim() -> None: """Require the write-capable OpenCode autofix agent to use NVIDIA NIM only.""" workflow = _workflow_text(AUTOFIX_WORKFLOW) required_fragments = ( - '"model": "nvidia-nim/mistralai/mistral-small-4-119b-2603"', - '"small_model": "nvidia-nim/nvidia/nemotron-3-nano-30b-a3b"', + '"model": "nvidia-nim/\\($model_id)"', + '"small_model": "nvidia-nim/\\($small_model_id)"', '"enabled_providers": ["nvidia-nim"]', '"nvidia-nim": {', - '"mistralai/mistral-small-4-119b-2603": {', + '($model_id): {', + '($small_model_id): {', '"reasoningEffort": "high"', '"npm": "@ai-sdk/openai-compatible"', '"baseURL": "https://integrate.api.nvidia.com/v1"', '"apiKey": "{env:NVIDIA_API_KEY}"', 'NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}', - 'MODEL: nvidia-nim/mistralai/mistral-small-4-119b-2603', + 'MODEL="nvidia-nim/${AUTOFIX_MODEL_ID}"', ) for fragment in required_fragments: assert fragment in workflow, fragment + agent_start = workflow.index('"ci-autofix": {') + provider_start = workflow.index('"provider":', agent_start) + assert '"reasoningEffort": "high"' not in workflow[agent_start:provider_start] forbidden_fragments = ( + 'mistralai/mistral-small-4-119b-2603', 'mistralai/mistral-nemotron', + 'meta/llama-3.3-70b-instruct', + 'meta/llama-3.1-8b-instruct', 'STRIX_GITHUB_MODELS_TOKEN:', 'MODEL: github-models/', 'USE_GITHUB_TOKEN:', @@ -99,22 +106,58 @@ def test_opencode_agent_denies_non_file_interactions() -> None: assert workflow.count(f'"{permission_name}": "deny"') == 2 -def test_nvidia_nim_secret_is_scoped_to_agent_execution_steps() -> None: - """Prevent the NVIDIA credential from leaking beyond the two OpenCode runs.""" +def test_nvidia_nim_secret_is_scoped_to_model_credential_steps() -> None: + """Confine the NVIDIA credential to model resolution and the two OpenCode runs. + + Model resolution needs the same credential because it asks the provider which + models are still served, so the allowed set is exactly three steps. + """ workflow = _workflow_text(AUTOFIX_WORKFLOW) binding = 'NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}' + resolve_start = workflow.index(" - name: Resolve live NVIDIA NIM autofix models") + resolve_end = workflow.index( + " - name: Prepare isolated OpenCode autofix workspace", resolve_start + ) ordinary_start = workflow.index(" - name: Run OpenCode review autofix") ordinary_end = workflow.index(" - name: Validate changed files", ordinary_start) conflict_start = workflow.index( " - name: Merge base branch and resolve conflicts with OpenCode" ) - assert workflow.count(binding) == 2 + assert workflow.count(binding) == 3 + assert binding in workflow[resolve_start:resolve_end] assert binding in workflow[ordinary_start:ordinary_end] assert binding in workflow[conflict_start:] - assert binding not in workflow[:ordinary_start] + assert binding not in workflow[:resolve_start] + assert binding not in workflow[resolve_end:ordinary_start] assert binding not in workflow[ordinary_end:conflict_start] +def test_model_ids_are_resolved_from_an_ordered_live_candidate_pool() -> None: + """Require run-time model resolution so a retired model cannot stop repairs. + + NVIDIA answers HTTP 410 Gone for a model past its end-of-life date, so the + worker resolves the first candidate the provider still serves and exports it + to the OpenCode configuration and both model invocations. + """ + workflow = _workflow_text(AUTOFIX_WORKFLOW) + resolve_start = workflow.index(" - name: Resolve live NVIDIA NIM autofix models") + resolve_end = workflow.index( + " - name: Prepare isolated OpenCode autofix workspace", resolve_start + ) + resolve = workflow[resolve_start:resolve_end] + + assert 'scripts/ci/select_nvidia_nim_model.py' in resolve + assert "--role primary" in resolve + assert "--role small" in resolve + assert "vars.NVIDIA_NIM_AUTOFIX_MODEL_CANDIDATES" in resolve + assert "vars.NVIDIA_NIM_AUTOFIX_SMALL_MODEL_CANDIDATES" in resolve + assert "meta/llama-3.3-70b-instruct" not in resolve + assert "meta/llama-3.1-8b-instruct" not in resolve + assert "AUTOFIX_MODEL_ID=%s" in resolve + assert "AUTOFIX_SMALL_MODEL_ID=%s" in resolve + assert resolve_end < workflow.index('--arg model_id "$AUTOFIX_MODEL_ID"') + + def test_model_subprocesses_receive_no_github_or_oidc_write_credentials() -> None: """Strip GitHub write and OIDC credentials from both OpenCode processes.""" workflow = _workflow_text(AUTOFIX_WORKFLOW) @@ -146,16 +189,37 @@ def test_missing_nvidia_nim_secret_fails_closed_before_model_execution() -> None " exit 1\n" " fi" ) + resolve_start = workflow.index(" - name: Resolve live NVIDIA NIM autofix models") + resolve_end = workflow.index( + " - name: Prepare isolated OpenCode autofix workspace", resolve_start + ) ordinary_start = workflow.index(" - name: Run OpenCode review autofix") ordinary_end = workflow.index(" - name: Validate changed files", ordinary_start) conflict_start = workflow.index( " - name: Merge base branch and resolve conflicts with OpenCode" ) - assert workflow.count(guard) == 2 + assert workflow.count(guard) == 3 + assert guard in workflow[resolve_start:resolve_end] assert guard in workflow[ordinary_start:ordinary_end] assert guard in workflow[conflict_start:] +def test_resolved_model_is_read_from_runner_environment() -> None: + """Use GITHUB_ENV output in the shell, not the pre-run env context.""" + workflow = _workflow_text(AUTOFIX_WORKFLOW) + + assert "nvidia-nim/${{ env.AUTOFIX_MODEL_ID }}" not in workflow + for step_name in ( + "Run OpenCode review autofix", + "Merge base branch and resolve conflicts with OpenCode", + ): + step_start = workflow.index(f" - name: {step_name}") + step_end = workflow.find("\n - name: ", step_start + 1) + step = workflow[step_start:] if step_end == -1 else workflow[step_start:step_end] + assert 'if [ -z "${AUTOFIX_MODEL_ID:-}" ]; then' in step + assert 'MODEL="nvidia-nim/${AUTOFIX_MODEL_ID}"' in step + + def test_independent_review_agent_key_system_is_unchanged() -> None: """Pin the existing read-only reviewer workflow byte-for-byte.""" result = subprocess.run( @@ -232,6 +296,11 @@ def test_operator_doctoring_and_changelog_record_exact_write_scope() -> None: assert "Git Project. (2026). *git-ls-files*" in doctoring assert "Git Project. (2026). *githooks*" in doctoring assert "OpenCode. (2026a). *Permissions*" in doctoring + assert "resolves the primary and small model ids" in doctoring + assert "reasoning-capable NVIDIA NIM models" in doctoring + assert "the exact Mistral Small 4 writer" not in doctoring + assert "the exact NVIDIA Mistral Small 4 writer" not in doctoring + assert "contracts asserted a fixed writer-model identity" in doctoring assert "ignored-path inventory" in changelog assert "model-mutable Git metadata" in changelog diff --git a/tests/test_pr_review_autofix_writer_security_contract.py b/tests/test_pr_review_autofix_writer_security_contract.py index 58ea05877..94a0a3480 100644 --- a/tests/test_pr_review_autofix_writer_security_contract.py +++ b/tests/test_pr_review_autofix_writer_security_contract.py @@ -6,7 +6,8 @@ _AUTOFIX_WORKFLOW = Path(".github/workflows/pr-review-autofix.yml") -_TARGET_MODEL = "nvidia-nim/mistralai/mistral-small-4-119b-2603" +_TARGET_MODEL = 'MODEL="nvidia-nim/${AUTOFIX_MODEL_ID}"' +_RETIRED_MODEL = "mistralai/mistral-small-4-119b-2603" def _workflow_text() -> str: @@ -30,14 +31,20 @@ def _step_header(workflow: str, step_name: str) -> str: return step[:run_start] -def test_writer_uses_supported_nvidia_mistral_small_with_high_reasoning() -> None: - """Pin the write-capable model and its deliberate high-reasoning budget.""" +def test_writer_resolves_a_live_model_with_high_reasoning() -> None: + """Pin the resolved-model indirection and its deliberate high-reasoning budget. + + The write-capable model id is resolved at run time from an ordered candidate + pool, because a hard-coded id becomes an HTTP 410 outage on the provider's + published end-of-life date. + """ workflow = _workflow_text() - assert f'"model": "{_TARGET_MODEL}"' in workflow - assert '"mistralai/mistral-small-4-119b-2603": {' in workflow - assert workflow.count(f"MODEL: {_TARGET_MODEL}") == 2 + assert '"model": "nvidia-nim/\\($model_id)"' in workflow + assert "($model_id): {" in workflow + assert workflow.count(_TARGET_MODEL) == 2 assert '"reasoningEffort": "high"' in workflow + assert _RETIRED_MODEL not in workflow assert "nvidia-nim/mistralai/mistral-nemotron" not in workflow assert "COPILOT_GITHUB_TOKEN" not in workflow diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py new file mode 100644 index 000000000..4d44d7abc --- /dev/null +++ b/tests/test_select_nvidia_nim_model.py @@ -0,0 +1,295 @@ +"""Tests for resolving a live NVIDIA NIM model from an ordered candidate pool.""" + +from __future__ import annotations + +import io +import http.client +import json +from pathlib import Path +import ssl +from typing import Any + +import pytest + +from scripts.ci import select_nvidia_nim_model as resolver + + +class _FakeResponse(io.BytesIO): + """Minimal context-managed HTTP response body for catalog stubs.""" + + status = 200 + + def __enter__(self) -> "_FakeResponse": + """Return the response itself, matching urlopen's context manager.""" + return self + + def __exit__(self, *_exc_info: object) -> bool: + """Close the buffer and never suppress an exception.""" + self.close() + return False + + +class _FakeConnection: + """Minimal non-context-managed HTTPS connection stub for catalog requests.""" + + def __init__( + self, + host: str, + port: int, + *, + timeout: float, + context: ssl.SSLContext, + response: _FakeResponse, + requests: list[Any], + ) -> None: + """Record the validated destination and canned response.""" + self.host = host + self.port = port + self.timeout = timeout + self.context = context + self.response = response + self.requests = requests + self.closed = False + + def close(self) -> None: + """Record explicit cleanup, matching ``HTTPSConnection.close``.""" + self.closed = True + + def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: + """Record one outbound request without opening a network socket.""" + self.requests.append((self, method, path, headers)) + + def getresponse(self) -> _FakeResponse: + """Return the canned provider response.""" + return self.response + + +def _catalog(*model_ids: str) -> bytes: + """Render an OpenAI-compatible model catalog payload for the given ids.""" + return json.dumps({"object": "list", "data": [{"id": model_id} for model_id in model_ids]}).encode("utf-8") + + +def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: + """Serve one canned catalog payload and record the issued requests.""" + requests: list[Any] = [] + + def fake_connection( + host: str, port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Return a canned HTTPS connection and record its destination.""" + return _FakeConnection( + host, + port, + timeout=timeout, + context=context, + response=_FakeResponse(payload), + requests=requests, + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + return requests + + +def test_catalog_sink_has_one_scoped_semgrep_exception_and_explicit_tls() -> None: + """Keep the reviewed HTTPS sink suppressed only for its known false positive.""" + source_text = Path(resolver.__file__).read_text(encoding="utf-8") + rule = "python.lang.security.audit.httpsconnection-detected.httpsconnection-detected" + sink_lines = [ + line for line in source_text.splitlines() if "http.client.HTTPSConnection(" in line + ] + + assert len(sink_lines) == 1 + assert f"# nosemgrep: {rule}" in sink_lines[0] + assert source_text.count(f"# nosemgrep: {rule}") == 1 + assert "context=ssl.create_default_context()" in source_text + + +def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: + """Operators may concatenate pools; order wins and repeats are dropped.""" + assert resolver.parse_candidates(" a/one\n b/two a/one ") == ["a/one", "b/two"] + assert resolver.parse_candidates(" ") == [] + + +@pytest.mark.parametrize( + ("base_url", "message"), + [ + ("http://integrate.api.nvidia.com/v1", "must use https"), + ("https://models.example.invalid/v1", "host is not allowed"), + ("https://integrate.api.nvidia.com:8443/v1", "default HTTPS port"), + ("https://user:pass@integrate.api.nvidia.com/v1", "must not embed credentials"), + ("https://integrate.api.nvidia.com/v1?mode=models", "query or fragment"), + ("https://integrate.api.nvidia.com/v1#models", "query or fragment"), + ], +) +def test_validate_catalog_base_url_refuses_untrusted_endpoints(base_url: str, message: str) -> None: + """A tampered base URL must never receive the provider API key.""" + with pytest.raises(ValueError, match=message): + resolver.validate_catalog_base_url(base_url) + + +def test_validate_catalog_base_url_normalizes_the_trusted_endpoint() -> None: + """The trusted endpoint is accepted with any trailing slash removed.""" + assert resolver.validate_catalog_base_url(f"{resolver.DEFAULT_BASE_URL}/") == resolver.DEFAULT_BASE_URL + + +def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """The resolver reads ids from the provider's OpenAI-compatible catalog.""" + requests = _stub_catalog(monkeypatch, _catalog("a/one", "b/two")) + + served = resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key", timeout_seconds=7.0) + + assert served == {"a/one", "b/two"} + connection, method, path, headers = requests[0] + assert connection.host == "integrate.api.nvidia.com" + assert connection.port == 443 + assert connection.timeout == 7.0 + assert connection.context.verify_mode == ssl.CERT_REQUIRED + assert connection.context.check_hostname is True + assert connection.closed is True + assert method == "GET" + assert path == "/v1/models" + assert headers["Authorization"] == "Bearer secret-key" + + +def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.MonkeyPatch) -> None: + """Entries without a usable string id cannot become selectable models.""" + payload = json.dumps({"data": [{"id": ""}, {"id": 7}, "not-an-object", {"id": "a/one"}]}).encode("utf-8") + _stub_catalog(monkeypatch, payload) + + assert resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") == {"a/one"} + + +@pytest.mark.parametrize( + ("error", "message"), + [ + (http.client.RemoteDisconnected("closed"), "unreachable"), + (OSError("dns"), "unreachable"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_transport_errors( + monkeypatch: pytest.MonkeyPatch, error: Exception, message: str +) -> None: + """A catalog outage is reported, never masked by guessing a model id.""" + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Raise the configured provider failure from the HTTP boundary.""" + del timeout + del context + raise error + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +def test_fetch_served_model_ids_reports_http_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Provider HTTP failures identify the status without exposing credentials.""" + response = _FakeResponse(b"{}") + response.status = 401 + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + """Return an unauthorized provider response.""" + return _FakeConnection( + "integrate.api.nvidia.com", + 443, + timeout=timeout, + context=context, + response=response, + requests=[], + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + with pytest.raises(RuntimeError, match="HTTP 401"): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + (b"maintenance", "non-JSON body"), + (b'{"object": "list"}', "no model list"), + (b'{"data": []}', "no usable model id"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_unusable_payloads( + monkeypatch: pytest.MonkeyPatch, payload: bytes, message: str +) -> None: + """Unparsable or empty catalogs are errors rather than silent fallbacks.""" + _stub_catalog(monkeypatch, payload) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +def test_select_model_prefers_the_first_served_candidate() -> None: + """A retired first choice transparently falls through to the next live one.""" + candidates = ["retired/model", "live/model", "other/model"] + + assert resolver.select_model(candidates, {"live/model", "other/model"}, role="primary") == "live/model" + + +def test_select_model_requires_a_configured_pool() -> None: + """An empty pool is a configuration error with the role named.""" + with pytest.raises(ValueError, match="no small NVIDIA NIM model candidates"): + resolver.select_model([], {"live/model"}, role="small") + + +def test_select_model_reports_a_fully_retired_pool() -> None: + """When no candidate is served, the message tells the operator what to do.""" + with pytest.raises(RuntimeError, match="Add a live model id to the candidate pool"): + resolver.select_model(["retired/model"], {"live/model"}, role="primary") + + +def test_main_prints_the_resolved_model_id( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The successful path prints exactly the resolved id for shell capture.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + exit_code = resolver.main(["--role", "primary", "--candidates", "retired/model live/model"]) + + assert exit_code == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_accepts_the_workflow_secret_name( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Either credential variable name works, so callers need no shim.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "live/model"]) == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_requires_a_provider_credential( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Without a credential the resolver fails closed with a CI annotation.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.delenv("NVIDIA_NIM_API_KEY", raising=False) + + assert resolver.main(["--candidates", "live/model"]) == 1 + assert "NVIDIA_API_KEY is required" in capsys.readouterr().err + + +def test_main_annotates_a_resolution_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Resolution failures surface as GitHub error annotations, not tracebacks.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "retired/model"]) == 1 + assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err