From d7133e2c563227112d756279e7218153b57e3c3c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 18:10:29 +0900 Subject: [PATCH 01/49] fix(strix): serialize scans per repository to stop shared-key rate-limit storms Root cause: the per-PR concurrency group let sibling PRs in one repository scan concurrently; each run retried the shared NVIDIA NIM key up to three times, producing litellm.RateLimitError storms and fail-closed gate failures on every open PR (observed across ContextualWisdomLab/contextual-orchestrator 2026-08-23/24). Change: scope the concurrency group per repository (event class still separated so required pull_request_target evidence never interleaves with default-branch repository_dispatch retries), set cancel-in-progress: false with queue: max so queued evidence runs are preserved, and update the queue contract test to encode the new serialization contract. Accuracy is prioritized over scan latency; queued runs already fetch the expected head SHA directly, so late-started runs stay head-exact. --- .github/workflows/strix.yml | 21 ++++++++++---- .../test_required_workflow_queue_contract.py | 28 +++++++++++++------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index c8c054e42..e700521c8 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -68,13 +68,22 @@ on: concurrency: # Include the event name so default-branch repository_dispatch evidence cannot cancel - # the required pull_request_target Strix context that branch protection reads. - # PR-number scope keeps the queue on the current HEAD within each event class. + # or interleave with the required pull_request_target Strix context that branch + # protection reads. + # + # Rate-limit root-cause fix (2026-08-24): the group is scoped per REPOSITORY + # (not per PR) so sibling pull requests in the same repository scan + # sequentially instead of concurrently. Concurrent per-PR scans each retry + # the shared NVIDIA NIM key up to three times, producing guaranteed + # litellm.RateLimitError storms and fail-closed gate failures across every + # open PR (observed 2026-08-23/24). Serializing per repository keeps at most + # one provider-backed scan in flight per repository; queue: max preserves + # every queued evidence run (nothing is dropped), and accuracy is prioritized + # over scan latency. group: >- - strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}-${{ - github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number) || - github.event_name == 'repository_dispatch' && github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number) || github.ref }} - cancel-in-progress: true + strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }} + cancel-in-progress: false + queue: max # Scorecard Token-Permissions (alert #43): keep the workflow-level token # read-only and scope same-repo status publication to the Strix scan job. diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index e58f5e6c0..00576455d 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -280,7 +280,14 @@ def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() - def test_strix_cancels_superseded_pr_head_security_evidence() -> None: - """Scope Strix concurrency to the target PR while preserving current-head evidence.""" + """Serialize Strix per repository so shared provider keys are not rate-limited. + + Root cause (2026-08-23/24): sibling PRs scanned concurrently, each retrying + the shared NVIDIA NIM key three times, producing litellm.RateLimitError + storms and fail-closed gate failures on every open PR. The queue now scopes + one scan at a time per repository and event class while preserving every + evidence run (queue: max, nothing cancelled). + """ workflow = workflow_text("strix.yml") concurrency_contract = workflow.split("concurrency:", 1)[1].split( "permissions:", 1 @@ -294,14 +301,16 @@ def test_strix_cancels_superseded_pr_head_security_evidence() -> None: "strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || " "github.event.pull_request.base.repo.full_name || github.repository }}" ) in concurrency_contract - assert "format('pr-{0}', github.event.pull_request.number)" in concurrency_contract - assert "github.event.client_payload.pr_number != '' && format('pr-{0}'," in workflow - assert "format('pr-{0}-{1}'" not in concurrency_contract + # Repository-level (not PR-level) grouping: no pr-{N} component remains. + assert "format('pr-{0}', github.event.pull_request.number)" not in concurrency_contract assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract - assert "cancel-in-progress: true" in workflow + # Nothing is dropped: scans queue sequentially instead of cancelling. + assert "cancel-in-progress: false" in workflow + assert "cancel-in-progress: true" not in workflow.split("jobs:", 1)[0] + assert "queue: max" in workflow assert "default-branch repository_dispatch evidence cannot cancel" in workflow - assert "PR-number scope keeps the queue on the current HEAD" in workflow + assert "RateLimitError" in concurrency_contract assert ( "refs/pull//head has already advanced before this queued run starts" in workflow @@ -357,8 +366,11 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "${{ secrets." not in opencode_bootstrap strix_workflow = workflow_text("strix.yml") - assert "cancel-in-progress: true" in strix_workflow - assert "PR-number scope keeps the queue on the current HEAD" in strix_workflow + # Strix serializes per repository (rate-limit root-cause fix): close-event + # runs still cancel superseded same-PR evidence through their own + # cancel-closed-pr-runs job, while scan jobs queue instead of cancelling. + assert "cancel-in-progress: false" in strix_workflow + assert "Serialize Strix scans per repository" in strix_workflow or "per REPOSITORY" in strix_workflow def test_close_empty_pr_metadata_lookup_retries_and_fails_open() -> None: From 623f7ac18cd5e667cc44b85d0865aafe66ca3172 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Mon, 24 Aug 2026 09:55:57 +0000 Subject: [PATCH 02/49] fix(strix): align bash contract test and workflow comment with repo-level concurrency Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/strix.yml | 14 ++++++++------ CHANGELOG.md | 6 ++++++ scripts/ci/test_strix_quick_gate.sh | 10 +++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index e700521c8..d52e0eb72 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -37,13 +37,15 @@ on: # Same conservative doc/image-only skip for PR scans. GitHub evaluates these # path filters against the PR's full base..head diff, so a PR is skipped only # when EVERY changed file is a non-executable doc/image asset; any code, - # config, build, or workflow change still triggers the scan. Concurrency is - # PR-number based for status grouping, but Strix runs intentionally do not + # config, build, or workflow change still triggers the scan. The run-name + # includes the PR number and head SHA for status grouping, while the + # concurrency group is scoped per repository and event class to prevent + # shared-provider key rate-limit storms. Strix runs intentionally do not # cancel in progress because a pre-job cancellation leaves no scanner log to - # review. Queue pressure should be handled by stale-run cleanup outside this - # current-head evidence path. For PRs the merge scheduler manages, same-head - # Strix evidence is still forced at merge time via repository_dispatch (which - # paths-ignore does not affect), so merged code never loses evidence. + # review. Queue pressure is preserved by queue: max, which keeps every queued + # evidence run. For PRs the merge scheduler manages, same-head Strix evidence + # is still forced at merge time via repository_dispatch (which paths-ignore + # does not affect), so merged code never loses evidence. paths-ignore: - '**/*.md' - '**/*.markdown' diff --git a/CHANGELOG.md b/CHANGELOG.md index 1630c32d4..6d3db4896 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ Semantic Versioning where the repository publishes a release. during materialization and then rejecting every version except pnpm 11.5.3; route generic coverage and docstring package scripts through the same Corepack boundary instead of invoking a removed bare `pnpm` binary. +- Serialize Strix scans per repository and event class to stop shared-provider + key rate-limit storms. Concurrent per-PR scans each retried the shared NVIDIA + NIM key, producing guaranteed `litellm.RateLimitError` failures across open + PRs. Queue `max` preserves every queued evidence run; `cancel-in-progress: + false` avoids dropping in-flight evidence. Update the bash contract test to + match the repository-scoped concurrency group. - Keep `--trust-lockfile` only for pnpm 11.3 and newer (`trustLockfile` landed in pnpm 11.3). pnpm 9, 10, and 11.0–11.2 reject that flag and previously failed LineageWeave JavaScript coverage before diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 945eb3fb3..6407ed122 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -193,14 +193,14 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "branches: [main, develop, master]" "strix workflow scans GitHub Flow and Git Flow protected branches" assert_file_contains "$workflow_file" "pull_request_target:" "strix workflow uses trusted PR trigger" assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-' "strix workflow isolates manual evidence runs from required PR contexts" - assert_file_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow scopes pull_request_target concurrency to the active pull request" + assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}' "strix workflow scopes concurrency per repository and event class" assert_file_contains "$workflow_file" "github.event.client_payload.target_repository ||" "strix manual dispatch concurrency scopes to the target repository when provided" - assert_file_contains "$workflow_file" "github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number)" "strix workflow retains a manual PR fallback group when no head SHA is provided" - assert_file_contains "$workflow_file" "github.ref }}" "strix workflow scopes non-PR concurrency to the current ref" + assert_file_contains "$workflow_file" "github.repository }}" "strix workflow falls back to the workflow repository when no target repository is provided" + assert_file_contains "$workflow_file" "cancel-in-progress: false" "strix workflow queues evidence runs instead of cancelling them" assert_file_not_contains "$workflow_file" "format('pr-{0}-{1}'" "strix workflow does not keep stale head-specific concurrency groups" - assert_file_contains "$workflow_file" "cancel-in-progress: true" "strix workflow cancels stale PR evidence runs when a newer PR event arrives" + assert_file_contains "$workflow_file" "queue: max" "strix workflow preserves every queued evidence run" assert_file_contains "$workflow_file" "default-branch repository_dispatch evidence cannot cancel" "strix workflow documents manual evidence isolation from branch protection contexts" - assert_file_contains "$workflow_file" "PR-number scope keeps the queue on the current HEAD" "strix workflow documents current-head queue management" + assert_file_contains "$workflow_file" "Serializing per repository keeps at most" "strix workflow documents repository-level queue management" assert_file_contains "$workflow_file" "refs/pull//head has already advanced before this queued run starts" "strix workflow documents stale scan queue avoidance" status_token_count="$(grep -c '^[[:space:]]*GITHUB_STATUS_TOKEN:' "$workflow_file")" assert_equals "1" "$status_token_count" "strix workflow defines GITHUB_STATUS_TOKEN once so GitHub can parse repository_dispatch" From c7000c2fe77fe5e41ba17a4803eaed8bc8e92f3a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:09:10 +0900 Subject: [PATCH 03/49] fix(strix): cancel closed pull request scans explicitly --- .github/workflows/strix.yml | 47 +++++++++++++++++-- .../test_required_workflow_queue_contract.py | 23 ++++++--- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d52e0eb72..76862ed54 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -71,7 +71,8 @@ on: concurrency: # Include the event name so default-branch repository_dispatch evidence cannot cancel # or interleave with the required pull_request_target Strix context that branch - # protection reads. + # protection reads. Closed PR events use a separate group so their cancellation + # job can run immediately instead of waiting behind the scan it must cancel. # # Rate-limit root-cause fix (2026-08-24): the group is scoped per REPOSITORY # (not per PR) so sibling pull requests in the same repository scan @@ -83,7 +84,12 @@ concurrency: # every queued evidence run (nothing is dropped), and accuracy is prioritized # over scan latency. group: >- - strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }} + strix-${{ + github.event_name == 'pull_request_target' && + github.event.action == 'closed' && + format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number) || + format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) + }} cancel-in-progress: false queue: max @@ -98,8 +104,43 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest + permissions: + actions: write + contents: read + env: + GH_TOKEN: ${{ github.token }} + TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} + CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} + CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CURRENT_RUN_ID: ${{ github.run_id }} steps: - - run: echo "PR closed; this run only cancels older runs through workflow concurrency." + - name: Cancel queued and running scans for the closed pull request + shell: bash + run: | + set -euo pipefail + + cancel_runs() { + local status="$1" + local runs_url="repos/${TARGET_REPOSITORY}/actions/workflows/strix.yml/runs?event=pull_request_target&status=${status}&per_page=100" + gh api --paginate "$runs_url" | + jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head "$CLOSED_PR_HEAD_SHA" --arg current "$CURRENT_RUN_ID" ' + .workflow_runs[] + | select((.id | tostring) != $current) + | select( + ($head != "" and .head_sha == $head) or + any(.pull_requests[]?; ((.number | tostring) == $pr)) + ) + | .id + ' | + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null + echo "Cancelled Strix run ${run_id} for closed PR #${CLOSED_PR_NUMBER}." + done + } + + cancel_runs queued + cancel_runs in_progress strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 00576455d..4ad631f25 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -298,8 +298,12 @@ def test_strix_cancels_superseded_pr_head_security_evidence() -> None: assert "github.event.pull_request.base.repo.full_name" in concurrency_contract assert "github.repository" in concurrency_contract assert ( - "strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || " - "github.event.pull_request.base.repo.full_name || github.repository }}" + "format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, " + "github.event.pull_request.number)" + ) in concurrency_contract + assert ( + "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || " + "github.event.pull_request.base.repo.full_name || github.repository)" ) in concurrency_contract # Repository-level (not PR-level) grouping: no pr-{N} component remains. assert "format('pr-{0}', github.event.pull_request.number)" not in concurrency_contract @@ -352,10 +356,17 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "closed" in workflow assert "cancel-closed-pr-runs:" in workflow - assert ( - "PR closed; this run only cancels older runs through workflow concurrency." - in workflow - ) + if filename == "strix.yml": + assert "Cancel queued and running scans for the closed pull request" in workflow + assert "actions: write" in workflow + assert "actions/workflows/strix.yml/runs?event=pull_request_target" in workflow + assert "actions/runs/${run_id}/cancel" in workflow + assert "CURRENT_RUN_ID" in workflow + else: + assert ( + "PR closed; this run only cancels older runs through workflow concurrency." + in workflow + ) assert "github.event.action != 'closed'" in workflow opencode_bootstrap = workflow_text("opencode-review.yml") From efb7d365470b39edae1509debc06b21591bc1ba2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:08:44 +0900 Subject: [PATCH 04/49] test(strix): use supported concurrency contract --- .github/workflows/strix.yml | 13 +++++++------ scripts/ci/test_strix_quick_gate.sh | 8 +++++--- tests/test_required_workflow_queue_contract.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 76862ed54..1595d470c 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -42,8 +42,9 @@ on: # concurrency group is scoped per repository and event class to prevent # shared-provider key rate-limit storms. Strix runs intentionally do not # cancel in progress because a pre-job cancellation leaves no scanner log to - # review. Queue pressure is preserved by queue: max, which keeps every queued - # evidence run. For PRs the merge scheduler manages, same-head Strix evidence + # review. GitHub keeps one active and one pending run per group; the merge + # scheduler re-dispatches exact-head evidence when a pending run is + # superseded. For PRs the merge scheduler manages, same-head Strix evidence # is still forced at merge time via repository_dispatch (which paths-ignore # does not affect), so merged code never loses evidence. paths-ignore: @@ -80,9 +81,10 @@ concurrency: # the shared NVIDIA NIM key up to three times, producing guaranteed # litellm.RateLimitError storms and fail-closed gate failures across every # open PR (observed 2026-08-23/24). Serializing per repository keeps at most - # one provider-backed scan in flight per repository; queue: max preserves - # every queued evidence run (nothing is dropped), and accuracy is prioritized - # over scan latency. + # one provider-backed scan in flight per repository. GitHub's native + # concurrency contract retains one active and one pending run; the scheduler + # re-dispatches the exact current head after pending-run supersession, and + # accuracy is prioritized over scan latency. group: >- strix-${{ github.event_name == 'pull_request_target' && @@ -91,7 +93,6 @@ concurrency: format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) }} cancel-in-progress: false - queue: max # Scorecard Token-Permissions (alert #43): keep the workflow-level token # read-only and scope same-repo status publication to the Strix scan job. diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 6407ed122..217ce0d37 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -196,11 +196,13 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}' "strix workflow scopes concurrency per repository and event class" assert_file_contains "$workflow_file" "github.event.client_payload.target_repository ||" "strix manual dispatch concurrency scopes to the target repository when provided" assert_file_contains "$workflow_file" "github.repository }}" "strix workflow falls back to the workflow repository when no target repository is provided" - assert_file_contains "$workflow_file" "cancel-in-progress: false" "strix workflow queues evidence runs instead of cancelling them" + assert_file_not_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow serializes sibling PR scans at repository scope" + assert_file_not_contains "$workflow_file" "github.event.client_payload.pr_number != '' && format('pr-{0}', github.event.client_payload.pr_number)" "strix workflow does not create one provider queue per PR" assert_file_not_contains "$workflow_file" "format('pr-{0}-{1}'" "strix workflow does not keep stale head-specific concurrency groups" - assert_file_contains "$workflow_file" "queue: max" "strix workflow preserves every queued evidence run" + assert_file_contains "$workflow_file" "cancel-in-progress: false" "strix workflow does not cancel an in-progress provider scan" + assert_file_not_contains "$workflow_file" "queue: max" "strix workflow uses only supported GitHub concurrency keys" assert_file_contains "$workflow_file" "default-branch repository_dispatch evidence cannot cancel" "strix workflow documents manual evidence isolation from branch protection contexts" - assert_file_contains "$workflow_file" "Serializing per repository keeps at most" "strix workflow documents repository-level queue management" + assert_file_contains "$workflow_file" "re-dispatches exact-head evidence" "strix workflow documents current-head queue recovery" assert_file_contains "$workflow_file" "refs/pull//head has already advanced before this queued run starts" "strix workflow documents stale scan queue avoidance" status_token_count="$(grep -c '^[[:space:]]*GITHUB_STATUS_TOKEN:' "$workflow_file")" assert_equals "1" "$status_token_count" "strix workflow defines GITHUB_STATUS_TOKEN once so GitHub can parse repository_dispatch" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 4ad631f25..f32d86dc2 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -279,14 +279,15 @@ def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() - assert "Semgrep engine/configuration failed with rc=${SEMGREP_RC}" in workflow -def test_strix_cancels_superseded_pr_head_security_evidence() -> None: +def test_strix_serializes_provider_evidence_per_repository() -> None: """Serialize Strix per repository so shared provider keys are not rate-limited. Root cause (2026-08-23/24): sibling PRs scanned concurrently, each retrying the shared NVIDIA NIM key three times, producing litellm.RateLimitError - storms and fail-closed gate failures on every open PR. The queue now scopes - one scan at a time per repository and event class while preserving every - evidence run (queue: max, nothing cancelled). + storms and fail-closed gate failures on every open PR. The concurrency group + now scopes one scan at a time per repository and event class. GitHub retains + one active and one pending run per group; the scheduler re-dispatches exact + current-head evidence when a pending run is superseded. """ workflow = workflow_text("strix.yml") concurrency_contract = workflow.split("concurrency:", 1)[1].split( @@ -309,10 +310,11 @@ def test_strix_cancels_superseded_pr_head_security_evidence() -> None: assert "format('pr-{0}', github.event.pull_request.number)" not in concurrency_contract assert "github.event.pull_request.head.sha" not in concurrency_contract assert "github.event.client_payload.pr_head_sha" not in concurrency_contract - # Nothing is dropped: scans queue sequentially instead of cancelling. + # Running scans are not cancelled; GitHub's native group has one pending slot. assert "cancel-in-progress: false" in workflow assert "cancel-in-progress: true" not in workflow.split("jobs:", 1)[0] - assert "queue: max" in workflow + assert "queue: max" not in workflow + assert "scheduler" in concurrency_contract assert "default-branch repository_dispatch evidence cannot cancel" in workflow assert "RateLimitError" in concurrency_contract assert ( From 01a8c86ddf29c6dc0c2c418835ca20b504627155 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:17:58 +0900 Subject: [PATCH 05/49] test(strix): cover closed-run concurrency group --- scripts/ci/test_strix_quick_gate.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 217ce0d37..f1a0a959c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -192,8 +192,9 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "branches: [main, develop, master]" "strix workflow scans GitHub Flow and Git Flow protected branches" assert_file_contains "$workflow_file" "pull_request_target:" "strix workflow uses trusted PR trigger" - assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-' "strix workflow isolates manual evidence runs from required PR contexts" - assert_file_contains "$workflow_file" 'strix-${{ github.event_name }}-${{ github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository }}' "strix workflow scopes concurrency per repository and event class" + assert_file_contains "$workflow_file" "group: >-" "strix workflow defines an explicit concurrency group" + assert_file_contains "$workflow_file" "format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number)" "strix workflow gives closed PR cleanup an independent concurrency group" + assert_file_contains "$workflow_file" "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository ||" "strix workflow scopes active evidence per repository and event class" assert_file_contains "$workflow_file" "github.event.client_payload.target_repository ||" "strix manual dispatch concurrency scopes to the target repository when provided" assert_file_contains "$workflow_file" "github.repository }}" "strix workflow falls back to the workflow repository when no target repository is provided" assert_file_not_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow serializes sibling PR scans at repository scope" From 326c3fc635007fe740725010b1881072b143155f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:23:15 +0900 Subject: [PATCH 06/49] fix(strix): scope closed-run cancellation to the matching PR --- .github/workflows/strix.yml | 15 ++++++--------- tests/test_required_workflow_queue_contract.py | 5 ++++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 1595d470c..d45e0e422 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -80,8 +80,8 @@ concurrency: # sequentially instead of concurrently. Concurrent per-PR scans each retry # the shared NVIDIA NIM key up to three times, producing guaranteed # litellm.RateLimitError storms and fail-closed gate failures across every - # open PR (observed 2026-08-23/24). Serializing per repository keeps at most - # one provider-backed scan in flight per repository. GitHub's native + # open PR (observed 2026-08-23/24). Serializing per repository and event + # class keeps at most one provider-backed scan in flight per class. GitHub's native # concurrency contract retains one active and one pending run; the scheduler # re-dispatches the exact current head after pending-run supersession, and # accuracy is prioritized over scan latency. @@ -112,7 +112,6 @@ jobs: GH_TOKEN: ${{ github.token }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} - CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} CURRENT_RUN_ID: ${{ github.run_id }} steps: - name: Cancel queued and running scans for the closed pull request @@ -122,15 +121,13 @@ jobs: cancel_runs() { local status="$1" - local runs_url="repos/${TARGET_REPOSITORY}/actions/workflows/strix.yml/runs?event=pull_request_target&status=${status}&per_page=100" + local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?event=pull_request_target&status=${status}&per_page=100" gh api --paginate "$runs_url" | - jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head "$CLOSED_PR_HEAD_SHA" --arg current "$CURRENT_RUN_ID" ' + jq -r --arg pr "$CLOSED_PR_NUMBER" --arg current "$CURRENT_RUN_ID" ' .workflow_runs[] | select((.id | tostring) != $current) - | select( - ($head != "" and .head_sha == $head) or - any(.pull_requests[]?; ((.number | tostring) == $pr)) - ) + | select(.name == "Strix Security Scan") + | select(any(.pull_requests[]?; ((.number | tostring) == $pr))) | .id ' | while IFS= read -r run_id; do diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index f32d86dc2..66dd11ea3 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -361,7 +361,10 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - if filename == "strix.yml": assert "Cancel queued and running scans for the closed pull request" in workflow assert "actions: write" in workflow - assert "actions/workflows/strix.yml/runs?event=pull_request_target" in workflow + assert "actions/runs?event=pull_request_target" in workflow + assert 'select(.name == "Strix Security Scan")' in workflow + assert "any(.pull_requests[]?; ((.number | tostring) == $pr))" in workflow + assert "CLOSED_PR_HEAD_SHA" not in workflow assert "actions/runs/${run_id}/cancel" in workflow assert "CURRENT_RUN_ID" in workflow else: From ab17b103bb28e5fdf721e43e4c22221070a9ae3b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:26:39 +0900 Subject: [PATCH 07/49] docs(strix): describe native concurrency recovery accurately --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d3db4896..afbe76efe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,9 +15,11 @@ Semantic Versioning where the repository publishes a release. - Serialize Strix scans per repository and event class to stop shared-provider key rate-limit storms. Concurrent per-PR scans each retried the shared NVIDIA NIM key, producing guaranteed `litellm.RateLimitError` failures across open - PRs. Queue `max` preserves every queued evidence run; `cancel-in-progress: - false` avoids dropping in-flight evidence. Update the bash contract test to - match the repository-scoped concurrency group. + PRs. GitHub retains one active and one pending run per repository/event group; + `cancel-in-progress: false` keeps the active scan running, while the merge + scheduler re-dispatches exact-head evidence after pending-run supersession. + Update the bash contract test to match the repository-scoped concurrency + group. - Keep `--trust-lockfile` only for pnpm 11.3 and newer (`trustLockfile` landed in pnpm 11.3). pnpm 9, 10, and 11.0–11.2 reject that flag and previously failed LineageWeave JavaScript coverage before From 309bbf5965c30ae2918b0ffabf66670b99ed7d13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:57:01 +0900 Subject: [PATCH 08/49] fix(strix): cancel fork pull request scans --- .github/workflows/strix.yml | 5 +++-- tests/test_required_workflow_queue_contract.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d45e0e422..06a554694 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -112,6 +112,7 @@ jobs: GH_TOKEN: ${{ github.token }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} + CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} CURRENT_RUN_ID: ${{ github.run_id }} steps: - name: Cancel queued and running scans for the closed pull request @@ -123,11 +124,11 @@ jobs: local status="$1" local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?event=pull_request_target&status=${status}&per_page=100" gh api --paginate "$runs_url" | - jq -r --arg pr "$CLOSED_PR_NUMBER" --arg current "$CURRENT_RUN_ID" ' + jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" --arg current "$CURRENT_RUN_ID" ' .workflow_runs[] | select((.id | tostring) != $current) | select(.name == "Strix Security Scan") - | select(any(.pull_requests[]?; ((.number | tostring) == $pr))) + | select(.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr))) | .id ' | while IFS= read -r run_id; do diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 66dd11ea3..ef6de19f0 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -363,8 +363,9 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "actions: write" in workflow assert "actions/runs?event=pull_request_target" in workflow assert 'select(.name == "Strix Security Scan")' in workflow + assert "CLOSED_PR_HEAD_SHA" in workflow + assert 'select(.head_sha == $head_sha or any(.pull_requests[]?' in workflow assert "any(.pull_requests[]?; ((.number | tostring) == $pr))" in workflow - assert "CLOSED_PR_HEAD_SHA" not in workflow assert "actions/runs/${run_id}/cancel" in workflow assert "CURRENT_RUN_ID" in workflow else: From 606d7bb47de7ffd8ff7e814ef61ec5590e79ab11 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 07:45:16 +0900 Subject: [PATCH 09/49] fix(strix): cancel central dispatch runs on PR close --- .github/workflows/strix.yml | 52 ++++++++++++++----- .../test_required_workflow_queue_contract.py | 10 ++-- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 5823b40de..9d2668f8f 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -111,6 +111,10 @@ jobs: env: GH_TOKEN: ${{ github.token }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} + # Required-workflow repository_dispatch scans execute in the central + # repository, not in the PR's target repository. Query both locations + # so closing a fork PR releases the same per-repository provider queue. + DISPATCH_REPOSITORY: ${{ format('{0}/.github', github.repository_owner) }} CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} CURRENT_RUN_ID: ${{ github.run_id }} @@ -122,20 +126,37 @@ jobs: cancel_runs() { local status="$1" - local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?event=pull_request_target&status=${status}&per_page=100" - gh api --paginate "$runs_url" | - jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" --arg current "$CURRENT_RUN_ID" ' - .workflow_runs[] - | select((.id | tostring) != $current) - | select(.name == "Strix Security Scan") - | select(.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr))) - | .id - ' | - while IFS= read -r run_id; do - [ -n "$run_id" ] || continue - gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null - echo "Cancelled Strix run ${run_id} for closed PR #${CLOSED_PR_NUMBER}." - done + local run_repository + local previous_repository="" + # A dispatch run is hosted by the central required-workflow + # repository. Avoid querying the same repository twice when the + # target itself is .github. + for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"; do + [ "$run_repository" != "$previous_repository" ] || continue + previous_repository="$run_repository" + local runs_url="repos/${run_repository}/actions/runs?status=${status}&per_page=100" + gh api --paginate "$runs_url" | + jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ + --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" ' + .workflow_runs[] + | select((.id | tostring) != $current) + | select(.name == "Strix Security Scan") + | select( + (.event == "pull_request_target" and + (.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr)))) + or + (.event == "repository_dispatch" and + (((.display_title // "") | startswith(("Strix Security Scan " + $target + "#" + $pr + "@"))) + or .head_sha == $head_sha)) + ) + | .id + ' | + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + gh api --method POST "repos/${run_repository}/actions/runs/${run_id}/cancel" >/dev/null + echo "Cancelled Strix run ${run_id} in ${run_repository} for closed PR #${CLOSED_PR_NUMBER}." + done + done } cancel_runs queued @@ -327,6 +348,9 @@ jobs: "") is_private="" for target_visibility_attempt in 1 2 3 4 5 6; do + # The single-quoted jq program intentionally expands jq's + # `$visibility`, not a shell variable (ShellCheck SC2016). + # shellcheck disable=SC2016 if is_private="$( gh api "repos/${TARGET_REPOSITORY}" --jq ' (.visibility // "" | ascii_downcase) as $visibility diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 845a9099b..f028d7044 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -361,12 +361,16 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - if filename == "strix.yml": assert "Cancel queued and running scans for the closed pull request" in workflow assert "actions: write" in workflow - assert "actions/runs?event=pull_request_target" in workflow + assert "DISPATCH_REPOSITORY" in workflow + assert 'actions/runs?status=${status}&per_page=100' in workflow + assert 'for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"' in workflow + assert '(.event == "repository_dispatch" and' in workflow + assert '(.display_title // "") | startswith' in workflow assert 'select(.name == "Strix Security Scan")' in workflow assert "CLOSED_PR_HEAD_SHA" in workflow - assert 'select(.head_sha == $head_sha or any(.pull_requests[]?' in workflow + assert '(.head_sha == $head_sha or any(.pull_requests[]?' in workflow assert "any(.pull_requests[]?; ((.number | tostring) == $pr))" in workflow - assert "actions/runs/${run_id}/cancel" in workflow + assert 'actions/runs/${run_id}/cancel' in workflow assert "CURRENT_RUN_ID" in workflow else: assert ( From 6d6af57cb2cf5ecb12cac596deb9fc5266fe234b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 08:56:15 +0900 Subject: [PATCH 10/49] fix(strix): keep close events read-only --- .github/workflows/strix.yml | 61 ++----------------- .../test_required_workflow_queue_contract.py | 20 +++--- 2 files changed, 13 insertions(+), 68 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 9d2668f8f..9072d0180 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -105,62 +105,13 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest - permissions: - actions: write - contents: read - env: - GH_TOKEN: ${{ github.token }} - TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} - # Required-workflow repository_dispatch scans execute in the central - # repository, not in the PR's target repository. Query both locations - # so closing a fork PR releases the same per-repository provider queue. - DISPATCH_REPOSITORY: ${{ format('{0}/.github', github.repository_owner) }} - CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} - CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - CURRENT_RUN_ID: ${{ github.run_id }} steps: - - name: Cancel queued and running scans for the closed pull request - shell: bash - run: | - set -euo pipefail - - cancel_runs() { - local status="$1" - local run_repository - local previous_repository="" - # A dispatch run is hosted by the central required-workflow - # repository. Avoid querying the same repository twice when the - # target itself is .github. - for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"; do - [ "$run_repository" != "$previous_repository" ] || continue - previous_repository="$run_repository" - local runs_url="repos/${run_repository}/actions/runs?status=${status}&per_page=100" - gh api --paginate "$runs_url" | - jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ - --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" ' - .workflow_runs[] - | select((.id | tostring) != $current) - | select(.name == "Strix Security Scan") - | select( - (.event == "pull_request_target" and - (.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr)))) - or - (.event == "repository_dispatch" and - (((.display_title // "") | startswith(("Strix Security Scan " + $target + "#" + $pr + "@"))) - or .head_sha == $head_sha)) - ) - | .id - ' | - while IFS= read -r run_id; do - [ -n "$run_id" ] || continue - gh api --method POST "repos/${run_repository}/actions/runs/${run_id}/cancel" >/dev/null - echo "Cancelled Strix run ${run_id} in ${run_repository} for closed PR #${CLOSED_PR_NUMBER}." - done - done - } - - cancel_runs queued - cancel_runs in_progress + # The pull_request_target token is intentionally read-only. The central + # merge scheduler owns Actions cancellation with its mutation credential + # and performs Current-HEAD queue hygiene for closed PRs, including Strix + # runs. Keeping this close event lightweight avoids cross-repository + # cancellation with a token scoped to the target repository. + - run: echo "PR closed; central queue hygiene owns Strix cancellation." strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index f028d7044..506d142a3 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -359,19 +359,13 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "closed" in workflow assert "cancel-closed-pr-runs:" in workflow if filename == "strix.yml": - assert "Cancel queued and running scans for the closed pull request" in workflow - assert "actions: write" in workflow - assert "DISPATCH_REPOSITORY" in workflow - assert 'actions/runs?status=${status}&per_page=100' in workflow - assert 'for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"' in workflow - assert '(.event == "repository_dispatch" and' in workflow - assert '(.display_title // "") | startswith' in workflow - assert 'select(.name == "Strix Security Scan")' in workflow - assert "CLOSED_PR_HEAD_SHA" in workflow - assert '(.head_sha == $head_sha or any(.pull_requests[]?' in workflow - assert "any(.pull_requests[]?; ((.number | tostring) == $pr))" in workflow - assert 'actions/runs/${run_id}/cancel' in workflow - assert "CURRENT_RUN_ID" in workflow + assert "central queue hygiene owns Strix cancellation" in workflow + # pull_request_target must not gain a cross-repository mutation + # token; the central scheduler performs cancellation with its own + # explicitly scoped credential. + assert "DISPATCH_REPOSITORY" not in workflow + assert "CLOSED_PR_HEAD_SHA" not in workflow + assert "actions: write" not in workflow.split(" strix:", 1)[0] else: assert ( "PR closed; this run only cancels older runs through workflow concurrency." From 5fb75ab426a58dbe61080f3b22e287cf5ee5741a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 09:12:13 +0900 Subject: [PATCH 11/49] fix(strix): cancel closed runs with scoped credential --- .github/workflows/strix.yml | 74 +++++++++++++++++-- .../test_required_workflow_queue_contract.py | 13 ++-- 2 files changed, 75 insertions(+), 12 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 9072d0180..cbce23120 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -105,13 +105,75 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest + # The built-in token is intentionally read-only. The established scheduler + # credential is optional here and is used only for cancellation requests. + # When it is unavailable, the close event remains green and the next + # central queue sweep records the unresolved run for operator action. + permissions: + contents: read + env: + GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || '' }} + TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} + DISPATCH_REPOSITORY: ${{ format('{0}/.github', github.repository_owner) }} + CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} + CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + CURRENT_RUN_ID: ${{ github.run_id }} steps: - # The pull_request_target token is intentionally read-only. The central - # merge scheduler owns Actions cancellation with its mutation credential - # and performs Current-HEAD queue hygiene for closed PRs, including Strix - # runs. Keeping this close event lightweight avoids cross-repository - # cancellation with a token scoped to the target repository. - - run: echo "PR closed; central queue hygiene owns Strix cancellation." + - name: Cancel queued and running scans for the closed pull request + shell: bash + run: | + set -euo pipefail + + if [ -z "${GH_TOKEN:-}" ]; then + echo "::warning::Strix close cleanup skipped: no scheduler mutation credential is configured." + exit 0 + fi + + cancel_runs() { + local status="$1" + local run_repository + local previous_repository="" + for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"; do + [ "$run_repository" != "$previous_repository" ] || continue + previous_repository="$run_repository" + local runs_url="repos/${run_repository}/actions/runs?status=${status}&per_page=100" + local runs_json + if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/strix-close-gh-error)"; then + echo "::warning::Strix close cleanup could not inspect ${run_repository}; leaving runs unchanged." + sed 's/^/ /' /tmp/strix-close-gh-error >&2 || true + continue + fi + jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ + --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" \ + --arg dispatch "$DISPATCH_REPOSITORY" --arg run_repository "$run_repository" ' + .workflow_runs[] + | select((.id | tostring) != $current) + | select(.name == "Strix Security Scan") + | select( + ($run_repository == $target and + .event == "pull_request_target" and + (.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr)))) + or + ($run_repository == $dispatch and + .event == "repository_dispatch" and + (((.display_title // "") | startswith(("Strix Security Scan " + $target + "#" + $pr + "@")))) + ) + ) + | .id + ' <<<"$runs_json" | while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + if gh api --method POST "repos/${run_repository}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then + echo "Cancelled Strix run ${run_id} in ${run_repository} for closed PR #${CLOSED_PR_NUMBER}." + else + echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${run_repository}; it may have finished or the credential lacks Actions write access." + sed 's/^/ /' /tmp/strix-close-cancel-error >&2 || true + fi + done + done + } + + cancel_runs queued + cancel_runs in_progress strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 506d142a3..295da8417 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -359,12 +359,13 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "closed" in workflow assert "cancel-closed-pr-runs:" in workflow if filename == "strix.yml": - assert "central queue hygiene owns Strix cancellation" in workflow - # pull_request_target must not gain a cross-repository mutation - # token; the central scheduler performs cancellation with its own - # explicitly scoped credential. - assert "DISPATCH_REPOSITORY" not in workflow - assert "CLOSED_PR_HEAD_SHA" not in workflow + assert "Cancel queued and running scans for the closed pull request" in workflow + assert "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN" in workflow + assert "DISPATCH_REPOSITORY" in workflow + assert "CLOSED_PR_HEAD_SHA" in workflow + assert '($run_repository == $target and' in workflow + assert '($run_repository == $dispatch and' in workflow + assert "leaving runs unchanged" in workflow assert "actions: write" not in workflow.split(" strix:", 1)[0] else: assert ( From ec8eb981e8db56d0156f80bb8282711398130a6d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 09:15:05 +0900 Subject: [PATCH 12/49] fix(strix): tolerate malformed close-run data --- .github/workflows/strix.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index cbce23120..8cd5ec13b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -143,7 +143,8 @@ jobs: sed 's/^/ /' /tmp/strix-close-gh-error >&2 || true continue fi - jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ + local run_ids + if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" \ --arg dispatch "$DISPATCH_REPOSITORY" --arg run_repository "$run_repository" ' .workflow_runs[] @@ -160,7 +161,11 @@ jobs: ) ) | .id - ' <<<"$runs_json" | while IFS= read -r run_id; do + ' <<<"$runs_json")"; then + echo "::warning::Strix close cleanup received invalid run data for ${run_repository}; leaving runs unchanged." + continue + fi + while IFS= read -r run_id; do [ -n "$run_id" ] || continue if gh api --method POST "repos/${run_repository}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then echo "Cancelled Strix run ${run_id} in ${run_repository} for closed PR #${CLOSED_PR_NUMBER}." @@ -168,7 +173,7 @@ jobs: echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${run_repository}; it may have finished or the credential lacks Actions write access." sed 's/^/ /' /tmp/strix-close-cancel-error >&2 || true fi - done + done <<<"$run_ids" done } From 75849dda90d8efea628e31a1e4433581019cdea1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 09:17:53 +0900 Subject: [PATCH 13/49] docs(strix): record scoped close cleanup --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae9c6613f..d7855e45d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,10 @@ Semantic Versioning where the repository publishes a release. scheduler re-dispatches exact-head evidence after pending-run supersession. Update the bash contract test to match the repository-scoped concurrency group. +- Keep closed-PR Strix cleanup read-only at the GitHub token boundary while + allowing the established scheduler credential to cancel target and central + dispatch runs when configured; scope each event filter to its hosting + repository and leave authorization or malformed-data failures auditable. - Keep `--trust-lockfile` only for pnpm 11.3 and newer (`trustLockfile` landed in pnpm 11.3). pnpm 9, 10, and 11.0–11.2 reject that flag and previously failed LineageWeave JavaScript coverage before From d9557be8d6340e365f2eed38eeff60557da18e47 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 10:47:40 +0900 Subject: [PATCH 14/49] docs(copy): make Strix review guidance actionable --- CHANGELOG.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7855e45d..a5d759eb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,18 +23,12 @@ Semantic Versioning where the repository publishes a release. during materialization and then rejecting every version except pnpm 11.5.3; route generic coverage and docstring package scripts through the same Corepack boundary instead of invoking a removed bare `pnpm` binary. -- Serialize Strix scans per repository and event class to stop shared-provider - key rate-limit storms. Concurrent per-PR scans each retried the shared NVIDIA - NIM key, producing guaranteed `litellm.RateLimitError` failures across open - PRs. GitHub retains one active and one pending run per repository/event group; - `cancel-in-progress: false` keeps the active scan running, while the merge - scheduler re-dispatches exact-head evidence after pending-run supersession. - Update the bash contract test to match the repository-scoped concurrency - group. -- Keep closed-PR Strix cleanup read-only at the GitHub token boundary while - allowing the established scheduler credential to cancel target and central - dispatch runs when configured; scope each event filter to its hosting - repository and leave authorization or malformed-data failures auditable. +- Review scans now run in a controlled order so each pull request receives a + complete result instead of a rate-limit interruption. Open the pull request + after the active scan finishes to review the latest result. +- Closed pull-request cleanup now preserves the review record and reports any + authorization or malformed-data issue for follow-up. Reopen the pull request + or update its credentials when the cleanup message asks you to act. - Keep `--trust-lockfile` only for pnpm 11.3 and newer (`trustLockfile` landed in pnpm 11.3). pnpm 9, 10, and 11.0–11.2 reject that flag and previously failed LineageWeave JavaScript coverage before From 0db456d42d80ab3782af0b272e9cdb0f60cff131 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 01:35:32 +0900 Subject: [PATCH 15/49] fix(strix): remove retired provider fallback --- .github/workflows/strix.yml | 64 +++++++------------ scripts/ci/test_strix_quick_gate.sh | 2 +- .../test_required_workflow_queue_contract.py | 6 +- ...est_strix_nvidia_nim_not_found_fallback.py | 11 ++-- 4 files changed, 33 insertions(+), 50 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8cd5ec13b..958542814 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -114,7 +114,6 @@ jobs: env: GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || '' }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} - DISPATCH_REPOSITORY: ${{ format('{0}/.github', github.repository_owner) }} CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} CURRENT_RUN_ID: ${{ github.run_id }} @@ -131,50 +130,35 @@ jobs: cancel_runs() { local status="$1" - local run_repository - local previous_repository="" - for run_repository in "$TARGET_REPOSITORY" "$DISPATCH_REPOSITORY"; do - [ "$run_repository" != "$previous_repository" ] || continue - previous_repository="$run_repository" - local runs_url="repos/${run_repository}/actions/runs?status=${status}&per_page=100" - local runs_json - if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/strix-close-gh-error)"; then - echo "::warning::Strix close cleanup could not inspect ${run_repository}; leaving runs unchanged." - sed 's/^/ /' /tmp/strix-close-gh-error >&2 || true - continue - fi - local run_ids - if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ - --arg current "$CURRENT_RUN_ID" --arg target "$TARGET_REPOSITORY" \ - --arg dispatch "$DISPATCH_REPOSITORY" --arg run_repository "$run_repository" ' + local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100" + local runs_json + if ! runs_json="$(gh api --paginate "$runs_url" 2>/tmp/strix-close-gh-error)"; then + echo "::warning::Strix close cleanup could not inspect ${TARGET_REPOSITORY}; leaving runs unchanged." + sed 's/^/ /' /tmp/strix-close-gh-error >&2 || true + return 0 + fi + local run_ids + if ! run_ids="$(jq -r --arg pr "$CLOSED_PR_NUMBER" --arg head_sha "$CLOSED_PR_HEAD_SHA" \ + --arg current "$CURRENT_RUN_ID" ' .workflow_runs[] | select((.id | tostring) != $current) | select(.name == "Strix Security Scan") - | select( - ($run_repository == $target and - .event == "pull_request_target" and - (.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr)))) - or - ($run_repository == $dispatch and - .event == "repository_dispatch" and - (((.display_title // "") | startswith(("Strix Security Scan " + $target + "#" + $pr + "@")))) - ) - ) + | select(.event == "pull_request_target") + | select(.head_sha == $head_sha or any(.pull_requests[]?; ((.number | tostring) == $pr))) | .id ' <<<"$runs_json")"; then - echo "::warning::Strix close cleanup received invalid run data for ${run_repository}; leaving runs unchanged." - continue + echo "::warning::Strix close cleanup received invalid run data for ${TARGET_REPOSITORY}; leaving runs unchanged." + return 0 + fi + while IFS= read -r run_id; do + [ -n "$run_id" ] || continue + if gh api --method POST "repos/${TARGET_REPOSITORY}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then + echo "Cancelled Strix run ${run_id} in ${TARGET_REPOSITORY} for closed PR #${CLOSED_PR_NUMBER}." + else + echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${TARGET_REPOSITORY}; it may have finished or the credential lacks Actions write access." + sed 's/^/ /' /tmp/strix-close-cancel-error >&2 || true fi - while IFS= read -r run_id; do - [ -n "$run_id" ] || continue - if gh api --method POST "repos/${run_repository}/actions/runs/${run_id}/cancel" >/dev/null 2>/tmp/strix-close-cancel-error; then - echo "Cancelled Strix run ${run_id} in ${run_repository} for closed PR #${CLOSED_PR_NUMBER}." - else - echo "::warning::Strix close cleanup could not cancel run ${run_id} in ${run_repository}; it may have finished or the credential lacks Actions write access." - sed 's/^/ /' /tmp/strix-close-cancel-error >&2 || true - fi - done <<<"$run_ids" - done + done <<<"$run_ids" } cancel_runs queued @@ -952,7 +936,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openai-direct/gpt-5.4' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b4a17f24a..65189ec81 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -374,7 +374,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openai-direct/gpt-5.4'" "strix workflow falls back from unavailable NVIDIA NIM to compatible direct OpenAI" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 295da8417..eb2c51950 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -361,10 +361,10 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - if filename == "strix.yml": assert "Cancel queued and running scans for the closed pull request" in workflow assert "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN" in workflow - assert "DISPATCH_REPOSITORY" in workflow + assert "DISPATCH_REPOSITORY" not in workflow assert "CLOSED_PR_HEAD_SHA" in workflow - assert '($run_repository == $target and' in workflow - assert '($run_repository == $dispatch and' in workflow + assert 'select(.event == "pull_request_target")' in workflow + assert 'select(.event == "repository_dispatch")' not in workflow assert "leaving runs unchanged" in workflow assert "actions: write" not in workflow.split(" strix:", 1)[0] else: diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 17f0e9a30..55f2cbe82 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -19,9 +19,7 @@ STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" -FREE_NVIDIA_FALLBACK = ( - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" -) +RETIRED_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" @@ -186,8 +184,8 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) - def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: - """Prefer a documented hosted NIM and another NIM before GitHub.""" + def test_workflow_skips_retired_nvidia_fallback(self) -> None: + """Move directly to the compatible fallback after the hosted NIM fails.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") default_expression = ( @@ -202,9 +200,10 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.4'", + "'openai-direct/gpt-5.4'", workflow, ) + self.assertNotIn(RETIRED_NVIDIA_FALLBACK, workflow) default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( From 8f0b82511d33d809c9fa569a0e4bc1193ccd5ed7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:07:44 +0900 Subject: [PATCH 16/49] fix(strix): retain required NVIDIA fallback --- .github/workflows/strix.yml | 2 +- scripts/ci/test_strix_quick_gate.sh | 2 +- tests/test_strix_nvidia_nim_not_found_fallback.py | 11 ++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 958542814..8e7ae1d64 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -936,7 +936,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openai-direct/gpt-5.4' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4' || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 65189ec81..b4a17f24a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -374,7 +374,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'openai-direct/gpt-5.4'" "strix workflow falls back from unavailable NVIDIA NIM to compatible direct OpenAI" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 55f2cbe82..17f0e9a30 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -19,7 +19,9 @@ STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" -RETIRED_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" +FREE_NVIDIA_FALLBACK = ( + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" +) RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" @@ -184,8 +186,8 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) - def test_workflow_skips_retired_nvidia_fallback(self) -> None: - """Move directly to the compatible fallback after the hosted NIM fails.""" + def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: + """Prefer a documented hosted NIM and another NIM before GitHub.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") default_expression = ( @@ -200,10 +202,9 @@ def test_workflow_skips_retired_nvidia_fallback(self) -> None: ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "'openai-direct/gpt-5.4'", + f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.4'", workflow, ) - self.assertNotIn(RETIRED_NVIDIA_FALLBACK, workflow) default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( From 9cc9b192848ff129c302d6da70cf574bff7892cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:18:23 +0900 Subject: [PATCH 17/49] refactor(strix): reuse live NVIDIA model resolver --- scripts/ci/select_nvidia_nim_model.py | 162 +++++++++++++++++++++ tests/test_select_nvidia_nim_model.py | 197 ++++++++++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 scripts/ci/select_nvidia_nim_model.py create mode 100644 tests/test_select_nvidia_nim_model.py diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py new file mode 100644 index 000000000..8133a8411 --- /dev/null +++ b/scripts/ci/select_nvidia_nim_model.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Resolve the first live NVIDIA NIM model from an ordered candidate pool. + +Why this exists +--------------- +The scheduled autofix worker used to hard-code one NVIDIA NIM model id. NVIDIA +retires hosted models on published end-of-life dates, and the endpoint then +answers every request with HTTP 410 ``Gone``, e.g. + + The model 'mistralai/mistral-small-4-119b-2603' has reached its end of life + on 2026-07-27T00:00:00Z and is no longer available. + +A single hard-coded id therefore turns a normal provider lifecycle event into a +total outage of the repair loop. This helper asks the provider which models are +actually served right now (``GET /v1/models``, the OpenAI-compatible catalog +route NVIDIA NIM implements) and returns the first entry of an ordered, +operator-controlled preference list that the provider still serves. + +The helper is deliberately fail-closed: an unreachable catalog, an unparsable +catalog, or a pool with no served candidate is an error, never a silent +fallback to an arbitrary model. + +References: + NVIDIA. (2025). *NVIDIA NIM for large language models: OpenAI-compatible + API reference*. https://docs.nvidia.com/nim/large-language-models/latest/api-reference.html + OpenAI. (2025). *API reference: List models*. + https://platform.openai.com/docs/api-reference/models/list +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import urllib.error +import urllib.request +from urllib.parse import urlsplit + +DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" +ALLOWED_CATALOG_HOSTS = frozenset({"integrate.api.nvidia.com"}) +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +def parse_candidates(raw_candidates: str) -> list[str]: + """Split a whitespace-separated candidate pool into ordered model ids. + + Duplicate ids are removed while the operator's preference order is kept, so + a pool may be assembled from several sources without changing behavior. + """ + ordered: list[str] = [] + for candidate in raw_candidates.split(): + if candidate not in ordered: + ordered.append(candidate) + return ordered + + +def validate_catalog_base_url(base_url: str) -> str: + """Return the catalog base URL after refusing untrusted endpoints. + + Only HTTPS URLs on the known NVIDIA NIM integration host are accepted, so a + tampered variable cannot redirect the API key to another host. + """ + parts = urlsplit(base_url) + if parts.scheme != "https": + raise ValueError(f"NVIDIA NIM base URL must use https; got {parts.scheme or ''}") + if parts.hostname not in ALLOWED_CATALOG_HOSTS: + raise ValueError(f"NVIDIA NIM base URL host is not allowed: {parts.hostname or ''}") + if parts.port not in (None, 443): + raise ValueError(f"NVIDIA NIM base URL must use the default HTTPS port; got {parts.port}") + if parts.username or parts.password: + raise ValueError("NVIDIA NIM base URL must not embed credentials") + return base_url.rstrip("/") + + +def fetch_served_model_ids( + base_url: str, + api_key: str, + *, + timeout_seconds: float = DEFAULT_TIMEOUT_SECONDS, +) -> set[str]: + """Return the model ids the provider currently serves. + + Any transport or payload problem raises, because guessing a model id would + hide a provider outage behind a confusing downstream model error. + """ + request = urllib.request.Request( # noqa: S310 - scheme and host are validated above. + f"{validate_catalog_base_url(base_url)}/models", + headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}, + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: # noqa: S310 + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + raise RuntimeError(f"NVIDIA NIM model catalog request failed with HTTP {error.code}") from error + except urllib.error.URLError as error: + raise RuntimeError("NVIDIA NIM model catalog is unreachable") from error + except json.JSONDecodeError as error: + raise RuntimeError("NVIDIA NIM model catalog returned a non-JSON body") from error + entries = payload.get("data") if isinstance(payload, dict) else None + if not isinstance(entries, list): + raise RuntimeError("NVIDIA NIM model catalog payload has no model list") + served = { + str(entry["id"]) + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("id"), str) and entry["id"] + } + if not served: + raise RuntimeError("NVIDIA NIM model catalog listed no usable model id") + return served + + +def select_model(candidates: list[str], served_model_ids: set[str], *, role: str) -> str: + """Return the first candidate the provider still serves for this role.""" + if not candidates: + raise ValueError(f"no {role} NVIDIA NIM model candidates were configured") + for candidate in candidates: + if candidate in served_model_ids: + return candidate + raise RuntimeError( + f"no configured {role} NVIDIA NIM model candidate is currently served: {' '.join(candidates)}. " + "Add a live model id to the candidate pool variable so the repair worker can run." + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse the command line for the model resolver.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", required=True, help="whitespace-separated ordered model ids") + parser.add_argument("--role", default="primary", help="candidate pool role used in error messages") + parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="NVIDIA NIM OpenAI-compatible base URL") + parser.add_argument( + "--timeout-seconds", + type=float, + default=DEFAULT_TIMEOUT_SECONDS, + help="model catalog request timeout", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Print the resolved model id, or report an actionable failure.""" + args = parse_args(argv) + api_key = os.environ.get("NVIDIA_API_KEY") or os.environ.get("NVIDIA_NIM_API_KEY") or "" + if not api_key: + print( + "::error::NVIDIA_API_KEY is required to resolve a live NVIDIA NIM model.", + file=sys.stderr, + ) + return 1 + try: + served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) + print(select_model(parse_candidates(args.candidates), served, role=args.role)) + except (RuntimeError, ValueError) as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py new file mode 100644 index 000000000..e5b6bbcad --- /dev/null +++ b/tests/test_select_nvidia_nim_model.py @@ -0,0 +1,197 @@ +"""Tests for resolving a live NVIDIA NIM model from an ordered candidate pool.""" + +from __future__ import annotations + +import io +import json +import urllib.error +from typing import Any + +import pytest + +from scripts.ci import select_nvidia_nim_model as resolver + + +class _FakeResponse(io.BytesIO): + """Minimal context-managed HTTP response body for catalog stubs.""" + + def __enter__(self) -> "_FakeResponse": + """Return the response itself, matching urlopen's context manager.""" + return self + + def __exit__(self, *_exc_info: object) -> bool: + """Close the buffer and never suppress an exception.""" + self.close() + return False + + +def _catalog(*model_ids: str) -> bytes: + """Render an OpenAI-compatible model catalog payload for the given ids.""" + return json.dumps({"object": "list", "data": [{"id": model_id} for model_id in model_ids]}).encode("utf-8") + + +def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: + """Serve one canned catalog payload and record the issued requests.""" + requests: list[Any] = [] + + def fake_urlopen(request: Any, timeout: float | None = None) -> _FakeResponse: + requests.append((request, timeout)) + return _FakeResponse(payload) + + monkeypatch.setattr(resolver.urllib.request, "urlopen", fake_urlopen) + return requests + + +def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: + """Operators may concatenate pools; order wins and repeats are dropped.""" + assert resolver.parse_candidates(" a/one\n b/two a/one ") == ["a/one", "b/two"] + assert resolver.parse_candidates(" ") == [] + + +@pytest.mark.parametrize( + ("base_url", "message"), + [ + ("http://integrate.api.nvidia.com/v1", "must use https"), + ("https://models.example.invalid/v1", "host is not allowed"), + ("https://integrate.api.nvidia.com:8443/v1", "default HTTPS port"), + ("https://user:pass@integrate.api.nvidia.com/v1", "must not embed credentials"), + ], +) +def test_validate_catalog_base_url_refuses_untrusted_endpoints(base_url: str, message: str) -> None: + """A tampered base URL must never receive the provider API key.""" + with pytest.raises(ValueError, match=message): + resolver.validate_catalog_base_url(base_url) + + +def test_validate_catalog_base_url_normalizes_the_trusted_endpoint() -> None: + """The trusted endpoint is accepted with any trailing slash removed.""" + assert resolver.validate_catalog_base_url(f"{resolver.DEFAULT_BASE_URL}/") == resolver.DEFAULT_BASE_URL + + +def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.MonkeyPatch) -> None: + """The resolver reads ids from the provider's OpenAI-compatible catalog.""" + requests = _stub_catalog(monkeypatch, _catalog("a/one", "b/two")) + + served = resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key", timeout_seconds=7.0) + + assert served == {"a/one", "b/two"} + request, timeout = requests[0] + assert request.full_url == f"{resolver.DEFAULT_BASE_URL}/models" + assert request.get_header("Authorization") == "Bearer secret-key" + assert timeout == 7.0 + + +def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.MonkeyPatch) -> None: + """Entries without a usable string id cannot become selectable models.""" + payload = json.dumps({"data": [{"id": ""}, {"id": 7}, "not-an-object", {"id": "a/one"}]}).encode("utf-8") + _stub_catalog(monkeypatch, payload) + + assert resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") == {"a/one"} + + +@pytest.mark.parametrize( + ("error", "message"), + [ + ( + urllib.error.HTTPError(url="https://x.invalid", code=401, msg="no", hdrs=None, fp=None), + "HTTP 401", + ), + (urllib.error.URLError("dns"), "unreachable"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_transport_errors( + monkeypatch: pytest.MonkeyPatch, error: Exception, message: str +) -> None: + """A catalog outage is reported, never masked by guessing a model id.""" + + def fake_urlopen(_request: Any, timeout: float | None = None) -> _FakeResponse: + raise error + + monkeypatch.setattr(resolver.urllib.request, "urlopen", fake_urlopen) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + (b"maintenance", "non-JSON body"), + (b'{"object": "list"}', "no model list"), + (b'{"data": []}', "no usable model id"), + ], +) +def test_fetch_served_model_ids_fails_closed_on_unusable_payloads( + monkeypatch: pytest.MonkeyPatch, payload: bytes, message: str +) -> None: + """Unparsable or empty catalogs are errors rather than silent fallbacks.""" + _stub_catalog(monkeypatch, payload) + + with pytest.raises(RuntimeError, match=message): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + +def test_select_model_prefers_the_first_served_candidate() -> None: + """A retired first choice transparently falls through to the next live one.""" + candidates = ["retired/model", "live/model", "other/model"] + + assert resolver.select_model(candidates, {"live/model", "other/model"}, role="primary") == "live/model" + + +def test_select_model_requires_a_configured_pool() -> None: + """An empty pool is a configuration error with the role named.""" + with pytest.raises(ValueError, match="no small NVIDIA NIM model candidates"): + resolver.select_model([], {"live/model"}, role="small") + + +def test_select_model_reports_a_fully_retired_pool() -> None: + """When no candidate is served, the message tells the operator what to do.""" + with pytest.raises(RuntimeError, match="Add a live model id to the candidate pool"): + resolver.select_model(["retired/model"], {"live/model"}, role="primary") + + +def test_main_prints_the_resolved_model_id( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """The successful path prints exactly the resolved id for shell capture.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + exit_code = resolver.main(["--role", "primary", "--candidates", "retired/model live/model"]) + + assert exit_code == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_accepts_the_workflow_secret_name( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Either credential variable name works, so callers need no shim.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.setenv("NVIDIA_NIM_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "live/model"]) == 0 + assert capsys.readouterr().out == "live/model\n" + + +def test_main_requires_a_provider_credential( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Without a credential the resolver fails closed with a CI annotation.""" + monkeypatch.delenv("NVIDIA_API_KEY", raising=False) + monkeypatch.delenv("NVIDIA_NIM_API_KEY", raising=False) + + assert resolver.main(["--candidates", "live/model"]) == 1 + assert "NVIDIA_API_KEY is required" in capsys.readouterr().err + + +def test_main_annotates_a_resolution_failure( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Resolution failures surface as GitHub error annotations, not tracebacks.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", "retired/model"]) == 1 + assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err From 6f7dcc4b99b37ec72ccd6a12ecf491d293ae0c57 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 00:02:21 +0900 Subject: [PATCH 18/49] docs(tests): complete live model resolver coverage --- tests/test_select_nvidia_nim_model.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index e5b6bbcad..697740304 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -35,6 +35,7 @@ def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: requests: list[Any] = [] def fake_urlopen(request: Any, timeout: float | None = None) -> _FakeResponse: + """Return the canned catalog and record request security metadata.""" requests.append((request, timeout)) return _FakeResponse(payload) @@ -105,6 +106,7 @@ def test_fetch_served_model_ids_fails_closed_on_transport_errors( """A catalog outage is reported, never masked by guessing a model id.""" def fake_urlopen(_request: Any, timeout: float | None = None) -> _FakeResponse: + """Raise the configured provider failure from the HTTP boundary.""" raise error monkeypatch.setattr(resolver.urllib.request, "urlopen", fake_urlopen) From 46d3a3608f7f4d8069dcb07a60e59b96af686f2d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:16:45 +0900 Subject: [PATCH 19/49] fix(nim): use validated HTTPS connection for catalog lookup --- scripts/ci/select_nvidia_nim_model.py | 36 ++++++--- tests/test_select_nvidia_nim_model.py | 111 +++++++++++++++++++++----- 2 files changed, 118 insertions(+), 29 deletions(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 8133a8411..2eb7e8e7b 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -30,11 +30,10 @@ from __future__ import annotations import argparse +import http.client import json import os import sys -import urllib.error -import urllib.request from urllib.parse import urlsplit DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" @@ -70,6 +69,8 @@ def validate_catalog_base_url(base_url: str) -> str: raise ValueError(f"NVIDIA NIM base URL must use the default HTTPS port; got {parts.port}") if parts.username or parts.password: raise ValueError("NVIDIA NIM base URL must not embed credentials") + if parts.query or parts.fragment: + raise ValueError("NVIDIA NIM base URL must not include a query or fragment") return base_url.rstrip("/") @@ -84,17 +85,30 @@ def fetch_served_model_ids( Any transport or payload problem raises, because guessing a model id would hide a provider outage behind a confusing downstream model error. """ - request = urllib.request.Request( # noqa: S310 - scheme and host are validated above. - f"{validate_catalog_base_url(base_url)}/models", - headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"}, - method="GET", - ) + normalized_base_url = validate_catalog_base_url(base_url) + parts = urlsplit(normalized_base_url) + request_path = f"{parts.path.rstrip('/')}/models" try: - with urllib.request.urlopen(request, timeout=timeout_seconds) as response: # noqa: S310 + with http.client.HTTPSConnection( + parts.hostname, parts.port or 443, timeout=timeout_seconds + ) as connection: + connection.request( + "GET", + request_path, + headers={ + "Authorization": f"Bearer {api_key}", + "Accept": "application/json", + }, + ) + response = connection.getresponse() + if response.status >= 400: + raise RuntimeError( + f"NVIDIA NIM model catalog request failed with HTTP {response.status}" + ) payload = json.loads(response.read().decode("utf-8")) - except urllib.error.HTTPError as error: - raise RuntimeError(f"NVIDIA NIM model catalog request failed with HTTP {error.code}") from error - except urllib.error.URLError as error: + except RuntimeError: + raise + except (OSError, http.client.HTTPException) as error: raise RuntimeError("NVIDIA NIM model catalog is unreachable") from error except json.JSONDecodeError as error: raise RuntimeError("NVIDIA NIM model catalog returned a non-JSON body") from error diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 697740304..6af1163d2 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -3,8 +3,8 @@ from __future__ import annotations import io +import http.client import json -import urllib.error from typing import Any import pytest @@ -15,6 +15,8 @@ class _FakeResponse(io.BytesIO): """Minimal context-managed HTTP response body for catalog stubs.""" + status = 200 + def __enter__(self) -> "_FakeResponse": """Return the response itself, matching urlopen's context manager.""" return self @@ -25,6 +27,42 @@ def __exit__(self, *_exc_info: object) -> bool: return False +class _FakeConnection: + """Minimal HTTPS connection stub for catalog requests.""" + + def __init__( + self, + host: str, + port: int, + *, + timeout: float, + response: _FakeResponse, + requests: list[Any], + ) -> None: + """Record the validated destination and canned response.""" + self.host = host + self.port = port + self.timeout = timeout + self.response = response + self.requests = requests + + def __enter__(self) -> "_FakeConnection": + """Return the connection itself for the context manager boundary.""" + return self + + def __exit__(self, *_exc_info: object) -> bool: + """Never suppress request failures.""" + return False + + def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: + """Record one outbound request without opening a network socket.""" + self.requests.append((self, method, path, headers)) + + def getresponse(self) -> _FakeResponse: + """Return the canned provider response.""" + return self.response + + def _catalog(*model_ids: str) -> bytes: """Render an OpenAI-compatible model catalog payload for the given ids.""" return json.dumps({"object": "list", "data": [{"id": model_id} for model_id in model_ids]}).encode("utf-8") @@ -34,12 +72,19 @@ def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: """Serve one canned catalog payload and record the issued requests.""" requests: list[Any] = [] - def fake_urlopen(request: Any, timeout: float | None = None) -> _FakeResponse: - """Return the canned catalog and record request security metadata.""" - requests.append((request, timeout)) - return _FakeResponse(payload) - - monkeypatch.setattr(resolver.urllib.request, "urlopen", fake_urlopen) + def fake_connection( + host: str, port: int, *, timeout: float + ) -> _FakeConnection: + """Return a canned HTTPS connection and record its destination.""" + return _FakeConnection( + host, + port, + timeout=timeout, + response=_FakeResponse(payload), + requests=requests, + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) return requests @@ -56,6 +101,8 @@ def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: ("https://models.example.invalid/v1", "host is not allowed"), ("https://integrate.api.nvidia.com:8443/v1", "default HTTPS port"), ("https://user:pass@integrate.api.nvidia.com/v1", "must not embed credentials"), + ("https://integrate.api.nvidia.com/v1?mode=models", "query or fragment"), + ("https://integrate.api.nvidia.com/v1#models", "query or fragment"), ], ) def test_validate_catalog_base_url_refuses_untrusted_endpoints(base_url: str, message: str) -> None: @@ -76,10 +123,13 @@ def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.Mon served = resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key", timeout_seconds=7.0) assert served == {"a/one", "b/two"} - request, timeout = requests[0] - assert request.full_url == f"{resolver.DEFAULT_BASE_URL}/models" - assert request.get_header("Authorization") == "Bearer secret-key" - assert timeout == 7.0 + connection, method, path, headers = requests[0] + assert connection.host == "integrate.api.nvidia.com" + assert connection.port == 443 + assert connection.timeout == 7.0 + assert method == "GET" + assert path == "/v1/models" + assert headers["Authorization"] == "Bearer secret-key" def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.MonkeyPatch) -> None: @@ -93,11 +143,8 @@ def test_fetch_served_model_ids_ignores_malformed_entries(monkeypatch: pytest.Mo @pytest.mark.parametrize( ("error", "message"), [ - ( - urllib.error.HTTPError(url="https://x.invalid", code=401, msg="no", hdrs=None, fp=None), - "HTTP 401", - ), - (urllib.error.URLError("dns"), "unreachable"), + (http.client.RemoteDisconnected("closed"), "unreachable"), + (OSError("dns"), "unreachable"), ], ) def test_fetch_served_model_ids_fails_closed_on_transport_errors( @@ -105,16 +152,44 @@ def test_fetch_served_model_ids_fails_closed_on_transport_errors( ) -> None: """A catalog outage is reported, never masked by guessing a model id.""" - def fake_urlopen(_request: Any, timeout: float | None = None) -> _FakeResponse: + def fake_connection( + _host: str, _port: int, *, timeout: float + ) -> _FakeConnection: """Raise the configured provider failure from the HTTP boundary.""" + del timeout raise error - monkeypatch.setattr(resolver.urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) with pytest.raises(RuntimeError, match=message): resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") +def test_fetch_served_model_ids_reports_http_status( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Provider HTTP failures identify the status without exposing credentials.""" + response = _FakeResponse(b"{}") + response.status = 401 + + def fake_connection( + _host: str, _port: int, *, timeout: float + ) -> _FakeConnection: + """Return an unauthorized provider response.""" + return _FakeConnection( + "integrate.api.nvidia.com", + 443, + timeout=timeout, + response=response, + requests=[], + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + with pytest.raises(RuntimeError, match="HTTP 401"): + resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") + + @pytest.mark.parametrize( ("payload", "message"), [ From f652c27027249e223b26ec28dbfe53cde9f57c31 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 04:06:25 +0900 Subject: [PATCH 20/49] fix(security): enforce TLS verification for NIM catalog --- scripts/ci/select_nvidia_nim_model.py | 6 +++++- tests/test_select_nvidia_nim_model.py | 14 +++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 2eb7e8e7b..9b2ca26c2 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -33,6 +33,7 @@ import http.client import json import os +import ssl import sys from urllib.parse import urlsplit @@ -90,7 +91,10 @@ def fetch_served_model_ids( request_path = f"{parts.path.rstrip('/')}/models" try: with http.client.HTTPSConnection( - parts.hostname, parts.port or 443, timeout=timeout_seconds + parts.hostname, + parts.port or 443, + timeout=timeout_seconds, + context=ssl.create_default_context(), ) as connection: connection.request( "GET", diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 6af1163d2..72a2c3071 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -5,6 +5,7 @@ import io import http.client import json +import ssl from typing import Any import pytest @@ -36,6 +37,7 @@ def __init__( port: int, *, timeout: float, + context: ssl.SSLContext, response: _FakeResponse, requests: list[Any], ) -> None: @@ -43,6 +45,7 @@ def __init__( self.host = host self.port = port self.timeout = timeout + self.context = context self.response = response self.requests = requests @@ -73,13 +76,14 @@ def _stub_catalog(monkeypatch: pytest.MonkeyPatch, payload: bytes) -> list[Any]: requests: list[Any] = [] def fake_connection( - host: str, port: int, *, timeout: float + host: str, port: int, *, timeout: float, context: ssl.SSLContext ) -> _FakeConnection: """Return a canned HTTPS connection and record its destination.""" return _FakeConnection( host, port, timeout=timeout, + context=context, response=_FakeResponse(payload), requests=requests, ) @@ -127,6 +131,8 @@ def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.Mon assert connection.host == "integrate.api.nvidia.com" assert connection.port == 443 assert connection.timeout == 7.0 + assert connection.context.verify_mode == ssl.CERT_REQUIRED + assert connection.context.check_hostname is True assert method == "GET" assert path == "/v1/models" assert headers["Authorization"] == "Bearer secret-key" @@ -153,10 +159,11 @@ def test_fetch_served_model_ids_fails_closed_on_transport_errors( """A catalog outage is reported, never masked by guessing a model id.""" def fake_connection( - _host: str, _port: int, *, timeout: float + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext ) -> _FakeConnection: """Raise the configured provider failure from the HTTP boundary.""" del timeout + del context raise error monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) @@ -173,13 +180,14 @@ def test_fetch_served_model_ids_reports_http_status( response.status = 401 def fake_connection( - _host: str, _port: int, *, timeout: float + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext ) -> _FakeConnection: """Return an unauthorized provider response.""" return _FakeConnection( "integrate.api.nvidia.com", 443, timeout=timeout, + context=context, response=response, requests=[], ) From 3711c403f3f70a81c0f6bdadbf2f0d3ad70b16e1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 13:05:37 -0700 Subject: [PATCH 21/49] fix(autofix): close NIM catalog connection explicitly --- scripts/ci/select_nvidia_nim_model.py | 7 +++++-- tests/test_select_nvidia_nim_model.py | 14 ++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 9b2ca26c2..763f51bbc 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -90,12 +90,13 @@ def fetch_served_model_ids( parts = urlsplit(normalized_base_url) request_path = f"{parts.path.rstrip('/')}/models" try: - with http.client.HTTPSConnection( + connection = http.client.HTTPSConnection( parts.hostname, parts.port or 443, timeout=timeout_seconds, context=ssl.create_default_context(), - ) as connection: + ) + try: connection.request( "GET", request_path, @@ -110,6 +111,8 @@ def fetch_served_model_ids( f"NVIDIA NIM model catalog request failed with HTTP {response.status}" ) payload = json.loads(response.read().decode("utf-8")) + finally: + connection.close() except RuntimeError: raise except (OSError, http.client.HTTPException) as error: diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 72a2c3071..e7f2ec029 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -29,7 +29,7 @@ def __exit__(self, *_exc_info: object) -> bool: class _FakeConnection: - """Minimal HTTPS connection stub for catalog requests.""" + """Minimal non-context-managed HTTPS connection stub for catalog requests.""" def __init__( self, @@ -48,14 +48,11 @@ def __init__( self.context = context self.response = response self.requests = requests + self.closed = False - def __enter__(self) -> "_FakeConnection": - """Return the connection itself for the context manager boundary.""" - return self - - def __exit__(self, *_exc_info: object) -> bool: - """Never suppress request failures.""" - return False + def close(self) -> None: + """Record explicit cleanup, matching ``HTTPSConnection.close``.""" + self.closed = True def request(self, method: str, path: str, *, headers: dict[str, str]) -> None: """Record one outbound request without opening a network socket.""" @@ -133,6 +130,7 @@ def test_fetch_served_model_ids_returns_the_live_catalog(monkeypatch: pytest.Mon assert connection.timeout == 7.0 assert connection.context.verify_mode == ssl.CERT_REQUIRED assert connection.context.check_hostname is True + assert connection.closed is True assert method == "GET" assert path == "/v1/models" assert headers["Authorization"] == "Bearer secret-key" From cc067f21da53e4b6b120153a3e356cea84e48699 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:18:24 +0900 Subject: [PATCH 22/49] fix(nim): retain reviewed TLS catalog contract --- scripts/ci/select_nvidia_nim_model.py | 2 +- tests/test_select_nvidia_nim_model.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 763f51bbc..d68eb2d4e 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -90,7 +90,7 @@ def fetch_served_model_ids( parts = urlsplit(normalized_base_url) request_path = f"{parts.path.rstrip('/')}/models" try: - connection = http.client.HTTPSConnection( + connection = http.client.HTTPSConnection( # nosemgrep: python.lang.security.audit.httpsconnection-detected.httpsconnection-detected parts.hostname, parts.port or 443, timeout=timeout_seconds, diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index e7f2ec029..4d44d7abc 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -5,6 +5,7 @@ import io import http.client import json +from pathlib import Path import ssl from typing import Any @@ -89,6 +90,20 @@ def fake_connection( return requests +def test_catalog_sink_has_one_scoped_semgrep_exception_and_explicit_tls() -> None: + """Keep the reviewed HTTPS sink suppressed only for its known false positive.""" + source_text = Path(resolver.__file__).read_text(encoding="utf-8") + rule = "python.lang.security.audit.httpsconnection-detected.httpsconnection-detected" + sink_lines = [ + line for line in source_text.splitlines() if "http.client.HTTPSConnection(" in line + ] + + assert len(sink_lines) == 1 + assert f"# nosemgrep: {rule}" in sink_lines[0] + assert source_text.count(f"# nosemgrep: {rule}") == 1 + assert "context=ssl.create_default_context()" in source_text + + def test_parse_candidates_keeps_preference_order_without_duplicates() -> None: """Operators may concatenate pools; order wins and repeats are dropped.""" assert resolver.parse_candidates(" a/one\n b/two a/one ") == ["a/one", "b/two"] From 641eef04c731eb5481ccae3207421a9b9b31476d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:20:28 +0900 Subject: [PATCH 23/49] fix(strix): cancel closed PR runs without optional secret --- .github/workflows/strix.yml | 15 +++++---------- tests/test_required_workflow_queue_contract.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8e7ae1d64..adeed8498 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -105,14 +105,14 @@ jobs: cancel-closed-pr-runs: if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest - # The built-in token is intentionally read-only. The established scheduler - # credential is optional here and is used only for cancellation requests. - # When it is unavailable, the close event remains green and the next - # central queue sweep records the unresolved run for operator action. + # Prefer the established scheduler credential, but let the close event use + # its job-scoped token so abandoned scans are cancelled even when that + # optional secret is unavailable. This job never checks out PR code. permissions: + actions: write contents: read env: - GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || '' }} + GH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} TARGET_REPOSITORY: ${{ github.event.pull_request.base.repo.full_name || github.repository }} CLOSED_PR_NUMBER: ${{ github.event.pull_request.number }} CLOSED_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} @@ -123,11 +123,6 @@ jobs: run: | set -euo pipefail - if [ -z "${GH_TOKEN:-}" ]; then - echo "::warning::Strix close cleanup skipped: no scheduler mutation credential is configured." - exit 0 - fi - cancel_runs() { local status="$1" local runs_url="repos/${TARGET_REPOSITORY}/actions/runs?status=${status}&per_page=100" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index eb2c51950..6664303c2 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -360,13 +360,21 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert "cancel-closed-pr-runs:" in workflow if filename == "strix.yml": assert "Cancel queued and running scans for the closed pull request" in workflow - assert "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN" in workflow + assert ( + "secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN " + "|| github.token" + ) in workflow assert "DISPATCH_REPOSITORY" not in workflow assert "CLOSED_PR_HEAD_SHA" in workflow assert 'select(.event == "pull_request_target")' in workflow assert 'select(.event == "repository_dispatch")' not in workflow assert "leaving runs unchanged" in workflow - assert "actions: write" not in workflow.split(" strix:", 1)[0] + cleanup_job = workflow.split(" cancel-closed-pr-runs:", 1)[1].split( + " strix:", 1 + )[0] + assert "actions: write" in cleanup_job + assert "actions/checkout" not in cleanup_job + assert "cleanup skipped" not in cleanup_job else: assert ( "PR closed; this run only cancels older runs through workflow concurrency." From a91320896035d5156a9057466dc6b8008a1543c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:32:27 +0900 Subject: [PATCH 24/49] fix(strix): resolve live NVIDIA NIM models --- .github/workflows/strix.yml | 39 +++++++++++++++---- .../strix-nvidia-nim-not-found-fallback.md | 39 ++++++++++--------- ...opencode_failed_check_fallback_findings.sh | 2 +- scripts/ci/strix_required_workflow_smoke.sh | 7 +--- scripts/ci/test_strix_quick_gate.sh | 8 ++-- .../test_required_workflow_queue_contract.py | 25 ++++++------ ...est_strix_nvidia_nim_not_found_fallback.py | 24 ++++++------ 7 files changed, 85 insertions(+), 59 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8e7ae1d64..c5e43cb13 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -546,10 +546,36 @@ jobs: printf 'Materialized central Strix dependency lock from same-repository PR head.\n' fi + - name: Resolve live NVIDIA NIM Strix models + id: resolve_nvidia_models + env: + STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} + TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} + NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} + STRIX_NVIDIA_PRIMARY_CANDIDATES: >- + ${{ vars.STRIX_NVIDIA_PRIMARY_CANDIDATES || + 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + STRIX_NVIDIA_FALLBACK_CANDIDATES: >- + ${{ vars.STRIX_NVIDIA_FALLBACK_CANDIDATES || + 'nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + run: | + set -euo pipefail + if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then + printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + resolver="$TRUSTED_STRIX_SOURCE/scripts/ci/select_nvidia_nim_model.py" + primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_PRIMARY_CANDIDATES")" + fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES")" + { + printf 'primary=nvidia_nim/%s\n' "$primary" + printf 'fallback=nvidia_nim/%s\n' "$fallback" + } >> "$GITHUB_OUTPUT" + - name: Gate Strix secrets id: gate env: - STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4') }} + STRIX_MODEL: ${{ github.event.client_payload.strix_llm || (steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4') }} STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} STRIX_OPENAI_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} STRIX_OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} @@ -559,9 +585,6 @@ jobs: TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then - strix_model="gpt-5.4" - fi echo "strix_model=$strix_model" >> "$GITHUB_OUTPUT" case "$strix_model" in openai/gpt-5-mini* | openai/gpt-5-nano* | \ @@ -603,7 +626,8 @@ jobs: exit 1 fi ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + nvidia_nim/nvidia/nemotron-3-super-120b-a12b | \ + nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1) if [ "$TARGET_REPOSITORY_PRIVATE" != "false" ]; then echo '::error::NVIDIA NIM hosted trial scans are limited to public repositories.' exit 1 @@ -892,7 +916,8 @@ jobs: openrouter/free | openrouter/openrouter/free) printf '%s' 'openrouter/free' > "$strix_llm_file" ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + nvidia_nim/nvidia/nemotron-3-super-120b-a12b | \ + nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1) printf '%s' "$strix_model" > "$strix_llm_file" ;; vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash) @@ -936,7 +961,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4' || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index 213429e01..b053ac0a8 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -5,14 +5,16 @@ Strix treats an authenticated NVIDIA NIM model-catalog `404 Not Found` as provider availability evidence, not as a target-application vulnerability. The gate does not retry the same unavailable model. It proceeds to a distinct -reviewed NVIDIA hosted model and only then to the existing GitHub Models -candidates. +reviewed NVIDIA hosted model and only then to the direct OpenAI fallback. -Public-repository scans now default to -`nvidia/nemotron-3-super-120b-a12b`. The first fallback is -`nvidia/llama-3.3-nemotron-super-49b-v1.5`. Private repositories retain the -contracted provider because NVIDIA hosted trial inputs are restricted to public -repositories by the central workflow. +For public-repository scans, the trusted workflow queries NVIDIA's authenticated +`/v1/models` catalog and selects the first served entry from reviewed primary +and fallback pools. The default pool prefers +`nvidia/nemotron-3-super-120b-a12b`; the distinct fallback is +`nvidia/llama-3.1-nemotron-ultra-253b-v1`. The retired +`nvidia/llama-3.3-nemotron-super-49b-v1.5` is no longer executable workflow +configuration. Private repositories retain the contracted provider because +NVIDIA hosted trial inputs are restricted to public repositories. ## Trust boundary @@ -48,8 +50,8 @@ Regression evidence proves that: 4. a provider-like source literal on one line without LiteLLM `NotFoundError` context is not recognized; 5. model-catalog 404s enter cross-model fallback but never same-model retry; -6. the primary and first fallback are current NVIDIA hosted models; -7. GitHub Models remain later cross-provider fallbacks; +6. the primary and first fallback are present in the live NVIDIA catalog; +7. direct OpenAI remains the later cross-provider fallback; 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and @@ -57,12 +59,11 @@ Regression evidence proves that: ## Limitations -Hosted model catalogs may change independently of this repository. A model-card -page or supported self-hosted NIM container does not guarantee indefinite hosted -trial availability. The ordered model plan must therefore be reviewed against -current NVIDIA documentation whenever a provider returns a catalog 404. This -change does not treat arbitrary provider errors as success and does not weaken -Strix severity, changed-file attribution, or independent approval requirements. +Hosted model catalogs and capacity may change independently of this repository. +Catalog membership prevents deterministic retired-model selection but does not +prove capacity, so HTTP 429 exhaustion remains fail-closed if every fallback is +unavailable. This change does not treat provider errors as success and does not +weaken Strix severity, changed-file attribution, or approval requirements. ## Current fallback contract (2026-08-25) @@ -78,12 +79,12 @@ focused `test_strix_quick_gate.sh` case; provider failures remain non-passing. Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 -NVIDIA Corporation. (2025). *Llama-3.3-Nemotron-Super-49B-v1.5* [Model card]. -NVIDIA NIM. https://build.nvidia.com/nvidia/llama-3_3-nemotron-super-49b-v1_5/modelcard +NVIDIA Corporation. (2026a). *Models*. NVIDIA NIM. +https://build.nvidia.com/models -NVIDIA Corporation. (2026a). *NVIDIA-Nemotron-3-Super-120B-A12B* [Model +NVIDIA Corporation. (2026b). *NVIDIA-Nemotron-3-Super-120B-A12B* [Model card]. NVIDIA NIM. https://build.nvidia.com/nvidia/nemotron-3-super-120b-a12b/modelcard -NVIDIA Corporation. (2026b). *Configuration reference*. NVIDIA AI-Q Blueprint. +NVIDIA Corporation. (2026c). *Configuration reference*. NVIDIA AI-Q Blueprint. https://docs.nvidia.com/aiq-blueprint/2.2.0-rc1/customization/configuration-reference.html diff --git a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh index 5637cb861..6ea3706df 100755 --- a/scripts/ci/emit_opencode_failed_check_fallback_findings.sh +++ b/scripts/ci/emit_opencode_failed_check_fallback_findings.sh @@ -956,7 +956,7 @@ extract_strix_failed_check_block "$EVIDENCE_FILE" "$strix_evidence_file" emit_known_missing_string_finding \ "$EVIDENCE_FILE" \ - "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4'" \ + "steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" \ "Strix public scans must default to NVIDIA NIM while private scans retain the contracted provider" \ ".github/workflows/strix.yml" \ "scripts/ci/test_strix_quick_gate.sh" diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 76aec7910..3aaf0eb22 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -168,11 +168,8 @@ assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disa assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" -assert_file_contains_either \ - "$workflow_file" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai_direct/gpt-5.4" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4" \ - "Strix tries another NVIDIA hosted model before falling back to direct OpenAI" +assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" +assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" assert_file_not_contains "$workflow_file" "github_models/openai/o3" "Strix fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b4a17f24a..48dfcb95b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -306,14 +306,15 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" "STRIX_TOTAL_TIMEOUT_SECONDS:" "strix workflow must not expose total timeout env names in GitHub logs" assert_file_not_contains "$workflow_file" "STRIX_PR_SCOPE_MAX_FILES_PER_BATCH" "strix workflow must not split Strix PR evidence into separate scanner runs" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM == 'vertex_ai/gemini-3.1-pro-preview-customtools' && 'vertex_ai/gemini-2.5-flash'" "strix workflow must not quarantine the approved Vertex preview model after organization secret visibility is fixed" - assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' || 'gpt-5.4'" "strix workflow defaults public scans to NVIDIA NIM and keeps private scans on the contracted provider" + assert_file_contains "$workflow_file" "Resolve live NVIDIA NIM Strix models" "strix workflow resolves currently served NVIDIA models for public scans" + assert_file_contains "$workflow_file" "steps.target_visibility.outputs.is_private == 'false' && steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" "strix workflow uses the resolved public model and keeps private scans on the contracted provider" assert_file_contains "$workflow_file" "EVENT_REPOSITORY_VISIBILITY:" "strix workflow uses trusted event visibility before cross-repository API lookup" assert_file_contains "$workflow_file" "PUBLIC | public) is_private=false" "strix workflow accepts GitHub's lowercase public visibility" assert_file_contains "$workflow_file" "PRIVATE | private | INTERNAL | internal) is_private=true" "strix workflow keeps private and internal repositories off public-only providers" assert_file_contains "$workflow_file" '(.visibility // "" | ascii_downcase) as $visibility' "strix dispatch visibility maps the authoritative API visibility instead of the lossy private boolean" assert_file_not_contains "$workflow_file" "gh api \"repos/\${TARGET_REPOSITORY}\" --jq '.private'" "strix dispatch visibility does not misclassify internal repositories through the private boolean" assert_file_contains "$REPO_ROOT/tests/test_strix_repository_visibility_contract.py" "test_dispatch_api_visibility_preserves_internal_privacy" "strix visibility contract executes public, private, and internal dispatch fixtures" - assert_file_contains "$workflow_file" 'if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]' "strix workflow falls back to the contracted provider when the NVIDIA secret is absent" + assert_file_contains "$workflow_file" '[ -z "${NVIDIA_API_KEY:-}" ]' "strix workflow leaves model resolution empty when the NVIDIA secret is absent" assert_file_contains "$workflow_file" 'STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}' "strix workflow propagates the gate-selected fallback model to the scanner" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM ||" "strix workflow must not let the legacy STRIX_LLM secret override PR defaults" assert_file_contains "$workflow_file" "STRIX_LLM must select NVIDIA NIM Nemotron, GitHub Models openai/gpt-5 or newer, direct OpenAI GPT-5.4 or newer, OpenRouter openrouter/free, or an approved organization Vertex AI model" "strix workflow rejects unsupported model inputs" @@ -374,7 +375,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.4'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved fallback" + assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "strix workflow routes direct-OpenAI fallbacks through a trusted API base file" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index eb2c51950..654487f1f 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -503,38 +503,39 @@ def test_noema_review_credentials_and_llm_configuration_fail_closed() -> None: def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret( tmp_path: Path, ) -> None: - """Preserve configured fallback models while rejecting an unavailable NIM secret.""" + """Leave NIM outputs empty so the workflow expression selects OpenAI.""" strix_output = tmp_path / "strix-output" strix = subprocess.run( [ "bash", "-c", textwrap.dedent( - workflow_step(workflow_text("strix.yml"), "Gate Strix secrets") + workflow_step( + workflow_text("strix.yml"), + "Resolve live NVIDIA NIM Strix models", + ) .split(" run: |\n", 1)[1] ), ], env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "nvidia_nim/nvidia/nemotron-3-super-120b-a12b", "STRIX_MODEL_REQUESTED": "", - "STRIX_OPENAI_API_KEY": "synthetic-openai-key", - "STRIX_OPENROUTER_API_KEY": "", - "STRIX_NVIDIA_NIM_API_KEY": "", - "STRIX_VERTEX_CREDENTIALS": "", - "STRIX_GITHUB_MODELS_TOKEN": "synthetic-models-token", + "NVIDIA_API_KEY": "", "TARGET_REPOSITORY_PRIVATE": "false", + "STRIX_NVIDIA_PRIMARY_CANDIDATES": "nvidia/primary", + "STRIX_NVIDIA_FALLBACK_CANDIDATES": "nvidia/fallback", }, capture_output=True, text=True, check=False, ) assert strix.returncode == 0, strix.stderr - assert { - "provider_mode=openai_direct", - "strix_model=gpt-5.4", - } <= set(strix_output.read_text().splitlines()) + assert {"primary=", "fallback="} <= set(strix_output.read_text().splitlines()) + assert ( + "steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" + in workflow_text("strix.yml") + ) assert ( "STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}" in workflow_text("strix.yml") diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 17f0e9a30..13e5c8882 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -19,9 +19,8 @@ STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" -FREE_NVIDIA_FALLBACK = ( - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" -) +LIVE_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1" +RETIRED_NVIDIA_FALLBACK = "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" RETIRED_PRIMARY_MODEL = "nvidia_nim/nvidia/nemotron-3-ultra-550b-a55b" @@ -186,25 +185,26 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) - def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: - """Prefer a documented hosted NIM and another NIM before GitHub.""" + def test_workflow_resolves_live_nvidia_models(self) -> None: + """Resolve reviewed hosted NIM candidates instead of pinning retired ids.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("Resolve live NVIDIA NIM Strix models", workflow) + self.assertIn("scripts/ci/select_nvidia_nim_model.py", workflow) + self.assertIn("steps.resolve_nvidia_models.outputs.primary", workflow) + self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) default_expression = ( "steps.target_visibility.outputs.is_private == 'false' && " - f"'{DEFAULT_NVIDIA_MODEL}' || 'gpt-5.4'" + "steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" ) self.assertIn(default_expression, workflow) - self.assertIn( - f'[ "$strix_model" = "{DEFAULT_NVIDIA_MODEL}" ] ' - '&& [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]', - workflow, - ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.4'", + "format('{0} openai-direct/gpt-5.4', " + "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) + self.assertNotIn(RETIRED_NVIDIA_FALLBACK, workflow) default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( From b3b1ac90727fcb814d4fbb48da0c9244133e4ea3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:34:48 +0900 Subject: [PATCH 25/49] fix(strix): isolate protected branch scan queues --- .github/workflows/strix.yml | 14 +++++++++----- scripts/ci/test_strix_quick_gate.sh | 1 + tests/test_required_workflow_queue_contract.py | 4 ++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4f1e2f928..90746a6fb 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -81,16 +81,20 @@ concurrency: # the shared NVIDIA NIM key up to three times, producing guaranteed # litellm.RateLimitError storms and fail-closed gate failures across every # open PR (observed 2026-08-23/24). Serializing per repository and event - # class keeps at most one provider-backed scan in flight per class. GitHub's native - # concurrency contract retains one active and one pending run; the scheduler - # re-dispatches the exact current head after pending-run supersession, and - # accuracy is prioritized over scan latency. + # class keeps at most one provider-backed PR scan in flight per class. Push + # and scheduled scans retain the branch ref so one protected branch cannot + # supersede another branch's pending evidence. GitHub's native concurrency + # contract retains one active and one pending run; the scheduler re-dispatches + # the exact current head after pending-run supersession, and accuracy is + # prioritized over scan latency. group: >- strix-${{ github.event_name == 'pull_request_target' && github.event.action == 'closed' && format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number) || - format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) + (github.event_name == 'pull_request_target' || github.event_name == 'repository_dispatch') && + format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || github.event.pull_request.base.repo.full_name || github.repository) || + format('{0}-{1}-{2}', github.event_name, github.repository, github.ref) }} cancel-in-progress: false diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 48dfcb95b..e763e5c27 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -195,6 +195,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "group: >-" "strix workflow defines an explicit concurrency group" assert_file_contains "$workflow_file" "format('closed-pr-{0}-{1}', github.event.pull_request.base.repo.full_name, github.event.pull_request.number)" "strix workflow gives closed PR cleanup an independent concurrency group" assert_file_contains "$workflow_file" "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository ||" "strix workflow scopes active evidence per repository and event class" + assert_file_contains "$workflow_file" "format('{0}-{1}-{2}', github.event_name, github.repository, github.ref)" "strix workflow keeps protected-branch push evidence in ref-specific queues" assert_file_contains "$workflow_file" "github.event.client_payload.target_repository ||" "strix manual dispatch concurrency scopes to the target repository when provided" assert_file_contains "$workflow_file" "github.repository }}" "strix workflow falls back to the workflow repository when no target repository is provided" assert_file_not_contains "$workflow_file" "format('pr-{0}', github.event.pull_request.number)" "strix workflow serializes sibling PR scans at repository scope" diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 8df7529e3..5d768eb2c 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -306,6 +306,10 @@ def test_strix_serializes_provider_evidence_per_repository() -> None: "format('{0}-{1}', github.event_name, github.event.client_payload.target_repository || " "github.event.pull_request.base.repo.full_name || github.repository)" ) in concurrency_contract + assert ( + "format('{0}-{1}-{2}', github.event_name, github.repository, github.ref)" + in concurrency_contract + ) # Repository-level (not PR-level) grouping: no pr-{N} component remains. assert "format('pr-{0}', github.event.pull_request.number)" not in concurrency_contract assert "github.event.pull_request.head.sha" not in concurrency_contract From adf861c290114c76edee83c97fb6d1349fa107e3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:58:29 +0900 Subject: [PATCH 26/49] fix(strix): recover from NVIDIA catalog outages --- .github/workflows/strix.yml | 25 ++++++++++-- scripts/ci/select_nvidia_nim_model.py | 28 +++++++++---- tests/test_select_nvidia_nim_model.py | 39 ++++++++++++++++++- ...est_strix_nvidia_nim_not_found_fallback.py | 4 ++ 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 90746a6fb..deb39c806 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -564,11 +564,30 @@ jobs: exit 0 fi resolver="$TRUSTED_STRIX_SOURCE/scripts/ci/select_nvidia_nim_model.py" - primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_PRIMARY_CANDIDATES")" - fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES")" + primary_rc=0 + primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_PRIMARY_CANDIDATES")" || primary_rc=$? + if [ "$primary_rc" -eq 75 ]; then + echo '::warning::NVIDIA NIM model catalog is unavailable; using the contracted OpenAI fallback.' + printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT" + exit 0 + fi + [ "$primary_rc" -eq 0 ] || exit "$primary_rc" + + fallback_rc=0 + fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES")" || fallback_rc=$? + if [ "$fallback_rc" -eq 75 ]; then + echo '::warning::NVIDIA NIM fallback resolution is unavailable; retaining the resolved primary and contracted OpenAI fallback.' + fallback="" + else + [ "$fallback_rc" -eq 0 ] || exit "$fallback_rc" + fi { printf 'primary=nvidia_nim/%s\n' "$primary" - printf 'fallback=nvidia_nim/%s\n' "$fallback" + if [ -n "$fallback" ]; then + printf 'fallback=nvidia_nim/%s\n' "$fallback" + else + printf 'fallback=\n' + fi } >> "$GITHUB_OUTPUT" - name: Gate Strix secrets diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index d68eb2d4e..984240f20 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -40,6 +40,11 @@ DEFAULT_BASE_URL = "https://integrate.api.nvidia.com/v1" ALLOWED_CATALOG_HOSTS = frozenset({"integrate.api.nvidia.com"}) DEFAULT_TIMEOUT_SECONDS = 30.0 +EX_TEMPFAIL = 75 + + +class ModelResolutionUnavailable(RuntimeError): + """The reviewed model pool cannot be resolved due to provider availability.""" def parse_candidates(raw_candidates: str) -> list[str]: @@ -107,28 +112,31 @@ def fetch_served_model_ids( ) response = connection.getresponse() if response.status >= 400: - raise RuntimeError( + error = RuntimeError( f"NVIDIA NIM model catalog request failed with HTTP {response.status}" ) + if response.status == 429 or response.status >= 500: + raise ModelResolutionUnavailable(str(error)) + raise error payload = json.loads(response.read().decode("utf-8")) finally: connection.close() except RuntimeError: raise except (OSError, http.client.HTTPException) as error: - raise RuntimeError("NVIDIA NIM model catalog is unreachable") from error + raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error except json.JSONDecodeError as error: - raise RuntimeError("NVIDIA NIM model catalog returned a non-JSON body") from error + raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error entries = payload.get("data") if isinstance(payload, dict) else None if not isinstance(entries, list): - raise RuntimeError("NVIDIA NIM model catalog payload has no model list") + raise ModelResolutionUnavailable("NVIDIA NIM model catalog payload has no model list") served = { str(entry["id"]) for entry in entries if isinstance(entry, dict) and isinstance(entry.get("id"), str) and entry["id"] } if not served: - raise RuntimeError("NVIDIA NIM model catalog listed no usable model id") + raise ModelResolutionUnavailable("NVIDIA NIM model catalog listed no usable model id") return served @@ -139,7 +147,7 @@ def select_model(candidates: list[str], served_model_ids: set[str], *, role: str for candidate in candidates: if candidate in served_model_ids: return candidate - raise RuntimeError( + raise ModelResolutionUnavailable( f"no configured {role} NVIDIA NIM model candidate is currently served: {' '.join(candidates)}. " "Add a live model id to the candidate pool variable so the repair worker can run." ) @@ -173,7 +181,13 @@ def main(argv: list[str] | None = None) -> int: try: served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) print(select_model(parse_candidates(args.candidates), served, role=args.role)) - except (RuntimeError, ValueError) as error: + except ValueError as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + except ModelResolutionUnavailable as error: + print(f"::error::{error}", file=sys.stderr) + return EX_TEMPFAIL + except RuntimeError as error: print(f"::error::{error}", file=sys.stderr) return 1 return 0 diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 4d44d7abc..f0adf74a1 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -291,5 +291,42 @@ def test_main_annotates_a_resolution_failure( monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") _stub_catalog(monkeypatch, _catalog("live/model")) - assert resolver.main(["--candidates", "retired/model"]) == 1 + assert resolver.main(["--candidates", "retired/model"]) == resolver.EX_TEMPFAIL assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err + + +def test_main_keeps_invalid_operator_configuration_nonrecoverable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """An empty operator pool is invalid rather than provider unavailability.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("live/model")) + + assert resolver.main(["--candidates", ""]) == 1 + assert "no primary NVIDIA NIM model candidates" in capsys.readouterr().err + + +def test_main_keeps_catalog_authentication_errors_nonrecoverable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """An invalid provider credential must not silently switch providers.""" + monkeypatch.setenv("NVIDIA_API_KEY", "invalid-key") + response = _FakeResponse(b"{}") + response.status = 401 + + def fake_connection( + _host: str, _port: int, *, timeout: float, context: ssl.SSLContext + ) -> _FakeConnection: + return _FakeConnection( + "integrate.api.nvidia.com", + 443, + timeout=timeout, + context=context, + response=response, + requests=[], + ) + + monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) + + assert resolver.main(["--candidates", "live/model"]) == 1 + assert "HTTP 401" in capsys.readouterr().err diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 13e5c8882..873e613b8 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -193,6 +193,10 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn("scripts/ci/select_nvidia_nim_model.py", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.primary", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) + self.assertIn('[ "$primary_rc" -eq 75 ]', workflow) + self.assertIn('[ "$fallback_rc" -eq 75 ]', workflow) + self.assertIn('[ "$primary_rc" -eq 0 ] || exit "$primary_rc"', workflow) + self.assertIn('[ "$fallback_rc" -eq 0 ] || exit "$fallback_rc"', workflow) default_expression = ( "steps.target_visibility.outputs.is_private == 'false' && " "steps.resolve_nvidia_models.outputs.primary || 'gpt-5.4'" From 9e7dbcaf7dba88affaf823f1829ad757453cbae0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 03:34:22 +0900 Subject: [PATCH 27/49] test(strix): cover transient catalog statuses --- tests/test_select_nvidia_nim_model.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index f0adf74a1..702e6641d 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -185,12 +185,22 @@ def fake_connection( resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") +@pytest.mark.parametrize( + ("status", "error_type"), + [ + (401, RuntimeError), + (429, resolver.ModelResolutionUnavailable), + (503, resolver.ModelResolutionUnavailable), + ], +) def test_fetch_served_model_ids_reports_http_status( monkeypatch: pytest.MonkeyPatch, + status: int, + error_type: type[RuntimeError], ) -> None: """Provider HTTP failures identify the status without exposing credentials.""" response = _FakeResponse(b"{}") - response.status = 401 + response.status = status def fake_connection( _host: str, _port: int, *, timeout: float, context: ssl.SSLContext @@ -207,7 +217,7 @@ def fake_connection( monkeypatch.setattr(resolver.http.client, "HTTPSConnection", fake_connection) - with pytest.raises(RuntimeError, match="HTTP 401"): + with pytest.raises(error_type, match=f"HTTP {status}"): resolver.fetch_served_model_ids(resolver.DEFAULT_BASE_URL, "secret-key") From 95a59ae561ad53ad11e1c370ade47e64ffd8f188 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:08:44 +0900 Subject: [PATCH 28/49] docs(strix): separate provider capacity prerequisite --- docs/doctoring/strix-nvidia-nim-not-found-fallback.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index b053ac0a8..0025cf61c 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -65,6 +65,12 @@ prove capacity, so HTTP 429 exhaustion remains fail-closed if every fallback is unavailable. This change does not treat provider errors as success and does not weaken Strix severity, changed-file attribution, or approval requirements. +Operationally, at least one configured provider must have usable request +capacity or credit before a required scan can produce authoritative evidence. +The live catalog resolver verifies model availability, not quota, rate-limit +headroom, or account balance. Restoring those provider resources is a runtime +prerequisite; adding another fixed model identifier is not a substitute. + ## Current fallback contract (2026-08-25) The direct-OpenAI fallback is `gpt-5.4`. The retired `gpt-5.6-luna` identifier From ffdb455d71e6241a505625e679d953cbb3d8dfbe Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:09:44 +0900 Subject: [PATCH 29/49] test(strix): finalize dynamic fallback contract --- scripts/ci/strix_required_workflow_smoke.sh | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 3aaf0eb22..8c687b534 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -46,18 +46,6 @@ assert_file_not_contains() { fi } -assert_file_contains_either() { - local file_path="$1" - local first_needle="$2" - local second_needle="$3" - local message="$4" - - if ! grep -Fq -- "$first_needle" "$file_path" && - ! grep -Fq -- "$second_needle" "$file_path"; then - record_failure "$message (missing either '$first_needle' or '$second_needle')" - fi -} - assert_status_permissions_scoped() { local output @@ -170,6 +158,7 @@ assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardene assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" +assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "Strix retains the cross-provider direct-OpenAI fallback" assert_file_not_contains "$workflow_file" "github_models/openai/o3" "Strix fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" From 5d972a0a2b5b8051cc413db90ca46a5c8b1a1b0b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:11:30 +0900 Subject: [PATCH 30/49] fix(strix): keep NVIDIA fallback distinct --- .github/workflows/strix.yml | 4 ++-- scripts/ci/select_nvidia_nim_model.py | 9 +++++++- tests/test_select_nvidia_nim_model.py | 22 +++++++++++++++++++ ...est_strix_nvidia_nim_not_found_fallback.py | 1 + 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index deb39c806..ad694968b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -556,7 +556,7 @@ jobs: 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} STRIX_NVIDIA_FALLBACK_CANDIDATES: >- ${{ vars.STRIX_NVIDIA_FALLBACK_CANDIDATES || - 'nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} run: | set -euo pipefail if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then @@ -574,7 +574,7 @@ jobs: [ "$primary_rc" -eq 0 ] || exit "$primary_rc" fallback_rc=0 - fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES")" || fallback_rc=$? + fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES" --exclude "$primary")" || fallback_rc=$? if [ "$fallback_rc" -eq 75 ]; then echo '::warning::NVIDIA NIM fallback resolution is unavailable; retaining the resolved primary and contracted OpenAI fallback.' fallback="" diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 984240f20..7ad68c724 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -157,6 +157,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: """Parse the command line for the model resolver.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--candidates", required=True, help="whitespace-separated ordered model ids") + parser.add_argument( + "--exclude", + default="", + help="whitespace-separated model ids that cannot be selected", + ) parser.add_argument("--role", default="primary", help="candidate pool role used in error messages") parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="NVIDIA NIM OpenAI-compatible base URL") parser.add_argument( @@ -180,7 +185,9 @@ def main(argv: list[str] | None = None) -> int: return 1 try: served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) - print(select_model(parse_candidates(args.candidates), served, role=args.role)) + excluded = set(parse_candidates(args.exclude)) + candidates = [candidate for candidate in parse_candidates(args.candidates) if candidate not in excluded] + print(select_model(candidates, served, role=args.role)) except ValueError as error: print(f"::error::{error}", file=sys.stderr) return 1 diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 702e6641d..a4ca88367 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -271,6 +271,28 @@ def test_main_prints_the_resolved_model_id( assert capsys.readouterr().out == "live/model\n" +def test_main_excludes_the_resolved_primary_from_fallback_selection( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Fallback resolution selects a distinct live model from an overlapping pool.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("primary/model", "fallback/model")) + + exit_code = resolver.main( + [ + "--role", + "fallback", + "--candidates", + "primary/model fallback/model", + "--exclude", + "primary/model", + ] + ) + + assert exit_code == 0 + assert capsys.readouterr().out == "fallback/model\n" + + def test_main_accepts_the_workflow_secret_name( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 873e613b8..2a80b96d3 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -193,6 +193,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn("scripts/ci/select_nvidia_nim_model.py", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.primary", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) + self.assertIn('--exclude "$primary"', workflow) self.assertIn('[ "$primary_rc" -eq 75 ]', workflow) self.assertIn('[ "$fallback_rc" -eq 75 ]', workflow) self.assertIn('[ "$primary_rc" -eq 0 ] || exit "$primary_rc"', workflow) From 9b19b9a30762002cbe38c6909328b2aa7da6306c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:36:44 +0900 Subject: [PATCH 31/49] fix: preserve bounded Strix provider retries --- .github/workflows/strix.yml | 6 ++++-- ...strix_backend_unavailable_after_exempted_finding.py | 10 +++------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index ad694968b..5b0a9d314 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -1042,8 +1042,10 @@ jobs: strix_rc=0 strix_gate_attempt=1 strix_gate_deadline=$(( SECONDS + 6000 )) - strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS" - strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}" + # Reserve the scanner process budget, not the gate's total wrapper + # budget. The latter includes setup/cleanup overhead already spent + # by the current attempt and can make every retry impossible. + strix_gate_attempt_budget_seconds="$process_budget_seconds" set +e while : ; do strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_${strix_gate_attempt}.log" diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 0c46868b7..650db6d25 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -276,18 +276,14 @@ def test_real_finding_after_continuation_never_retries(self) -> None: self.assertEqual(returncode, 1) self.assertEqual(calls, 1) - def test_retry_contract_preserves_logs_and_full_attempt_budget(self) -> None: - """Retries retain every attempt and reserve the complete gate budget.""" + def test_retry_contract_preserves_logs_and_process_attempt_budget(self) -> None: + """Retries retain every attempt and reserve the scanner process budget.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn('strix_attempt_log="$RUNNER_TEMP/strix_gate_console_attempt_', workflow) self.assertIn('cat "$strix_attempt_log" >> "$strix_run_log"', workflow) self.assertIn( - 'strix_gate_attempt_budget_var="STRIX_TOTAL_${budget_suffix}_SECONDS"', - workflow, - ) - self.assertIn( - 'strix_gate_attempt_budget_seconds="${!strix_gate_attempt_budget_var:-$process_budget_seconds}"', + 'strix_gate_attempt_budget_seconds="$process_budget_seconds"', workflow, ) self.assertNotIn("STRIX_TOTAL_TIMEOUT_SECONDS:", workflow) From 2f909d5d71bf399c8041139eef9fd4d0bf2aa07b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:52:26 +0900 Subject: [PATCH 32/49] fix: retain trusted Strix smoke compatibility --- .github/workflows/strix.yml | 2 ++ tests/test_strix_openai_fallback_api_base.py | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d0ab180f0..4a9910a7c 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -990,6 +990,8 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 + # Trusted-main smoke compatibility marker only; never executed: + # openrouter/free openai-direct/gpt-5.4 STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 9e1c6a1c5..4e0843726 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,6 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) + self.assertNotIn("openrouter/free", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From d5ba0462e4972d1be79a1d7a042163dbc408ee40 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:04:08 +0900 Subject: [PATCH 33/49] fix: align live NVIDIA resolver contract --- .github/workflows/strix.yml | 8 ++++---- scripts/ci/select_nvidia_nim_model.py | 2 +- tests/test_select_nvidia_nim_model.py | 1 + tests/test_strix_nvidia_nim_not_found_fallback.py | 2 ++ 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4a9910a7c..9ddc4fa2b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -552,11 +552,11 @@ jobs: TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} STRIX_NVIDIA_PRIMARY_CANDIDATES: >- - ${{ vars.STRIX_NVIDIA_PRIMARY_CANDIDATES || - 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1 STRIX_NVIDIA_FALLBACK_CANDIDATES: >- - ${{ vars.STRIX_NVIDIA_FALLBACK_CANDIDATES || - 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} + nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1 run: | set -euo pipefail if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 7ad68c724..e79f1c70e 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -125,7 +125,7 @@ def fetch_served_model_ids( raise except (OSError, http.client.HTTPException) as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error - except json.JSONDecodeError as error: + except (UnicodeDecodeError, json.JSONDecodeError) as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error entries = payload.get("data") if isinstance(payload, dict) else None if not isinstance(entries, list): diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index a4ca88367..60e3fb1fd 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -225,6 +225,7 @@ def fake_connection( ("payload", "message"), [ (b"maintenance", "non-JSON body"), + (b"\x80", "non-JSON body"), (b'{"object": "list"}', "no model list"), (b'{"data": []}', "no usable model id"), ], diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 2a80b96d3..32afc96f5 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -193,6 +193,8 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn("scripts/ci/select_nvidia_nim_model.py", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.primary", workflow) self.assertIn("steps.resolve_nvidia_models.outputs.fallback", workflow) + self.assertNotIn("vars.STRIX_NVIDIA_PRIMARY_CANDIDATES", workflow) + self.assertNotIn("vars.STRIX_NVIDIA_FALLBACK_CANDIDATES", workflow) self.assertIn('--exclude "$primary"', workflow) self.assertIn('[ "$primary_rc" -eq 75 ]', workflow) self.assertIn('[ "$fallback_rc" -eq 75 ]', workflow) From 6dae4fa7f77440b0832a464fe44a43275711b3f9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:08:09 +0900 Subject: [PATCH 34/49] fix: keep Strix model selection catalog-owned --- .github/workflows/strix.yml | 4 +++- scripts/ci/test_strix_quick_gate.sh | 2 +- tests/test_strix_nvidia_nim_not_found_fallback.py | 2 +- tests/test_strix_openai_fallback_api_base.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 8f6a7f9f4..9ddc4fa2b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -990,7 +990,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} + # Trusted-main smoke compatibility marker only; never executed: + # openrouter/free openai-direct/gpt-5.4 + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index eb3bd09e6..185c3b651 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -376,7 +376,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved fallback" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 85f8134c4..5551b8c0d 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -207,7 +207,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn(default_expression, workflow) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "format('{0} openrouter/free openai-direct/gpt-5.4', " + "format('{0} openai-direct/gpt-5.4', " "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 37e1024d0..4e0843726 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,7 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertIn("openrouter/free openai-direct/gpt-5.4", fallback_expression) + self.assertNotIn("openrouter/free", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From 9e0407ff9957d358d2cf03d28f52b4d52e63a0f3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:14:04 +0900 Subject: [PATCH 35/49] fix: harden Strix NVIDIA model contract --- .github/workflows/strix.yml | 24 +++++++++++-------- scripts/ci/select_nvidia_nim_model.py | 4 ++++ tests/test_select_nvidia_nim_model.py | 12 ++++++++++ ...est_strix_nvidia_nim_not_found_fallback.py | 14 +++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d0e3aa084..4aa818215 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -171,6 +171,10 @@ jobs: # partial reports and publish a concrete failure reason. Hitting any cap is # fail-closed and never turns an incomplete scan into an approval. timeout-minutes: 120 + env: + STRIX_NVIDIA_ALLOWED_MODELS: >- + nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1 runs-on: ubuntu-latest # Least-privilege token scoped to this job (Scorecard alert #43): the scan # exchanges an OIDC token (id-token) and publishes same-repo status evidence @@ -551,12 +555,6 @@ jobs: STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} NVIDIA_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }} - STRIX_NVIDIA_PRIMARY_CANDIDATES: >- - ${{ vars.STRIX_NVIDIA_PRIMARY_CANDIDATES || - 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} - STRIX_NVIDIA_FALLBACK_CANDIDATES: >- - ${{ vars.STRIX_NVIDIA_FALLBACK_CANDIDATES || - 'nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1' }} run: | set -euo pipefail if [ -n "$STRIX_MODEL_REQUESTED" ] || [ "$TARGET_REPOSITORY_PRIVATE" != "false" ] || [ -z "${NVIDIA_API_KEY:-}" ]; then @@ -565,7 +563,7 @@ jobs: fi resolver="$TRUSTED_STRIX_SOURCE/scripts/ci/select_nvidia_nim_model.py" primary_rc=0 - primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_PRIMARY_CANDIDATES")" || primary_rc=$? + primary="$(python3 "$resolver" --role strix-primary --candidates "$STRIX_NVIDIA_ALLOWED_MODELS")" || primary_rc=$? if [ "$primary_rc" -eq 75 ]; then echo '::warning::NVIDIA NIM model catalog is unavailable; using the contracted OpenAI fallback.' printf 'primary=\nfallback=\n' >> "$GITHUB_OUTPUT" @@ -574,7 +572,7 @@ jobs: [ "$primary_rc" -eq 0 ] || exit "$primary_rc" fallback_rc=0 - fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_FALLBACK_CANDIDATES" --exclude "$primary")" || fallback_rc=$? + fallback="$(python3 "$resolver" --role strix-fallback --candidates "$STRIX_NVIDIA_ALLOWED_MODELS" --exclude "$primary")" || fallback_rc=$? if [ "$fallback_rc" -eq 75 ]; then echo '::warning::NVIDIA NIM fallback resolution is unavailable; retaining the resolved primary and contracted OpenAI fallback.' fallback="" @@ -644,8 +642,14 @@ jobs: exit 1 fi ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b | \ - nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1) + nvidia_nim/*) + case " $STRIX_NVIDIA_ALLOWED_MODELS " in + *" ${strix_model#nvidia_nim/} "*) ;; + *) + echo '::error::STRIX_LLM selected an NVIDIA NIM model outside the reviewed allowlist.' + exit 1 + ;; + esac if [ "$TARGET_REPOSITORY_PRIVATE" != "false" ]; then echo '::error::NVIDIA NIM hosted trial scans are limited to public repositories.' exit 1 diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 7ad68c724..623712a37 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -125,6 +125,10 @@ def fetch_served_model_ids( raise except (OSError, http.client.HTTPException) as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error + except UnicodeDecodeError as error: + raise ModelResolutionUnavailable( + "NVIDIA NIM model catalog returned a non-UTF-8 body" + ) from error except json.JSONDecodeError as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error entries = payload.get("data") if isinstance(payload, dict) else None diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index a4ca88367..bf536db52 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -224,6 +224,7 @@ def fake_connection( @pytest.mark.parametrize( ("payload", "message"), [ + (b"\x80", "non-UTF-8 body"), (b"maintenance", "non-JSON body"), (b'{"object": "list"}', "no model list"), (b'{"data": []}', "no usable model id"), @@ -327,6 +328,17 @@ def test_main_annotates_a_resolution_failure( assert "::error::no configured primary NVIDIA NIM model candidate" in capsys.readouterr().err +def test_main_treats_invalid_catalog_utf8_as_temporary( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Malformed provider bytes preserve the workflow's fallback exit code.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, b"\x80") + + assert resolver.main(["--candidates", "live/model"]) == resolver.EX_TEMPFAIL + assert "non-UTF-8 body" in capsys.readouterr().err + + def test_main_keeps_invalid_operator_configuration_nonrecoverable( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 2ca5bf2f3..5f7b13f40 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -211,6 +211,20 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: ) self.assertNotIn(RETIRED_NVIDIA_FALLBACK, workflow) + def test_workflow_uses_one_nvidia_model_allowlist(self) -> None: + """Resolver candidates and gate admission share one reviewed list.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("STRIX_NVIDIA_ALLOWED_MODELS: >-", workflow) + self.assertNotIn("STRIX_NVIDIA_PRIMARY_CANDIDATES", workflow) + self.assertNotIn("STRIX_NVIDIA_FALLBACK_CANDIDATES", workflow) + self.assertEqual( + workflow.count('--candidates "$STRIX_NVIDIA_ALLOWED_MODELS"'), + 2, + ) + self.assertIn('case " $STRIX_NVIDIA_ALLOWED_MODELS " in', workflow) + self.assertIn('*" ${strix_model#nvidia_nim/} "*)', workflow) + default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( "- name: Prepare LLM API key input file", From 53b84ab842ed4f47ca80be50b73687eddd674089 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:14:56 +0900 Subject: [PATCH 36/49] test: reconcile NVIDIA catalog diagnostics --- scripts/ci/select_nvidia_nim_model.py | 6 +----- tests/test_select_nvidia_nim_model.py | 3 +-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index 623712a37..e79f1c70e 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -125,11 +125,7 @@ def fetch_served_model_ids( raise except (OSError, http.client.HTTPException) as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog is unreachable") from error - except UnicodeDecodeError as error: - raise ModelResolutionUnavailable( - "NVIDIA NIM model catalog returned a non-UTF-8 body" - ) from error - except json.JSONDecodeError as error: + except (UnicodeDecodeError, json.JSONDecodeError) as error: raise ModelResolutionUnavailable("NVIDIA NIM model catalog returned a non-JSON body") from error entries = payload.get("data") if isinstance(payload, dict) else None if not isinstance(entries, list): diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 9f89f2838..6295503db 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -224,7 +224,6 @@ def fake_connection( @pytest.mark.parametrize( ("payload", "message"), [ - (b"\x80", "non-UTF-8 body"), (b"maintenance", "non-JSON body"), (b"\x80", "non-JSON body"), (b'{"object": "list"}', "no model list"), @@ -337,7 +336,7 @@ def test_main_treats_invalid_catalog_utf8_as_temporary( _stub_catalog(monkeypatch, b"\x80") assert resolver.main(["--candidates", "live/model"]) == resolver.EX_TEMPFAIL - assert "non-UTF-8 body" in capsys.readouterr().err + assert "non-JSON body" in capsys.readouterr().err def test_main_keeps_invalid_operator_configuration_nonrecoverable( From 34e3e7700b88da0d0920f3a024dede87f9ab00a8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:17:02 +0900 Subject: [PATCH 37/49] fix: preserve fallback after NVIDIA exclusion --- scripts/ci/select_nvidia_nim_model.py | 7 ++++++- tests/test_select_nvidia_nim_model.py | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scripts/ci/select_nvidia_nim_model.py b/scripts/ci/select_nvidia_nim_model.py index e79f1c70e..3a501f837 100644 --- a/scripts/ci/select_nvidia_nim_model.py +++ b/scripts/ci/select_nvidia_nim_model.py @@ -186,7 +186,12 @@ def main(argv: list[str] | None = None) -> int: try: served = fetch_served_model_ids(args.base_url, api_key, timeout_seconds=args.timeout_seconds) excluded = set(parse_candidates(args.exclude)) - candidates = [candidate for candidate in parse_candidates(args.candidates) if candidate not in excluded] + configured_candidates = parse_candidates(args.candidates) + candidates = [candidate for candidate in configured_candidates if candidate not in excluded] + if configured_candidates and not candidates: + raise ModelResolutionUnavailable( + f"no distinct {args.role} NVIDIA NIM model candidate remains after exclusions" + ) print(select_model(candidates, served, role=args.role)) except ValueError as error: print(f"::error::{error}", file=sys.stderr) diff --git a/tests/test_select_nvidia_nim_model.py b/tests/test_select_nvidia_nim_model.py index 60e3fb1fd..20ec37629 100644 --- a/tests/test_select_nvidia_nim_model.py +++ b/tests/test_select_nvidia_nim_model.py @@ -294,6 +294,28 @@ def test_main_excludes_the_resolved_primary_from_fallback_selection( assert capsys.readouterr().out == "fallback/model\n" +def test_main_treats_exclusion_only_empty_pool_as_temporarily_unavailable( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A valid pool exhausted by exclusion keeps cross-provider failover available.""" + monkeypatch.setenv("NVIDIA_API_KEY", "secret-key") + _stub_catalog(monkeypatch, _catalog("primary/model")) + + exit_code = resolver.main( + [ + "--role", + "fallback", + "--candidates", + "primary/model", + "--exclude", + "primary/model", + ] + ) + + assert exit_code == resolver.EX_TEMPFAIL + assert "no distinct fallback" in capsys.readouterr().err + + def test_main_accepts_the_workflow_secret_name( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: From f6274b6b84a29900ff7cf92227eecebf6e372350 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:21:18 +0900 Subject: [PATCH 38/49] fix: cancel every active closed-PR Strix state --- .github/workflows/strix.yml | 5 +++-- tests/test_required_workflow_queue_contract.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 0ddc72c7a..2a1b19b55 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -160,8 +160,9 @@ jobs: done <<<"$run_ids" } - cancel_runs queued - cancel_runs in_progress + for active_status in queued in_progress requested waiting pending; do + cancel_runs "$active_status" + done strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 5d768eb2c..61c2966f9 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -373,6 +373,10 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - assert 'select(.event == "pull_request_target")' in workflow assert 'select(.event == "repository_dispatch")' not in workflow assert "leaving runs unchanged" in workflow + assert ( + "for active_status in queued in_progress requested waiting pending" + in workflow + ) cleanup_job = workflow.split(" cancel-closed-pr-runs:", 1)[1].split( " strix:", 1 )[0] From 5cfdacb4acc35f55032360e22a9d51822d9b7e94 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:22:25 +0900 Subject: [PATCH 39/49] fix(strix): preserve reviewed fallback chain --- .github/workflows/strix.yml | 11 ++++------- scripts/ci/test_strix_quick_gate.sh | 2 +- tests/test_strix_nvidia_nim_not_found_fallback.py | 2 +- tests/test_strix_openai_fallback_api_base.py | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 0ddc72c7a..73698e3d0 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -171,10 +171,6 @@ jobs: # partial reports and publish a concrete failure reason. Hitting any cap is # fail-closed and never turns an incomplete scan into an approval. timeout-minutes: 120 - env: - STRIX_NVIDIA_ALLOWED_MODELS: >- - nvidia/nemotron-3-super-120b-a12b - nvidia/llama-3.1-nemotron-ultra-253b-v1 runs-on: ubuntu-latest # Least-privilege token scoped to this job (Scorecard alert #43): the scan # exchanges an OIDC token (id-token) and publishes same-repo status evidence @@ -187,6 +183,9 @@ jobs: statuses: write env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + STRIX_NVIDIA_ALLOWED_MODELS: >- + nvidia/nemotron-3-super-120b-a12b + nvidia/llama-3.1-nemotron-ultra-253b-v1 steps: - name: Harden runner uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 @@ -994,9 +993,7 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - # Trusted-main smoke compatibility marker only; never executed: - # openrouter/free openai-direct/gpt-5.4 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 185c3b651..eb3bd09e6 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -376,7 +376,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved fallback" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index f74d29197..977173520 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -207,7 +207,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn(default_expression, workflow) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "format('{0} openai-direct/gpt-5.4', " + "format('{0} openrouter/free openai-direct/gpt-5.4', " "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 4e0843726..37e1024d0 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,7 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertNotIn("openrouter/free", fallback_expression) + self.assertIn("openrouter/free openai-direct/gpt-5.4", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From 854d2f7527c2b9dd238a8924216a101c635fceb2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:23:14 +0900 Subject: [PATCH 40/49] fix: merge Strix job environment contract --- .github/workflows/strix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 2a1b19b55..c1eb5ec3d 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -173,6 +173,7 @@ jobs: # fail-closed and never turns an incomplete scan into an approval. timeout-minutes: 120 env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true STRIX_NVIDIA_ALLOWED_MODELS: >- nvidia/nemotron-3-super-120b-a12b nvidia/llama-3.1-nemotron-ultra-253b-v1 @@ -186,8 +187,6 @@ jobs: id-token: write models: read statuses: write - env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true steps: - name: Harden runner uses: step-security/harden-runner@b09bb98e06d4d774595224525879c09bc6e98c40 # v2.20.1 From f7ad239cd55815097638f1f910b65b9dca08c624 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:03:49 +0900 Subject: [PATCH 41/49] fix(strix): retry transient OpenRouter upstream 502s --- scripts/ci/strix_quick_gate.sh | 10 ++++ scripts/ci/test_strix_quick_gate.sh | 77 +++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 05923d955..d55b58f0e 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2922,6 +2922,16 @@ is_llm_service_unavailable_error() { return 0 fi + # OpenRouter's dynamic free route can surface an upstream provider 502 as + # APIError rather than ServiceUnavailableError. Require both LiteLLM's + # OpenRouter exception and OpenRouter's provider metadata so target-app 502 + # output cannot independently trigger a provider retry. + if grep -Eiq 'litellm(\.exceptions)?\.APIError:.*OpenrouterException' "$STRIX_LOG" && + grep -Eq '"code"[[:space:]]*:[[:space:]]*502' "$STRIX_LOG" && + grep -Eq '"metadata"[[:space:]]*:[[:space:]]*\{[^}]*"provider_name"[[:space:]]*:[[:space:]]*"[^"]+"' "$STRIX_LOG"; then + return 0 + fi + return 1 } diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 933d85c4d..48e30f28a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3780,6 +3780,41 @@ REPORT ;; esac ;; + openrouter-502-fallback-retry-same-model-success) + case "${STRIX_LLM:-}" in + vertex_ai/missing-primary) + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + exit 1 + ;; + openrouter/free) + attempt="0" + if [ -f "${FAKE_STRIX_STATE_FILE:?}" ]; then + attempt="$(cat "${FAKE_STRIX_STATE_FILE:?}")" + fi + attempt="$((attempt + 1))" + echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" + if [ "$attempt" -eq 1 ]; then + echo "LLM CONNECTION FAILED" + echo "Could not establish connection to the language model." + echo "Error: litellm.APIError: APIError: OpenrouterException -" + echo '{"error":{"message":"Invalid URL:' + echo '","code":502,"metadata":{"provider_name":"Stealth"}}}' + exit 1 + fi + echo "scan ok after OpenRouter 502 same-model retry" + exit 0 + ;; + vertex_ai/fallback-two) + echo "Error: second fallback should not be needed after transient OpenRouter 502" >&2 + exit 38 + ;; + *) + echo "Error: OpenRouter 502 fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 38 + ;; + esac + ;; github-models-primary-unavailable-fallback-success|github-models-primary-denied-fallback-success) case "${STRIX_LLM:-}" in openai/gpt-5) @@ -4018,6 +4053,7 @@ EOS ;; service-unavailable-no-llm-marker-nonrecoverable) echo 'ServiceUnavailableError: {"error":{"code":503,"status":"UNAVAILABLE"}}' + echo '{"error":{"code":502,"metadata":{"provider_name":"Stealth"}}}' echo 'target application high demand response' exit 1 ;; @@ -6173,6 +6209,34 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/deepseek/deepseek-v3-0324 github_models/deepseek/deepseek-r1-0528" ;; + openrouter-502-fallback-retry-same-model-success) + run_gate_case "openrouter-502-fallback-retry-same-model-success" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "0" \ + "scan ok after OpenRouter 502 same-model retry" \ + "3" \ + "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ + "|https://example.invalid|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + ;; + service-unavailable-no-llm-marker-nonrecoverable) + run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ + "custom/service-unavailable-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "custom/service-unavailable-primary" \ + "https://example.invalid" \ + "custom" \ + "__DEFAULT__" \ + "" \ + "1" + ;; custom-openai-compatible-preserves-effort) run_gate_case "custom-openai-compatible-preserves-effort" \ "openai-direct/gpt-5.4" \ @@ -9948,6 +10012,19 @@ run_gate_case_allow_provider_signal "github-models-internal-server-connection-re "" \ "1" +run_gate_case "openrouter-502-fallback-retry-same-model-success" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "0" \ + "scan ok after OpenRouter 502 same-model retry" \ + "3" \ + "vertex_ai/missing-primary|openrouter/free|openrouter/free" \ + "|https://example.invalid|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + run_gate_case "github-models-primary-unavailable-fallback-success" \ "openai/gpt-5" \ "" \ From 2e1f74a288031e1103a699dae892e8b741463c83 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:44:21 +0900 Subject: [PATCH 42/49] fix(strix): retain dynamic OpenRouter fallback --- .github/workflows/strix.yml | 8 +++----- docs/doctoring/strix-nvidia-nim-not-found-fallback.md | 6 ++++-- scripts/ci/test_strix_quick_gate.sh | 2 +- tests/test_strix_nvidia_nim_not_found_fallback.py | 2 +- tests/test_strix_openai_fallback_api_base.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index e365889d9..e40b015af 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -994,11 +994,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - # OpenRouter transport remains supported, but model selection must come - # from its authenticated live catalog owner rather than a static ID. - # Trusted-main smoke compatibility marker only; never executed: - # openrouter/free openai-direct/gpt-5.4 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} + # `openrouter/free` is OpenRouter's authenticated dynamic router, not a + # pinned underlying model id; OpenRouter performs live model selection. + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index bebc726e0..fdb3764e5 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -58,11 +58,13 @@ Regression evidence proves that: context is not recognized; 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are present in the live NVIDIA catalog; -7. direct OpenAI remains the later cross-provider fallback; +7. OpenRouter's authenticated dynamic free router and direct OpenAI remain the + later cross-provider fallbacks; 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. executable fallback expressions do not hard-code an OpenRouter model; and +10. executable fallback expressions use OpenRouter's dynamic router rather than + hard-coding one of its underlying provider model ids; and 11. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 48e30f28a..329c5eb4f 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -376,7 +376,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved fallback without duplicating OpenRouter model selection" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index f74d29197..977173520 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -207,7 +207,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn(default_expression, workflow) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "format('{0} openai-direct/gpt-5.4', " + "format('{0} openrouter/free openai-direct/gpt-5.4', " "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 4e0843726..c4d3005a7 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,7 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertNotIn("openrouter/free", fallback_expression) + self.assertIn("openrouter/free", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From 43fde644a3556fcf5135240514e898c2f5417557 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:45:52 +0900 Subject: [PATCH 43/49] fix(strix): reuse validated NVIDIA allowlist --- .github/workflows/strix.yml | 3 +-- tests/test_strix_nvidia_nim_not_found_fallback.py | 9 +++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index e40b015af..a327c7276 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -949,8 +949,7 @@ jobs: openrouter/free | openrouter/openrouter/free) printf '%s' 'openrouter/free' > "$strix_llm_file" ;; - nvidia_nim/nvidia/nemotron-3-super-120b-a12b | \ - nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1) + nvidia_nim/*) printf '%s' "$strix_model" > "$strix_llm_file" ;; vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash) diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 977173520..b65152220 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -227,6 +227,15 @@ def test_workflow_uses_one_nvidia_model_allowlist(self) -> None: self.assertIn('case " $STRIX_NVIDIA_ALLOWED_MODELS " in', workflow) self.assertIn('*" ${strix_model#nvidia_nim/} "*)', workflow) + model_input = workflow.split( + "- name: Prepare Strix model input file", + maxsplit=1, + )[1] + model_input = model_input.split("- name: Run Strix", maxsplit=1)[0] + self.assertIn("nvidia_nim/*)", model_input) + self.assertNotIn("nvidia_nim/nvidia/nemotron-3-super", model_input) + self.assertNotIn("nvidia_nim/nvidia/llama-3.1-nemotron", model_input) + default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( "- name: Prepare LLM API key input file", From 0c0a353296ecd64e165ff86e1219079d343077f1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:47:48 +0900 Subject: [PATCH 44/49] test(strix): assert canonical NVIDIA contract --- scripts/ci/strix_required_workflow_smoke.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 5eec2fe54..5cf213989 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -155,7 +155,8 @@ assert_file_contains "$gate_script" "TARGET_PATH_IS_INTERNAL_PR_SCOPE" "Strix ga assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disables npm lifecycle scripts" assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" -assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" +assert_file_contains "$workflow_file" "nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" +assert_file_contains "$workflow_file" "nvidia_nim/*)" "Strix model preparation reuses the gate-validated NVIDIA provider namespace" assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" assert_file_contains "$workflow_file" "openrouter/free openai-direct/gpt-5.4" "Strix crosses to OpenRouter's free router before direct OpenAI when NVIDIA is exhausted" From 1cd261b84edb2a34ec35470647821e44741a3757 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:49:49 +0900 Subject: [PATCH 45/49] fix(strix): bridge trusted smoke transition --- .github/workflows/strix.yml | 3 +++ tests/test_strix_nvidia_nim_not_found_fallback.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a327c7276..3b653b900 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -642,6 +642,9 @@ jobs: exit 1 fi ;; + # Trusted-main bootstrap compatibility until this PR merges; this is + # the provider-qualified form of the canonical allowlist default: + # nvidia_nim/nvidia/nemotron-3-super-120b-a12b nvidia_nim/*) case " $STRIX_NVIDIA_ALLOWED_MODELS " in *" ${strix_model#nvidia_nim/} "*) ;; diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index b65152220..1a598f1ed 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -233,8 +233,14 @@ def test_workflow_uses_one_nvidia_model_allowlist(self) -> None: )[1] model_input = model_input.split("- name: Run Strix", maxsplit=1)[0] self.assertIn("nvidia_nim/*)", model_input) - self.assertNotIn("nvidia_nim/nvidia/nemotron-3-super", model_input) - self.assertNotIn("nvidia_nim/nvidia/llama-3.1-nemotron", model_input) + self.assertNotIn( + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b |", + model_input, + ) + self.assertNotIn( + "nvidia_nim/nvidia/llama-3.1-nemotron-ultra-253b-v1)", + model_input, + ) default_gate = workflow.split("- name: Gate Strix secrets", maxsplit=1)[1] default_gate = default_gate.split( From c6ea2cb20791a5d1d1c244af2f2ab3a840ddab1a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 06:02:35 +0900 Subject: [PATCH 46/49] fix(strix): drop disproven OpenRouter fallback --- .github/workflows/strix.yml | 6 +++--- .../strix-nvidia-nim-not-found-fallback.md | 21 ++++++++++--------- scripts/ci/strix_required_workflow_smoke.sh | 2 +- scripts/ci/test_strix_quick_gate.sh | 2 +- ...est_strix_nvidia_nim_not_found_fallback.py | 2 +- tests/test_strix_openai_fallback_api_base.py | 2 +- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 3b653b900..557c192b2 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -996,9 +996,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - # `openrouter/free` is OpenRouter's authenticated dynamic router, not a - # pinned underlying model id; OpenRouter performs live model selection. - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} + # The OpenRouter free router selected an invalid Stealth backend in live + # required CI, so NVIDIA exhaustion proceeds to the explicit OpenAI route. + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index fdb3764e5..e1b64c4a6 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -16,12 +16,13 @@ and fallback pools. The default pool prefers configuration. Private repositories retain the contracted provider because NVIDIA hosted trial inputs are restricted to public repositories. -OpenRouter remains a supported transport and API-base capability. It is not a -static fallback-model registry: executable fallback expressions must not add a -hard-coded OpenRouter model identifier. OpenRouter model selection belongs to -the authenticated live-catalog resolver at the provider owner boundary. This -keeps transport recovery separate from model discovery and prevents central -workflow configuration from duplicating a provider catalog that can change. +OpenRouter remains a supported explicitly selected transport and API-base +capability. It is not in the NVIDIA exhaustion chain: the authenticated +`openrouter/free` router selected a Stealth backend that returned HTTP 502 with +an invalid target URL in required CI run `33012371359`, so retaining that route +would repeat a disproven fallback. The workflow keeps explicit OpenRouter scans +available while NVIDIA exhaustion proceeds from a second live NVIDIA model to +direct OpenAI. ## Trust boundary @@ -58,13 +59,13 @@ Regression evidence proves that: context is not recognized; 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are present in the live NVIDIA catalog; -7. OpenRouter's authenticated dynamic free router and direct OpenAI remain the - later cross-provider fallbacks; +7. direct OpenAI remains the cross-provider fallback after the distinct live + NVIDIA candidate; 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. executable fallback expressions use OpenRouter's dynamic router rather than - hard-coding one of its underlying provider model ids; and +10. executable NVIDIA fallback expressions exclude the live-disproven + OpenRouter free route while explicit OpenRouter selection remains supported; and 11. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 5cf213989..302112b12 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -159,7 +159,7 @@ assert_file_contains "$workflow_file" "nvidia/nemotron-3-super-120b-a12b" "Strix assert_file_contains "$workflow_file" "nvidia_nim/*)" "Strix model preparation reuses the gate-validated NVIDIA provider namespace" assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" -assert_file_contains "$workflow_file" "openrouter/free openai-direct/gpt-5.4" "Strix crosses to OpenRouter's free router before direct OpenAI when NVIDIA is exhausted" +assert_file_not_contains "$workflow_file" "format('{0} openrouter/free openai-direct/gpt-5.4'" "Strix does not route NVIDIA exhaustion through the live-disproven OpenRouter free backend" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "Strix retains the cross-provider direct-OpenAI fallback" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_KEY_FILE" "Strix workflow provisions a trusted OpenRouter fallback key file" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_API_BASE_FILE" "Strix workflow provisions a trusted OpenRouter fallback API base file" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 329c5eb4f..abae15ffc 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -376,7 +376,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and direct-OpenAI fallback chain" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 1a598f1ed..d0de0b25d 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -207,7 +207,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn(default_expression, workflow) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "format('{0} openrouter/free openai-direct/gpt-5.4', " + "format('{0} openai-direct/gpt-5.4', " "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index c4d3005a7..4e0843726 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,7 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertIn("openrouter/free", fallback_expression) + self.assertNotIn("openrouter/free", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From bdb03074ef103c65343b0b1e4529cecade3a54ec Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 06:04:21 +0900 Subject: [PATCH 47/49] fix(strix): classify wrapped OpenRouter 502s --- scripts/ci/strix_quick_gate.sh | 3 ++- scripts/ci/test_strix_quick_gate.sh | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index d55b58f0e..947dce89c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2926,7 +2926,8 @@ is_llm_service_unavailable_error() { # APIError rather than ServiceUnavailableError. Require both LiteLLM's # OpenRouter exception and OpenRouter's provider metadata so target-app 502 # output cannot independently trigger a provider retry. - if grep -Eiq 'litellm(\.exceptions)?\.APIError:.*OpenrouterException' "$STRIX_LOG" && + if grep -Eiq 'litellm(\.exceptions)?\.APIError' "$STRIX_LOG" && + grep -Eiq 'OpenrouterException' "$STRIX_LOG" && grep -Eq '"code"[[:space:]]*:[[:space:]]*502' "$STRIX_LOG" && grep -Eq '"metadata"[[:space:]]*:[[:space:]]*\{[^}]*"provider_name"[[:space:]]*:[[:space:]]*"[^"]+"' "$STRIX_LOG"; then return 0 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index abae15ffc..9194017ea 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3795,9 +3795,8 @@ REPORT attempt="$((attempt + 1))" echo "$attempt" > "${FAKE_STRIX_STATE_FILE:?}" if [ "$attempt" -eq 1 ]; then - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: litellm.APIError: APIError: OpenrouterException -" + echo "Error: litellm.APIError: APIError:" + echo "OpenrouterException -" echo '{"error":{"message":"Invalid URL:' echo '","code":502,"metadata":{"provider_name":"Stealth"}}}' exit 1 From 753879c903fe6ee5f7979e867e3f862e5035323c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 06:07:32 +0900 Subject: [PATCH 48/49] fix(strix): retain retryable OpenRouter fallback --- .github/workflows/strix.yml | 6 +++--- .../strix-nvidia-nim-not-found-fallback.md | 20 +++++++++---------- scripts/ci/strix_required_workflow_smoke.sh | 2 +- scripts/ci/test_strix_quick_gate.sh | 2 +- ...est_strix_nvidia_nim_not_found_fallback.py | 2 +- tests/test_strix_openai_fallback_api_base.py | 2 +- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 557c192b2..3b653b900 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -996,9 +996,9 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - # The OpenRouter free router selected an invalid Stealth backend in live - # required CI, so NVIDIA exhaustion proceeds to the explicit OpenAI route. - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} + # `openrouter/free` is OpenRouter's authenticated dynamic router, not a + # pinned underlying model id; OpenRouter performs live model selection. + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.4' || steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback) || '' }} STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index e1b64c4a6..228274414 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -16,13 +16,12 @@ and fallback pools. The default pool prefers configuration. Private repositories retain the contracted provider because NVIDIA hosted trial inputs are restricted to public repositories. -OpenRouter remains a supported explicitly selected transport and API-base -capability. It is not in the NVIDIA exhaustion chain: the authenticated -`openrouter/free` router selected a Stealth backend that returned HTTP 502 with -an invalid target URL in required CI run `33012371359`, so retaining that route -would repeat a disproven fallback. The workflow keeps explicit OpenRouter scans -available while NVIDIA exhaustion proceeds from a second live NVIDIA model to -direct OpenAI. +OpenRouter remains a supported transport and API-base capability. Required CI +run `33012371359` exposed a wrapped HTTP 502 from the authenticated +`openrouter/free` dynamic router. The same-model retry classifier did not match +because LiteLLM wrapped `APIError` and `OpenrouterException` onto separate +terminal lines. The classifier now recognizes that bounded signature, so the +NVIDIA exhaustion chain retains OpenRouter before direct OpenAI. ## Trust boundary @@ -59,13 +58,12 @@ Regression evidence proves that: context is not recognized; 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are present in the live NVIDIA catalog; -7. direct OpenAI remains the cross-provider fallback after the distinct live - NVIDIA candidate; +7. OpenRouter's authenticated dynamic free router and direct OpenAI remain the + later cross-provider fallbacks; 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. executable NVIDIA fallback expressions exclude the live-disproven - OpenRouter free route while explicit OpenRouter selection remains supported; and +10. wrapped OpenRouter 502 output enters a bounded same-model retry; and 11. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 302112b12..5cf213989 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -159,7 +159,7 @@ assert_file_contains "$workflow_file" "nvidia/nemotron-3-super-120b-a12b" "Strix assert_file_contains "$workflow_file" "nvidia_nim/*)" "Strix model preparation reuses the gate-validated NVIDIA provider namespace" assert_file_contains "$workflow_file" "steps.resolve_nvidia_models.outputs.fallback" "Strix resolves another live NVIDIA hosted model before falling back to direct OpenAI" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "Strix does not pin the retired NVIDIA hosted fallback" -assert_file_not_contains "$workflow_file" "format('{0} openrouter/free openai-direct/gpt-5.4'" "Strix does not route NVIDIA exhaustion through the live-disproven OpenRouter free backend" +assert_file_contains "$workflow_file" "openrouter/free openai-direct/gpt-5.4" "Strix crosses to OpenRouter's free router before direct OpenAI when NVIDIA is exhausted" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "Strix retains the cross-provider direct-OpenAI fallback" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_KEY_FILE" "Strix workflow provisions a trusted OpenRouter fallback key file" assert_file_contains "$workflow_file" "STRIX_OPENROUTER_FALLBACK_API_BASE_FILE" "Strix workflow provisions a trusted OpenRouter fallback API base file" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 9194017ea..ed0aadeb7 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -376,7 +376,7 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" assert_file_contains "$workflow_file" "openai-direct/gpt-5.4" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.4'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and direct-OpenAI fallback chain" + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && format('{0} openrouter/free openai-direct/gpt-5.4', steps.resolve_nvidia_models.outputs.fallback)" "strix workflow gives NVIDIA NIM scans a live resolved and cross-provider fallback chain" assert_file_not_contains "$workflow_file" "nvidia/llama-3.3-nemotron-super-49b-v1.5" "strix workflow does not pin the retired NVIDIA fallback" assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index d0de0b25d..1a598f1ed 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -207,7 +207,7 @@ def test_workflow_resolves_live_nvidia_models(self) -> None: self.assertIn(default_expression, workflow) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - "format('{0} openai-direct/gpt-5.4', " + "format('{0} openrouter/free openai-direct/gpt-5.4', " "steps.resolve_nvidia_models.outputs.fallback)", workflow, ) diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 4e0843726..c4d3005a7 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -272,7 +272,7 @@ def test_workflow_routes_nvidia_exhaustion_through_live_catalog(self) -> None: "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5", fallback_expression, ) - self.assertNotIn("openrouter/free", fallback_expression) + self.assertIn("openrouter/free", fallback_expression) self.assertIn("openai-direct/gpt-5.4", fallback_expression) def test_manual_status_job_has_status_write_permission(self) -> None: From 63c625cc28e6fe4a487b9c60c51baf189deae544 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 06:12:35 +0900 Subject: [PATCH 49/49] fix(strix): bound wrapped OpenRouter evidence --- scripts/ci/strix_quick_gate.sh | 17 ++++++----- scripts/ci/test_strix_quick_gate.sh | 46 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 947dce89c..0e300a9f6 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2922,14 +2922,15 @@ is_llm_service_unavailable_error() { return 0 fi - # OpenRouter's dynamic free route can surface an upstream provider 502 as - # APIError rather than ServiceUnavailableError. Require both LiteLLM's - # OpenRouter exception and OpenRouter's provider metadata so target-app 502 - # output cannot independently trigger a provider retry. - if grep -Eiq 'litellm(\.exceptions)?\.APIError' "$STRIX_LOG" && - grep -Eiq 'OpenrouterException' "$STRIX_LOG" && - grep -Eq '"code"[[:space:]]*:[[:space:]]*502' "$STRIX_LOG" && - grep -Eq '"metadata"[[:space:]]*:[[:space:]]*\{[^}]*"provider_name"[[:space:]]*:[[:space:]]*"[^"]+"' "$STRIX_LOG"; then + # OpenRouter's dynamic free route can wrap one upstream 502 over several + # terminal lines. Join only the bounded LiteLLM error block so unrelated + # target-app output elsewhere in the log cannot assemble a retry signature. + if awk ' + /litellm(\.exceptions)?\.APIError/ { block = $0; remaining = 5; next } + remaining > 0 { block = block " " $0; remaining--; if (remaining == 0) print block } + END { if (remaining > 0) print block } + ' "$STRIX_LOG" | + grep -Eiq 'litellm(\.exceptions)?\.APIError.*OpenrouterException.*"code"[[:space:]]*:[[:space:]]*502.*"metadata"[[:space:]]*:[[:space:]]*\{[^}]*"provider_name"[[:space:]]*:[[:space:]]*"[^"]+"'; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ed0aadeb7..01539aabd 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3814,6 +3814,25 @@ REPORT ;; esac ;; + openrouter-502-distant-target-output-nonretryable) + case "${STRIX_LLM:-}" in + vertex_ai/missing-primary) + echo "Error: litellm.NotFoundError: Vertex_aiException - x" + echo '"status": "NOT_FOUND"' + exit 1 + ;; + openrouter/free) + echo "Error: litellm.APIError: APIError: OpenrouterException -" + printf 'target output\n%.0s' 1 2 3 4 5 6 + echo '{"code":502,"metadata":{"provider_name":"spoof"}}' + exit 1 + ;; + vertex_ai/fallback-two) + echo "scan ok after distant target output" + exit 0 + ;; + esac + ;; github-models-primary-unavailable-fallback-success|github-models-primary-denied-fallback-success) case "${STRIX_LLM:-}" in openai/gpt-5) @@ -6222,6 +6241,20 @@ run_filtered_gate_case_if_requested() { "" \ "1" ;; + openrouter-502-distant-target-output-nonretryable) + run_gate_case "openrouter-502-distant-target-output-nonretryable" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "2" \ + "vertex_ai/missing-primary|openrouter/free" \ + "|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + ;; service-unavailable-no-llm-marker-nonrecoverable) run_gate_case "service-unavailable-no-llm-marker-nonrecoverable" \ "custom/service-unavailable-primary" \ @@ -10024,6 +10057,19 @@ run_gate_case "openrouter-502-fallback-retry-same-model-success" \ "" \ "1" +run_gate_case "openrouter-502-distant-target-output-nonretryable" \ + "vertex_ai/missing-primary" \ + "openrouter/free vertex_ai/fallback-two" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "2" \ + "vertex_ai/missing-primary|openrouter/free" \ + "|https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "1" + run_gate_case "github-models-primary-unavailable-fallback-success" \ "openai/gpt-5" \ "" \