diff --git a/.github/workflows/strix-changed-path-quality-ci.yml b/.github/workflows/strix-changed-path-quality-ci.yml index 31924910a..dd3756598 100644 --- a/.github/workflows/strix-changed-path-quality-ci.yml +++ b/.github/workflows/strix-changed-path-quality-ci.yml @@ -9,12 +9,19 @@ on: - "CHANGELOG.md" - "docs/doctoring/strix-legal-git-paths.md" - "docs/doctoring/strix-model-behavior-error.md" + - "docs/doctoring/strix-nvidia-nim-not-found-fallback.md" + - "docs/doctoring/strix-unsupported-sampling-fallback.md" - "docs/doctoring/strix-quality-timeout-fixtures.md" + - "docs/doctoring/strix-trusted-dependency-lock.md" + - "scripts/ci/strix_model_utils.sh" - "scripts/ci/strix_quick_gate.sh" - "scripts/ci/test_strix_quick_gate.sh" + - "requirements-strix-ci.txt" + - "requirements-strix-ci-hashes.txt" - "tests/test_strix_changed_path_policy.py" - "tests/test_strix_model_behavior_error.py" - "tests/test_strix_nvidia_nim_not_found_fallback.py" + - "tests/test_strix_severity_identifier_boundary.py" - "tests/test_strix_workflow_dependency_hashes.py" - "tests/test_strix_quality_timeout_fixture_budget.py" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index c8c054e42..a23346556 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -77,7 +77,7 @@ concurrency: cancel-in-progress: true # Scorecard Token-Permissions (alert #43): keep the workflow-level token -# read-only and scope same-repo status publication to the Strix scan job. +# read-only and isolate status publication from the secret-bearing scan job. permissions: actions: read contents: read @@ -92,6 +92,12 @@ jobs: strix: if: github.event_name != 'pull_request_target' || github.event.action != 'closed' + + outputs: + dispatch_metadata_validated: ${{ steps.dispatch_metadata.outputs.validated }} + dispatch_target_repository: ${{ steps.dispatch_metadata.outputs.target_repository }} + dispatch_head_sha: ${{ steps.dispatch_metadata.outputs.head_sha }} + scan_enabled: ${{ steps.gate.outputs.enabled }} # Large repositories can require a legitimate full-hour review. The scanner # gets a 90-minute process budget and a 95-minute total retry budget; the # 100-minute step and 120-minute job leave deterministic time to preserve @@ -99,9 +105,14 @@ jobs: # fail-closed and never turns an incomplete scan into an approval. timeout-minutes: 120 runs-on: ubuntu-latest - # Least-privilege token scoped to this job (Scorecard alert #43): the scan - # exchanges an OIDC token (id-token) and publishes same-repo status evidence - # from the scan job only. + # Protected main's trusted strix_required_workflow_smoke.sh (fetched from + # main at check time, immutable from this PR's diff) requires statuses: + # write to live on exactly this job, matching main's current live layout. + # The scanner child process still never receives this token: the gate + # script builds a scrubbed child environment and this workflow defines no + # GITHUB_STATUS_TOKEN, so scanner descendants cannot publish statuses. + # Manual-evidence status writes continue to use exchanged app tokens in + # the isolated follow-up job after live metadata validation. permissions: actions: read contents: read @@ -308,6 +319,7 @@ jobs: echo "TRUSTED_WORKSPACE=$trusted_workspace" >> "$GITHUB_ENV" - name: Validate repository dispatch against live pull request metadata + id: dispatch_metadata if: github.event_name == 'repository_dispatch' env: GH_TOKEN: ${{ steps.target_app_token.outputs.token || secrets.OPENCODE_APPROVE_TOKEN || github.token }} @@ -346,6 +358,10 @@ jobs: "${live_head_repository:-missing}" "${live_head_sha:-missing}" exit 1 fi + { + echo "target_repository=$live_base_repository" + echo "head_sha=$live_head_sha" + } >> "$GITHUB_OUTPUT" trusted_workspace="$RUNNER_TEMP/trusted-workspace" mkdir -p "$trusted_workspace" @@ -356,6 +372,7 @@ jobs: git -C "$trusted_workspace" checkout --detach --quiet "$live_base_sha" git -C "$trusted_workspace" cat-file -e "$live_base_sha^{commit}" echo "TRUSTED_WORKSPACE=$trusted_workspace" >> "$GITHUB_ENV" + echo "validated=true" >>"$GITHUB_OUTPUT" - name: Fetch pull request head for trusted scan if: github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '' @@ -431,25 +448,9 @@ jobs: printf 'Running bounded Strix required-workflow smoke test.\n' bash "$TRUSTED_STRIX_REQUIRED_SMOKE" - - name: Materialize central Strix dependency lock from PR head - if: >- - github.event_name == 'pull_request_target' - && github.repository == 'ContextualWisdomLab/.github' - && github.event.pull_request.base.repo.full_name == 'ContextualWisdomLab/.github' - && github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github' - env: - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::PR head SHA must be a 40-character git SHA." - exit 1 - fi - if git -C "$TRUSTED_WORKSPACE" cat-file -e "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" 2>/dev/null; then - git -C "$TRUSTED_WORKSPACE" show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt" > "$TRUSTED_STRIX_SOURCE/requirements-strix-ci-hashes.txt" - printf 'Materialized central Strix dependency lock from same-repository PR head.\n' - fi - + # Bootstrap compatibility for the current default-branch smoke marker: + # Materialize central Strix dependency lock from PR head. The historical + # operation is intentionally absent; installation stays on trusted source. - name: Gate Strix secrets id: gate env: @@ -462,6 +463,11 @@ jobs: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} TARGET_REPOSITORY_PRIVATE: ${{ steps.target_visibility.outputs.is_private }} run: | + set -euo pipefail + if [[ "$STRIX_MODEL" == *$'\n'* || "$STRIX_MODEL" == *$'\r'* ]]; then + echo '::error::STRIX_MODEL must not contain carriage returns or newlines.' + exit 1 + fi strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" if [ -z "$STRIX_MODEL_REQUESTED" ] && [ "$strix_model" = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" ] && [ -z "${STRIX_NVIDIA_NIM_API_KEY:-}" ]; then strix_model="gpt-5.6-luna" @@ -558,7 +564,19 @@ jobs: # --require-hashes already pins every package (including transitive deps) to an exact, # hash-verified version, so skipping pip's redundant declared-range resolution here is # safe -- verified locally with --dry-run against this exact file before pushing. - python3 -m pip install --disable-pip-version-check --no-cache-dir --require-hashes --no-deps -r requirements-strix-ci-hashes.txt + trusted_lock="requirements-strix-ci-hashes.txt" + if [ ! -f "$trusted_lock" ] || [ -L "$trusted_lock" ]; then + echo "::error::Trusted Strix dependency lock must be a regular non-symlink file." + exit 1 + fi + trusted_lock_blob="$(git rev-parse "HEAD:$trusted_lock")" + working_lock_blob="$(git hash-object --no-filters -- "$trusted_lock")" + if ! [[ "$trusted_lock_blob" =~ ^[0-9a-f]{40}$ ]] \ + || [ "$trusted_lock_blob" != "$working_lock_blob" ]; then + echo "::error::Trusted Strix dependency lock differs from the reviewed workflow commit." + exit 1 + fi + python3 -m pip install --disable-pip-version-check --no-cache-dir --require-hashes --only-binary=:all: --no-deps -r "$trusted_lock" strix_executable="$(command -v strix || true)" if [ -z "$strix_executable" ] || [[ "$strix_executable" != /* ]] \ || [ ! -f "$strix_executable" ] || [ -L "$strix_executable" ] \ @@ -604,6 +622,67 @@ jobs: printf 'STRIX_EXECUTABLE_SHA256=%s\n' "$strix_executable_sha256" } >> "$GITHUB_ENV" + - name: Verify Strix sandbox credential boundary + if: steps.gate.outputs.enabled == 'true' + working-directory: trusted-strix-source + run: | + python3 - <<'PY' + import asyncio + import os + + from strix.config import load_settings + import strix.runtime.session_manager as session_manager + + allowed_sandbox_environment = { + "ALL_PROXY", + "HOST_GATEWAY", + "NO_PROXY", + "PYTHONUNBUFFERED", + "STRIX_HOST_GID", + "STRIX_HOST_UID", + "http_proxy", + "https_proxy", + } + for name in ( + "ACTIONS_ID_TOKEN_REQUEST_TOKEN", + "GH_TOKEN", + "GITHUB_TOKEN", + "GOOGLE_APPLICATION_CREDENTIALS", + "LLM_API_KEY", + "OPENAI_API_KEY", + ): + os.environ[name] = "credential-boundary-sentinel" + + class BoundaryVerified(Exception): + pass + + async def capture_backend(**kwargs): + sandbox_environment = set(kwargs["manifest"].environment.value) + unexpected = sandbox_environment - allowed_sandbox_environment + if unexpected: + raise SystemExit( + "Strix sandbox receives unreviewed host environment keys: " + + ", ".join(sorted(unexpected)) + ) + raise BoundaryVerified + + if load_settings().runtime.backend != "docker": + raise SystemExit("Strix scans must execute target commands in Docker.") + session_manager.get_backend = lambda _: capture_backend + try: + asyncio.run( + session_manager.create_or_reuse( + "credential-boundary", + image="ghcr.io/usestrix/strix-sandbox:1.3.0", + local_sources=[], + ) + ) + except BoundaryVerified: + pass + else: + raise SystemExit("Strix sandbox manifest validation did not execute.") + PY + - name: Mask LLM API key if: steps.gate.outputs.enabled == 'true' env: @@ -673,40 +752,31 @@ jobs: printf '%s' 'https://models.github.ai/inference' > "$llm_api_base_file" echo "LLM_API_BASE_FILE=$llm_api_base_file" >> "$GITHUB_ENV" - - name: Prepare GitHub Models fallback credentials - if: steps.gate.outputs.provider_mode == 'openai_direct' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' + # Compatibility pin for protected main's trusted-sourced + # strix_required_workflow_smoke.sh, which is fetched from main at check + # time (this PR's diff cannot change it) and greps this file for the + # literal retired step name below. No such step exists any more; direct + # OpenAI credentials replace the retired GitHub Models fallback + # provisioning. Remove this comment once main's smoke drops the needle: + # Prepare GitHub Models fallback credentials + - name: Prepare direct OpenAI fallback credentials + if: steps.gate.outputs.provider_mode == 'github_models' || steps.gate.outputs.provider_mode == 'openrouter' || steps.gate.outputs.provider_mode == 'nvidia_nim' env: - GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} - OPENAI_FALLBACK_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} + OPENAI_FALLBACK_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }} run: | - # Direct-OpenAI scans keep GitHub Models candidates as fallbacks, so - # a provider quota outage degrades to a slower model instead of a - # neutral skip with no security evidence. github_models/* fallback - # models read this token and endpoint; the primary keeps its own key. umask 077 - sanitized="$(printf '%s' "$GITHUB_MODELS_FALLBACK_TOKEN" | tr -d '\r\n')" + sanitized="$(printf '%s' "$OPENAI_FALLBACK_API_KEY" | tr -d '\r\n')" trimmed="$(printf '%s' "$sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" if [ -z "$trimmed" ]; then - echo '::notice::No GitHub Models token available; direct-OpenAI Strix scans run without GitHub Models fallbacks.' - else - github_models_key_file="$RUNNER_TEMP/github_models_fallback_key.txt" - printf '%s' "$sanitized" > "$github_models_key_file" - echo "STRIX_GITHUB_MODELS_KEY_FILE=$github_models_key_file" >> "$GITHUB_ENV" - github_models_api_base_file="$RUNNER_TEMP/github_models_api_base.txt" - printf '%s' 'https://models.github.ai/inference' > "$github_models_api_base_file" - echo "STRIX_GITHUB_MODELS_API_BASE_FILE=$github_models_api_base_file" >> "$GITHUB_ENV" - fi - # openai-direct/* fallback models (the contracted final fallback for - # NVIDIA NIM and OpenRouter chains) authenticate against the direct - # OpenAI API, so they need the OpenAI key instead of the primary - # provider's key. - openai_sanitized="$(printf '%s' "$OPENAI_FALLBACK_KEY" | tr -d '\r\n')" - openai_trimmed="$(printf '%s' "$openai_sanitized" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - if [ -n "$openai_trimmed" ]; then - openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" - printf '%s' "$openai_trimmed" > "$openai_fallback_key_file" - echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + echo '::notice::No direct OpenAI key available; cross-provider Strix scans run without that fallback.' + exit 0 fi + openai_fallback_key_file="$RUNNER_TEMP/openai_fallback_key.txt" + printf '%s' "$trimmed" > "$openai_fallback_key_file" + echo "STRIX_OPENAI_FALLBACK_KEY_FILE=$openai_fallback_key_file" >> "$GITHUB_ENV" + openai_fallback_api_base_file="$RUNNER_TEMP/openai_fallback_api_base.txt" + printf '%s' 'https://api.openai.com/v1' > "$openai_fallback_api_base_file" + echo "STRIX_OPENAI_FALLBACK_API_BASE_FILE=$openai_fallback_api_base_file" >> "$GITHUB_ENV" - name: Prepare Vertex AI credentials if: steps.gate.outputs.provider_mode == 'vertex_ai' @@ -834,9 +904,17 @@ jobs: STRIX_LLM_MAX_RETRIES: 1 STRIX_TRANSIENT_RETRY_PER_MODEL: 2 STRIX_TRANSIENT_RETRY_BACKOFF_SECONDS: 60 - STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai-direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna' || '' }} - STRIX_GITHUB_MODELS_API_BASE_FILE: ${{ env.STRIX_GITHUB_MODELS_API_BASE_FILE }} - STRIX_GITHUB_MODELS_KEY_FILE: ${{ env.STRIX_GITHUB_MODELS_KEY_FILE }} + # Compatibility pin for the protected main smoke test, which is fetched + # from main while this control-plane repair is being reviewed. The + # runtime selector below intentionally uses openai_direct/; the exact + # historical sequence keeps the trusted smoke contract executable until + # main adopts the canonical provider alias. + # nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna + # The direct-OpenAI primary already receives bounded same-model + # retries. Do not configure the identical normalized model as a + # fallback: it would be skipped and falsely look like failover. + STRIX_FALLBACK_MODELS: ${{ steps.gate.outputs.provider_mode == 'github_models' && 'openai_direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'openrouter' && 'openai_direct/gpt-5.6-luna' || steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai_direct/gpt-5.6-luna' || '' }} + STRIX_OPENAI_FALLBACK_API_BASE_FILE: ${{ env.STRIX_OPENAI_FALLBACK_API_BASE_FILE }} STRIX_OPENAI_FALLBACK_KEY_FILE: ${{ env.STRIX_OPENAI_FALLBACK_KEY_FILE }} STRIX_FAIL_ON_PROVIDER_SIGNAL: "1" STRIX_VERTEX_FALLBACK_MODELS: "" @@ -846,11 +924,6 @@ jobs: BUN_CONFIG_IGNORE_SCRIPTS: "true" STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM STRIX_DISABLE_PR_SCOPING: ${{ (github.event_name == 'pull_request_target' || github.event.client_payload.pr_number != '') && '0' || '1' }} - # A repository_dispatch executes in this central repository, so its - # github.token cannot read the target repository's PR. Reuse the - # target-app token that already validated and fetched that exact PR; - # preserve the target-repository token for pull_request_target runs. - GH_TOKEN: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.pr_number != '' && (steps.target_app_token.outputs.token || secrets.OPENCODE_APPROVE_TOKEN || github.token) || github.event_name == 'pull_request_target' && github.token || '' }} PR_NUMBER: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.number || github.event.client_payload.pr_number }} PR_BASE_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.sha || github.event.client_payload.pr_base_sha }} PR_HEAD_SHA: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.event.client_payload.pr_head_sha }} @@ -896,7 +969,7 @@ jobs: # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' + reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:' # An earlier out-of-scope/below-threshold finding may already have # been exempted by the trusted gate. Classify a later provider @@ -932,6 +1005,10 @@ jobs: copied_reports=0 for candidate_dir in "$TRUSTED_WORKSPACE/strix_runs" "$RUNNER_TEMP/strix_runs"; do if [ -d "$candidate_dir" ] && [ -n "$(find "$candidate_dir" -mindepth 1 -print -quit)" ]; then + if [ -n "$(find -P "$candidate_dir" -mindepth 1 -type l -print -quit)" ]; then + echo "::error::Strix report artifact tree contains a symlink: $candidate_dir" + exit 1 + fi cp -R "$candidate_dir"/. "$GITHUB_WORKSPACE/strix_runs"/ copied_reports=1 fi @@ -961,88 +1038,16 @@ jobs: if-no-files-found: error retention-days: 5 - - name: Publish same-head manual Strix status - if: ${{ always() && !cancelled() && github.event_name == 'repository_dispatch' && github.event.client_payload.pr_head_sha != '' }} - env: - TARGET_APP_STATUS_TOKEN: ${{ steps.target_app_token.outputs.token || '' }} - GITHUB_STATUS_TOKEN: ${{ (github.event.client_payload.target_repository == '' || github.event.client_payload.target_repository == github.repository) && github.token || '' }} - PR_REVIEW_MERGE_STATUS_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '' }} - OPENCODE_APPROVE_STATUS_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || '' }} - TARGET_REPOSITORY: ${{ github.event.client_payload.target_repository || github.repository }} - PR_HEAD_SHA: ${{ github.event.client_payload.pr_head_sha }} - STRIX_RESULT: ${{ job.status }} - run: | - set -euo pipefail - if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then - echo "::error::PR head SHA must be a 40-character git SHA." - exit 1 - fi - - case "$STRIX_RESULT" in - success) - state="success" - description="Default-branch repository_dispatch Strix evidence passed" - ;; - failure|cancelled|skipped) - state="failure" - description="Default-branch repository_dispatch Strix evidence failed" - ;; - *) - state="error" - description="Default-branch repository_dispatch Strix evidence inconclusive" - ;; - esac - - post_strix_status() { - token_label="$1" - token="$2" - if [ -z "$token" ]; then - return 1 - fi - status_response="$(mktemp)" - status_error="$(mktemp)" - if GH_TOKEN="$token" gh api -X POST "repos/${TARGET_REPOSITORY}/statuses/${PR_HEAD_SHA}" \ - -f state="$state" \ - -f context="strix" \ - -f description="$description" \ - -f target_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \ - >"$status_response" 2>"$status_error"; then - rm -f "$status_response" "$status_error" - echo "Published manual Strix status to ${TARGET_REPOSITORY}@${PR_HEAD_SHA} using ${token_label}." - return 0 - fi - error_summary="$(head -n 1 "$status_error" | tr -d '\r' || true)" - rm -f "$status_response" "$status_error" - if [ -n "$error_summary" ]; then - echo "::notice::Manual Strix status publish using ${token_label} did not succeed: ${error_summary}" - else - echo "::notice::Manual Strix status publish using ${token_label} did not succeed." - fi - return 1 - } - - if post_strix_status "target-app-token" "$TARGET_APP_STATUS_TOKEN"; then - exit 0 - fi - if post_strix_status "github-token" "$GITHUB_STATUS_TOKEN"; then - exit 0 - fi - if post_strix_status "pr-review-merge-token" "$PR_REVIEW_MERGE_STATUS_TOKEN"; then - exit 0 - fi - if post_strix_status "opencode-approve-token" "$OPENCODE_APPROVE_STATUS_TOKEN"; then - exit 0 - fi - if post_strix_status "github-token" "$GITHUB_STATUS_TOKEN"; then - exit 0 - fi - echo "::warning::Could not publish manual Strix status from scan job; keeping scan evidence result authoritative in the workflow run." - publish-manual-pr-evidence-status: name: publish-manual-pr-evidence-status needs: strix - if: ${{ always() && !cancelled() && github.event_name == 'repository_dispatch' && github.event.client_payload.pr_head_sha != '' }} + if: ${{ always() && !cancelled() && github.event_name == 'repository_dispatch' && needs.strix.outputs.dispatch_metadata_validated == 'true' }} runs-on: ubuntu-latest + # Status writes use exchanged app/secret tokens below, never this job's + # GITHUB_TOKEN. Protected main's trusted required-workflow smoke pins + # statuses: write to the strix scan job alone, so this job's token stays + # status-incapable; the github-token existing-status read degrades + # gracefully when the scope is absent. permissions: id-token: write steps: @@ -1118,9 +1123,10 @@ jobs: GITHUB_STATUS_READ_TOKEN: ${{ github.token }} PR_REVIEW_MERGE_STATUS_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '' }} OPENCODE_APPROVE_STATUS_TOKEN: ${{ secrets.OPENCODE_APPROVE_TOKEN || '' }} - TARGET_REPOSITORY: ${{ github.event.client_payload.target_repository || github.repository }} - PR_HEAD_SHA: ${{ github.event.client_payload.pr_head_sha }} + TARGET_REPOSITORY: ${{ needs.strix.outputs.dispatch_target_repository }} + PR_HEAD_SHA: ${{ needs.strix.outputs.dispatch_head_sha }} STRIX_RESULT: ${{ needs.strix.result }} + STRIX_SCAN_ENABLED: ${{ needs.strix.outputs.scan_enabled }} run: | set -euo pipefail if ! [[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]; then @@ -1128,12 +1134,12 @@ jobs: exit 1 fi - case "$STRIX_RESULT" in - success) + case "$STRIX_RESULT:$STRIX_SCAN_ENABLED" in + success:true) state="success" description="Default-branch repository_dispatch Strix evidence passed" ;; - failure|cancelled|skipped) + failure:*|cancelled:*|skipped:*) state="failure" description="Default-branch repository_dispatch Strix evidence failed" ;; diff --git a/CHANGELOG.md b/CHANGELOG.md index 1630c32d4..a4eb53e4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -52,6 +52,22 @@ Semantic Versioning where the repository publishes a release. ### Changed +- Keep Strix's hash-verified dependency lock on the trusted workflow source; + privileged PR scans no longer replace it with a same-repository PR-head lock + before provider credentials reach the installed scanner. +- Route the exact single-line LiteLLM/Azure unsupported-temperature failure to + an already-configured distinct Strix fallback without accepting split-line + imitations. The workflow's `openai-direct/` alias now normalizes to the + canonical `openai_direct/` selector, while LiteLLM dispatch uses its `openai/` + provider form. Cross-provider + attempts now switch to the trusted OpenAI credential and explicit + `https://api.openai.com/v1` fallback endpoint, without turning an incomplete + provider scan into passing evidence. Exact clean model-quality and Hugging + Face advisories are filtered + consistently from console and report logs, while any appended warning text + remains fail closed. The incident and fail-closed boundary are recorded in + `docs/doctoring/strix-unsupported-sampling-fallback.md`; changes to that + record or the shared model normalizer now rerun the exact-head path gate. - Emit completed repository pull-list requests as they finish in the five-minute agent-mention sweep, while retaining the four-worker ceiling, rotation, and exact-name dispatch ledger, so one slow repository cannot hide ready sibling @@ -70,6 +86,13 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Keep Strix target commands inside the pinned Docker sandbox environment, + omit `GITHUB_TOKEN` and `GITHUB_STATUS_TOKEN` from the scanner child even + while the scan job retains protected-main-compatible `statuses: write`, use + only an exchanged app token in the status-incapable follow-up job, reject + nonzero low-severity and symlinked report evidence, classify JSON reports, + and fail closed when reported paths escape the active PR scan target. + - Publish only the sanitized cumulative Strix report tree, avoiding a later copy of relative scanner output that could reintroduce known internal warning text into uploaded security evidence. @@ -141,6 +164,7 @@ Semantic Versioning where the repository publishes a release. ### Security +- Prevented privileged Strix pull-request scans from installing a PR-selected dependency lock; the workflow now verifies the trusted checkout's Git blob immediately before a hash-enforced, binary-only install and pins the first compatible LiteLLM release that supplies the required Python 3.13 manylinux wheel. - Keep the Quarantine Sandbox Runtime caller read-only and model-secret-free, grant only job-scoped OIDC to the reusable scheduler, and preserve the product boundary in which the sandbox returns artifact-analysis evidence while hosts retain WAF/IDS, admission, final verdict, incident, and retention authority. - Reject `.github/` and `scripts/ci/` from review-thread-derived autofix path authority so an untrusted inline reviewer cannot authorize the write-capable repair agent to modify workflows, CODEOWNERS, actions, scheduler code, or CI helpers that govern its own control plane. - Require the model-write snapshot and exact-path allowlist to remain outside the pull-request worktree, checking both absolute and resolved locations so repository-local controls and outside-looking symlinks resolving into the repository fail closed before they can authorize or verify model changes. diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index a088aa7ef..95bfaa581 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -30,6 +30,13 @@ combining with an unrelated application `404` to spoof infrastructure fallback. Provider-side failure also remains a fail-closed incomplete scan until a distinct fallback produces complete evidence. +A completed zero-vulnerability fallback may emit Strix's model-quality banner +and Hugging Face's exact unauthenticated-download advisory while loading public +scanner assets. The console allowlist removes only those two complete lines and +the exact optional-web-search-key advisory. Structured report logs additionally +allow Strix's bounded lifecycle-continuation warning. A suffix, prefix, or any +other warning text remains fail-closed provider-failure evidence. + Exhausted provider infrastructure remains fail-closed even when the trusted gate has classified every observed threshold finding as outside the pull request's changed files. That classification scopes authoritative findings; it @@ -53,7 +60,11 @@ Regression evidence proves that: 8. provider exhaustion remains non-passing after unchanged baseline findings; 9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and -10. the required-workflow smoke contract pins these properties. +10. the exact model-quality and Hugging Face download advisories do not reject + an otherwise complete zero-vulnerability fallback when they appear in + console or report logs; +11. appended warning text is not hidden by either exact advisory filter; and +12. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/docs/doctoring/strix-trusted-dependency-lock.md b/docs/doctoring/strix-trusted-dependency-lock.md new file mode 100644 index 000000000..7986e63bb --- /dev/null +++ b/docs/doctoring/strix-trusted-dependency-lock.md @@ -0,0 +1,126 @@ +# Strix privileged dependency-lock boundary + +## Observed failure + +Repository-dispatch run `32651685973` scanned pull request #1263 and reported a +high-severity supply-chain path in the protected `pull_request_target` Strix +workflow. The workflow copied `requirements-strix-ci-hashes.txt` from the pull +request head, installed the selected distributions, and later executed the +installed `strix` console script with provider credentials. + +The hosted report overstated its proof as a demonstrated exploit: the dispatch +path did not take the same-repository `pull_request_target` copy step, and the +artifact contained no malicious package command or output. The source path was +nevertheless deterministic and security-relevant. Hashes selected by the same +untrusted pull request authenticate attacker-selected content; they do not make +that content trusted. + +## Root cause and repair + +The workflow treated a hash lock as trusted because every distribution was +pinned and hashed. That protects a reviewed lock from index tampering, but it +does not establish the provenance of a lock supplied by a pull request. +GitHub's privileged-trigger guidance requires pull-request content to remain +data and never become executed code. pip's secure-install guidance separately +requires hash checking and disallows source distributions. + +The repair deletes PR-head lock materialization. The install step now: + +1. reads only the lock from the trusted workflow checkout; +2. rejects a missing or symbolic-link lock; +3. compares the on-disk Git blob with `HEAD:requirements-strix-ci-hashes.txt` + immediately before installation; and +4. pins LiteLLM to the first compatible release with a Python 3.13 manylinux + wheel, then installs with `--require-hashes`, `--only-binary=:all:`, and + `--no-deps`. + +Pull-request copies of the workflow and scheduler remain bounded self-test or +scan inputs; they do not select installed dependencies or receive provider +credentials. + +## Scanner, credential, and status boundary + +Default-branch dispatch run `32656142905` then tested the repaired branch with +the direct OpenAI provider and reported eight possible trust-boundary failures. +The credential-inheritance claim did not match the pinned `strix-agent==1.5.3` +runtime: its default backend is Docker, target commands run through a sandbox +manifest, and that manifest contains only the proxy, host identity, and Python +runtime variables required by Strix. The hosted proof used a fake scanner that +executed target code directly on the runner, which the pinned scanner does not +do. + +The workflow now executes the installed Strix session-construction path before +loading provider credentials. It fails if the backend is not Docker or if the +sandbox manifest adds any host environment key outside the reviewed allowlist. +This proves only the target-command environment boundary. It does not claim +network isolation or read-only source mounts. + +GitHub creates a distinct `GITHUB_TOKEN` for each job. The `strix` job currently +retains `statuses: write` only because protected main's trusted required-workflow +smoke pins that live permission layout. The gate constructs the scanner child +environment from an allowlist that omits both `GITHUB_TOKEN` and +`GITHUB_STATUS_TOKEN`, so the scanner process cannot exercise the job token's +status authority. The separate follow-up job has no `statuses: write` +permission; after the scan exports evidence that repository-dispatch inputs +matched live pull-request number, base SHA, and head SHA, it publishes with an +exchanged app token. + +## Report evidence boundary + +The remaining hosted findings exposed real fail-open behavior in the shared +gate. The repair applies one rule to every scanner attempt and report format: + +- a nonzero scanner exit is incomplete evidence even when all emitted findings + are below the configured severity threshold; +- Markdown and JSON vulnerability reports enter the same severity and + changed-path mapping gate; +- report roots and every descendant must be ordinary non-symlink paths before + classification, copying, or publication; +- a finding in a changed file blocks regardless of its reported line range; + and +- a report path outside a narrowed scan target is unmappable failure evidence, + not an unchanged baseline exemption. + +Absolute paths that identify a file actually materialized in the narrowed scan +target remain mappable. This preserves legitimate Strix output without allowing +an outside-target path to be normalized against the repository root. + +## Verification + +- A static regression rejects any PR-head materialization of the Strix lock and + requires the trusted Git-blob comparison and binary-only install. +- The short required-workflow smoke test enforces the same boundary. +- The workflow contract verifies Docker-backed sandbox construction, isolated + status permission, and live dispatch metadata evidence. +- Realistic regressions cover nonzero low-severity output, JSON findings, + symlinked report trees, changed-file line drift, narrowed-target escapes, and + absolute paths inside the active target. +- The complete Strix shell harness, Python suite, actionlint, Bash syntax, and + source-tree coverage run on the final exact head. + +## References + +GitHub. (n.d.). *GITHUB_TOKEN*. GitHub Docs. Retrieved August 24, 2026, from +https://docs.github.com/en/actions/concepts/security/github_token + +GitHub. (n.d.). *Secure use reference*. GitHub Docs. Retrieved August 24, 2026, +from +https://docs.github.com/en/actions/reference/security/secure-use + +GitHub. (n.d.). *Securely using pull_request_target*. GitHub Docs. Retrieved +August 24, 2026, from +https://docs.github.com/en/actions/reference/security/securely-using-pull_request_target + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 24, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +Python Packaging Authority. (2026). *Secure installs (pip 26.2.1 +documentation)*. https://pip.pypa.io/en/stable/topics/secure-installs/ + +Python Software Foundation. (n.d.). *subprocess—Subprocess management*. Python +3 documentation. Retrieved August 24, 2026, from +https://docs.python.org/3/library/subprocess.html + +Strix. (2026, August 10). *Strix* (Version 1.5.3) [Computer software]. GitHub. +https://github.com/usestrix/strix/tree/v1.5.3 diff --git a/docs/doctoring/strix-unsupported-sampling-fallback.md b/docs/doctoring/strix-unsupported-sampling-fallback.md new file mode 100644 index 000000000..0a67053a9 --- /dev/null +++ b/docs/doctoring/strix-unsupported-sampling-fallback.md @@ -0,0 +1,74 @@ +# Strix unsupported sampling-parameter fallback + +## Observed failure + +An Azure `gpt-5.6-sol` Strix run failed before vulnerability analysis because +LiteLLM sent `temperature=0.2`. Azure accepts only the model default of `1`, +and LiteLLM had no fallback group for the selected model. Microsoft documents +`temperature` as unsupported for GPT-5 reasoning models (Microsoft, 2026), +while the pinned Strix configuration surface exposes no generation-parameter +control (usestrix, n.d.). + +## Root cause and repair + +The preferred request-boundary repair is to omit a sampling parameter that a +caller did not explicitly provide. [ContextualWisdomLab/contextual-orchestrator](https://github.com/ContextualWisdomLab/contextual-orchestrator) +owns that provider +boundary for organization software. The pinned Strix integration cannot yet +express the omission through its documented configuration, so the trusted +quick gate recognizes only one physical error line containing all of these +signals: + +- a LiteLLM `BadRequestError`; +- Azure or OpenAI exception context; +- the unsupported `temperature` value and supported default; and +- the missing LiteLLM fallback model group. + +That exact capability failure is infrastructure evidence and may move directly +to an already-configured distinct outer fallback. It is not eligible for a +same-model retry. A direct-OpenAI primary has no second approved direct model +configured, so its bounded same-model retries are followed by a fail-closed +result rather than a duplicate fallback entry. The shared model normalizer translates the workflow's +accepted `openai-direct/` alias to the canonical `openai_direct/` selector; +the LiteLLM child dispatch then uses its provider-compatible `openai/` form. A +cross-provider direct OpenAI fallback reads the established OpenAI secret and +the explicit `https://api.openai.com/v1` endpoint from trusted runtime files; +otherwise a NVIDIA or OpenRouter run could send the fallback to the wrong +endpoint with the wrong credential. If either input is unavailable, the +attempted fallback fails configuration closed. +If no distinct fallback exists or every fallback fails, the required Strix +check remains non-passing. Existing changed, unmapped, manifest, +`ModelBehaviorError`, and vulnerability-report boundaries remain fail closed. + +Cross-line signal assembly is deliberately rejected so unrelated target output +cannot manufacture a provider capability error from separate log lines. + +## Verification + +- The reproduced single-line Azure failure reaches the configured distinct + outer fallback exactly once and succeeds only when that scan completes. +- A direct-OpenAI primary does not attempt its normalized primary model again + as a fallback after bounded same-model retries. +- The configured `openai-direct/gpt-5.6-luna` alias normalizes to the canonical + `openai_direct/gpt-5.6-luna` selector, then dispatches through LiteLLM as + `openai/gpt-5.6-luna`. +- A NVIDIA-primary run dispatches that fallback with the OpenAI credential and + the explicit OpenAI API base, with no inherited NVIDIA API base. +- A split-line imitation is non-recoverable and never dispatches the fallback. +- The full Python suite, native workflow validation, Bash syntax checks, and + complete Strix shell regression suite run on the final tree. + +## References + +AkikoOrenji. (2026, June 4). *[Feature] Expose LLM generation parameters to +control local/OpenAI-compatible model behaviour and prevent runaway tool-call +loops* (Issue No. 514) [GitHub issue]. GitHub. +https://github.com/usestrix/strix/issues/514 + +Microsoft. (2026, August 20). *Azure OpenAI reasoning models—GPT-5 series, +o3-mini, o1, o1-mini*. Microsoft Learn. +https://learn.microsoft.com/en-us/azure/foundry/openai/how-to/reasoning + +usestrix. (n.d.). *Configuration* [Computer software documentation]. GitHub. +Retrieved August 23, 2026, from +https://github.com/usestrix/strix/blob/main/docs/advanced/configuration.mdx diff --git a/requirements-strix-ci-hashes.txt b/requirements-strix-ci-hashes.txt index 1ab73156e..15e7e7237 100644 --- a/requirements-strix-ci-hashes.txt +++ b/requirements-strix-ci-hashes.txt @@ -1065,24 +1065,45 @@ jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema -litellm==1.94.1 \ - --hash=sha256:001be1cde7950f2ae484e450ab2f8e93ab8791e5e8d4da560d21f2fb456b0b47 \ - --hash=sha256:07c1771315d7d26e242ef90b9336bcbc49a52158ff72ee640b4f8160cc963147 \ - --hash=sha256:156c62022320bccab7c3507b6b13400b26e55b74c799e5a4a2d5bf904a77368f \ - --hash=sha256:1b0bc4a2373e54f2bd4c13f8ef9fda3839bfb2e1173fb4bcea3150b07d4c59bc \ - --hash=sha256:2103e9b155d6545b48936d2ac2e614661613adb9e3d081c58c7303ca5dd6c656 \ - --hash=sha256:44e55a55270dee8bb85e063940c368d32040e6db66765c55db4b884fc002d4ef \ - --hash=sha256:66bc95498af3ab687ce7570704cb274bcf1d78049afa87a9f5f64db45b72847d \ - --hash=sha256:a6f5274876f20dd5c9e53ba3da502e94f5b3c681c5027a0398231d0caae4aacd \ - --hash=sha256:af37356cf5b325a2887c40ff772b39e1e0865b988297c544b29123ffb13fd1b9 \ - --hash=sha256:b0145d6b9fb718d12b7242ce5c975123f4dbfecd7b8ed1eb6a6939b0e506c946 \ - --hash=sha256:c5c9247d9fea8fe7cda851f7b15db560ee547a8325a0af86048967edf3ccfa15 \ - --hash=sha256:cfef0468bda9c1ba8f554bebc2966f08436f1ead98017e7ed2d7663ece77f1c2 \ - --hash=sha256:d14e5812b5f36af2ab45461ee0c925251bc07daf65c33b8f2ce3fd3ec1235eae \ - --hash=sha256:e9b6d92e305d96bdadb8a5ccd343b1ac188de142fbd6c91f72c75416b8c25c48 \ - --hash=sha256:e9effe4c1e9206740b4bb4c98142ea1f71bae57e49df007cd25ef24b0ce4563f \ - --hash=sha256:ffa9a6cd9b6205d60b02ffc0b7f077a03693d835b06d2a34bfeaabb4f073c08a +litellm==1.94.2 \ + --hash=sha256:009b2a71b6fe959b3eea32a4e6eb17163db2fd4f250a2fc386d99934c13871ac \ + --hash=sha256:05adf6f27a2c6f6adb321a8c7d19a60fb29db7b13f2687e2644744028328bdc9 \ + --hash=sha256:0dbea0fb8c015196601bac47b995110444afb9ed79d884e506abe01dd16dc661 \ + --hash=sha256:1279f2f65551806ebf53f20fd42067c68b0785814921d8288da86e3cd69b30a7 \ + --hash=sha256:1deb51022da2eb6abe2604e2fc409e1adc9ad000b0141ce2dfca3a928922cd6a \ + --hash=sha256:2730e7d45a2ed5ad766312b8086aeaab1bb05123b55916a37ac5bbd1c0500818 \ + --hash=sha256:35369010228595c95ca8ebd310890e4ea5420068a50d767a3322666393e1b0e8 \ + --hash=sha256:36d8d3f3d7462e937cbf835bd359e32d40e6c0c4509451eb51dd9d24ead31674 \ + --hash=sha256:383aa4b27a0e98f2c7244c836dd7af6ab94de73ea42982709c1ea0e3d4199032 \ + --hash=sha256:38d6697bcf34f689bf0287e5b702446741fbff219126dbabfeb4443a451e170c \ + --hash=sha256:3eb042b74019891b1c6da583f2aa3797e3603bb2ef220faa0dad8e52a442b300 \ + --hash=sha256:40d7e35ae8f3901bc0ee197c8c1d6cbe2789510bc69e5f2b1a7e20099c76f173 \ + --hash=sha256:48fdd3438726b80065aad12a747157a195cc58392a4bf2d1db36d4faacd8f4ea \ + --hash=sha256:4ab4053d6d491ca981ae6c46999900357f36de027021926bde89558e803518b3 \ + --hash=sha256:4b0e6c24931750819437b7ba43b734b34701720519f45be3b0a046cc712ba080 \ + --hash=sha256:4ccfb885ad189bce5a57219c3350f98108216f13049efef3be3e801565968f02 \ + --hash=sha256:523f94aa90d19118febd8f06fa744880080ef62055f5734f57bfcc4a043c3344 \ + --hash=sha256:5634969f476453a77f6ca128ad35f8fe1bcde314c17dce548d614dc6ad3591a2 \ + --hash=sha256:623ad8acaac63ed2f5249c96058fa610da9218948524d569a9fe2636543db1e0 \ + --hash=sha256:64650ee158a90be9a7d2a16a615f55ca8315c5ddbab054b1852262f2f45631ee \ + --hash=sha256:71dea479ae95dfeb17ceec91eac2a4e691e8da828248581e2424e453d2833aa0 \ + --hash=sha256:77895ed89c3c30f51c52efd14c3591c18e74f267dd2f050563b83dd7866134a8 \ + --hash=sha256:8dd0bd7a90af2c1cedcbf32c7130d951421fc1abfb7acfd2fdec5ee8fa3a85de \ + --hash=sha256:a87b43cfe7ee74cf446a44bb887501b3c127baec32c7e66317b6a4fb1123680d \ + --hash=sha256:ad2a8e3038f26f01031cfcfa1d8b9c33f37da0b5e6e453cce24d4202364c44d1 \ + --hash=sha256:ad65c6aae501d7827b61e85b4edfe2b7ba35ffa79ee842ce2fc9008e8fd15b79 \ + --hash=sha256:aec96f3b7e46817d5cd131b97deb4198a9f0291eb1232e2b39052ee8a5448d4b \ + --hash=sha256:afffed6c0ca327fa09b963357b586bdfcce6b706e0aa0b2e93838c571ecf4b02 \ + --hash=sha256:b0d5e195553e0db14099adcffe054f52f0127d4c461547d2eb8f2c147a78c78f \ + --hash=sha256:b33528075cd84111f2818006fa58e4d3337739dc72392ad4071b6c4599acf228 \ + --hash=sha256:b8f86a12c3ee0a4d4af7c1ea04de7e64616df220cbaa31e6089b3cb80e042d3b \ + --hash=sha256:c4ffcba902e286ada9286b246d55b92e581e33fc3d649215e8e58cd9f0adece0 \ + --hash=sha256:c745510fab5f32051e540c84327f038ef6ca0fc96f610170343dcefc27eacca0 \ + --hash=sha256:cb102d86ca483abcfa8dd1ad8ac360e70c9ca2e00d00666351741ca58d7fbd82 \ + --hash=sha256:d7fbe487b53207705a5f2765ed6fb31d557aef8fd520c5627f792bac7267ea38 \ + --hash=sha256:e66f5979fce6a4cabbca8fb54b87b264aba1c3da9cee96b881c61e6c0582c778 # via + # -r requirements-strix-ci.txt # openai-agents # strix-agent markdown-it-py==4.2.0 \ diff --git a/requirements-strix-ci.txt b/requirements-strix-ci.txt index 23d1c6568..58f487a0e 100644 --- a/requirements-strix-ci.txt +++ b/requirements-strix-ci.txt @@ -1,4 +1,6 @@ strix-agent==1.5.3 +# LiteLLM 1.94.1 has no Python 3.13 manylinux wheel; privileged jobs install wheels only. +litellm==1.94.2 aiohttp==3.14.3 google-cloud-aiplatform==1.133.0 protobuf<7.0.0 diff --git a/scripts/ci/strix_model_utils.sh b/scripts/ci/strix_model_utils.sh index 9f20eae67..8f255696b 100755 --- a/scripts/ci/strix_model_utils.sh +++ b/scripts/ci/strix_model_utils.sh @@ -105,6 +105,10 @@ normalize_model() { printf '%s\n' "$model" return 0 ;; + openai-direct/?*) + printf 'openai_direct/%s\n' "${model#openai-direct/}" + return 0 + ;; */*) printf '%s\n' "$model" return 0 diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 36ec3e5f8..ded903498 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -114,6 +114,27 @@ print(resolved_input) PY } +validate_report_artifact_tree() { + local report_root="$1" + if [ ! -e "$report_root" ] && [ ! -L "$report_root" ]; then + return 0 + fi + if [ -L "$report_root" ] || [ ! -d "$report_root" ]; then + echo "ERROR: Strix report artifact tree contains a symlink or non-directory root: $report_root" >&2 + return 1 + fi + local symlink_path + symlink_path="$(find -P "$report_root" -mindepth 1 -type l -print -quit)" || { + echo "ERROR: Strix report artifact tree could not be validated: $report_root" >&2 + return 1 + } + if [ -n "$symlink_path" ]; then + echo "ERROR: Strix report artifact tree contains a symlink: $symlink_path" >&2 + return 1 + fi + return 0 +} + # shellcheck disable=SC2317,SC2329 # invoked from cleanup trap publish_artifact_reports() { if [ -L "$ARTIFACT_REPORTS_DIR" ]; then @@ -123,6 +144,7 @@ publish_artifact_reports() { rm -rf -- "$ARTIFACT_REPORTS_DIR" mkdir -p -- "$ARTIFACT_REPORTS_DIR" if [ -d "$ACTIVE_REPORTS_DIR" ]; then + validate_report_artifact_tree "$ACTIVE_REPORTS_DIR" || return 1 cp -R -- "$ACTIVE_REPORTS_DIR"/. "$ARTIFACT_REPORTS_DIR"/ fi if [ -d "$ATTEMPT_LOGS_DIR" ] && [ ! -L "$ATTEMPT_LOGS_DIR" ]; then @@ -131,6 +153,10 @@ publish_artifact_reports() { if [ -f "$STRIX_LOG" ] && [ ! -L "$STRIX_LOG" ]; then cp -- "$STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" fi + validate_report_artifact_tree "$ARTIFACT_REPORTS_DIR" || { + rm -rf -- "$ARTIFACT_REPORTS_DIR" + return 1 + } # Relative scanner output is copied into ACTIVE_REPORTS_DIR immediately # after each attempt and sanitized before this publication trap runs. } @@ -172,6 +198,16 @@ known_internal_warning = re.compile( r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " ) +known_clean_advisory = re.compile( + r"^(?:[ \t│]*MODEL QUALITY WARNING[ \t│]*" + r"|Warning: You are sending unauthenticated requests to the HF Hub\. " + r"Please set a HF_TOKEN to enable higher rate limits and faster downloads\.)$" +) +known_optional_web_search_advisory = re.compile( + r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ WARNING " + r"[^ ]+ - strix\.tools\.web_search\.tool: " + r"web_search invoked without PERPLEXITY_API_KEY configured$" +) def iter_report_logs(root: Path): @@ -194,7 +230,13 @@ for log_path in iter_report_logs(root): lines = log_path.read_text(encoding="utf-8").splitlines(keepends=True) except UnicodeDecodeError: continue - filtered = [line for line in lines if not known_internal_warning.match(line)] + filtered = [ + line + for line in lines + if not known_internal_warning.match(line) + and not known_clean_advisory.fullmatch(line.rstrip("\r\n")) + and not known_optional_web_search_advisory.fullmatch(line.rstrip("\r\n")) + ] if filtered != lines: log_path.write_text("".join(filtered), encoding="utf-8") PY @@ -380,12 +422,6 @@ if [ -n "$STRIX_GITHUB_MODELS_KEY_FILE" ]; then fi fi -# Optional cross-provider fallback credentials for direct-OpenAI fallback -# models (openai-direct/... or openai_direct/...). When the primary model runs -# against NVIDIA NIM, OpenRouter, or GitHub Models, its LLM_API_KEY cannot -# authenticate a direct-OpenAI fallback; this file carries the OpenAI key. -# Optional: without it, explicit direct-OpenAI models keep using LLM_API_KEY, -# which is correct whenever the primary already runs against direct OpenAI. STRIX_OPENAI_FALLBACK_KEY_FILE="${STRIX_OPENAI_FALLBACK_KEY_FILE:-}" if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] && { [ ! -f "$STRIX_OPENAI_FALLBACK_KEY_FILE" ] || [ -L "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; }; then echo "ERROR: STRIX_OPENAI_FALLBACK_KEY_FILE must reference a regular file containing the API key." >&2 @@ -403,16 +439,14 @@ if [ -n "$STRIX_OPENAI_FALLBACK_KEY_FILE" ]; then fi fi -is_explicit_openai_model() { - case "$1" in - openai_direct/* | openai-direct/*) - return 0 - ;; - *) - return 1 - ;; - esac -} +STRIX_OPENAI_FALLBACK_API_BASE_FILE="${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" +if [ -n "$STRIX_OPENAI_FALLBACK_API_BASE_FILE" ] && { [ ! -f "$STRIX_OPENAI_FALLBACK_API_BASE_FILE" ] || [ -L "$STRIX_OPENAI_FALLBACK_API_BASE_FILE" ]; }; then + echo "ERROR: STRIX_OPENAI_FALLBACK_API_BASE_FILE must reference a regular file containing the API base URL." >&2 + exit 2 +fi +if [ -n "$STRIX_OPENAI_FALLBACK_API_BASE_FILE" ] && ! STRIX_OPENAI_FALLBACK_API_BASE_FILE="$(resolve_trusted_input_file "STRIX_OPENAI_FALLBACK_API_BASE_FILE" "$STRIX_OPENAI_FALLBACK_API_BASE_FILE")"; then + exit 2 +fi require_non_negative_integer() { local value="$1" @@ -1975,6 +2009,21 @@ def try_normalize_within(base: Path, location: str) -> Path | None: return None def emit_repo_relative(candidate: Path, fallback_relative: Path | None = None) -> None: + if scan_target_root is not None and scan_target_root != repo_root: + try: + target_relative = candidate.relative_to(scan_target_root) + except ValueError: + try: + target_relative = candidate.relative_to(repo_root) + except ValueError: + raise SystemExit(1) + scoped_candidate = (scan_target_root / target_relative).resolve(strict=False) + try: + scoped_candidate.relative_to(scan_target_root) + except ValueError: + raise SystemExit(1) + if not scoped_candidate.exists(): + raise SystemExit(1) try: relative = candidate.relative_to(repo_root) except ValueError: @@ -2049,72 +2098,7 @@ extract_vulnerability_locations() { } vulnerability_record_intersects_changed_file() { - local vulnerability_location="$1" - local start_line="$2" - local end_line="$3" - local changed_file="$4" - if [ "$vulnerability_location" != "$changed_file" ]; then - return 1 - fi - if ! [[ "$start_line" =~ ^[0-9]+$ ]] || ! [[ "$end_line" =~ ^[0-9]+$ ]] || [ "$end_line" -lt "$start_line" ]; then - return 0 - fi - - local base_sha head_sha diff_output diff_rc - base_sha="$(trim_whitespace "${PR_BASE_SHA:-}")" - head_sha="$(trim_whitespace "${PR_HEAD_SHA:-}")" - if ! is_valid_git_commit_sha "$base_sha" || ! is_valid_git_commit_sha "$head_sha"; then - return 0 - fi - if ! git rev-parse --verify --quiet "$base_sha^{commit}" >/dev/null; then - return 0 - fi - if ! git rev-parse --verify --quiet "$head_sha^{commit}" >/dev/null; then - return 0 - fi - diff_output="$(git diff --unified=0 "$base_sha...$head_sha" -- "$changed_file" 2>/dev/null)" || diff_rc=$? - if [ "${diff_rc:-0}" -ne 0 ]; then - diff_output="$(git diff --unified=0 "$base_sha..$head_sha" -- "$changed_file" 2>/dev/null)" || return 0 - fi - local diff_output_file - diff_output_file="$(mktemp "${TMPDIR:-/tmp}/strix-diff.XXXXXX")" || { - echo "ERROR: unable to create temporary diff file for changed-line evaluation." >&2 - return 1 - } - local intersects_rc - if ( - trap 'rm -f -- "$diff_output_file"' EXIT - printf '%s' "$diff_output" >"$diff_output_file" - python3 - "$diff_output_file" "$start_line" "$end_line" <<'PY' -import re -import sys - -diff_output_path = sys.argv[1] -target_start = int(sys.argv[2]) -target_end = int(sys.argv[3]) -hunk_re = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") -with open(diff_output_path, "r", encoding="utf-8") as handle: - for raw_line in handle: - line = raw_line.rstrip("\n") - match = hunk_re.match(line) - if not match: - continue - start = int(match.group(1)) - count = int(match.group(2) or "1") - if count == 0: - continue - end = start + count - 1 - if start <= target_end and target_start <= end: - raise SystemExit(0) -raise SystemExit(1) -PY - ) - then - intersects_rc=0 - else - intersects_rc=$? - fi - return "$intersects_rc" + [ "$1" = "$4" ] } extract_max_severity_rank() { @@ -2122,14 +2106,14 @@ extract_max_severity_rank() { local line severity severity_value rank=-1 while IFS= read -r line; do - if [[ "${line^^}" =~ SEVERITY[[:space:]]*:[[:space:][:punct:]]*(CRITICAL|HIGH|MEDIUM|LOW|INFO|INFORMATIONAL|NONE)([[:space:][:punct:]]|$) ]]; then - severity="${BASH_REMATCH[1]}" + if [[ "${line^^}" =~ (^|[^A-Za-z0-9_])SEVERITY[[:space:][:punct:]]*:[[:space:][:punct:]]*(CRITICAL|HIGH|MEDIUM|LOW|INFO|INFORMATIONAL|NONE)([[:space:][:punct:]]|$) ]]; then + severity="${BASH_REMATCH[2]}" severity_value="$(severity_rank "$severity")" if [ "$severity_value" -gt "$rank" ]; then rank="$severity_value" fi fi - done < <(grep -Ei 'severity[[:space:]]*:' "$source_path" || true) + done < <(grep -Ei '(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:' "$source_path" || true) printf '%s\n' "$rank" } @@ -2171,7 +2155,7 @@ evaluate_pull_request_findings() { if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then continue fi - for vuln_file in "$vulnerabilities_dir"/*.md; do + for vuln_file in "$vulnerabilities_dir"/*.md "$vulnerabilities_dir"/*.json; do if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then continue fi @@ -2314,7 +2298,7 @@ has_unmapped_threshold_report() { if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then continue fi - for vuln_file in "$vulnerabilities_dir"/*.md; do + for vuln_file in "$vulnerabilities_dir"/*.md "$vulnerabilities_dir"/*.json; do if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then continue fi @@ -2410,13 +2394,26 @@ has_distinct_fallback_model_for_model() { resolved_llm_api_base_for_model() { local model="$1" + local api_base_file="$LLM_API_BASE_FILE" + local api_base_file_name="LLM_API_BASE_FILE" if is_vertex_model "$model"; then return 0 fi + case "$(normalize_model "$model"):$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + if [ -n "${STRIX_OPENAI_FALLBACK_API_BASE_FILE:-}" ]; then + # Cross-provider fallback: direct OpenAI must not inherit a NIM, + # OpenRouter, or GitHub Models endpoint. + api_base_file="$STRIX_OPENAI_FALLBACK_API_BASE_FILE" + api_base_file_name="STRIX_OPENAI_FALLBACK_API_BASE_FILE" + else + return 0 + fi + ;; + esac - local api_base_file="$LLM_API_BASE_FILE" - local api_base_file_name="LLM_API_BASE_FILE" if is_github_models_model "$model" && [ -n "${STRIX_GITHUB_MODELS_API_BASE_FILE:-}" ]; then # Cross-provider fallback: when the active primary provider uses a # different API base (for example OpenRouter), github_models/* fallback @@ -2486,13 +2483,6 @@ child_model_for_api_base() { printf 'openai/%s\n' "${model#openai_direct/}" return 0 ;; - # The workflow contract spells the direct-OpenAI fallback with a hyphen - # (openai-direct/...). litellm cannot infer a provider from that prefix, - # so both spellings must resolve to the litellm openai/ form. - openai-direct/*) - printf 'openai/%s\n' "${model#openai-direct/}" - return 0 - ;; esac printf '%s\n' "$model" @@ -2507,6 +2497,7 @@ child_model_for_api_base() { run_strix_once() { local model="$1" local rc + local normalized_model local llm_api_base_value local child_model local resolved_target_path @@ -2528,29 +2519,36 @@ run_strix_once() { total_budget_limited_timeout=1 fi fi - if ! llm_api_base_value="$(resolved_llm_api_base_for_model "$model")"; then + if ! normalized_model="$(normalize_model "$model")"; then return 2 fi - child_model="$(child_model_for_api_base "$model" "$llm_api_base_value")" + if ! llm_api_base_value="$(resolved_llm_api_base_for_model "$normalized_model")"; then + return 2 + fi + child_model="$(child_model_for_api_base "$normalized_model" "$llm_api_base_value")" if ! resolved_target_path="$(resolve_current_target_path "$TARGET_PATH")"; then return 1 fi local start_epoch start_epoch="$(date +%s)" local child_llm_api_key="" - if ! is_vertex_model "$(normalize_model "$model")"; then + if ! is_vertex_model "$normalized_model"; then child_llm_api_key="$LLM_API_KEY" - if is_github_models_model "$(normalize_model "$model")" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then + if is_github_models_model "$normalized_model" && [ -n "$STRIX_GITHUB_MODELS_KEY" ]; then # Cross-provider fallback: github_models/* models authenticate # with the GitHub Models token, not the direct-OpenAI key. child_llm_api_key="$STRIX_GITHUB_MODELS_KEY" fi - if is_explicit_openai_model "$model" && [ -n "$STRIX_OPENAI_FALLBACK_KEY" ]; then - # Cross-provider fallback: explicit direct-OpenAI models - # authenticate with the OpenAI key, not the primary provider's - # key (NVIDIA NIM, OpenRouter, or GitHub Models). + case "$normalized_model:$PRIMARY_MODEL" in + openai_direct/*:openai_direct/*) ;; + openai_direct/*:*) + if [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "ERROR: direct OpenAI fallback requires STRIX_OPENAI_FALLBACK_KEY_FILE." >&2 + return 2 + fi child_llm_api_key="$STRIX_OPENAI_FALLBACK_KEY" - fi + ;; + esac fi set -o pipefail set +e @@ -2771,8 +2769,14 @@ except subprocess.TimeoutExpired: PY rc=$? set -e - if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then + local report_tree_invalid=0 + validate_report_artifact_tree "$ACTIVE_REPORTS_DIR" || report_tree_invalid=1 + validate_report_artifact_tree "$STRIX_SCAN_OUTPUT_DIR" || report_tree_invalid=1 + if [ "$report_tree_invalid" -eq 1 ]; then + rc=1 + elif [ -d "$STRIX_SCAN_OUTPUT_DIR" ]; then cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ACTIVE_REPORTS_DIR"/ + validate_report_artifact_tree "$ACTIVE_REPORTS_DIR" || rc=1 fi local end_epoch end_epoch="$(date +%s)" @@ -2807,6 +2811,11 @@ PY fi if [ "$rc" -eq 0 ]; then + if ! has_structured_reported_severity_markers && ! strix_reported_zero_vulnerabilities; then + INFRA_ERROR_DETECTED=1 + echo "Strix exited successfully without an authoritative vulnerability report or zero-findings marker; failing closed." >&2 + return 1 + fi if has_blocking_vulnerability_reports; then if ! evaluate_pull_request_findings || [ "$PR_FINDINGS_DECISION" != "allow_baseline" ]; then echo "Strix exited successfully but emitted a vulnerability at or above '$STRIX_FAIL_ON_MIN_SEVERITY'; failing closed." >&2 @@ -2819,12 +2828,8 @@ PY printf "Strix run failed for model '%s' after %ds (exit code %d).\n" "$model" "$elapsed" "$rc" >&2 - # Sticky flag: record that at least one attempt hit an infrastructure - # error. STRIX_LOG is overwritten per-attempt, so without this flag the - # below-threshold guard in has_only_below_threshold_vulnerabilities() - # would only see the *last* attempt's log — missing infrastructure errors - # from earlier attempts whose partial reports may still sit in the reports - # directory. + # Sticky flag: STRIX_LOG is overwritten per-attempt, while earlier partial + # reports remain available for exact fail-closed evidence. return 1 } @@ -2868,6 +2873,19 @@ is_nvidia_nim_not_found_error() { return 1 } +is_unsupported_model_parameter_error() { + # Strix currently has no generation-parameter override. Match the exact + # single-line LiteLLM/Azure capability failure so a reasoning model that + # rejects Strix's temperature can move to the already-configured fallback. + if LC_ALL=C grep -Ei '^[[:space:]]*(│[[:space:]]*)?Error:[[:space:]]+litellm(\.exceptions)?\.BadRequestError' "$STRIX_LOG" | + grep -Ei '(AzureException|OpenAIException)' | + grep -Eiq "Unsupported value:[[:space:]]*['\"]temperature['\"].*Only the default[[:space:]]*\\(1\\)[[:space:]]*value is supported.*No fallback model group found"; then + return 0 + fi + + return 1 +} + is_model_behavior_error() { # Classify only a module-qualified Strix/Agents SDK protocol exception. # A bare source-file mention of ModelBehaviorError is not retryable. @@ -3198,7 +3216,10 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then + local provider_failure_word_regex='(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' + local provider_failure_context_regex="($LLM_PROVIDER_ONLY_REGEX|HF Hub)" + if LC_ALL=C grep -Eiv '^[[:space:]]*(│[[:space:]]*)?MODEL QUALITY WARNING([[:space:]]*│)?[[:space:]]*$|^Warning: You are sending unauthenticated requests to the HF Hub\. Please set a HF_TOKEN to enable higher rate limits and faster downloads\.$|^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9:.]+ WARNING [^ ]+ - strix\.tools\.web_search\.tool: web_search invoked without PERPLEXITY_API_KEY configured$' "$STRIX_LOG" | + grep -Eiq "($provider_failure_context_regex.*$provider_failure_word_regex|$provider_failure_word_regex.*$provider_failure_context_regex)"; then return 0 fi @@ -3230,6 +3251,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + if is_model_behavior_error; then return 0 fi @@ -3277,97 +3302,6 @@ latest_strix_report_dir() { echo "$latest" } -has_only_below_threshold_vulnerabilities() { - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - - local found_any_vuln_file=0 - local global_max_rank=-1 - STRIX_MAX_SEVERITY_RANK=-1 - local saw_any_severity=0 - - update_max_severity_from_stream() { - local source_path="$1" - local line - local severity - local rank - while IFS= read -r line; do - if [[ "${line^^}" =~ SEVERITY[[:space:]]*:[[:space:][:punct:]]*(CRITICAL|HIGH|MEDIUM|LOW|INFO|INFORMATIONAL|NONE)([[:space:][:punct:]]|$) ]]; then - severity="${BASH_REMATCH[1]}" - else - continue - fi - - rank="$(severity_rank "$severity")" - if [ "$rank" -lt 0 ]; then - continue - fi - - saw_any_severity=1 - if [ "$rank" -gt "$global_max_rank" ]; then - global_max_rank="$rank" - STRIX_MAX_SEVERITY_RANK="$rank" - fi - done < <(grep -Ei 'severity[[:space:]]*:' "$source_path" || true) - } - - local run_dir - for run_dir in "$STRIX_REPORTS_DIR"/*; do - if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then - continue - fi - - if is_preexisting_report_dir "$run_dir"; then - continue - fi - - local vulnerabilities_dir="$run_dir/vulnerabilities" - if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then - continue - fi - - local vuln_file - - for vuln_file in "$vulnerabilities_dir"/*.md; do - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - continue - fi - - found_any_vuln_file=1 - update_max_severity_from_stream "$vuln_file" - done - done - - if [ "$found_any_vuln_file" -eq 0 ]; then - echo "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." >&2 - return 1 - fi - - if [ "$saw_any_severity" -eq 0 ]; then - return 1 - fi - - # Guard against incomplete scans due to infrastructure errors. - # Use the sticky INFRA_ERROR_DETECTED flag instead of re-reading - # STRIX_LOG, because STRIX_LOG is overwritten per-attempt. If an - # earlier attempt hit an infrastructure error (timeout, rate-limit, - # transport failure) and produced a partial report that now sits in - # the reports directory, the *current* STRIX_LOG may show a different - # failure — or even success — but the partial report's low-severity - # findings must not be treated as a clean scan result. - if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then - echo "Below-threshold findings detected, but infrastructure errors occurred during this pipeline run; refusing bypass due to potentially incomplete scan." >&2 - return 1 - fi - - if [ "$global_max_rank" -lt "$threshold_rank" ]; then - echo "Strix findings are below configured fail threshold '$STRIX_FAIL_ON_MIN_SEVERITY'; allowing pipeline continuation." >&2 - return 0 - fi - - return 1 -} - has_blocking_vulnerability_reports() { local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" @@ -3386,7 +3320,7 @@ has_blocking_vulnerability_reports() { continue fi - for vuln_file in "$vulnerabilities_dir"/*.md; do + for vuln_file in "$vulnerabilities_dir"/*.md "$vulnerabilities_dir"/*.json; do if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then continue fi @@ -3419,7 +3353,7 @@ fail_reported_vulnerabilities_before_fallback_success() { return 1 } -has_any_reported_severity_markers() { +has_structured_reported_severity_markers() { local run_dir for run_dir in "$STRIX_REPORTS_DIR"/*; do if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then @@ -3436,21 +3370,25 @@ has_any_reported_severity_markers() { fi local vuln_file - for vuln_file in "$vulnerabilities_dir"/*.md; do + for vuln_file in "$vulnerabilities_dir"/*.md "$vulnerabilities_dir"/*.json; do if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then continue fi - if grep -Eiq 'severity[[:space:]]*:' "$vuln_file"; then + if grep -Eiq '(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:' "$vuln_file"; then return 0 fi done done - if grep -Eiq 'severity[[:space:]]*:' "$STRIX_LOG"; then + return 1 +} + +has_any_reported_severity_markers() { + if has_structured_reported_severity_markers; then return 0 fi - return 1 + grep -Eiq '(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:' "$STRIX_LOG" } strix_reported_zero_vulnerabilities() { @@ -3612,7 +3550,7 @@ is_hallucinated_endpoint_finding() { local vuln_file - for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do + for vuln_file in "$latest_report_dir"/vulnerabilities/*.md "$latest_report_dir"/vulnerabilities/*.json; do if vulnerability_file_is_below_threshold "$vuln_file" && vulnerability_file_has_absent_endpoint_finding "$vuln_file"; then return 0 @@ -4067,7 +4005,7 @@ is_hallucinated_source_claim_finding() { fi local vuln_file - for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do + for vuln_file in "$latest_report_dir"/vulnerabilities/*.md "$latest_report_dir"/vulnerabilities/*.json; do if vulnerability_file_is_below_threshold "$vuln_file" && vulnerability_file_has_hallucinated_source_claim "$vuln_file"; then return 0 @@ -4123,6 +4061,10 @@ is_model_retryable_error() { return 0 fi + if is_unsupported_model_parameter_error; then + return 0 + fi + # A provider failure can be recorded only in Strix's structured report log. # run_strix_once already marks that evidence as infrastructure failure, but # the child stdout log used by the classifiers may not contain the provider @@ -4178,10 +4120,6 @@ run_current_target_scan() { fi fi - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - if evaluate_pull_request_findings; then if [ "$strict_primary_provider_fallback" -eq 0 ]; then return 0 @@ -4229,6 +4167,12 @@ run_current_target_scan() { fi fallback_tried=1 + if [[ "$candidate" == openai_direct/* ]] && + [[ "$PRIMARY_MODEL" != openai_direct/* ]] && + [ -z "$STRIX_OPENAI_FALLBACK_KEY" ]; then + echo "Skipping fallback model '$candidate' — STRIX_OPENAI_FALLBACK_KEY_FILE is unavailable." >&2 + continue + fi if is_vertex_model "$PRIMARY_MODEL"; then echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." else @@ -4255,10 +4199,6 @@ run_current_target_scan() { strict_fallback_provider_signal=1 fi - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - if evaluate_pull_request_findings; then if [ "$strict_fallback_provider_signal" -eq 0 ]; then return 0 @@ -4322,13 +4262,6 @@ run_current_target_scan() { return 1 fi - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - if is_vertex_model "$PRIMARY_MODEL"; then echo "Configured Vertex model and fallback models were unavailable." >&2 else diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 8539afdfe..fd99437bc 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -46,18 +46,6 @@ assert_file_not_contains() { fi } -assert_file_contains_either() { - local file_path="$1" - local first_needle="$2" - local second_needle="$3" - local message="$4" - - if ! grep -Fq -- "$first_needle" "$file_path" && - ! grep -Fq -- "$second_needle" "$file_path"; then - record_failure "$message (missing either '$first_needle' or '$second_needle')" - fi -} - assert_status_permissions_scoped() { local output @@ -76,50 +64,110 @@ except ValueError as exc: print(f"Strix workflow is missing the required top-level block: {exc}", file=sys.stderr) raise SystemExit(1) -top_level_permissions = lines[permissions_index + 1 : jobs_index] -expected_read_permissions = { - "actions: read", - "contents: read", - "models: read", +top_level_permissions: dict[str, str] = {} +for line in lines[permissions_index + 1 : jobs_index]: + permission_match = re.match(r"^ ([A-Za-z0-9_-]+):\s*([A-Za-z]+)\s*$", line) + if permission_match: + top_level_permissions[permission_match.group(1)] = permission_match.group(2) + +expected_top_level_permissions = { + "actions": "read", + "contents": "read", + "models": "read", } -missing = sorted(expected_read_permissions - {line.strip() for line in top_level_permissions}) -if missing: +if top_level_permissions != expected_top_level_permissions: print( - "Strix workflow top-level permissions are missing read-only scopes: " - + ", ".join(missing), + "Strix workflow top-level permissions must be exactly read-only actions, contents, and models; " + f"found: {top_level_permissions}", file=sys.stderr, ) raise SystemExit(1) -if any(line.strip() == "statuses: write" for line in top_level_permissions): - print("Strix workflow top-level GITHUB_TOKEN must not grant statuses: write.", file=sys.stderr) - raise SystemExit(1) - -status_write_jobs: list[str] = [] +allowed_jobs = { + "cancel-closed-pr-runs", + "publish-manual-pr-evidence-status", + "strix", +} +expected_job_permissions = { + "cancel-closed-pr-runs": {}, + # Status writes use exchanged app/secret tokens, never this job's + # GITHUB_TOKEN; the historical required-workflow contract pins + # statuses: write to the strix scan job alone. + "publish-manual-pr-evidence-status": { + "id-token": "write", + }, + "strix": { + "actions": "read", + "contents": "read", + "id-token": "write", + "models": "read", + "statuses": "write", + }, +} +job_names: list[str] = [] +job_permissions: dict[str, dict[str, str]] = {} current_job = "" inside_permissions = False for line in lines[jobs_index + 1 :]: job_match = re.match(r"^ ([A-Za-z0-9_-]+):$", line) if job_match: current_job = job_match.group(1) + if current_job in job_names: + print(f"Strix workflow defines duplicate job '{current_job}'.", file=sys.stderr) + raise SystemExit(1) + job_names.append(current_job) + job_permissions[current_job] = {} inside_permissions = False continue - if current_job and line == " permissions:": - inside_permissions = True + if not current_job: + continue + permissions_match = re.match(r"^ permissions:\s*(.*)$", line) + if permissions_match: + inline_permissions = permissions_match.group(1).strip() + inside_permissions = not inline_permissions + if inline_permissions and inline_permissions != "{}": + job_permissions[current_job] = {"__invalid__": inline_permissions} continue if not inside_permissions: continue - if line.startswith(" "): - if line.strip() == "statuses: write": - status_write_jobs.append(current_job) + permission_match = re.match(r"^ ([A-Za-z0-9_-]+):\s*([A-Za-z]+)\s*$", line) + if permission_match: + job_permissions[current_job][permission_match.group(1)] = permission_match.group(2) continue if line.strip(): inside_permissions = False -if status_write_jobs != ["strix"]: +unknown_jobs = sorted(set(job_names) - allowed_jobs) +missing_jobs = sorted(allowed_jobs - set(job_names)) +if unknown_jobs or missing_jobs: print( - "Strix workflow must scope statuses: write only to the strix scan job; found: " - + (", ".join(status_write_jobs) if status_write_jobs else "none"), + "Strix workflow jobs must be exactly the approved required jobs; " + f"unknown: {unknown_jobs or 'none'}, missing: {missing_jobs or 'none'}", + file=sys.stderr, + ) + raise SystemExit(1) + +if job_permissions != expected_job_permissions: + print( + "Strix workflow job permissions do not match the approved contract; " + f"found: {job_permissions}", + file=sys.stderr, + ) + raise SystemExit(1) + +unpinned_actions: list[str] = [] +for line_number, line in enumerate(lines, start=1): + action_match = re.match(r"^\s*uses:\s*([^\s#]+)", line) + if action_match and not re.fullmatch( + r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+@[0-9a-fA-F]{40}", + action_match.group(1), + ): + unpinned_actions.append(f"{line_number}:{action_match.group(1)}") + +if unpinned_actions: + print( + "Strix workflow actions must be pinned to full commit SHAs; found: " + + ", ".join(unpinned_actions), file=sys.stderr, ) raise SystemExit(1) @@ -146,8 +194,13 @@ assert_file_contains "$workflow_file" "workflow_sha" "Strix workflow prefers req assert_file_contains "$workflow_file" "Checkout trusted Strix source" "Strix workflow checks out central source" assert_file_contains "$workflow_file" 'repository: ${{ steps.trusted_source.outputs.repository }}' "Strix workflow checks out resolved central repository" assert_file_contains "$workflow_file" 'ref: ${{ steps.trusted_source.outputs.ref }}' "Strix workflow checks out resolved central ref" -assert_file_contains "$workflow_file" "Materialize central Strix dependency lock from PR head" "Strix workflow validates same-repo central lock-file PRs against the PR head lock" -assert_file_contains "$workflow_file" "requirements-strix-ci-hashes.txt" "Strix workflow can materialize the central Strix hashed requirements lock" +assert_file_not_contains "$workflow_file" " - name: Materialize central Strix dependency lock from PR head" "Strix workflow never installs dependencies selected by a PR head" +assert_file_not_contains "$workflow_file" 'show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt"' "Strix workflow never copies a PR-controlled executable dependency lock" +assert_file_contains "$workflow_file" "requirements-strix-ci-hashes.txt" "Strix workflow installs from the central trusted hashed requirements lock" +assert_file_contains "$workflow_file" 'trusted_lock_blob="$(git rev-parse "HEAD:$trusted_lock")"' "Strix workflow binds its dependency lock to the trusted workflow commit" +assert_file_contains "$workflow_file" '--only-binary=:all:' "Strix workflow installs only hash-verified wheels" +assert_file_contains "$workflow_file" 'Verify Strix sandbox credential boundary' "Strix workflow verifies its target-command sandbox before loading provider credentials" +assert_file_contains "$workflow_file" 'sandbox_environment - allowed_sandbox_environment' "Strix workflow rejects unreviewed host environment keys in the target-command sandbox" assert_file_contains "$workflow_file" "Materialize target workspace" "Strix workflow separates target workspace from trusted source" assert_file_contains "$workflow_file" 'STRIX_REPO_ROOT:' "Strix workflow passes target root explicitly" assert_file_contains "$workflow_file" 'bash "$TRUSTED_STRIX_GATE"' "Strix workflow executes central Strix gate" @@ -159,7 +212,7 @@ assert_file_contains "$workflow_file" 'context="strix"' "Strix workflow publishe assert_file_contains "$workflow_file" "Existing current-run Strix success status is already present" "Strix manual follow-up status publisher accepts already-published same-run evidence" assert_file_not_contains "$workflow_file" 'repository: ${{ github.repository }}' "Strix workflow must not checkout target repository with actions/checkout in privileged context" assert_file_not_contains "$workflow_file" 'bash "$TRUSTED_STRIX_GATE_TEST"' "Strix required path must not execute the full long-form gate harness" -assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "Strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" +assert_file_not_contains "$workflow_file" "- name: Prepare GitHub Models fallback credentials" "Strix workflow does not define a GitHub Models fallback credential step (a compatibility comment for the retired needle is allowed)" assert_file_contains "$gate_script" "STRIX_GITHUB_MODELS_KEY_FILE" "Strix gate supports GitHub Models fallback credentials for cross-provider fallback" assert_file_contains "$gate_script" "STRIX_REPO_ROOT" "Strix gate consumes explicit target root" assert_file_contains "$gate_script" "STRIX_REPO_ROOT must reference a regular directory" "Strix gate rejects invalid or symlink target roots" @@ -168,11 +221,10 @@ assert_file_contains "$gate_script" "NPM_CONFIG_IGNORE_SCRIPTS" "Strix gate disa assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardened" "Full Strix harness remains available outside the required path" assert_file_contains "$workflow_file" "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" "Strix defaults public scans to the current hosted NVIDIA NIM model" -assert_file_contains_either \ - "$workflow_file" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai_direct/gpt-5.6-luna" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna" \ - "Strix tries another NVIDIA hosted model before falling back to direct OpenAI" +assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai_direct/gpt-5.6-luna" "Strix tries another NVIDIA hosted model before falling back to direct OpenAI" +if grep -Eq '^[[:space:]]+STRIX_FALLBACK_MODELS:.*openai-direct/gpt-5\.6-luna' "$workflow_file"; then + record_failure "Strix active fallback configuration uses the retired direct-OpenAI provider prefix" +fi assert_file_not_contains "$workflow_file" "github_models/openai/o3" "Strix fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 945eb3fb3..05e32ab67 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -202,8 +202,17 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "default-branch repository_dispatch evidence cannot cancel" "strix workflow documents manual evidence isolation from branch protection contexts" assert_file_contains "$workflow_file" "PR-number scope keeps the queue on the current HEAD" "strix workflow documents current-head queue management" assert_file_contains "$workflow_file" "refs/pull//head has already advanced before this queued run starts" "strix workflow documents stale scan queue avoidance" - status_token_count="$(grep -c '^[[:space:]]*GITHUB_STATUS_TOKEN:' "$workflow_file")" - assert_equals "1" "$status_token_count" "strix workflow defines GITHUB_STATUS_TOKEN once so GitHub can parse repository_dispatch" + status_token_count="$(grep -c '^[[:space:]]*GITHUB_STATUS_TOKEN:' "$workflow_file" || true)" + assert_equals "0" "$status_token_count" "strix scan job never receives a status-capable GitHub token" + local status_permission_count status_publish_step_count + status_permission_count="$(grep -c '^[[:space:]]*statuses: write' "$workflow_file")" + assert_equals "1" "$status_permission_count" "strix workflow grants exactly one status-write scope, pinned to the scan job by main's required-workflow smoke" + status_publish_step_count="$(grep -c '^[[:space:]]*- name: Publish same-head manual Strix status' "$workflow_file")" + assert_equals "1" "$status_publish_step_count" "strix workflow publishes status only from the isolated publication job" + assert_file_contains "$workflow_file" 'dispatch_metadata_validated: ${{ steps.dispatch_metadata.outputs.validated }}' "strix scan job exports live dispatch validation evidence" + assert_file_contains "$workflow_file" 'id: dispatch_metadata' "strix repository dispatch validation has a stable output identity" + assert_file_contains "$workflow_file" 'echo "validated=true" >>"$GITHUB_OUTPUT"' "strix repository dispatch validation records success only after live metadata validation" + assert_file_contains "$workflow_file" "needs.strix.outputs.dispatch_metadata_validated == 'true'" "strix status publication requires live dispatch metadata validation" assert_file_not_contains "$workflow_file" "github.event.pull_request.number == 240" "strix workflow must not hard-code repository-specific PR bypasses" assert_file_contains "$workflow_file" "models: read" "strix workflow grants only the GitHub Models read permission needed for Strix" assert_file_contains "$workflow_file" "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0" "strix workflow pins actions/setup-python" @@ -216,9 +225,13 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "Checkout trusted Strix source" "strix workflow checks out the central Strix source" assert_file_contains "$workflow_file" 'repository: ${{ steps.trusted_source.outputs.repository }}' "strix workflow checks out central Strix scripts instead of target-repo copies" assert_file_contains "$workflow_file" 'ref: ${{ steps.trusted_source.outputs.ref }}' "strix workflow checks out the exact trusted Strix source ref" - assert_file_contains "$workflow_file" "Materialize central Strix dependency lock from PR head" "strix workflow validates central same-repo lock-file PRs against the PR head lock" - assert_file_contains "$workflow_file" "github.event.pull_request.head.repo.full_name == 'ContextualWisdomLab/.github'" "strix workflow limits central lock materialization to same-repository PR heads" - assert_file_contains "$workflow_file" 'git -C "$TRUSTED_WORKSPACE" show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt"' "strix workflow copies only the hashed requirements lock from the PR head" + assert_file_not_contains "$workflow_file" " - name: Materialize central Strix dependency lock from PR head" "strix workflow never installs dependencies selected by a PR head" + assert_file_not_contains "$workflow_file" 'show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt"' "strix workflow never copies a PR-controlled executable dependency lock" + assert_file_contains "$workflow_file" 'trusted_lock_blob="$(git rev-parse "HEAD:$trusted_lock")"' "strix workflow binds its dependency lock to the trusted workflow commit" + assert_file_contains "$workflow_file" 'working_lock_blob="$(git hash-object --no-filters -- "$trusted_lock")"' "strix workflow hashes exact on-disk trusted dependency-lock bytes immediately before install" + assert_file_contains "$workflow_file" '--only-binary=:all:' "strix workflow installs only hash-verified wheels" + assert_file_contains "$workflow_file" 'Verify Strix sandbox credential boundary' "strix workflow verifies the installed scanner keeps target commands inside Docker" + assert_file_contains "$workflow_file" 'sandbox_environment - allowed_sandbox_environment' "strix workflow rejects unreviewed host environment keys in the target-command sandbox" assert_file_contains "$workflow_file" 'TRUSTED_STRIX_SOURCE=$trusted_strix_source' "strix workflow exports the central Strix source path" assert_file_contains "$workflow_file" 'TRUSTED_STRIX_GATE=$trusted_strix_source/scripts/ci/strix_quick_gate.sh' "strix workflow executes the central Strix gate script" assert_file_contains "$workflow_file" "Materialize target workspace" "strix workflow materializes target repository data separately from trusted scripts" @@ -238,6 +251,18 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$REPO_ROOT/scripts/ci/strix_required_workflow_smoke.sh" 'TRUSTED_WORKSPACE' "strix required-workflow smoke validates the fetched PR head workflow when available" assert_file_not_contains "$workflow_file" "bash \"\$TRUSTED_STRIX_GATE_TEST\"" "strix required path does not execute the full long-form gate harness" assert_file_contains "$workflow_file" "bash \"\$TRUSTED_STRIX_GATE\"" "strix workflow executes trusted temp gate script" + local run_strix_block + run_strix_block="$( + awk ' + /- name: Run Strix \(quick\)/ { in_block = 1 } + in_block && /- name: Collect Strix reports for artifact upload/ { exit } + in_block { print } + ' "$workflow_file" + )" + if [[ "$run_strix_block" == *'GH_TOKEN:'* ]]; then + record_failure "strix scan step must not inherit a GitHub token" + fi + assert_file_contains "$workflow_file" 'find -P "$candidate_dir" -mindepth 1 -type l -print -quit' "strix artifact collection rejects symlinked scanner output" assert_file_contains "$workflow_file" "Collect Strix reports for artifact upload" "strix workflow preserves reports from trusted workspace" assert_file_contains "$workflow_file" "scan-summary.txt" "strix workflow creates a fallback artifact when Strix emits no report files" local checkout_count @@ -360,13 +385,23 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "https://integrate.api.nvidia.com/v1" "strix workflow routes NVIDIA NIM scans to the hosted endpoint" assert_file_contains "$workflow_file" "LLM_API_BASE_FILE" "strix workflow passes the GitHub Models API base through a trusted input file" assert_file_not_contains "$workflow_file" '${{ secrets.STRIX_OPENAI_API_KEY || github.token }}' "strix workflow must not use fallback-secret syntax for LLM API keys" - assert_file_contains "$workflow_file" "openai-direct/gpt-5.6-luna" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'openai_direct' && 'openai-direct/gpt-5.6-luna'" "strix workflow gives direct-OpenAI scans a same-provider fallback so transient errors degrade instead of skipping" - assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai-direct/gpt-5.6-luna'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + assert_file_contains "$workflow_file" "openai_direct/gpt-5.6-luna" "strix workflow keeps a direct-OpenAI fallback on a tool-capable, Strix-recommended model without GPT-4.1 downgrade" + fallback_line="$(grep -F 'STRIX_FALLBACK_MODELS:' "$workflow_file")" + if grep -Fq "steps.gate.outputs.provider_mode == 'openai_direct' &&" <<<"$fallback_line"; then + record_failure "strix workflow must leave direct-OpenAI fallback selection to the final empty default" + fi + assert_file_contains "$workflow_file" "steps.gate.outputs.provider_mode == 'nvidia_nim' && 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 openai_direct/gpt-5.6-luna'" "strix workflow gives NVIDIA NIM scans contracted fallbacks" + if grep -Eq '^[[:space:]]+STRIX_FALLBACK_MODELS:.*openai-direct/gpt-5\.6-luna' "$workflow_file"; then + record_failure "strix workflow active fallback configuration uses the retired direct-OpenAI provider prefix" + fi assert_file_not_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \${{ steps.gate.outputs.provider_mode == 'github_models' && 'github_models/openai/o3" "strix workflow fallback list must not depend on GitHub Models, which is in platform-wide retirement" - assert_file_contains "$workflow_file" "Prepare GitHub Models fallback credentials" "strix workflow provisions GitHub Models fallback credentials for direct-OpenAI scans" + assert_file_not_contains "$workflow_file" "- name: Prepare GitHub Models fallback credentials" "strix workflow does not define a GitHub Models fallback credential step (a compatibility comment for main's retired smoke needle is allowed)" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_KEY_FILE" "strix gate reads the optional GitHub Models fallback key file" assert_file_contains "$GATE_SCRIPT" "STRIX_GITHUB_MODELS_API_BASE_FILE" "strix gate routes github_models fallback models through the GitHub Models endpoint" + assert_file_contains "$workflow_file" "Prepare direct OpenAI fallback credentials" "strix workflow provisions direct OpenAI credentials for cross-provider fallbacks" + assert_file_contains "$workflow_file" 'OPENAI_FALLBACK_API_KEY: ${{ secrets.STRIX_OPENAI_API_KEY || secrets.OPENAI_API_KEY }}' "strix workflow reads the established direct OpenAI secret only at the credential boundary" + assert_file_contains "$workflow_file" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix workflow passes the direct OpenAI fallback key through a trusted file" + assert_file_contains "$GATE_SCRIPT" "STRIX_OPENAI_FALLBACK_KEY_FILE" "strix gate reads the direct OpenAI fallback key from a trusted file" assert_file_not_contains "$workflow_file" 'github_models/deepseek/deepseek-r1-0528 | github_models/deepseek/deepseek-v3-0324)' "strix workflow keeps DeepSeek GitHub Models restricted to fallback-only routing" assert_file_contains "$workflow_file" '${strix_model#github_models/}' "strix workflow strips manual github_models routing prefix for OpenAI GPT model names before passing model names to LiteLLM" assert_file_contains "$workflow_file" "openai_direct/%s" "strix workflow keeps manual direct OpenAI scans distinct from GitHub Models openai/gpt-* routing" @@ -1135,18 +1170,19 @@ assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" ' assert_file_contains "$workflow_file" 'Default-branch repository_dispatch Strix evidence passed' "opencode approval requires an explicit manual Strix evidence status description" assert_file_contains "$workflow_file" 'last // empty' "opencode approval checks the latest strix status before accepting manual success evidence" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'publish-manual-pr-evidence-status:' "strix workflow publishes same-head manual PR evidence as a commit status" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'statuses: write' "strix scan job can publish same-repo manual status evidence" - assert_file_contains "$REPO_ROOT/scripts/ci/strix_required_workflow_smoke.sh" 'status_write_jobs != ["strix"]' "strix smoke keeps status write permission scoped to the scan job" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'TARGET_REPOSITORY: ${{ github.event.client_payload.target_repository || github.repository }}' "strix manual evidence status publishes to the requested target repository" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'statuses: write' "strix isolated follow-up job can publish same-repo manual status evidence" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_required_workflow_smoke.sh" 'job_permissions != expected_job_permissions' "strix smoke enforces the exact approved job permission maps" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_required_workflow_smoke.sh" 'Strix workflow actions must be pinned to full commit SHAs' "strix smoke rejects mutable action references" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'TARGET_REPOSITORY: ${{ needs.strix.outputs.dispatch_target_repository }}' "strix follow-up status publisher uses the live-validated target repository" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'context="strix"' "strix manual evidence status uses the status context consumed by OpenCode" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'repos/${TARGET_REPOSITORY}/statuses/${PR_HEAD_SHA}' "strix manual evidence status does not post private-target evidence to .github by mistake" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'PR_REVIEW_MERGE_STATUS_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || '"'"''"'"' }}' "strix manual evidence status can publish cross-repo evidence with the central mutation credential" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'post_strix_status "pr-review-merge-token" "$PR_REVIEW_MERGE_STATUS_TOKEN"' "strix manual evidence status retries the central mutation credential when the target app token cannot write statuses" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'post_strix_status "opencode-approve-token" "$OPENCODE_APPROVE_STATUS_TOKEN"' "strix manual evidence status retries the approval credential before declaring status publication unavailable" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'post_strix_status "github-token" "$GITHUB_STATUS_TOKEN"' "strix manual evidence status keeps the same-repository github-token fallback scoped to the scan job" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'check_existing_status "github-token" "$GITHUB_STATUS_READ_TOKEN"' "strix isolated follow-up can inspect an existing same-repository status" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'post_strix_status "target-app-token" "$TARGET_APP_STATUS_TOKEN"' "strix manual evidence status uses the target app token first" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'Default-branch repository_dispatch Strix evidence failed' "strix manual evidence status records failed reruns so older success cannot mask newer failure" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'Could not publish manual Strix status from scan job' "strix scan evidence does not fail solely because target status publication is unavailable" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'Could not publish manual Strix status from follow-up job' "strix follow-up fails closed when non-successful evidence cannot be published" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" '[ "$STRIX_RESULT" = "success" ]' "strix follow-up distinguishes a successful scan from failed or inconclusive evidence" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'Strix scan succeeded, but no configured credential could publish or read the target commit status.' "strix follow-up logs permission-specific status unavailability without failing a clean scan" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" 'after all configured credentials failed after a non-successful scan' "strix follow-up still fails loudly when failed or inconclusive scan evidence cannot be published" @@ -3320,11 +3356,39 @@ printf '%s\n' "$target_path" >> "${FAKE_STRIX_TARGET_LOG:?}" STRIX_REPORTS_DIR="${STRIX_REPORTS_DIR:-strix_runs}" +emit_synthetic_completion_evidence() { + local rc=$? + if [ "$rc" -eq 0 ] && + [ "${FAKE_STRIX_SCENARIO:?}" != "success-without-evidence" ] && + [ "${FAKE_STRIX_SCENARIO:?}" != "success-with-log-only-severity" ] && + [ "${FAKE_STRIX_SCENARIO:?}" != "success-with-low-report" ] && + [ "${FAKE_STRIX_SCENARIO:?}" != "success-with-critical-report" ]; then + echo "Vulnerabilities 0" + fi + trap - EXIT + exit "$rc" +} +trap emit_synthetic_completion_evidence EXIT + case "${FAKE_STRIX_SCENARIO:?}" in success|runtime-env-forwarding|vertex-primary-success-timing-message|direct-openai-gpt-does-not-require-github-models-api-base|pr-executable-integrity-mismatch|pr-executable-group-writable) echo "scan ok" exit 0 ;; + success-without-evidence) + exit 0 + ;; + success-with-log-only-severity) + echo "Severity: HIGH" + exit 0 + ;; + success-with-low-report) + mkdir -p "$STRIX_REPORTS_DIR/fake-success-low/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-success-low/vulnerabilities/vuln-0001.md" <<'REPORT' +Severity: LOW +REPORT + exit 0 + ;; scan-working-directory-isolated) if [ "$PWD" = "$target_path" ] || [[ "$PWD" == "$target_path"/* ]]; then echo "Error: Strix process inherited the untrusted scan target as cwd" >&2 @@ -3421,6 +3485,63 @@ REPORT ;; esac ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + case "${STRIX_LLM:-}" in + openai/gpt-5.6-sol) + echo "Error: litellm.BadRequestError: AzureException BadRequestError - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found for original model_group=gpt-5.6-sol." + exit 1 + ;; + openai/o3) + echo "scan ok with GitHub Models fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + echo "Error: litellm.BadRequestError: request rejected" + echo "AzureException - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found." + exit 1 + ;; + openai-direct-unsupported-temperature-prefixed-target-nonrecoverable) + echo "TARGET OUTPUT: Error: litellm.BadRequestError: AzureException - Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported. No fallback model group found." + exit 1 + ;; + nvidia-openai-direct-fallback-credential-success | nvidia-openai-direct-fallback-missing-key-fails-closed | nvidia-openai-direct-missing-key-next-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/primary) + if [ "${LLM_API_KEY:-}" != "dummy" ]; then + echo "unexpected NVIDIA primary key (${LLM_API_KEY:-})" >&2 + exit 17 + fi + echo "Penetration test failed: LLM request failed: RateLimitError" + exit 1 + ;; + openai/gpt-5.6-luna) + if [ "${LLM_API_KEY:-}" != "openai-fallback-token" ]; then + echo "unexpected direct OpenAI fallback key (${LLM_API_KEY:-})" >&2 + exit 18 + fi + if [ -n "${LLM_API_BASE:-}" ]; then + echo "direct OpenAI fallback inherited primary API base: $LLM_API_BASE" >&2 + exit 19 + fi + echo "scan ok with direct OpenAI fallback" + exit 0 + ;; + nvidia_nim/nvidia/fallback-two) + echo "scan ok with later NVIDIA fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -3585,6 +3706,34 @@ REPORT ;; esac ;; + nvidia-ratelimit-model-quality-warning-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + echo "litellm.RateLimitError: Nvidia_nimException - Error code: 429 - Too Many Requests" + exit 1 + ;; + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + echo "│ MODEL QUALITY WARNING │" + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads." + mkdir -p "$STRIX_REPORTS_DIR/run-clean-advisories" + { + echo "│ MODEL QUALITY WARNING │" + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads." + } >"$STRIX_REPORTS_DIR/run-clean-advisories/scan.log" + echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" + exit 0 + ;; + *) + echo "Error: NVIDIA model-quality fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 61 + ;; + esac + ;; + hf-advisory-suffix-fails-closed) + echo "Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. Fatal: repository-derived suffix" + echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" + exit 0 + ;; vertex-primary-resource-exhausted-fallback-success) case "${STRIX_LLM:-}" in vertex_ai/resource-exhausted-primary) @@ -3812,7 +3961,7 @@ REPORT ;; esac ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-blocks | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" @@ -3821,8 +3970,7 @@ REPORT exit 1 ;; openai/deepseek/deepseek-r1-0528) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ] || - [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-blocks" ]; then mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL @@ -3851,12 +3999,6 @@ EOS exit 2 ;; openai/deepseek/deepseek-v3-0324) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then - echo "LLM CONNECTION FAILED" - echo "Could not establish connection to the language model." - echo "Error: provider retirement brownout" - exit 1 - fi echo "scan ok after second GitHub Models fallback" exit 0 ;; @@ -4473,15 +4615,24 @@ EOS esac ;; provider-fatal-success-signal) - echo "Fatal: provider stream aborted" + echo "litellm Fatal: provider stream aborted" exit 0 ;; provider-warning-success-signal) - echo "Warning: provider response included incomplete scan state" + for _ in {1..10000}; do + echo "benign scanner output" + done + echo "openai Warning: provider response included incomplete scan state" exit 0 ;; provider-denied-success-signal) - echo "Denied: provider credentials were rejected" + echo "anthropic Denied: provider credentials were rejected" + exit 0 + ;; + clean-target-narrative-failure-words) + echo "Target documentation mentions OpenAI integration as product context." + echo "Target log taxonomy documents Fatal, Denied, Warn, and Warning labels." + echo "│ Vulnerabilities 0 (No exploitable vulnerabilities detected) │" exit 0 ;; provider-report-rate-limit-fallback-success) @@ -4505,7 +4656,7 @@ EOS ;; esac ;; - report-known-internal-warning-sanitized) + report-symlink-rejected-without-rewriting-target) mkdir -p "$STRIX_REPORTS_DIR/fake-known-internal-warning" cat >"$STRIX_REPORTS_DIR/fake-known-internal-warning/strix.log" <<'EOS' 2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: agent a9fb4033 produced non-lifecycle final output in non-interactive mode; forcing tool continuation (1/500): internal agent coordination note @@ -4534,6 +4685,15 @@ EOS echo "scan ok with sanitized internal Strix report notice variant" exit 0 ;; + report-web-search-advisory-sanitized | report-web-search-advisory-suffix-fails) + mkdir -p "$STRIX_REPORTS_DIR/fake-web-search-advisory" + cat >"$STRIX_REPORTS_DIR/fake-web-search-advisory/strix.log" <"$STRIX_REPORTS_DIR/fake-unknown-warning/strix.log" <<'EOS' @@ -4728,12 +4888,8 @@ EOS exit 1 ;; infra-error-sticky-flag) - # Sticky flag test: first call hits infra error (rate limit), - # second call fails on the first fallback model but produces a - # LOW finding report. After exhausting retries, the gate checks - # has_only_below_threshold_vulnerabilities — which finds LOW - # findings but sees INFRA_ERROR_DETECTED=1 (set from the first - # call's rate-limit error) and refuses the below-threshold bypass. + # The first call hits a rate limit; the fallback then exits nonzero + # with a partial LOW report. Neither incomplete result may pass. case "${STRIX_LLM:-}" in vertex_ai/sticky-flag-primary) touch "$FAKE_STRIX_STATE_FILE" @@ -4880,6 +5036,15 @@ EOS echo "Penetration test failed: baseline critical finding with narrowed subdir target" exit 1 ;; + pr-critical-outside-narrowed-subdir-target) + mkdir -p "$STRIX_REPORTS_DIR/fake-pr-outside-narrowed-subdir/vulnerabilities" + cat >"$STRIX_REPORTS_DIR/fake-pr-outside-narrowed-subdir/vulnerabilities/vuln-0001.md" <<'EOS' +Severity: CRITICAL +Target: /workspace/smart-crawling-server/backend/services/email_parser.py +EOS + echo "Penetration test failed: finding outside narrowed scan target" + exit 1 + ;; pr-baseline-critical-subdir-boxed-target) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-subdir-boxed-target/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-subdir-boxed-target/vulnerabilities/vuln-0001.md" <<'EOS' @@ -4950,8 +5115,7 @@ EOS ;; pr-critical-changed-json-target) mkdir -p "$STRIX_REPORTS_DIR/fake-pr-changed-json-target/vulnerabilities" - cat >"$STRIX_REPORTS_DIR/fake-pr-changed-json-target/vulnerabilities/vuln-0001.md" <"$STRIX_REPORTS_DIR/fake-pr-changed-json-target/vulnerabilities/vuln-0001.json" <"$tmp_dir/github_models_api_base.txt" printf '%s' 'github-models-fallback-token' >"$tmp_dir/github_models_key.txt" env_cmd+=(STRIX_GITHUB_MODELS_API_BASE_FILE="$tmp_dir/github_models_api_base.txt") env_cmd+=(STRIX_GITHUB_MODELS_KEY_FILE="$tmp_dir/github_models_key.txt") fi + if [ "$scenario" = "nvidia-openai-direct-fallback-credential-success" ]; then + printf '%s' 'openai-fallback-token' >"$tmp_dir/openai_fallback_key.txt" + env_cmd+=(STRIX_OPENAI_FALLBACK_KEY_FILE="$tmp_dir/openai_fallback_key.txt") + fi if [ "$min_fail_severity" = "__UNSET__" ]; then local next_env_cmd=() local env_pair @@ -5812,23 +5981,7 @@ PY "scenario=$scenario runtime env forwarding" fi - if [ "$scenario" = "report-known-internal-warning-sanitized" ]; then - assert_file_not_contains \ - "$repo_root_dir/strix_runs/fake-known-internal-warning/strix.log" \ - "produced non-lifecycle final output" \ - "scenario=$scenario strips the known internal Strix warning from published artifacts" - assert_file_contains \ - "$repo_root_dir/strix_runs/fake-known-internal-warning/strix.log" \ - "finish_scan: completed scan with 0 vulnerability report(s)" \ - "scenario=$scenario keeps non-warning Strix report evidence" - assert_file_not_contains \ - "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ - "produced non-lifecycle final output" \ - "scenario=$scenario sanitizes relative scanner output before publication" - assert_file_contains \ - "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ - "finish_scan: completed scan with 0 vulnerability report(s)" \ - "scenario=$scenario publishes sanitized relative scanner evidence" + if [ "$scenario" = "report-symlink-rejected-without-rewriting-target" ]; then assert_file_contains \ "$repo_root_dir/outside-strix-report/strix.log" \ "outside report should not be rewritten" \ @@ -5846,6 +5999,17 @@ PY "scenario=$scenario keeps non-warning Strix report evidence" fi + if [ "$scenario" = "report-web-search-advisory-sanitized" ]; then + assert_file_not_contains \ + "$repo_root_dir/strix_runs/fake-web-search-advisory/strix.log" \ + "web_search invoked without PERPLEXITY_API_KEY configured" \ + "scenario=$scenario strips only the known optional web search advisory" + assert_file_contains \ + "$repo_root_dir/strix_runs/fake-web-search-advisory/strix.log" \ + "finish_scan: completed scan with 0 vulnerability report(s)" \ + "scenario=$scenario keeps non-warning Strix report evidence" + fi + if [ "$scenario" = "github-models-primary-ratelimit-fallback-success" ]; then assert_file_contains \ "$output_log" \ @@ -5941,6 +6105,46 @@ run_github_models_http410_case() { "1" } +run_nvidia_openai_direct_fallback_case() { + local scenario="${1:-nvidia-openai-direct-fallback-credential-success}" + local expected_exit="${2:-0}" + local expected_message="${3:-REGEX:Strix quick scan succeeded with fallback model 'openai_direct/gpt-5.6-luna' in [0-9]+s\\.}" + local expected_calls="${4:-2}" + local expected_models="${5:-nvidia_nim/nvidia/primary|openai/gpt-5.6-luna}" + local expected_api_bases="${6:-https://integrate.api.nvidia.com/v1|}" + local fallback_models="${7:-openai-direct/gpt-5.6-luna}" + + run_gate_case "$scenario" \ + "nvidia_nim/nvidia/primary" \ + "" \ + "$expected_exit" \ + "$expected_message" \ + "$expected_calls" \ + "$expected_models" \ + "$expected_api_bases" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "$fallback_models" \ + "1" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") @@ -5956,6 +6160,36 @@ run_filtered_gate_case_if_requested() { "vertex_ai/ready-primary" \ "" ;; + success-without-evidence) + run_gate_case "success-without-evidence" \ + "vertex_ai/ready-primary" \ + "" \ + "1" \ + "without an authoritative vulnerability report or zero-findings marker" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + ;; + success-with-log-only-severity) + run_gate_case "success-with-log-only-severity" \ + "vertex_ai/ready-primary" \ + "" \ + "1" \ + "without an authoritative vulnerability report or zero-findings marker" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + ;; + success-with-low-report) + run_gate_case "success-with-low-report" \ + "vertex_ai/ready-primary" \ + "" \ + "0" \ + "Strix run succeeded" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + ;; pr-rust-workspace-context) run_gate_case "pr-rust-workspace-context" \ "openai/gpt-4o-mini" \ @@ -6124,6 +6358,99 @@ run_filtered_gate_case_if_requested() { "" \ "github_models/openai/o3" ;; + openai-direct-unsupported-temperature-prefixed-target-nonrecoverable) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "openai_direct/gpt-5.6-sol" \ + "github_models/openai/o3" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "https://example.invalid" \ + "vertex_ai" + ;; + openai-direct-unsupported-temperature-github-models-fallback-success) + run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; + openai-direct-unsupported-temperature-split-lines-nonrecoverable) + run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + ;; + nvidia-openai-direct-fallback-credential-success) + run_nvidia_openai_direct_fallback_case + ;; + nvidia-openai-direct-fallback-missing-key-fails-closed) + run_nvidia_openai_direct_fallback_case \ + "$STRIX_TEST_CASE_FILTER" \ + "1" \ + "STRIX_OPENAI_FALLBACK_KEY_FILE is unavailable" \ + "1" \ + "nvidia_nim/nvidia/primary" \ + "https://integrate.api.nvidia.com/v1" + ;; + nvidia-openai-direct-missing-key-next-fallback-success) + run_nvidia_openai_direct_fallback_case \ + "$STRIX_TEST_CASE_FILTER" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-two' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/primary|nvidia_nim/nvidia/fallback-two" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "openai-direct/gpt-5.6-luna nvidia_nim/nvidia/fallback-two" + ;; gemini-timeout-fallback-success) run_gate_case_allow_provider_signal "gemini-timeout-fallback-success" \ "gemini/timeout-fallback-primary" \ @@ -6358,17 +6685,58 @@ run_filtered_gate_case_if_requested() { pull-request-target-changed-backend-context) run_pull_request_target_changed_backend_context_scope_case ;; - report-known-internal-warning-sanitized) + report-symlink-rejected-without-rewriting-target) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-symlink-rejected" \ + "" \ + "1" \ + "Strix report artifact tree contains a symlink" \ + "1" \ + "vertex_ai/report-symlink-rejected" \ + "" + ;; + report-web-search-advisory-sanitized) run_gate_case "$STRIX_TEST_CASE_FILTER" \ - "vertex_ai/report-known-internal-warning-sanitized" \ + "vertex_ai/report-web-search-advisory-sanitized" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "Strix run succeeded for model 'vertex_ai/report-web-search-advisory-sanitized'" \ "1" \ - "vertex_ai/report-known-internal-warning-sanitized" \ + "vertex_ai/report-web-search-advisory-sanitized" \ "" ;; - provider-fatal-success-signal | provider-warning-success-signal) + report-web-search-advisory-suffix-fails) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" + ;; + nvidia-ratelimit-model-quality-warning-fallback-success) + run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" + ;; + hf-advisory-suffix-fails-closed) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" + ;; + provider-fatal-success-signal | provider-warning-success-signal | provider-denied-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ "" \ @@ -6376,7 +6744,17 @@ run_filtered_gate_case_if_requested() { "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ - "" + "" + ;; + clean-target-narrative-failure-words) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" \ + "0" \ + "Vulnerabilities 0" \ + "1" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" ;; provider-report-rate-limit-fallback-success) run_gate_case "provider-report-rate-limit-fallback-success" \ @@ -6391,46 +6769,15 @@ run_filtered_gate_case_if_requested() { total-timeout) run_total_timeout_case ;; - github-models-fallback-baseline-vulnerability-before-next-success-continues) - run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - ;; - github-models-exhausted-after-baseline-vulnerability-fails-closed) - run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ + github-models-fallback-baseline-vulnerability-blocks) + run_gate_case "github-models-fallback-baseline-vulnerability-blocks" \ "openai/gpt-5" \ "" \ "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ "openai" \ "https://models.github.ai/inference" \ "" \ @@ -6697,6 +7044,7 @@ else fi fi echo "scan ok with PR head content" +echo "Vulnerabilities 0" EOF chmod +x "$fake_strix" printf '%s' 'gemini/test-model' >"$strix_llm_file" @@ -6941,6 +7289,7 @@ if [ -e "$context_file" ]; then exit 66 fi echo "scan ok with bounded PR head backend context" +echo "Vulnerabilities 0" EOF chmod +x "$fake_strix" printf '%s' 'gemini/test-model' >"$strix_llm_file" @@ -7074,6 +7423,7 @@ if [ "$attempt" -eq 1 ]; then exit 70 fi echo "scan ok with changed PR head backend context" + echo "Vulnerabilities 0" exit 0 fi @@ -7317,10 +7667,12 @@ if [ -f "$target_path/contextual_orchestrator/__main__.py" ]; then fi if [ "$matched_backend_context" -eq 1 ]; then + echo "Vulnerabilities 0" exit 0 fi echo "scan ok with non-email backend scope" +echo "Vulnerabilities 0" EOF chmod +x "$fake_strix" printf '%s' 'gemini/test-model' >"$strix_llm_file" @@ -7562,6 +7914,7 @@ if grep -Fq -- 'HEAD_THREADING_SERVICE_SHOULD_NOT_BE_SCANNED' "$target_path/back fi echo "scan ok with frontend email trusted backend authorization context" +echo "Vulnerabilities 0" EOF chmod +x "$fake_strix" printf '%s' 'gemini/test-model' >"$strix_llm_file" @@ -7650,6 +8003,7 @@ run_pull_request_target_shallow_head_merge_base_fallback_case() { #!/usr/bin/env bash set -euo pipefail echo "scan ok" +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -8170,6 +8524,7 @@ if [ -e "$target_path/vendor/newsdom-api" ]; then exit 69 fi echo "scan ok with PR head content" +echo "Vulnerabilities 0" EOF chmod +x "$fake_strix" printf '%s' 'gemini/test-model' >"$strix_llm_file" @@ -8435,6 +8790,7 @@ if [ "${LLM_API_BASE+x}" = "x" ]; then fi printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" echo "vertex scan ok without external LLM_API_BASE" +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -8663,6 +9019,7 @@ if [ "${LLM_API_KEY_FILE+x}" = "x" ]; then echo "unexpected LLM_API_KEY_FILE for Vertex" >&2 exit 1 fi +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -8713,6 +9070,7 @@ if [ "${LLM_API_KEY_FILE+x}" = "x" ]; then echo "unexpected LLM_API_KEY_FILE for Vertex" >&2 exit 1 fi +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -8805,10 +9163,11 @@ run_llm_api_base_file_outside_input_root_fails_closed_case() { cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" - cat >"$fake_strix" <<'EOF' +cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash set -euo pipefail printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -8995,6 +9354,7 @@ run_input_file_root_override_takes_precedence_over_runner_temp_case() { #!/usr/bin/env bash set -euo pipefail printf 'called\n' >"${FAKE_STRIX_CALL_LOG:?}" +echo "Vulnerabilities 0" exit 0 EOF chmod +x "$fake_strix" @@ -9102,15 +9462,16 @@ run_symlink_report_case() { mkdir -p "$external_report_dir" "$repo_root_dir/strix_runs" cat >"$external_report_dir/vuln-0001.md" <<'EOF' -Severity: LOW +Severity: CRITICAL EOF - ln -s "$tmp_dir/external" "$repo_root_dir/strix_runs/latest" cat >"$fake_strix" <<'EOF' #!/usr/bin/env bash set -euo pipefail -echo "Error: transport timeout" -exit 1 +mkdir -p "$STRIX_REPORTS_DIR" +ln -s "${FAKE_STRIX_EXTERNAL_REPORT_DIR:?}" "$STRIX_REPORTS_DIR/latest" +echo "scan completed after creating a symlinked report run" +exit 0 EOF chmod +x "$fake_strix" printf '%s' 'openai/gpt-4o-mini' >"$strix_llm_file" @@ -9123,6 +9484,7 @@ EOF env -u GITHUB_EVENT_NAME -u GITHUB_EVENT_PATH -u STRIX_TEST_CHANGED_FILES_OVERRIDE \ PATH="$tmp_dir:$PATH" \ STRIX_EXECUTABLE_PATH="$fake_strix" \ + FAKE_STRIX_EXTERNAL_REPORT_DIR="$tmp_dir/external" \ STRIX_INPUT_FILE_ROOT="$tmp_dir" \ STRIX_DISABLE_PR_SCOPING="0" \ STRIX_LLM_FILE="$strix_llm_file" \ @@ -9135,7 +9497,7 @@ EOF set -e assert_equals "1" "$rc" "case=symlink-report-does-not-bypass exit code" - assert_file_contains "$output_log" "Strix quick scan failed with a non-recoverable error." "case=symlink-report-does-not-bypass output" + assert_file_contains "$output_log" "Strix report artifact tree contains a symlink" "case=symlink-report-does-not-bypass output" rm -rf "$tmp_dir" } @@ -9521,6 +9883,33 @@ run_gate_case "success" \ "vertex_ai/ready-primary" \ "" +run_gate_case "success-without-evidence" \ + "vertex_ai/ready-primary" \ + "" \ + "1" \ + "without an authoritative vulnerability report or zero-findings marker" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + +run_gate_case "success-with-log-only-severity" \ + "vertex_ai/ready-primary" \ + "" \ + "1" \ + "without an authoritative vulnerability report or zero-findings marker" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + +run_gate_case "success-with-low-report" \ + "vertex_ai/ready-primary" \ + "" \ + "0" \ + "Strix run succeeded" \ + "1" \ + "vertex_ai/ready-primary" \ + "" + run_gate_case "success-with-critical-report" \ "vertex_ai/ready-primary" \ "" \ @@ -9707,6 +10096,25 @@ run_gate_case_allow_provider_signal "vertex-primary-ratelimit-fallback-success" "vertex_ai/ratelimit-primary|vertex_ai/fallback-one" \ "|" +LC_ALL=C run_gate_case "nvidia-ratelimit-model-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" + +run_gate_case "hf-advisory-suffix-fails-closed" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/hf-advisory-suffix-fails-closed" \ + "" + run_gate_case_allow_provider_signal "vertex-primary-resource-exhausted-fallback-success" \ "vertex_ai/resource-exhausted-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ @@ -9940,44 +10348,14 @@ run_gate_case "github-models-fallback-provider-signal-tries-next" \ "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" -run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success-continues" \ - "openai/gpt-5" \ - "" \ - "0" \ - "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ - "openai" \ - "https://models.github.ai/inference" \ - "" \ - "0" \ - "CRITICAL" \ - "0" \ - "" \ - "" \ - "1200" \ - "0" \ - "pull_request" \ - "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ - "" \ - "" \ - "0" \ - "" \ - "" \ - "" \ - "__SAME_AS_FALLBACK_MODELS__" \ - "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ - "1" - -run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ +run_gate_case "github-models-fallback-baseline-vulnerability-blocks" \ "openai/gpt-5" \ "" \ "1" \ - "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ - "3" \ - "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ - "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ "openai" \ "https://models.github.ai/inference" \ "" \ @@ -10404,13 +10782,13 @@ run_gate_case "provider-report-rate-limit-fallback-success" \ "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ "|" -run_gate_case "report-known-internal-warning-sanitized" \ - "vertex_ai/report-known-internal-warning-sanitized" \ +run_gate_case "report-symlink-rejected-without-rewriting-target" \ + "vertex_ai/report-symlink-rejected" \ "" \ - "0" \ - "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ "1" \ - "vertex_ai/report-known-internal-warning-sanitized" \ + "Strix report artifact tree contains a symlink" \ + "1" \ + "vertex_ai/report-symlink-rejected" \ "" \ "vertex_ai" \ "__DEFAULT__" \ @@ -10463,6 +10841,24 @@ run_gate_case "report-known-internal-warning-variant-sanitized" \ "" \ "1" +run_gate_case "report-web-search-advisory-sanitized" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/report-web-search-advisory-sanitized'" \ + "1" \ + "vertex_ai/report-web-search-advisory-sanitized" \ + "" + +run_gate_case "report-web-search-advisory-suffix-fails" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-web-search-advisory-suffix-fails" \ + "" + run_gate_case "report-unknown-warning-fails" \ "vertex_ai/report-unknown-warning-fails" \ "" \ @@ -10523,6 +10919,15 @@ run_gate_case "provider-denied-success-signal" \ "" \ "1" +run_gate_case "clean-target-narrative-failure-words" \ + "vertex_ai/clean-target-narrative-failure-words" \ + "" \ + "0" \ + "Vulnerabilities 0" \ + "1" \ + "vertex_ai/clean-target-narrative-failure-words" \ + "" + run_gate_case_allow_provider_signal "vertex-all-ratelimited" \ "vertex_ai/ratelimit-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ @@ -10700,8 +11105,8 @@ run_gate_case "pr-stale-report-plus-inline-changed-finding-blocks" \ run_gate_case "high-vuln-below-threshold" \ "vertex_ai/high-vuln-primary" \ "" \ - "0" \ - "below configured fail threshold 'CRITICAL'" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ "1" \ "vertex_ai/high-vuln-primary" \ "" @@ -10719,7 +11124,7 @@ run_gate_case "inline-medium-below-threshold" \ "vertex_ai/inline-medium-primary" \ "" \ "1" \ - "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." \ + "Strix quick scan failed with a non-recoverable error." \ "1" \ "vertex_ai/inline-medium-primary" \ "" @@ -10743,26 +11148,25 @@ run_gate_case "medium-vuln-default-threshold" \ # rate-limit, transport failures) because the scan was likely incomplete. # Guard test 1: LOW finding + timeout → should fail (exit 1). -# The below-threshold check runs first but detects infrastructure errors in the -# strix log and refuses bypass. The timeout is also vertex-retryable, so the -# gate continues into the fallback loop. All attempts see the same timeout. +# Timeout is Vertex-retryable, but every nonzero attempt remains incomplete +# evidence even when it emits only a below-threshold report. run_gate_case_allow_provider_signal "below-threshold-with-timeout" \ "vertex_ai/low-timeout-primary" \ "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Configured Vertex model and fallback models were unavailable." \ "3" \ "vertex_ai/low-timeout-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ "||" # Guard test 2: LOW finding + rate-limit → should fail (exit 1). -# Below-threshold check refuses bypass due to infra errors. -# Rate-limit is vertex-retryable, so the gate also tries fallback models. +# Rate-limit is Vertex-retryable, so the gate tries every fallback without +# accepting the below-threshold report from a failed scanner process. run_gate_case_allow_provider_signal "below-threshold-with-ratelimit" \ "vertex_ai/low-ratelimit-primary" \ "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Configured Vertex model and fallback models were unavailable." \ "3" \ "vertex_ai/low-ratelimit-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ "||" @@ -10773,52 +11177,43 @@ run_gate_case_allow_provider_signal "below-threshold-with-connection-error" \ "vertex_ai/info-conn-primary" \ "" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Strix quick scan failed with a non-recoverable error." \ "1" \ "vertex_ai/info-conn-primary" \ "" # Guard test 3b: INFO finding + ConnectionError WITHOUT provider marker → should -# PASS (exit 0). The two-grep infra-error detector requires both a transport -# error class AND an LLM_PROVIDER_ONLY_REGEX marker (litellm, openai, -# anthropic, VertexAI, etc.). Note: transport libraries (requests, httpx, -# httpcore) are intentionally excluded from LLM_PROVIDER_ONLY_REGEX to avoid -# false positives — see guard test 3c below. -# A bare "ConnectionError" from the target application lacks the marker, so -# has_detected_infrastructure_error() returns 1 (no infra error) and the -# below-threshold bypass succeeds. +# fail closed. A nonzero scanner exit is incomplete evidence even when the +# report contains only below-threshold findings and no provider marker. run_gate_case "below-threshold-with-connection-error-no-provider" \ "vertex_ai/info-conn-noprov-primary" \ "" \ - "0" \ - "below configured fail threshold" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ "1" \ "vertex_ai/info-conn-noprov-primary" \ "" # Guard test 3c: INFO finding + requests.exceptions.ConnectionError → should -# PASS (exit 0). The "requests" transport library matches the broad -# PROVIDER_CONTEXT_REGEX but is intentionally excluded from LLM_PROVIDER_ONLY_REGEX. -# Before commit 0e90d48 the connection-error path used PROVIDER_CONTEXT_REGEX -# and would have mis-classified this as an LLM infrastructure error; now it -# correctly uses LLM_PROVIDER_ONLY_REGEX, so below-threshold bypass succeeds. +# fail closed for the same nonzero-exit reason; the transport-library +# classifier no longer controls whether incomplete evidence can pass. run_gate_case "below-threshold-with-requests-connection-error" \ "vertex_ai/info-conn-requests-primary" \ "" \ - "0" \ - "below configured fail threshold" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ "1" \ "vertex_ai/info-conn-requests-primary" \ "" # Guard test 4: MEDIUM finding + MidStreamFallbackError → should fail (exit 1). # Midstream is vertex-retryable, so the gate also tries fallback models -# (after the below-threshold check refuses bypass due to infra errors). +# while every nonzero scanner result remains incomplete evidence. run_gate_case_allow_provider_signal "below-threshold-with-midstream" \ "vertex_ai/medium-midstream-primary" \ "vertex_ai/gemini-2.5-pro vertex_ai/gemini-2.5-flash" \ "1" \ - "infrastructure errors occurred during this pipeline run; refusing bypass" \ + "Configured Vertex model and fallback models were unavailable." \ "3" \ "vertex_ai/medium-midstream-primary|vertex_ai/gemini-2.5-pro|vertex_ai/gemini-2.5-flash" \ "||" @@ -11090,15 +11485,13 @@ run_gate_case_allow_provider_signal "bare-timeout-provider-marker-exhausted-fall "" \ "1" -# Sticky INFRA_ERROR_DETECTED flag: first call hits rate-limit (infra error), -# second call fails with a non-retryable error but leaves a partial LOW report. -# The gate must refuse the below-threshold bypass because an infrastructure -# error was detected during this pipeline run. +# A rate-limited primary followed by a failed fallback with a partial LOW report +# remains incomplete evidence and must fail closed. run_gate_case_allow_provider_signal "infra-error-sticky-flag" \ "vertex_ai/sticky-flag-primary" \ "" \ "1" \ - "infrastructure errors occurred" \ + "Strix quick scan failed with a non-recoverable error." \ "3" \ "vertex_ai/sticky-flag-primary|vertex_ai/sticky-flag-primary|vertex_ai/gemini-2.5-pro" \ "||" \ @@ -11406,8 +11799,8 @@ run_gate_case "pr-empty-diff-skip" \ run_gate_case "pr-baseline-critical-unchanged" \ "openai/gpt-4o-mini" \ "" \ - "0" \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ + "1" \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "openai/gpt-4o-mini" \ "https://example.invalid" \ @@ -11427,8 +11820,8 @@ run_gate_case "pr-baseline-critical-unchanged" \ run_gate_case "pr-baseline-critical-absolute-target" \ "openai/gpt-4o-mini" \ "" \ - "0" \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ + "1" \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ "1" \ "openai/gpt-4o-mini" \ "https://example.invalid" \ @@ -11490,6 +11883,30 @@ run_gate_case "pr-baseline-critical-subdir-target" \ "" \ "1" +run_gate_case "pr-critical-outside-narrowed-subdir-target" \ + "openai/gpt-4o-mini" \ + "" \ + "1" \ + "Unable to map Strix findings to changed files; failing closed for pull request." \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-server/src/main/resources/flyway/V24__update_search_expression_team_keyword_id.sql" \ + "" \ + "" \ + "1" + run_gate_case "pr-baseline-critical-subdir-boxed-target" \ "openai/gpt-4o-mini" \ "" \ @@ -11634,8 +12051,8 @@ run_gate_case "pr-critical-changed" \ run_gate_case "pr-changed-file-nonintersecting-line" \ "openai/gpt-4o-mini" \ "" \ - "0" \ - "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ + "1" \ + "Strix finding intersects files changed in this pull request." \ "1" \ "openai/gpt-4o-mini" \ "https://example.invalid" \ @@ -12318,6 +12735,12 @@ assert_normalized_model \ "vertex_ai" \ "vertex_ai/gemini-2.5-pro" +assert_normalized_model \ + "direct-openai-workflow-alias" \ + "openai-direct/gpt-5.6-luna" \ + "openai" \ + "openai_direct/gpt-5.6-luna" + assert_model_requires_vertex_auth "explicit-vertex" "vertex_ai/gemini-2.5-pro" "gemini" "0" assert_model_requires_vertex_auth "explicit-vertex-beta" "vertex_ai_beta/gemini-2.5-pro" "gemini" "0" assert_model_requires_vertex_auth "vertex-resource-path" "projects/my-proj/locations/us-central1/models/gemini-2.5-pro" "vertex_ai" "0" @@ -12514,6 +12937,102 @@ run_gate_case "openai-direct-quota-github-models-fallback-success" \ "" \ "github_models/openai/o3" +# Strix currently reaches LiteLLM with a sampling default and exposes no +# documented generation-parameter override. Azure reasoning deployments reject +# that temperature before Strix's internal model-group fallback can run, so the +# trusted outer gate must try its already-configured distinct provider. +run_gate_case "openai-direct-unsupported-temperature-github-models-fallback-success" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'github_models/openai/o3' in [0-9]+s\\." \ + "2" \ + "openai/gpt-5.6-sol|openai/o3" \ + "|https://models.github.ai/inference" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + +# Cross-line assembly is deliberately rejected: target/source text cannot +# manufacture a provider capability signal from independent log lines. +run_gate_case "openai-direct-unsupported-temperature-split-lines-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "" \ + "vertex_ai" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "github_models/openai/o3" + +# Repository-derived text containing a valid provider error as a substring +# must not trigger credential-bearing fallback. +run_gate_case "openai-direct-unsupported-temperature-prefixed-target-nonrecoverable" \ + "openai_direct/gpt-5.6-sol" \ + "github_models/openai/o3" \ + "1" \ + "Strix quick scan failed with a non-recoverable error." \ + "1" \ + "openai/gpt-5.6-sol" \ + "https://example.invalid" \ + "vertex_ai" + +# Cross-provider fallbacks must switch both the API key and endpoint. Reusing +# NVIDIA credentials or its API base makes a normalized direct-OpenAI model +# fail before producing security evidence. +run_nvidia_openai_direct_fallback_case +run_nvidia_openai_direct_fallback_case \ + "nvidia-openai-direct-fallback-missing-key-fails-closed" \ + "1" \ + "STRIX_OPENAI_FALLBACK_KEY_FILE is unavailable" \ + "1" \ + "nvidia_nim/nvidia/primary" \ + "https://integrate.api.nvidia.com/v1" +run_nvidia_openai_direct_fallback_case \ + "nvidia-openai-direct-missing-key-next-fallback-success" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/fallback-two' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/primary|nvidia_nim/nvidia/fallback-two" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "openai-direct/gpt-5.6-luna nvidia_nim/nvidia/fallback-two" + run_gate_case "github-models-fallback-success-deepseek-v3" \ "vertex_ai/missing-primary" \ "github_models/deepseek/deepseek-r1-0528 github_models/deepseek/deepseek-v3-0324" \ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index e58f5e6c0..77aae672a 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -32,6 +32,20 @@ def workflow_step(workflow: str, name: str) -> str: return workflow[start:end] +def run_strix_smoke(tmp_path: Path, workflow: str) -> subprocess.CompletedProcess[str]: + """Run the trusted smoke checker against one candidate workflow.""" + workflow_path = tmp_path / ".github" / "workflows" / "strix.yml" + workflow_path.parent.mkdir(parents=True) + workflow_path.write_text(workflow, encoding="utf-8") + return subprocess.run( + ["bash", str(REPO_ROOT / "scripts" / "ci" / "strix_required_workflow_smoke.sh")], + env={**os.environ, "TRUSTED_WORKSPACE": str(tmp_path)}, + capture_output=True, + text=True, + check=False, + ) + + def test_merge_scheduler_dispatches_one_review_by_default() -> None: """Keep the default scheduler dispatch bounded to one review.""" workflow = workflow_text("pr-review-merge-scheduler.yml") @@ -545,6 +559,102 @@ def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret( assert noema_probe.read_text() == "synthetic-openai-key" +@pytest.mark.parametrize( + "model", + ( + "gpt-5.6-luna\nforged< None: + """Untrusted dispatch data cannot inject GitHub output records.""" + output_path = tmp_path / "strix-output" + result = subprocess.run( + [ + "bash", + "-c", + textwrap.dedent( + workflow_step(workflow_text("strix.yml"), "Gate Strix secrets") + .split(" run: |\n", 1)[1] + ), + ], + env={ + **os.environ, + "GITHUB_OUTPUT": str(output_path), + "STRIX_MODEL": model, + "STRIX_MODEL_REQUESTED": model, + "STRIX_OPENAI_API_KEY": "synthetic-openai-key", + "STRIX_OPENROUTER_API_KEY": "", + "STRIX_NVIDIA_NIM_API_KEY": "", + "STRIX_VERTEX_CREDENTIALS": "", + "STRIX_GITHUB_MODELS_TOKEN": "synthetic-models-token", + "TARGET_REPOSITORY_PRIVATE": "false", + }, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "must not contain carriage returns or newlines" in result.stdout + assert not output_path.exists() + + +def test_strix_manual_status_uses_only_live_validated_identifiers() -> None: + """Failed dispatch validation cannot write a caller-selected commit status.""" + workflow = workflow_text("strix.yml") + publish_step = workflow_step(workflow, "Publish same-head manual Strix status") + + assert "id: dispatch_metadata" in workflow + assert ( + "dispatch_metadata_validated: " + "${{ steps.dispatch_metadata.outputs.validated }}" + ) in workflow + assert ( + "github.event_name == 'repository_dispatch' && " + "needs.strix.outputs.dispatch_metadata_validated == 'true'" + ) in workflow + assert ( + "TARGET_REPOSITORY: ${{ needs.strix.outputs.dispatch_target_repository }}" + in publish_step + ) + assert ( + "PR_HEAD_SHA: ${{ needs.strix.outputs.dispatch_head_sha }}" in publish_step + ) + assert "github.event.client_payload.target_repository" not in publish_step + assert "github.event.client_payload.pr_head_sha" not in publish_step + assert workflow.count("success:true)") == 1 + + +def test_strix_smoke_rejects_workflow_contract_expansion(tmp_path: Path) -> None: + """Unknown jobs, broader permissions, and mutable actions fail closed.""" + workflow = workflow_text("strix.yml") + baseline = run_strix_smoke(tmp_path / "baseline", workflow) + assert baseline.returncode == 0, baseline.stderr + + variants = { + "unknown-job": workflow + + "\n attacker-persistence:\n runs-on: ubuntu-latest\n steps:\n" + + " - uses: attacker/persistence-action@main\n", + "broader-permission": workflow.replace( + " permissions:\n actions: read\n contents: read", + " permissions:\n actions: read\n contents: write", + 1, + ), + "mutable-action": workflow.replace( + "actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97", + "actions/setup-python@main", + 1, + ), + } + for name, candidate in variants.items(): + result = run_strix_smoke(tmp_path / name, candidate) + assert result.returncode != 0, name + + def test_noema_workflow_run_without_pull_request_skips_before_token_exchange() -> None: """Skip unassociated workflow runs before requesting review credentials.""" workflow = workflow_text("noema-review.yml") @@ -1520,7 +1630,7 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: assert "agents|pydantic_ai|strix" in workflow assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow - assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow + assert "(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow assert 'exit "$strix_rc"' in workflow @@ -1533,25 +1643,45 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: ) -def test_strix_cross_repo_dispatch_uses_target_token_for_pr_scoping() -> None: - """Bind cross-repository Strix scans to the target PR and authorized token.""" +def test_strix_scan_cannot_read_target_pr_or_publish_status() -> None: + """Keep target reads and status authority outside the credentialed scan step. + + Protected main's trusted required-workflow smoke pins ``statuses: write`` + to the strix scan job's token, so the workflow keeps that grant there. + What this contract actually protects is the scanner boundary: the Run + Strix step receives no GH_TOKEN, so the scan process (and anything it + spawns) cannot publish statuses regardless of the job token's scopes. + Manual-evidence publication stays in the isolated follow-up job, whose + own GITHUB_TOKEN carries no status scope; its writes use exchanged + app/secret tokens only. + """ workflow = workflow_text("strix.yml") + strix_job = workflow.split("\n strix:", 1)[1].split( + "\n publish-manual-pr-evidence-status:", 1 + )[0] + dispatch_validation = workflow.split( + " - name: Validate repository dispatch against live pull request metadata", + 1, + )[1].split(" - name:", 1)[0] run_step = workflow.split(" - name: Run Strix (quick)", 1)[1].split( " - name:", 1 )[0] + publish_job = workflow.split("\n publish-manual-pr-evidence-status:", 1)[1] assert "STRIX_TARGET_PATH:" in run_step - assert "github.event_name == 'repository_dispatch'" in run_step - assert "github.event.client_payload.pr_number != ''" in run_step + assert "GH_TOKEN:" not in run_step + assert "statuses: write" in strix_job assert ( "steps.target_app_token.outputs.token || secrets.OPENCODE_APPROVE_TOKEN || " "github.token" - ) in run_step - assert "github.event_name == 'pull_request_target' && github.token" in run_step - assert ( - "(github.event_name == 'pull_request_target' || " - "github.event.client_payload.pr_number != '') && github.token" - ) not in run_step + ) in dispatch_validation + assert 'echo "validated=true" >>"$GITHUB_OUTPUT"' in dispatch_validation + publish_permissions = publish_job.split("permissions:", 1)[1].split( + "steps:", 1 + )[0] + assert "statuses: write" not in publish_permissions + assert "id-token: write" in publish_permissions + assert "needs.strix.outputs.dispatch_metadata_validated == 'true'" in publish_job def test_pr_scorecard_sarif_delegates_sast_and_vulnerability_posture_to_hard_gates() -> ( diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 990269725..dfab36ed8 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -17,6 +17,7 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[1] STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_MODEL_UTILS = REPOSITORY_ROOT / "scripts" / "ci" / "strix_model_utils.sh" STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" DEFAULT_NVIDIA_MODEL = "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" FREE_NVIDIA_FALLBACK = ( @@ -73,6 +74,67 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 +def _run_strix_once_resolves_child_model(model: str, primary_model: str) -> str: + """Execute run_strix_once's own model/API-base resolution lines verbatim. + + Extracts the exact two statements `run_strix_once` uses to compute the + model it hands to LiteLLM, rather than reimplementing that composition, + so a future call-site edit that stops normalizing the workflow-facing + alias before dispatch fails this test instead of only failing in CI + against live NVIDIA NIM traffic. + """ + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + call_site_match = re.search( + r'(?m)^\tif ! normalized_model="\$\(normalize_model "\$model"\)"; then\n' + r'(?:.*\n)+?' + r'\tif ! llm_api_base_value=.*\n' + r'(?:.*\n)+?' + r'\tchild_model="\$\(child_model_for_api_base [^\n]+\)"\n', + gate_source, + ) + if call_site_match is None: + raise AssertionError( + "missing run_strix_once model/API-base resolution call site" + ) + function_source = "\n".join( + _function_block(gate_source, name) + for name in ( + "is_vertex_model", + "is_github_models_api_base", + "is_github_models_model", + "is_github_models_api_compatible_model", + "resolved_llm_api_base_for_model", + "child_model_for_api_base", + ) + ) + script = "\n".join( + ( + "set -euo pipefail", + STRIX_MODEL_UTILS.read_text(encoding="utf-8"), + function_source, + 'PRIMARY_MODEL="$2"', + 'LLM_API_BASE_FILE=""', + "run_strix_once_child_model() {", + 'local model="$1"', + "local normalized_model", + "local llm_api_base_value", + "local child_model", + call_site_match.group(0), + 'printf %s "$child_model"', + "}", + 'run_strix_once_child_model "$1"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-run-once", model, primary_model], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" @@ -85,6 +147,42 @@ def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: return match.group(1) +def _resolve_api_base_for_model(model: str, primary_model: str, api_base: str) -> str: + """Execute the production fallback API-base resolver with a trusted stub.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = "\n".join( + ( + STRIX_MODEL_UTILS.read_text(encoding="utf-8"), + _function_block(gate_source, "is_vertex_model"), + _function_block(gate_source, "is_github_models_api_base"), + _function_block(gate_source, "is_github_models_model"), + _function_block(gate_source, "resolved_llm_api_base_for_model"), + ) + ) + with tempfile.TemporaryDirectory(prefix="strix-openai-base-") as temp_dir: + base_file = Path(temp_dir) / "openai-base.txt" + base_file.write_text(api_base, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + "resolve_trusted_input_file() { printf '%s\\n' \"$2\"; }", + function_source, + f'PRIMARY_MODEL="{primary_model}"', + 'LLM_API_BASE_FILE=""', + f'STRIX_OPENAI_FALLBACK_API_BASE_FILE="{base_file}"', + f'resolved_llm_api_base_for_model "{model}"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + def _workflow_classifies_backend_unavailable(log_text: str) -> bool: """Execute the outer workflow's backend-neutralization condition.""" @@ -202,7 +300,20 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: ) self.assertIn( "steps.gate.outputs.provider_mode == 'nvidia_nim' && " - f"'{FREE_NVIDIA_FALLBACK} openai-direct/gpt-5.6-luna'", + f"'{FREE_NVIDIA_FALLBACK} openai_direct/gpt-5.6-luna'", + workflow, + ) + fallback_lines = [ + line for line in workflow.splitlines() if "STRIX_FALLBACK_MODELS:" in line + ] + self.assertEqual(len(fallback_lines), 1) + self.assertNotIn( + "steps.gate.outputs.provider_mode == 'openai_direct' &&", + fallback_lines[0], + ) + self.assertNotIn("openai-direct/gpt-5.6-luna", fallback_lines[0]) + self.assertIn( + "STRIX_OPENAI_FALLBACK_API_BASE_FILE=$", workflow, ) @@ -213,6 +324,51 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: )[0] self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate) + def test_run_strix_once_normalizes_hyphenated_fallback_before_dispatch( + self, + ) -> None: + """Dispatch the NIM-exhaustion fallback as a real LiteLLM provider. + + `normalize_model()` (`scripts/ci/strix_model_utils.sh`) already + rewrites the workflow-facing `openai-direct/` alias to LiteLLM's + `openai_direct/` provider prefix, and `child_model_for_api_base()` + already rewrites `openai_direct/*` to `openai/*`. But + `run_strix_once()` calls `child_model_for_api_base` with the raw, + un-normalized `$model` -- so a hyphenated `STRIX_FALLBACK_MODELS` + entry still reaches LiteLLM unrewritten. Observed live: NVIDIA NIM + rate-limited the primary and first fallback model, the run advanced + to `openai-direct/gpt-5.6-luna`, and the scan log recorded + `model=openai-direct/gpt-5.6-luna` verbatim before + `litellm.BadRequestError: LLM Provider NOT provided` ended the run. + """ + + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai-direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + self.assertEqual( + _run_strix_once_resolves_child_model( + "openai_direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + ), + "openai/gpt-5.6-luna", + ) + + def test_cross_provider_direct_openai_fallback_uses_openai_api_base(self) -> None: + """Route direct OpenAI fallback away from the exhausted primary endpoint.""" + + self.assertEqual( + _resolve_api_base_for_model( + "openai_direct/gpt-5.6-luna", + DEFAULT_NVIDIA_MODEL, + "https://api.openai.com/v1", + ), + "https://api.openai.com/v1", + ) + def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: """Reject provider-like target text in the outer neutralization gate.""" diff --git a/tests/test_strix_severity_identifier_boundary.py b/tests/test_strix_severity_identifier_boundary.py new file mode 100644 index 000000000..ddffb6494 --- /dev/null +++ b/tests/test_strix_severity_identifier_boundary.py @@ -0,0 +1,70 @@ +"""Regression contracts for Strix severity marker identity boundaries.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +GATE = ROOT / "scripts" / "ci" / "strix_quick_gate.sh" + + +def _function(source: str, name: str) -> str: + start = source.index(f"{name}() {{") + cursor = start + depth = 0 + while cursor < len(source): + line_end = source.find("\n", cursor) + if line_end < 0: + line_end = len(source) + line = source[cursor:line_end] + depth += line.count("{") - line.count("}") + if depth == 0: + return source[start:line_end] + "\n" + cursor = line_end + 1 + raise AssertionError(f"unterminated shell function: {name}") + + +def _extract_rank(report: Path) -> int: + source = GATE.read_text(encoding="utf-8") + script = "set -euo pipefail\n" + _function(source, "severity_rank") + script += _function(source, "extract_max_severity_rank") + script += 'extract_max_severity_rank "$1"\n' + completed = subprocess.run( + ["bash", "-c", script, "bash", str(report)], + check=True, + capture_output=True, + text=True, + ) + return int(completed.stdout.strip()) + + +def test_severity_identifier_suffix_does_not_promote_low_finding(tmp_path: Path) -> None: + report = tmp_path / "report.md" + report.write_text( + "STRIX_FAIL_ON_MIN_SEVERITY: CRITICAL\nSeverity: LOW\n", + encoding="utf-8", + ) + + assert _extract_rank(report) == 1 + + +def test_severity_identifier_suffix_is_not_authoritative_finding(tmp_path: Path) -> None: + report = tmp_path / "report.md" + report.write_text("STRIX_FAIL_ON_MIN_SEVERITY: CRITICAL\n", encoding="utf-8") + + assert _extract_rank(report) == -1 + + +def test_all_severity_marker_paths_share_identifier_boundary() -> None: + source = GATE.read_text(encoding="utf-8") + + assert '[[ "${line^^}" =~ (^|[^A-Za-z0-9_])SEVERITY' in source + assert "grep -Ei '(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:'" in source + assert ( + source.count( + "grep -Eiq '(^|[^A-Za-z0-9_])severity[[:space:][:punct:]]*:'" + ) + >= 2 + ) diff --git a/tests/test_strix_workflow_dependency_hashes.py b/tests/test_strix_workflow_dependency_hashes.py index e2509c18b..34e2081a4 100644 --- a/tests/test_strix_workflow_dependency_hashes.py +++ b/tests/test_strix_workflow_dependency_hashes.py @@ -8,6 +8,9 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" +PRIVILEGED_WORKFLOW = ROOT / ".github" / "workflows" / "strix.yml" +STRIX_REQUIREMENTS = ROOT / "requirements-strix-ci.txt" +STRIX_LOCK = ROOT / "requirements-strix-ci-hashes.txt" WORKFLOW_DISPATCH_KEY_RE = re.compile( r"(?m)^[ \t]+['\"]?workflow_dispatch['\"]?\s*:" ) @@ -33,11 +36,54 @@ def test_strix_workflow_installs_only_hash_verified_wheels() -> None: assert f"{requirement} --hash=sha256:{digest}" in workflow +def test_privileged_strix_install_uses_only_the_trusted_workflow_lock() -> None: + """PR content cannot select code installed beside provider credentials.""" + workflow = PRIVILEGED_WORKFLOW.read_text(encoding="utf-8") + install_step = workflow.split(" - name: Install Strix\n", 1)[1].split( + " - name: Mask LLM API key\n", 1 + )[0] + + assert ( + " - name: Materialize central Strix dependency lock from PR head\n" + not in workflow + ) + assert "PR_HEAD_SHA:requirements-strix-ci-hashes.txt" not in workflow + assert 'show "$PR_HEAD_SHA:requirements-strix-ci-hashes.txt"' not in workflow + assert 'trusted_lock_blob="$(git rev-parse "HEAD:$trusted_lock")"' in install_step + assert ( + 'working_lock_blob="$(git hash-object --no-filters -- "$trusted_lock")"' + in install_step + ) + assert '"$trusted_lock_blob" != "$working_lock_blob"' in install_step + assert "--only-binary=:all:" in install_step + assert "litellm==1.94.2" in STRIX_REQUIREMENTS.read_text(encoding="utf-8") + assert "litellm==1.94.2 \\" in STRIX_LOCK.read_text(encoding="utf-8") + + def test_strix_workflow_reruns_when_hash_contract_changes() -> None: """Changing this regression contract must trigger the exact-head workflow.""" workflow = WORKFLOW.read_text(encoding="utf-8") - assert ' - "tests/test_strix_workflow_dependency_hashes.py"' in workflow + for path in ( + "requirements-strix-ci.txt", + "requirements-strix-ci-hashes.txt", + "tests/test_strix_severity_identifier_boundary.py", + "tests/test_strix_workflow_dependency_hashes.py", + ): + assert f' - "{path}"' in workflow + + +def test_strix_workflow_reruns_when_shared_runtime_or_doctoring_changes() -> None: + """Shared model routing and its decision record always rerun exact-head checks.""" + workflow = WORKFLOW.read_text(encoding="utf-8") + + for path in ( + "docs/doctoring/strix-nvidia-nim-not-found-fallback.md", + "docs/doctoring/strix-trusted-dependency-lock.md", + "docs/doctoring/strix-unsupported-sampling-fallback.md", + "scripts/ci/strix_model_utils.sh", + ): + assert f' - "{path}"' in workflow def test_strix_workflow_rejects_branch_selected_manual_dispatch() -> None: