diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 337373001..ae324f44c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -221,7 +221,8 @@ has_strix_report_failure_signal() { report_root="$newest_report_root" fi while IFS= read -r -d '' report_log; do - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then + if has_unexpected_strix_failure_signal_file "$report_log" || + grep -Eiq '(^|[^[:alpha:]])Timeout([^[:alpha:]]|$)' "$report_log"; then return 0 fi done < <(find "$report_root" -type f -name '*.log' -print0) @@ -3150,8 +3151,29 @@ is_llm_token_limit_error() { # errors (timeout, rate-limit, transport failures) that indicate the scan # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. +has_unexpected_strix_failure_signal_file() { + local signal_file="$1" + local signal_line + while IFS= read -r signal_line; do + # Strix emits this exact decorative banner for supported models that are + # not on its recommended frontier list. It is a quality advisory, not + # evidence that provider execution or the scan was incomplete. + if printf '%s\n' "$signal_line" | + grep -Eq '^[^[:alnum:]]*MODEL QUALITY WARNING[^[:alnum:]]*$'; then + continue + fi + return 0 + done < <(grep -Ei '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$signal_file" || true) + + return 1 +} + +has_unexpected_strix_console_failure_signal() { + has_unexpected_strix_failure_signal_file "$STRIX_LOG" +} + has_detected_infrastructure_error() { - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then + if has_unexpected_strix_console_failure_signal; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index bf0a8693e..cb6abea20 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4467,6 +4467,37 @@ EOS ;; esac ;; + nim-primary-rate-limit-clean-fallback-quality-warning|nim-primary-rate-limit-clean-fallback-quality-report-warning) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + echo "LLM CONNECTION FAILED" + echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429" + exit 1 + ;; + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" + echo "│ MODEL QUALITY WARNING │" + echo "│ 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' is not a recommended │" + echo "│ frontier model for Strix. │" + echo "│ You can continue, but weaker models may miss vulnerabilities or produce │" + echo "│ lower-quality findings. │" + if [ "${FAKE_STRIX_SCENARIO:?}" = "nim-primary-rate-limit-clean-fallback-quality-report-warning" ]; then + mkdir -p "$STRIX_REPORTS_DIR/fake-nim-quality-advisory" + cat >"$STRIX_REPORTS_DIR/fake-nim-quality-advisory/strix.log" <<'EOS' +│ MODEL QUALITY WARNING │ +2026-08-23 12:32:17.984 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) +EOS + fi + echo "│ Vulnerabilities 0 │" + echo "╰──────────────────────────────────────────────────────────────────────────────╯" + exit 0 + ;; + *) + echo "Error: NIM quality-warning fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 62 + ;; + esac + ;; provider-fatal-success-signal) echo "Fatal: provider stream aborted" exit 0 @@ -6363,6 +6394,37 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + nim-primary-rate-limit-clean-fallback-quality-warning | nim-primary-rate-limit-clean-fallback-quality-report-warning) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -10330,6 +10392,66 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \ "" \ "1" +run_gate_case "nim-primary-rate-limit-clean-fallback-quality-warning" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" + +run_gate_case "nim-primary-rate-limit-clean-fallback-quality-report-warning" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" + run_gate_case "provider-fatal-success-signal" \ "vertex_ai/provider-fatal-success-signal" \ "" \