diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 337373001..1e85b8661 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -221,7 +221,8 @@ has_strix_report_failure_signal() { report_root="$newest_report_root" fi while IFS= read -r -d '' report_log; do - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then + if grep -Eiv '^[[:space:]]*[^[:alnum:]]*[[:space:]]*MODEL QUALITY WARNING[[:space:]]*[^[:alnum:]]*[[:space:]]*$' "$report_log" | + grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)'; then return 0 fi done < <(find "$report_root" -type f -name '*.log' -print0) @@ -3151,7 +3152,8 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then + if grep -Eiv '^[[:space:]]*[^[:alnum:]]*[[:space:]]*MODEL QUALITY WARNING[[:space:]]*[^[:alnum:]]*[[:space:]]*$' "$STRIX_LOG" | + grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)'; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index bf0a8693e..fd39a44c5 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3416,6 +3416,23 @@ REPORT ;; esac ;; + nvidia-nim-quality-warning-fallback-success) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429" + exit 1 + ;; + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + echo "│ MODEL QUALITY WARNING" + echo "scan ok with NVIDIA NIM fallback" + exit 0 + ;; + *) + echo "unexpected model ${STRIX_LLM:-}" >&2 + exit 9 + ;; + esac + ;; vertex-all-notfound) echo "Error: litellm.NotFoundError: Vertex_aiException - x" echo '"status": "NOT_FOUND"' @@ -5951,6 +5968,18 @@ run_filtered_gate_case_if_requested() { "vertex_ai/ready-primary" \ "" ;; + nvidia-nim-quality-warning-fallback-success) + run_gate_case "nvidia-nim-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" \ + "" + ;; pr-rust-workspace-context) run_gate_case "pr-rust-workspace-context" \ "openai/gpt-4o-mini" \ @@ -12370,6 +12399,17 @@ run_gate_case "direct-openai-gpt-does-not-require-github-models-api-base" \ "openai" \ "" +run_gate_case "nvidia-nim-quality-warning-fallback-success" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "|" \ + "openai" \ + "" + run_gate_case "github-models-model-prefix-with-api-base-succeeds" \ "openai/gpt-5" \ "" \