Skip to content

fix(strix): accept complete scans with benign model-quality advisory #1255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
seonghobae wants to merge 7 commits into from
Closed

fix(strix): accept complete scans with benign model-quality advisory #1255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
585710e
test(strix): reproduce benign quality warning false negative
seonghobae Aug 23, 2026
f68a060
test(strix): register quality-warning focused regression
seonghobae Aug 23, 2026
e21e421
test(strix): exercise the governed NIM fallback lane
seonghobae Aug 23, 2026
17c5395
fix(strix): distinguish model-quality advisory from provider failure
seonghobae Aug 23, 2026
5bf2230
fix(strix): restore complete quality-advisory classifier
seonghobae Aug 23, 2026
09cb76e
test(strix): reproduce report-log quality advisory false negative
seonghobae Aug 23, 2026
a1fc43a
fix(strix): classify quality advisory consistently across evidence files
seonghobae Aug 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions scripts/ci/strix_quick_gate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,7 +221,8 @@ has_strix_report_failure_signal() {
report_root="$newest_report_root"
fi
while IFS= read -r -d '' report_log; do
if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then
if has_unexpected_strix_failure_signal_file "$report_log" ||
grep -Eiq '(^|[^[:alpha:]])Timeout([^[:alpha:]]|$)' "$report_log"; then
return 0
fi
done < <(find "$report_root" -type f -name '*.log' -print0)
Expand Down Expand Up @@ -3150,8 +3151,29 @@ is_llm_token_limit_error() {
# errors (timeout, rate-limit, transport failures) that indicate the scan
# was interrupted or incomplete. Used as a guard to prevent the
# below-threshold override from silently passing an aborted scan.
has_unexpected_strix_failure_signal_file() {
local signal_file="$1"
local signal_line
while IFS= read -r signal_line; do
# Strix emits this exact decorative banner for supported models that are
# not on its recommended frontier list. It is a quality advisory, not
# evidence that provider execution or the scan was incomplete.
if printf '%s\n' "$signal_line" |
grep -Eq '^[^[:alnum:]]*MODEL QUALITY WARNING[^[:alnum:]]*$'; then
continue
fi
return 0
done < <(grep -Ei '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$signal_file" || true)
Comment thread
seonghobae marked this conversation as resolved.
Comment thread
seonghobae marked this conversation as resolved.

return 1
}

has_unexpected_strix_console_failure_signal() {
has_unexpected_strix_failure_signal_file "$STRIX_LOG"
}

has_detected_infrastructure_error() {
if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then
if has_unexpected_strix_console_failure_signal; then
return 0
fi

Expand Down
122 changes: 122 additions & 0 deletions scripts/ci/test_strix_quick_gate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4467,6 +4467,37 @@ EOS
;;
esac
;;
nim-primary-rate-limit-clean-fallback-quality-warning|nim-primary-rate-limit-clean-fallback-quality-report-warning)
case "${STRIX_LLM:-}" in
nvidia_nim/nvidia/nemotron-3-super-120b-a12b)
echo "LLM CONNECTION FAILED"
echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429"
exit 1
;;
nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5)
echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮"
echo "│ MODEL QUALITY WARNING │"
echo "│ 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' is not a recommended │"
echo "│ frontier model for Strix. │"
echo "│ You can continue, but weaker models may miss vulnerabilities or produce │"
echo "│ lower-quality findings. │"
if [ "${FAKE_STRIX_SCENARIO:?}" = "nim-primary-rate-limit-clean-fallback-quality-report-warning" ]; then
mkdir -p "$STRIX_REPORTS_DIR/fake-nim-quality-advisory"
cat >"$STRIX_REPORTS_DIR/fake-nim-quality-advisory/strix.log" <<'EOS'
│ MODEL QUALITY WARNING │
2026-08-23 12:32:17.984 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s)
EOS
fi
echo "│ Vulnerabilities 0 │"
echo "╰──────────────────────────────────────────────────────────────────────────────╯"
exit 0
;;
*)
echo "Error: NIM quality-warning fallback path unexpected (${STRIX_LLM:-})" >&2
exit 62
;;
esac
;;
provider-fatal-success-signal)
echo "Fatal: provider stream aborted"
exit 0
Expand Down Expand Up @@ -6363,6 +6394,37 @@ run_filtered_gate_case_if_requested() {
"vertex_ai/report-known-internal-warning-sanitized" \
"<unset>"
;;
nim-primary-rate-limit-clean-fallback-quality-warning | nim-primary-rate-limit-clean-fallback-quality-report-warning)
run_gate_case "$STRIX_TEST_CASE_FILTER" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \
"" \
"0" \
"REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \
"2" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \
"nvidia_nim" \
"https://integrate.api.nvidia.com/v1" \
"" \
"0" \
"CRITICAL" \
"0" \
"" \
"" \
"1200" \
"0" \
"" \
"" \
"" \
"" \
"0" \
"" \
"" \
"" \
"__SAME_AS_FALLBACK_MODELS__" \
"nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"1"
;;
provider-fatal-success-signal | provider-warning-success-signal)
run_gate_case "$STRIX_TEST_CASE_FILTER" \
"vertex_ai/$STRIX_TEST_CASE_FILTER" \
Expand Down Expand Up @@ -10330,6 +10392,66 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \
"" \
"1"

run_gate_case "nim-primary-rate-limit-clean-fallback-quality-warning" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \
"" \
"0" \
"REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \
"2" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \
"nvidia_nim" \
"https://integrate.api.nvidia.com/v1" \
"" \
"0" \
"CRITICAL" \
"0" \
"" \
"" \
"1200" \
"0" \
"" \
"" \
"" \
"" \
"0" \
"" \
"" \
"" \
"__SAME_AS_FALLBACK_MODELS__" \
"nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"1"

run_gate_case "nim-primary-rate-limit-clean-fallback-quality-report-warning" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \
"" \
"0" \
"REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \
"2" \
"nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \
"nvidia_nim" \
"https://integrate.api.nvidia.com/v1" \
"" \
"0" \
"CRITICAL" \
"0" \
"" \
"" \
"1200" \
"0" \
"" \
"" \
"" \
"" \
"0" \
"" \
"" \
"" \
"__SAME_AS_FALLBACK_MODELS__" \
"nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \
"1"

run_gate_case "provider-fatal-success-signal" \
"vertex_ai/provider-fatal-success-signal" \
"" \
Expand Down
Loading