From 585710e5c35edbd27a7789b17493da4dfd642e5f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:29:11 -0700 Subject: [PATCH 1/7] test(strix): reproduce benign quality warning false negative --- scripts/ci/test_strix_quick_gate.sh | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index bf0a8693e..fbd4ecc0b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4467,6 +4467,30 @@ EOS ;; esac ;; + nim-primary-rate-limit-clean-fallback-quality-warning) + case "${STRIX_LLM:-}" in + nvidia_nim/nvidia/nemotron-3-super-120b-a12b) + echo "LLM CONNECTION FAILED" + echo "Error: litellm.RateLimitError: Nvidia_nimException - Error code: 429" + exit 1 + ;; + nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5) + echo "╭─ STRIX ──────────────────────────────────────────────────────────────────────╮" + echo "│ MODEL QUALITY WARNING │" + echo "│ 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' is not a recommended │" + echo "│ frontier model for Strix. │" + echo "│ You can continue, but weaker models may miss vulnerabilities or produce │" + echo "│ lower-quality findings. │" + echo "│ Vulnerabilities 0 │" + echo "╰──────────────────────────────────────────────────────────────────────────────╯" + exit 0 + ;; + *) + echo "Error: NIM quality-warning fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 62 + ;; + esac + ;; provider-fatal-success-signal) echo "Fatal: provider stream aborted" exit 0 @@ -10330,6 +10354,16 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \ "" \ "1" +run_gate_case "nim-primary-rate-limit-clean-fallback-quality-warning" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" + run_gate_case "provider-fatal-success-signal" \ "vertex_ai/provider-fatal-success-signal" \ "" \ From f68a0602885c5db58123aa6c3fb856b4f13902ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:34:07 -0700 Subject: [PATCH 2/7] test(strix): register quality-warning focused regression --- scripts/ci/test_strix_quick_gate.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index fbd4ecc0b..440dda8a7 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -6387,6 +6387,17 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + nim-primary-rate-limit-clean-fallback-quality-warning) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ From e21e421c3f9bc9a6034fcc7e5aeca7abdf7ec8f7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:35:53 -0700 Subject: [PATCH 3/7] test(strix): exercise the governed NIM fallback lane --- scripts/ci/test_strix_quick_gate.sh | 48 ++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 440dda8a7..5807a9f81 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -6390,13 +6390,33 @@ run_filtered_gate_case_if_requested() { nim-primary-rate-limit-clean-fallback-quality-warning) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ "0" \ "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ "2" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ @@ -10367,13 +10387,33 @@ run_gate_case "strict-zero-findings-timeout-fails-pr" \ run_gate_case "nim-primary-rate-limit-clean-fallback-quality-warning" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ - "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "" \ "0" \ "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ "2" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ - "nvidia_nim" + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" run_gate_case "provider-fatal-success-signal" \ "vertex_ai/provider-fatal-success-signal" \ From 17c5395bc0e470dc1b21ae8f16668c01a3b61bbb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:37:23 -0700 Subject: [PATCH 4/7] fix(strix): distinguish model-quality advisory from provider failure --- scripts/ci/strix_quick_gate.sh | 1165 +++++++++++++++++++++++++++++++- 1 file changed, 1164 insertions(+), 1 deletion(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 337373001..01656bb3f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3150,8 +3150,1171 @@ is_llm_token_limit_error() { # errors (timeout, rate-limit, transport failures) that indicate the scan # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. +has_unexpected_strix_console_failure_signal() { + local signal_line + while IFS= read -r signal_line; do + # Strix emits this exact decorative banner for supported models that are + # not on its recommended frontier list. It is a quality advisory, not + # evidence that provider execution or the scan was incomplete. + if printf '%s\n' "$signal_line" | + grep -Eq '^[^[:alnum:]]*MODEL QUALITY WARNING[^[:alnum:]]* + + if is_timeout_error; then + return 0 + fi + + if is_rate_limit_error; then + return 0 + fi + + if is_llm_token_limit_error; then + return 0 + fi + + if is_midstream_fallback_error; then + return 0 + fi + + if is_llm_api_connection_error; then + return 0 + fi + + if is_llm_service_unavailable_error; then + return 0 + fi + + if is_nvidia_nim_not_found_error; then + return 0 + fi + + if is_model_behavior_error; then + return 0 + fi + + if is_caido_bootstrap_timing_error; then + return 0 + fi + + # Generic strix non-zero exit with known transport/connection errors + # that don't fall into the specific categories above. + # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid + # false positives: PROVIDER_CONTEXT_REGEX includes httpx/httpcore/requests + # which would self-match on e.g. "requests.exceptions.ConnectionError" + # from target-application logs. + if grep -Eiq '(ConnectionError|ConnectionRefusedError|ConnectionResetError|SSLError|ProxyError|NetworkError)' "$STRIX_LOG" && + grep -Eiq "$LLM_PROVIDER_ONLY_REGEX" "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + +latest_strix_report_dir() { + local latest="" + local run_dir + + for run_dir in "$STRIX_REPORTS_DIR"/*; do + if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then + continue + fi + + if is_preexisting_report_dir "$run_dir"; then + continue + fi + + if [ -z "$latest" ] || [ "$run_dir" -nt "$latest" ]; then + latest="$run_dir" + fi + done + + if [ -z "$latest" ]; then + return 1 + fi + + echo "$latest" +} + +has_only_below_threshold_vulnerabilities() { + local threshold_rank + threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" + + local found_any_vuln_file=0 + local global_max_rank=-1 + STRIX_MAX_SEVERITY_RANK=-1 + local saw_any_severity=0 + + update_max_severity_from_stream() { + local source_path="$1" + local line + local severity + local rank + while IFS= read -r line; do + if [[ "${line^^}" =~ SEVERITY[[:space:]]*:[[:space:][:punct:]]*(CRITICAL|HIGH|MEDIUM|LOW|INFO|INFORMATIONAL|NONE)([[:space:][:punct:]]|$) ]]; then + severity="${BASH_REMATCH[1]}" + else + continue + fi + + rank="$(severity_rank "$severity")" + if [ "$rank" -lt 0 ]; then + continue + fi + + saw_any_severity=1 + if [ "$rank" -gt "$global_max_rank" ]; then + global_max_rank="$rank" + STRIX_MAX_SEVERITY_RANK="$rank" + fi + done < <(grep -Ei 'severity[[:space:]]*:' "$source_path" || true) + } + + local run_dir + for run_dir in "$STRIX_REPORTS_DIR"/*; do + if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then + continue + fi + + if is_preexisting_report_dir "$run_dir"; then + continue + fi + + local vulnerabilities_dir="$run_dir/vulnerabilities" + if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then + continue + fi + + local vuln_file + + for vuln_file in "$vulnerabilities_dir"/*.md; do + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + continue + fi + + found_any_vuln_file=1 + update_max_severity_from_stream "$vuln_file" + done + done + + if [ "$found_any_vuln_file" -eq 0 ]; then + echo "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." >&2 + return 1 + fi + + if [ "$saw_any_severity" -eq 0 ]; then + return 1 + fi + + # Guard against incomplete scans due to infrastructure errors. + # Use the sticky INFRA_ERROR_DETECTED flag instead of re-reading + # STRIX_LOG, because STRIX_LOG is overwritten per-attempt. If an + # earlier attempt hit an infrastructure error (timeout, rate-limit, + # transport failure) and produced a partial report that now sits in + # the reports directory, the *current* STRIX_LOG may show a different + # failure — or even success — but the partial report's low-severity + # findings must not be treated as a clean scan result. + if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + echo "Below-threshold findings detected, but infrastructure errors occurred during this pipeline run; refusing bypass due to potentially incomplete scan." >&2 + return 1 + fi + + if [ "$global_max_rank" -lt "$threshold_rank" ]; then + echo "Strix findings are below configured fail threshold '$STRIX_FAIL_ON_MIN_SEVERITY'; allowing pipeline continuation." >&2 + return 0 + fi + + return 1 +} + +has_blocking_vulnerability_reports() { + local threshold_rank + threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" + + local run_dir vulnerabilities_dir vuln_file rank + for run_dir in "$STRIX_REPORTS_DIR"/*; do + if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then + continue + fi + if is_preexisting_report_dir "$run_dir"; then + continue + fi + + vulnerabilities_dir="$run_dir/vulnerabilities" + if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then + continue + fi + + for vuln_file in "$vulnerabilities_dir"/*.md; do + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + continue + fi + if vulnerability_file_is_retryable_model_inconsistency "$vuln_file"; then + continue + fi + + rank="$(extract_max_severity_rank "$vuln_file")" + if [ "$rank" -lt 0 ] || [ "$rank" -ge "$threshold_rank" ]; then + return 0 + fi + done + done + + return 1 +} + +fail_reported_vulnerabilities_before_fallback_success() { + case "$PR_FINDINGS_DECISION" in + allow_baseline) + return 1 + ;; + esac + + if has_blocking_vulnerability_reports; then + echo "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." >&2 + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 0 + fi + return 1 +} + +has_any_reported_severity_markers() { + local run_dir + for run_dir in "$STRIX_REPORTS_DIR"/*; do + if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then + continue + fi + + if is_preexisting_report_dir "$run_dir"; then + continue + fi + + local vulnerabilities_dir="$run_dir/vulnerabilities" + if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then + continue + fi + + local vuln_file + for vuln_file in "$vulnerabilities_dir"/*.md; do + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + continue + fi + if grep -Eiq 'severity[[:space:]]*:' "$vuln_file"; then + return 0 + fi + done + done + + if grep -Eiq 'severity[[:space:]]*:' "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + +strix_reported_zero_vulnerabilities() { + if [ "$ZERO_FINDINGS_REPORTED" -eq 1 ]; then + return 0 + fi + + strix_reported_zero_vulnerabilities_in_file "$STRIX_LOG" +} + +strix_reported_zero_vulnerabilities_in_file() { + local source_path="$1" + grep -Eq 'Vulnerabilities[[:space:]]+0([^0-9]|$)' "$source_path" +} + +should_fail_pull_request_infra_zero_findings() { + if ! is_pull_request_event; then + return 1 + fi + + if [ "$INFRA_ERROR_DETECTED" -ne 1 ]; then + return 1 + fi + + if has_any_reported_severity_markers; then + return 1 + fi + + if ! strix_reported_zero_vulnerabilities; then + return 1 + fi + + echo "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." >&2 + return 0 +} + +vulnerability_file_has_absent_endpoint_finding() { + local vuln_file="$1" + # Configurable list of source directories to check for endpoints. + # Defaults to "." (i.e. TARGET_PATH itself) so that both + # STRIX_TARGET_PATH=./ and STRIX_TARGET_PATH=./src work correctly + # without producing bogus double-nested paths like ./src/src. + # Set STRIX_SOURCE_DIRS (space-separated) to override. + local source_dirs_raw="${STRIX_SOURCE_DIRS:-.}" + local resolved_target_root="" + local resolved_dirs=() + local dir_entry + if ! resolved_target_root="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null)"; then + return 1 + fi + + # Disable globbing so that entries like "*" or "[" in STRIX_SOURCE_DIRS + # are not expanded by pathname expansion during word-splitting. + set -f + for dir_entry in $source_dirs_raw; do + local candidate="${resolved_target_root%/}/$dir_entry" + if [ -d "$candidate" ] && [ ! -L "$candidate" ]; then + resolved_dirs+=("$candidate") + fi + done + set +f + + if [ "${#resolved_dirs[@]}" -eq 0 ]; then + return 1 + fi + + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + return 1 + fi + + local endpoint_seen=0 + local endpoint_present_in_source=0 + local endpoint + + while IFS= read -r endpoint; do + if [ -z "$endpoint" ]; then + continue + fi + + endpoint_seen=1 + local search_dir + for search_dir in "${resolved_dirs[@]}"; do + # Exclude the strix reports directory and common non-source + # directories from the source search to prevent accidental + # matches and reduce runtime (especially when STRIX_TARGET_PATH=./). + # + # Each exclude-dir: + # STRIX_REPORTS_DIR — strix output itself (would always match). + # Both the full path and basename are excluded so that + # nested paths like "reports/strix_runs" are also caught. + # .git — VCS internals + # node_modules — JS/TS dependencies (may contain API strings) + # vendor — Go/PHP vendored deps + # __pycache__ — Python bytecode cache + # .venv — Python virtualenv + # target — Rust/Java build artifacts + # .mypy_cache — mypy type-check cache + # .pytest_cache — pytest result cache + # dist — common build output directory + # build — common build output directory + # .tox — Python tox test environments + # .ruff_cache — Ruff linter cache + if grep -r -Fq \ + --exclude-dir="$STRIX_REPORTS_DIR" \ + --exclude-dir="$(basename "$STRIX_REPORTS_DIR")" \ + --exclude-dir=".git" \ + --exclude-dir="node_modules" \ + --exclude-dir="vendor" \ + --exclude-dir="__pycache__" \ + --exclude-dir=".venv" \ + --exclude-dir="target" \ + --exclude-dir=".mypy_cache" \ + --exclude-dir=".pytest_cache" \ + --exclude-dir="dist" \ + --exclude-dir="build" \ + --exclude-dir=".tox" \ + --exclude-dir=".ruff_cache" \ + -- "$endpoint" "$search_dir"; then + endpoint_present_in_source=1 + break + fi + done + if [ "$endpoint_present_in_source" -eq 1 ]; then + break + fi + done < <(python3 - "$vuln_file" <<'PY' +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") +endpoints = set() +for line in text.splitlines(): + if not re.search(r"\bEndpoint\b", line, re.IGNORECASE): + continue + endpoints.update(re.findall(r"/api/[A-Za-z0-9_./-]+", line)) +for endpoint in sorted(endpoints): + print(endpoint) +PY + ) + + if [ "$endpoint_seen" -eq 0 ]; then + return 1 + fi + + if [ "$endpoint_present_in_source" -eq 1 ]; then + return 1 + fi + + echo "Detected Strix report endpoint(s) absent from source; treating as retryable model inconsistency." >&2 + return 0 +} + +is_hallucinated_endpoint_finding() { + local latest_report_dir + if ! latest_report_dir="$(latest_strix_report_dir)"; then + return 1 + fi + + local vuln_file + + for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do + if vulnerability_file_is_below_threshold "$vuln_file" && + vulnerability_file_has_absent_endpoint_finding "$vuln_file"; then + return 0 + fi + done + + return 1 +} + +vulnerability_file_has_absent_source_snippets() { + local vuln_file="$1" + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + return 1 + fi + + local location_records_file + location_records_file="$(mktemp)" + extract_vulnerability_location_records "$vuln_file" >"$location_records_file" || true + if [ ! -s "$location_records_file" ]; then + rm -f "$location_records_file" + return 1 + fi + + local resolved_scan_target="" + resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" + if python3 - "$vuln_file" "$REPO_ROOT" "$resolved_scan_target" "$location_records_file" <<'PY' +from pathlib import Path +import re +import sys + +vuln_path = Path(sys.argv[1]) +repo_root = Path(sys.argv[2]) +scan_target = Path(sys.argv[3]) if sys.argv[3] else None +records_path = Path(sys.argv[4]) + +record_paths = { + line.split("\t", 1)[0].strip().replace("\\", "/") + for line in records_path.read_text(encoding="utf-8", errors="replace").splitlines() + if line.strip() +} +if not record_paths: + raise SystemExit(1) + +# ⚡ Bolt: Pre-compile regex for path normalization to optimize inner loop scanning +location_suffix_re = re.compile(r":\d+(?:-\d+)?$") + +def normalize_report_path(raw: str) -> str | None: + value = raw.strip().strip("`").replace("\\", "/") + value = location_suffix_re.sub("", value) + for record_path in record_paths: + if value == record_path or value.endswith("/" + record_path): + return record_path + return value if value in record_paths else None + + +def source_lines_for(path: str) -> set[str] | None: + candidates = [] + if scan_target is not None: + candidates.append(scan_target / path) + candidates.append(repo_root / path) + for candidate in candidates: + try: + if candidate.is_file() and not candidate.is_symlink(): + return { + line.strip() + for line in candidate.read_text( + encoding="utf-8", errors="replace" + ).splitlines() + if line.strip() + } + except OSError: + continue + return None + + +source_by_path = { + path: lines + for path in record_paths + if (lines := source_lines_for(path)) is not None +} +if not source_by_path: + raise SystemExit(1) + +text_lines = vuln_path.read_text(encoding="utf-8", errors="replace").splitlines() +in_code_analysis = False +current_file: str | None = None +in_fence = False +fence_file: str | None = None +fence_lang = "" +fence_lines: list[str] = [] +blocks: list[tuple[str, str, list[str]]] = [] +location_re = re.compile(r"Location\s+\d+:.*?`([^`]+)`", re.IGNORECASE) + +for raw_line in text_lines: + stripped = raw_line.strip() + if re.match(r"^##\s+Code Analysis\b", stripped, re.IGNORECASE): + in_code_analysis = True + current_file = None + continue + if stripped.startswith("## ") and not re.match( + r"^##\s+Code Analysis\b", stripped, re.IGNORECASE + ): + in_code_analysis = False + current_file = None + continue + if not in_code_analysis: + continue + + location_match = location_re.search(raw_line) + if location_match: + current_file = normalize_report_path(location_match.group(1)) + + if stripped.startswith("```"): + if in_fence: + if fence_file: + blocks.append((fence_file, fence_lang, fence_lines)) + in_fence = False + fence_file = None + fence_lang = "" + fence_lines = [] + else: + in_fence = True + fence_file = current_file + fence_lang = stripped[3:].strip().casefold() + fence_lines = [] + continue + + if in_fence: + fence_lines.append(raw_line) + + +def meaningful_lines(lang: str, raw_lines: list[str]) -> list[str]: + result: list[str] = [] + for line in raw_lines: + value = line.strip() + if not value: + continue + if lang == "diff": + if value.startswith("---") or value.startswith("+++"): + continue + if not value.startswith("-"): + continue + value = value[1:].strip() + if not value or value in {"{", "}", "(", ")", "):"}: + continue + result.append(value) + return list(dict.fromkeys(result)) + + +checked_blocks = 0 +stale_blocks = 0 +for source_path, lang, raw_lines in blocks: + source_lines = source_by_path.get(source_path) + if source_lines is None: + continue + snippet_lines = meaningful_lines(lang, raw_lines) + if len(snippet_lines) < 2: + continue + checked_blocks += 1 + present = sum(1 for line in snippet_lines if line in source_lines) + if present * 2 < len(snippet_lines): + stale_blocks += 1 + +if checked_blocks > 0 and stale_blocks == checked_blocks: + raise SystemExit(0) +raise SystemExit(1) +PY + then + rm -f "$location_records_file" + echo "Detected Strix report source snippets absent from scanned source; treating as retryable model inconsistency." >&2 + return 0 + fi + + rm -f "$location_records_file" + return 1 +} + +source_file_has_encrypted_runner_registration_token() { + local source_file="$1" + python3 - "$source_file" <<'PY' +from pathlib import Path +import re +import sys + +source_path = Path(sys.argv[1]) +text = source_path.read_text(encoding="utf-8", errors="replace") +class_match = re.search( + r"^class\s+WorkspaceRunnerConfig\b[\s\S]*?(?=^class\s+\w|\Z)", + text, + re.MULTILINE, +) +if not class_match: + raise SystemExit(1) +class_body = class_match.group(0) +encrypted_registration_token = re.search( + r"registration_token[\s\S]{0,260}mapped_column\(\s*EncryptedString\b", + class_body, +) +raise SystemExit(0 if encrypted_registration_token else 1) +PY +} + +report_claims_plain_runner_registration_token() { + local vuln_file="$1" + python3 - "$vuln_file" <<'PY' +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") +if "WorkspaceRunnerConfig" not in text or "registration_token" not in text: + raise SystemExit(1) +if "backend/db/models.py" not in text: + raise SystemExit(1) +plain_string_claim = re.search( + r"registration_token[\s\S]{0,500}mapped_column\(\s*String\b", + text, +) +plain_text_claim = re.search( + r"registration_token[\s\S]{0,500}(plain text|plain string|stored as a plain)", + text, + re.IGNORECASE, +) +raise SystemExit(0 if plain_string_claim or plain_text_claim else 1) +PY +} + +runner_registration_token_source_candidates() { + local resolved_scan_target="" + resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" + + if [ -n "$resolved_scan_target" ]; then + printf '%s\n' "$resolved_scan_target/backend/db/models.py" + fi + if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then + return 0 + fi + printf '%s\n' "$REPO_ROOT/backend/db/models.py" +} + +vulnerability_file_has_hallucinated_source_claim() { + local vuln_file="$1" + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + return 1 + fi + if ! report_claims_plain_runner_registration_token "$vuln_file"; then + return 1 + fi + + local source_file + while IFS= read -r source_file; do + if [ -z "$source_file" ]; then + continue + fi + if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then + continue + fi + if source_file_has_encrypted_runner_registration_token "$source_file"; then + echo "Detected Strix report contradicting scanned runner registration token encryption; treating as retryable model inconsistency." >&2 + return 0 + fi + done < <(runner_registration_token_source_candidates) + + return 1 +} + +opencode_config_source_candidates() { + local resolved_scan_target="" + resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" + + if [ -n "$resolved_scan_target" ]; then + printf '%s\n' "$resolved_scan_target/.github/workflows/opencode-review-dispatch.yml" + printf '%s\n' "$resolved_scan_target/.github/workflows/opencode-review.yml" + printf '%s\n' "$resolved_scan_target/opencode.jsonc" + fi + if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then + return 0 + fi + printf '%s\n' "$REPO_ROOT/.github/workflows/opencode-review-dispatch.yml" + printf '%s\n' "$REPO_ROOT/.github/workflows/opencode-review.yml" + printf '%s\n' "$REPO_ROOT/opencode.jsonc" +} + +source_file_uses_documented_opencode_env_api_key_reference() { + local source_file="$1" + python3 - "$source_file" <<'PY' +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") +documented_reference = re.search( + r'"apiKey"\s*:\s*"\{env:STRIX_GITHUB_MODELS_TOKEN\}"', + text, +) +raise SystemExit(0 if documented_reference else 1) +PY +} + +vulnerability_file_reports_documented_opencode_env_api_key_reference() { + local vuln_file="$1" + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + return 1 + fi + if ! grep -Fq "Secret templating in configuration file" "$vuln_file"; then + return 1 + fi + if ! grep -Fq '"apiKey": "{env:STRIX_GITHUB_MODELS_TOKEN}"' "$vuln_file"; then + return 1 + fi + + local source_file + while IFS= read -r source_file; do + if [ -z "$source_file" ]; then + continue + fi + if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then + continue + fi + if source_file_uses_documented_opencode_env_api_key_reference "$source_file"; then + echo "Detected Strix report treating OpenCode's documented env apiKey reference as secret material; treating as retryable model inconsistency." >&2 + return 0 + fi + done < <(opencode_config_source_candidates) + + return 1 +} + +github_actions_workflow_source_candidates() { + local resolved_scan_target="" + resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" + + if [ -n "$resolved_scan_target" ]; then + printf '%s\n' "$resolved_scan_target/.github/workflows/strix.yml" + fi + if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then + return 0 + fi + printf '%s\n' "$REPO_ROOT/.github/workflows/strix.yml" +} + +source_file_refutes_generic_github_actions_workflow_insecurity() { + local source_file="$1" + python3 - "$source_file" <<'PY' +from pathlib import Path +import re +import sys + +text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") +permissions_block = re.search(r"(?ms)^permissions:\n(?:(?:[ \t]+[A-Za-z-]+:[ \t]+read[ \t]*\n)+)", text) +if not permissions_block: + raise SystemExit(1) +permissions_text = permissions_block.group(0) +required_permissions = {"actions", "contents", "models"} +observed_permissions = set(re.findall(r"^[ \t]+([A-Za-z-]+):[ \t]+read[ \t]*$", permissions_text, re.MULTILINE)) +if not required_permissions.issubset(observed_permissions): + raise SystemExit(1) +if re.search( + r"(?m)^[ \t]*(?:write-all|(?:actions|contents|models|pull-requests|issues|checks|deployments):[ \t]+write)\b", + text, +): + raise SystemExit(1) + +counterevidence = [ + 'echo "::add-mask::${sanitized}"', + "umask 077", + '[[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]', + '[[ "$PR_BASE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]', + "STRIX_LLM must select GitHub Models openai/gpt-5 or newer", +] +if not all(needle in text for needle in counterevidence): + raise SystemExit(1) + +raise SystemExit(0) +PY +} + +vulnerability_file_reports_generic_github_actions_workflow_insecurity() { + local vuln_file="$1" + if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then + return 1 + fi + if ! grep -Fq "Insecure Configurations in GitHub Actions Workflows" "$vuln_file"; then + return 1 + fi + if ! grep -Fq ".github/workflows/strix.yml" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "Full file content" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "Current content" "$vuln_file" || ! grep -Fq "Secured version" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "Secrets are written to temporary files without proper access controls" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "API keys are passed through environment variables without adequate masking" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "Excessive permissions granted to workflows" "$vuln_file"; then + return 1 + fi + if ! grep -Fq "Insufficient input validation for workflow parameters" "$vuln_file"; then + return 1 + fi + + local source_file + while IFS= read -r source_file; do + if [ -z "$source_file" ]; then + continue + fi + if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then + continue + fi + if source_file_refutes_generic_github_actions_workflow_insecurity "$source_file"; then + echo "Detected Strix report making a generic GitHub Actions workflow security claim contradicted by the scanned workflow; treating as retryable model inconsistency." >&2 + return 0 + fi + done < <(github_actions_workflow_source_candidates) + + return 1 +} + +vulnerability_file_is_retryable_model_inconsistency() { + local vuln_file="$1" + if ! vulnerability_file_is_below_threshold "$vuln_file"; then + return 1 + fi + if vulnerability_file_has_absent_endpoint_finding "$vuln_file"; then + return 0 + fi + if vulnerability_file_has_absent_source_snippets "$vuln_file"; then + return 0 + fi + if vulnerability_file_has_hallucinated_source_claim "$vuln_file"; then + return 0 + fi + if vulnerability_file_reports_documented_opencode_env_api_key_reference "$vuln_file"; then + return 0 + fi + if vulnerability_file_reports_generic_github_actions_workflow_insecurity "$vuln_file"; then + return 0 + fi + return 1 +} + +is_hallucinated_source_claim_finding() { + local latest_report_dir + if ! latest_report_dir="$(latest_strix_report_dir)"; then + return 1 + fi + + local vuln_file + for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do + if vulnerability_file_is_below_threshold "$vuln_file" && + vulnerability_file_has_hallucinated_source_claim "$vuln_file"; then + return 0 + fi + done + + return 1 +} + +is_model_retryable_error() { + local model="$1" + + if is_vertex_model "$model" && is_vertex_not_found_error; then + return 0 + fi + + if is_nvidia_nim_not_found_error; then + return 0 + fi + + if is_model_behavior_error; then + return 0 + fi + + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then + return 0 + fi + + if is_rate_limit_error; then + return 0 + fi + + if is_llm_token_limit_error; then + return 0 + fi + + if is_timeout_error; then + # Process and provider timeouts are not clean evidence, but they are + # recoverable across distinct fallback models. Strict provider-signal + # mode still fails closed after fallback exhaustion or partial reports. + return 0 + fi + + if is_midstream_fallback_error; then + return 0 + fi + + if is_llm_api_connection_error; then + return 0 + fi + + if is_llm_service_unavailable_error; then + return 0 + fi + + # A provider failure can be recorded only in Strix's structured report log. + # run_strix_once already marks that evidence as infrastructure failure, but + # the child stdout log used by the classifiers may not contain the provider + # exception. In strict mode, let configured distinct fallbacks run instead of + # treating the report-only signal as a non-recoverable source failure. + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled && + has_strix_report_provider_failure_signal "$ACTIVE_REPORTS_DIR" "${TARGET_PATH%/}/strix_runs"; then + return 0 + fi + + if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then + return 0 + fi + + if is_pull_request_event; then + return 1 + fi + + if is_hallucinated_endpoint_finding; then + return 0 + fi + + if is_hallucinated_source_claim_finding; then + return 0 + fi + + return 1 +} + +run_current_target_scan() { + INFRA_ERROR_DETECTED=0 + ZERO_FINDINGS_REPORTED=0 + + local primary_scan_rc=0 + run_strix_with_transient_retry "$PRIMARY_MODEL" || primary_scan_rc=$? + if [ "$primary_scan_rc" -eq 0 ]; then + return 0 + fi + if [ "$primary_scan_rc" -eq 2 ]; then + return 2 + fi + if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then + return 1 + fi + + local strict_primary_provider_fallback=0 + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then + if is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then + strict_primary_provider_fallback=1 + else + echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 + return 1 + fi + fi + + if has_only_below_threshold_vulnerabilities; then + return 0 + fi + + if evaluate_pull_request_findings; then + if [ "$strict_primary_provider_fallback" -eq 0 ]; then + return 0 + fi + fi + + case "$PR_FINDINGS_DECISION" in + block_changed | block_unmapped | block_manifest_unverified) + if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then + return 1 + fi + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + ;; + esac + if fail_unmapped_threshold_report; then + return 1 + fi + + if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then + return 1 + fi + + if ! is_model_retryable_error "$PRIMARY_MODEL"; then + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + fi + + FALLBACK_MODELS_RAW="$(fallback_models_raw_for_model "$PRIMARY_MODEL")" + FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\r'/ }" + FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\n'/ }" + read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" + + fallback_tried=0 + for candidate_raw in "${FALLBACK_MODELS[@]}"; do + candidate="$(normalize_model "$candidate_raw")" + if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then + if [ -n "$candidate" ]; then + echo "Skipping fallback model '$candidate' — same as primary model." >&2 + fi + continue + fi + if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then + return 1 + fi + + fallback_tried=1 + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." + else + echo "Primary model unavailable; retrying with fallback '$candidate'." + fi + local fallback_scan_rc=0 + local fallback_start_epoch + fallback_start_epoch="$(date +%s)" + run_strix_with_transient_retry "$candidate" || fallback_scan_rc=$? + local fallback_elapsed=$(( $(date +%s) - fallback_start_epoch )) + if [ "$fallback_scan_rc" -eq 0 ]; then + if fail_reported_vulnerabilities_before_fallback_success; then + return 1 + fi + echo "Strix quick scan succeeded with fallback model '$candidate' in ${fallback_elapsed}s." >&2 + return 0 + fi + if [ "$fallback_scan_rc" -eq 2 ]; then + return 2 + fi + + local strict_fallback_provider_signal=0 + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then + strict_fallback_provider_signal=1 + fi + + if has_only_below_threshold_vulnerabilities; then + return 0 + fi + + if evaluate_pull_request_findings; then + if [ "$strict_fallback_provider_signal" -eq 0 ]; then + return 0 + fi + fi + + case "$PR_FINDINGS_DECISION" in + block_changed | block_unmapped | block_manifest_unverified) + if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then + return 1 + fi + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + ;; + esac + if fail_unmapped_threshold_report; then + return 1 + fi + + if fail_reported_vulnerabilities_before_fallback_success; then + return 1 + fi + + if [ "$strict_fallback_provider_signal" -eq 1 ]; then + if is_model_retryable_error "$candidate"; then + continue + fi + echo "Strix fallback model '$candidate' emitted provider infrastructure or failure-signal output; trying next configured fallback if available." >&2 + continue + fi + + if ! is_model_retryable_error "$candidate"; then + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + fi + done + + if should_fail_pull_request_infra_zero_findings; then + return 1 + fi + + if [ "$fallback_tried" -eq 0 ]; then + local fallback_config_name + fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" + local configured_fallback_count=0 + for candidate_raw in "${FALLBACK_MODELS[@]}"; do + candidate="$(normalize_model "$candidate_raw")" + [ -n "$candidate" ] && configured_fallback_count=$((configured_fallback_count + 1)) + done + if [ "$configured_fallback_count" -eq 0 ]; then + echo "ERROR: No fallback models configured ($fallback_config_name is empty). Configure distinct models." >&2 + else + echo "ERROR: All configured fallback models are the same as the primary model" >&2 + fi + return 1 + fi + + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && + [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then + echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 + return 1 + fi + + local threshold_rank + threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" + if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then + echo "Strix quick scan failed with a non-recoverable error." >&2 + return 1 + fi + + if is_vertex_model "$PRIMARY_MODEL"; then + echo "Configured Vertex model and fallback models were unavailable." >&2 + else + echo "Configured model and fallback models were unavailable." >&2 + fi + return 1 +} + +prepare_pull_request_scan_scope +if [ "$TARGET_PATH_REQUESTS_PR_SCOPE" -eq 1 ] && + [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -ne 1 ]; then + echo "ERROR: STRIX_TARGET_PATH=$PR_SCOPE_TARGET_SENTINEL did not produce a PR scan scope." >&2 + exit 2 +fi + +scan_rc=0 +run_current_target_scan || scan_rc=$? +exit "$scan_rc" +; then + continue + fi + return 0 + done < <(grep -Ei '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG" || true) + + return 1 +} + has_detected_infrastructure_error() { - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then + if has_unexpected_strix_console_failure_signal; then return 0 fi From 5bf2230c349da9f7882e34aa217c05df891891c7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:38:57 -0700 Subject: [PATCH 5/7] fix(strix): restore complete quality-advisory classifier --- scripts/ci/strix_quick_gate.sh | 1149 +------------------------------- 1 file changed, 1 insertion(+), 1148 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 01656bb3f..cff0f27e3 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3157,1154 +3157,7 @@ has_unexpected_strix_console_failure_signal() { # not on its recommended frontier list. It is a quality advisory, not # evidence that provider execution or the scan was incomplete. if printf '%s\n' "$signal_line" | - grep -Eq '^[^[:alnum:]]*MODEL QUALITY WARNING[^[:alnum:]]* - - if is_timeout_error; then - return 0 - fi - - if is_rate_limit_error; then - return 0 - fi - - if is_llm_token_limit_error; then - return 0 - fi - - if is_midstream_fallback_error; then - return 0 - fi - - if is_llm_api_connection_error; then - return 0 - fi - - if is_llm_service_unavailable_error; then - return 0 - fi - - if is_nvidia_nim_not_found_error; then - return 0 - fi - - if is_model_behavior_error; then - return 0 - fi - - if is_caido_bootstrap_timing_error; then - return 0 - fi - - # Generic strix non-zero exit with known transport/connection errors - # that don't fall into the specific categories above. - # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid - # false positives: PROVIDER_CONTEXT_REGEX includes httpx/httpcore/requests - # which would self-match on e.g. "requests.exceptions.ConnectionError" - # from target-application logs. - if grep -Eiq '(ConnectionError|ConnectionRefusedError|ConnectionResetError|SSLError|ProxyError|NetworkError)' "$STRIX_LOG" && - grep -Eiq "$LLM_PROVIDER_ONLY_REGEX" "$STRIX_LOG"; then - return 0 - fi - - return 1 -} - -latest_strix_report_dir() { - local latest="" - local run_dir - - for run_dir in "$STRIX_REPORTS_DIR"/*; do - if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then - continue - fi - - if is_preexisting_report_dir "$run_dir"; then - continue - fi - - if [ -z "$latest" ] || [ "$run_dir" -nt "$latest" ]; then - latest="$run_dir" - fi - done - - if [ -z "$latest" ]; then - return 1 - fi - - echo "$latest" -} - -has_only_below_threshold_vulnerabilities() { - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - - local found_any_vuln_file=0 - local global_max_rank=-1 - STRIX_MAX_SEVERITY_RANK=-1 - local saw_any_severity=0 - - update_max_severity_from_stream() { - local source_path="$1" - local line - local severity - local rank - while IFS= read -r line; do - if [[ "${line^^}" =~ SEVERITY[[:space:]]*:[[:space:][:punct:]]*(CRITICAL|HIGH|MEDIUM|LOW|INFO|INFORMATIONAL|NONE)([[:space:][:punct:]]|$) ]]; then - severity="${BASH_REMATCH[1]}" - else - continue - fi - - rank="$(severity_rank "$severity")" - if [ "$rank" -lt 0 ]; then - continue - fi - - saw_any_severity=1 - if [ "$rank" -gt "$global_max_rank" ]; then - global_max_rank="$rank" - STRIX_MAX_SEVERITY_RANK="$rank" - fi - done < <(grep -Ei 'severity[[:space:]]*:' "$source_path" || true) - } - - local run_dir - for run_dir in "$STRIX_REPORTS_DIR"/*; do - if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then - continue - fi - - if is_preexisting_report_dir "$run_dir"; then - continue - fi - - local vulnerabilities_dir="$run_dir/vulnerabilities" - if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then - continue - fi - - local vuln_file - - for vuln_file in "$vulnerabilities_dir"/*.md; do - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - continue - fi - - found_any_vuln_file=1 - update_max_severity_from_stream "$vuln_file" - done - done - - if [ "$found_any_vuln_file" -eq 0 ]; then - echo "No Strix vulnerability report artifact was produced; log-only severity markers are incomplete evidence, so the scan is failing closed." >&2 - return 1 - fi - - if [ "$saw_any_severity" -eq 0 ]; then - return 1 - fi - - # Guard against incomplete scans due to infrastructure errors. - # Use the sticky INFRA_ERROR_DETECTED flag instead of re-reading - # STRIX_LOG, because STRIX_LOG is overwritten per-attempt. If an - # earlier attempt hit an infrastructure error (timeout, rate-limit, - # transport failure) and produced a partial report that now sits in - # the reports directory, the *current* STRIX_LOG may show a different - # failure — or even success — but the partial report's low-severity - # findings must not be treated as a clean scan result. - if [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then - echo "Below-threshold findings detected, but infrastructure errors occurred during this pipeline run; refusing bypass due to potentially incomplete scan." >&2 - return 1 - fi - - if [ "$global_max_rank" -lt "$threshold_rank" ]; then - echo "Strix findings are below configured fail threshold '$STRIX_FAIL_ON_MIN_SEVERITY'; allowing pipeline continuation." >&2 - return 0 - fi - - return 1 -} - -has_blocking_vulnerability_reports() { - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - - local run_dir vulnerabilities_dir vuln_file rank - for run_dir in "$STRIX_REPORTS_DIR"/*; do - if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then - continue - fi - if is_preexisting_report_dir "$run_dir"; then - continue - fi - - vulnerabilities_dir="$run_dir/vulnerabilities" - if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then - continue - fi - - for vuln_file in "$vulnerabilities_dir"/*.md; do - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - continue - fi - if vulnerability_file_is_retryable_model_inconsistency "$vuln_file"; then - continue - fi - - rank="$(extract_max_severity_rank "$vuln_file")" - if [ "$rank" -lt 0 ] || [ "$rank" -ge "$threshold_rank" ]; then - return 0 - fi - done - done - - return 1 -} - -fail_reported_vulnerabilities_before_fallback_success() { - case "$PR_FINDINGS_DECISION" in - allow_baseline) - return 1 - ;; - esac - - if has_blocking_vulnerability_reports; then - echo "Strix model reported threshold vulnerabilities before fallback success; failing closed so every model-reported vulnerability is reviewed." >&2 - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 0 - fi - return 1 -} - -has_any_reported_severity_markers() { - local run_dir - for run_dir in "$STRIX_REPORTS_DIR"/*; do - if [ ! -d "$run_dir" ] || [ -L "$run_dir" ]; then - continue - fi - - if is_preexisting_report_dir "$run_dir"; then - continue - fi - - local vulnerabilities_dir="$run_dir/vulnerabilities" - if [ ! -d "$vulnerabilities_dir" ] || [ -L "$vulnerabilities_dir" ]; then - continue - fi - - local vuln_file - for vuln_file in "$vulnerabilities_dir"/*.md; do - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - continue - fi - if grep -Eiq 'severity[[:space:]]*:' "$vuln_file"; then - return 0 - fi - done - done - - if grep -Eiq 'severity[[:space:]]*:' "$STRIX_LOG"; then - return 0 - fi - - return 1 -} - -strix_reported_zero_vulnerabilities() { - if [ "$ZERO_FINDINGS_REPORTED" -eq 1 ]; then - return 0 - fi - - strix_reported_zero_vulnerabilities_in_file "$STRIX_LOG" -} - -strix_reported_zero_vulnerabilities_in_file() { - local source_path="$1" - grep -Eq 'Vulnerabilities[[:space:]]+0([^0-9]|$)' "$source_path" -} - -should_fail_pull_request_infra_zero_findings() { - if ! is_pull_request_event; then - return 1 - fi - - if [ "$INFRA_ERROR_DETECTED" -ne 1 ]; then - return 1 - fi - - if has_any_reported_severity_markers; then - return 1 - fi - - if ! strix_reported_zero_vulnerabilities; then - return 1 - fi - - echo "Strix reported zero vulnerabilities before provider infrastructure failure; failing closed because provider infrastructure failures are not clean scan evidence." >&2 - return 0 -} - -vulnerability_file_has_absent_endpoint_finding() { - local vuln_file="$1" - # Configurable list of source directories to check for endpoints. - # Defaults to "." (i.e. TARGET_PATH itself) so that both - # STRIX_TARGET_PATH=./ and STRIX_TARGET_PATH=./src work correctly - # without producing bogus double-nested paths like ./src/src. - # Set STRIX_SOURCE_DIRS (space-separated) to override. - local source_dirs_raw="${STRIX_SOURCE_DIRS:-.}" - local resolved_target_root="" - local resolved_dirs=() - local dir_entry - if ! resolved_target_root="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null)"; then - return 1 - fi - - # Disable globbing so that entries like "*" or "[" in STRIX_SOURCE_DIRS - # are not expanded by pathname expansion during word-splitting. - set -f - for dir_entry in $source_dirs_raw; do - local candidate="${resolved_target_root%/}/$dir_entry" - if [ -d "$candidate" ] && [ ! -L "$candidate" ]; then - resolved_dirs+=("$candidate") - fi - done - set +f - - if [ "${#resolved_dirs[@]}" -eq 0 ]; then - return 1 - fi - - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - return 1 - fi - - local endpoint_seen=0 - local endpoint_present_in_source=0 - local endpoint - - while IFS= read -r endpoint; do - if [ -z "$endpoint" ]; then - continue - fi - - endpoint_seen=1 - local search_dir - for search_dir in "${resolved_dirs[@]}"; do - # Exclude the strix reports directory and common non-source - # directories from the source search to prevent accidental - # matches and reduce runtime (especially when STRIX_TARGET_PATH=./). - # - # Each exclude-dir: - # STRIX_REPORTS_DIR — strix output itself (would always match). - # Both the full path and basename are excluded so that - # nested paths like "reports/strix_runs" are also caught. - # .git — VCS internals - # node_modules — JS/TS dependencies (may contain API strings) - # vendor — Go/PHP vendored deps - # __pycache__ — Python bytecode cache - # .venv — Python virtualenv - # target — Rust/Java build artifacts - # .mypy_cache — mypy type-check cache - # .pytest_cache — pytest result cache - # dist — common build output directory - # build — common build output directory - # .tox — Python tox test environments - # .ruff_cache — Ruff linter cache - if grep -r -Fq \ - --exclude-dir="$STRIX_REPORTS_DIR" \ - --exclude-dir="$(basename "$STRIX_REPORTS_DIR")" \ - --exclude-dir=".git" \ - --exclude-dir="node_modules" \ - --exclude-dir="vendor" \ - --exclude-dir="__pycache__" \ - --exclude-dir=".venv" \ - --exclude-dir="target" \ - --exclude-dir=".mypy_cache" \ - --exclude-dir=".pytest_cache" \ - --exclude-dir="dist" \ - --exclude-dir="build" \ - --exclude-dir=".tox" \ - --exclude-dir=".ruff_cache" \ - -- "$endpoint" "$search_dir"; then - endpoint_present_in_source=1 - break - fi - done - if [ "$endpoint_present_in_source" -eq 1 ]; then - break - fi - done < <(python3 - "$vuln_file" <<'PY' -from pathlib import Path -import re -import sys - -text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") -endpoints = set() -for line in text.splitlines(): - if not re.search(r"\bEndpoint\b", line, re.IGNORECASE): - continue - endpoints.update(re.findall(r"/api/[A-Za-z0-9_./-]+", line)) -for endpoint in sorted(endpoints): - print(endpoint) -PY - ) - - if [ "$endpoint_seen" -eq 0 ]; then - return 1 - fi - - if [ "$endpoint_present_in_source" -eq 1 ]; then - return 1 - fi - - echo "Detected Strix report endpoint(s) absent from source; treating as retryable model inconsistency." >&2 - return 0 -} - -is_hallucinated_endpoint_finding() { - local latest_report_dir - if ! latest_report_dir="$(latest_strix_report_dir)"; then - return 1 - fi - - local vuln_file - - for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do - if vulnerability_file_is_below_threshold "$vuln_file" && - vulnerability_file_has_absent_endpoint_finding "$vuln_file"; then - return 0 - fi - done - - return 1 -} - -vulnerability_file_has_absent_source_snippets() { - local vuln_file="$1" - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - return 1 - fi - - local location_records_file - location_records_file="$(mktemp)" - extract_vulnerability_location_records "$vuln_file" >"$location_records_file" || true - if [ ! -s "$location_records_file" ]; then - rm -f "$location_records_file" - return 1 - fi - - local resolved_scan_target="" - resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" - if python3 - "$vuln_file" "$REPO_ROOT" "$resolved_scan_target" "$location_records_file" <<'PY' -from pathlib import Path -import re -import sys - -vuln_path = Path(sys.argv[1]) -repo_root = Path(sys.argv[2]) -scan_target = Path(sys.argv[3]) if sys.argv[3] else None -records_path = Path(sys.argv[4]) - -record_paths = { - line.split("\t", 1)[0].strip().replace("\\", "/") - for line in records_path.read_text(encoding="utf-8", errors="replace").splitlines() - if line.strip() -} -if not record_paths: - raise SystemExit(1) - -# ⚡ Bolt: Pre-compile regex for path normalization to optimize inner loop scanning -location_suffix_re = re.compile(r":\d+(?:-\d+)?$") - -def normalize_report_path(raw: str) -> str | None: - value = raw.strip().strip("`").replace("\\", "/") - value = location_suffix_re.sub("", value) - for record_path in record_paths: - if value == record_path or value.endswith("/" + record_path): - return record_path - return value if value in record_paths else None - - -def source_lines_for(path: str) -> set[str] | None: - candidates = [] - if scan_target is not None: - candidates.append(scan_target / path) - candidates.append(repo_root / path) - for candidate in candidates: - try: - if candidate.is_file() and not candidate.is_symlink(): - return { - line.strip() - for line in candidate.read_text( - encoding="utf-8", errors="replace" - ).splitlines() - if line.strip() - } - except OSError: - continue - return None - - -source_by_path = { - path: lines - for path in record_paths - if (lines := source_lines_for(path)) is not None -} -if not source_by_path: - raise SystemExit(1) - -text_lines = vuln_path.read_text(encoding="utf-8", errors="replace").splitlines() -in_code_analysis = False -current_file: str | None = None -in_fence = False -fence_file: str | None = None -fence_lang = "" -fence_lines: list[str] = [] -blocks: list[tuple[str, str, list[str]]] = [] -location_re = re.compile(r"Location\s+\d+:.*?`([^`]+)`", re.IGNORECASE) - -for raw_line in text_lines: - stripped = raw_line.strip() - if re.match(r"^##\s+Code Analysis\b", stripped, re.IGNORECASE): - in_code_analysis = True - current_file = None - continue - if stripped.startswith("## ") and not re.match( - r"^##\s+Code Analysis\b", stripped, re.IGNORECASE - ): - in_code_analysis = False - current_file = None - continue - if not in_code_analysis: - continue - - location_match = location_re.search(raw_line) - if location_match: - current_file = normalize_report_path(location_match.group(1)) - - if stripped.startswith("```"): - if in_fence: - if fence_file: - blocks.append((fence_file, fence_lang, fence_lines)) - in_fence = False - fence_file = None - fence_lang = "" - fence_lines = [] - else: - in_fence = True - fence_file = current_file - fence_lang = stripped[3:].strip().casefold() - fence_lines = [] - continue - - if in_fence: - fence_lines.append(raw_line) - - -def meaningful_lines(lang: str, raw_lines: list[str]) -> list[str]: - result: list[str] = [] - for line in raw_lines: - value = line.strip() - if not value: - continue - if lang == "diff": - if value.startswith("---") or value.startswith("+++"): - continue - if not value.startswith("-"): - continue - value = value[1:].strip() - if not value or value in {"{", "}", "(", ")", "):"}: - continue - result.append(value) - return list(dict.fromkeys(result)) - - -checked_blocks = 0 -stale_blocks = 0 -for source_path, lang, raw_lines in blocks: - source_lines = source_by_path.get(source_path) - if source_lines is None: - continue - snippet_lines = meaningful_lines(lang, raw_lines) - if len(snippet_lines) < 2: - continue - checked_blocks += 1 - present = sum(1 for line in snippet_lines if line in source_lines) - if present * 2 < len(snippet_lines): - stale_blocks += 1 - -if checked_blocks > 0 and stale_blocks == checked_blocks: - raise SystemExit(0) -raise SystemExit(1) -PY - then - rm -f "$location_records_file" - echo "Detected Strix report source snippets absent from scanned source; treating as retryable model inconsistency." >&2 - return 0 - fi - - rm -f "$location_records_file" - return 1 -} - -source_file_has_encrypted_runner_registration_token() { - local source_file="$1" - python3 - "$source_file" <<'PY' -from pathlib import Path -import re -import sys - -source_path = Path(sys.argv[1]) -text = source_path.read_text(encoding="utf-8", errors="replace") -class_match = re.search( - r"^class\s+WorkspaceRunnerConfig\b[\s\S]*?(?=^class\s+\w|\Z)", - text, - re.MULTILINE, -) -if not class_match: - raise SystemExit(1) -class_body = class_match.group(0) -encrypted_registration_token = re.search( - r"registration_token[\s\S]{0,260}mapped_column\(\s*EncryptedString\b", - class_body, -) -raise SystemExit(0 if encrypted_registration_token else 1) -PY -} - -report_claims_plain_runner_registration_token() { - local vuln_file="$1" - python3 - "$vuln_file" <<'PY' -from pathlib import Path -import re -import sys - -text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") -if "WorkspaceRunnerConfig" not in text or "registration_token" not in text: - raise SystemExit(1) -if "backend/db/models.py" not in text: - raise SystemExit(1) -plain_string_claim = re.search( - r"registration_token[\s\S]{0,500}mapped_column\(\s*String\b", - text, -) -plain_text_claim = re.search( - r"registration_token[\s\S]{0,500}(plain text|plain string|stored as a plain)", - text, - re.IGNORECASE, -) -raise SystemExit(0 if plain_string_claim or plain_text_claim else 1) -PY -} - -runner_registration_token_source_candidates() { - local resolved_scan_target="" - resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" - - if [ -n "$resolved_scan_target" ]; then - printf '%s\n' "$resolved_scan_target/backend/db/models.py" - fi - if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then - return 0 - fi - printf '%s\n' "$REPO_ROOT/backend/db/models.py" -} - -vulnerability_file_has_hallucinated_source_claim() { - local vuln_file="$1" - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - return 1 - fi - if ! report_claims_plain_runner_registration_token "$vuln_file"; then - return 1 - fi - - local source_file - while IFS= read -r source_file; do - if [ -z "$source_file" ]; then - continue - fi - if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then - continue - fi - if source_file_has_encrypted_runner_registration_token "$source_file"; then - echo "Detected Strix report contradicting scanned runner registration token encryption; treating as retryable model inconsistency." >&2 - return 0 - fi - done < <(runner_registration_token_source_candidates) - - return 1 -} - -opencode_config_source_candidates() { - local resolved_scan_target="" - resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" - - if [ -n "$resolved_scan_target" ]; then - printf '%s\n' "$resolved_scan_target/.github/workflows/opencode-review-dispatch.yml" - printf '%s\n' "$resolved_scan_target/.github/workflows/opencode-review.yml" - printf '%s\n' "$resolved_scan_target/opencode.jsonc" - fi - if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then - return 0 - fi - printf '%s\n' "$REPO_ROOT/.github/workflows/opencode-review-dispatch.yml" - printf '%s\n' "$REPO_ROOT/.github/workflows/opencode-review.yml" - printf '%s\n' "$REPO_ROOT/opencode.jsonc" -} - -source_file_uses_documented_opencode_env_api_key_reference() { - local source_file="$1" - python3 - "$source_file" <<'PY' -from pathlib import Path -import re -import sys - -text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") -documented_reference = re.search( - r'"apiKey"\s*:\s*"\{env:STRIX_GITHUB_MODELS_TOKEN\}"', - text, -) -raise SystemExit(0 if documented_reference else 1) -PY -} - -vulnerability_file_reports_documented_opencode_env_api_key_reference() { - local vuln_file="$1" - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - return 1 - fi - if ! grep -Fq "Secret templating in configuration file" "$vuln_file"; then - return 1 - fi - if ! grep -Fq '"apiKey": "{env:STRIX_GITHUB_MODELS_TOKEN}"' "$vuln_file"; then - return 1 - fi - - local source_file - while IFS= read -r source_file; do - if [ -z "$source_file" ]; then - continue - fi - if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then - continue - fi - if source_file_uses_documented_opencode_env_api_key_reference "$source_file"; then - echo "Detected Strix report treating OpenCode's documented env apiKey reference as secret material; treating as retryable model inconsistency." >&2 - return 0 - fi - done < <(opencode_config_source_candidates) - - return 1 -} - -github_actions_workflow_source_candidates() { - local resolved_scan_target="" - resolved_scan_target="$(resolve_current_target_path "$TARGET_PATH" 2>/dev/null || true)" - - if [ -n "$resolved_scan_target" ]; then - printf '%s\n' "$resolved_scan_target/.github/workflows/strix.yml" - fi - if pull_request_head_blob_required || [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -eq 1 ]; then - return 0 - fi - printf '%s\n' "$REPO_ROOT/.github/workflows/strix.yml" -} - -source_file_refutes_generic_github_actions_workflow_insecurity() { - local source_file="$1" - python3 - "$source_file" <<'PY' -from pathlib import Path -import re -import sys - -text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") -permissions_block = re.search(r"(?ms)^permissions:\n(?:(?:[ \t]+[A-Za-z-]+:[ \t]+read[ \t]*\n)+)", text) -if not permissions_block: - raise SystemExit(1) -permissions_text = permissions_block.group(0) -required_permissions = {"actions", "contents", "models"} -observed_permissions = set(re.findall(r"^[ \t]+([A-Za-z-]+):[ \t]+read[ \t]*$", permissions_text, re.MULTILINE)) -if not required_permissions.issubset(observed_permissions): - raise SystemExit(1) -if re.search( - r"(?m)^[ \t]*(?:write-all|(?:actions|contents|models|pull-requests|issues|checks|deployments):[ \t]+write)\b", - text, -): - raise SystemExit(1) - -counterevidence = [ - 'echo "::add-mask::${sanitized}"', - "umask 077", - '[[ "$PR_HEAD_SHA" =~ ^[0-9a-fA-F]{40}$ ]]', - '[[ "$PR_BASE_SHA" =~ ^[0-9a-fA-F]{40}$ ]]', - "STRIX_LLM must select GitHub Models openai/gpt-5 or newer", -] -if not all(needle in text for needle in counterevidence): - raise SystemExit(1) - -raise SystemExit(0) -PY -} - -vulnerability_file_reports_generic_github_actions_workflow_insecurity() { - local vuln_file="$1" - if [ ! -f "$vuln_file" ] || [ -L "$vuln_file" ]; then - return 1 - fi - if ! grep -Fq "Insecure Configurations in GitHub Actions Workflows" "$vuln_file"; then - return 1 - fi - if ! grep -Fq ".github/workflows/strix.yml" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "Full file content" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "Current content" "$vuln_file" || ! grep -Fq "Secured version" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "Secrets are written to temporary files without proper access controls" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "API keys are passed through environment variables without adequate masking" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "Excessive permissions granted to workflows" "$vuln_file"; then - return 1 - fi - if ! grep -Fq "Insufficient input validation for workflow parameters" "$vuln_file"; then - return 1 - fi - - local source_file - while IFS= read -r source_file; do - if [ -z "$source_file" ]; then - continue - fi - if [ ! -f "$source_file" ] || [ -L "$source_file" ]; then - continue - fi - if source_file_refutes_generic_github_actions_workflow_insecurity "$source_file"; then - echo "Detected Strix report making a generic GitHub Actions workflow security claim contradicted by the scanned workflow; treating as retryable model inconsistency." >&2 - return 0 - fi - done < <(github_actions_workflow_source_candidates) - - return 1 -} - -vulnerability_file_is_retryable_model_inconsistency() { - local vuln_file="$1" - if ! vulnerability_file_is_below_threshold "$vuln_file"; then - return 1 - fi - if vulnerability_file_has_absent_endpoint_finding "$vuln_file"; then - return 0 - fi - if vulnerability_file_has_absent_source_snippets "$vuln_file"; then - return 0 - fi - if vulnerability_file_has_hallucinated_source_claim "$vuln_file"; then - return 0 - fi - if vulnerability_file_reports_documented_opencode_env_api_key_reference "$vuln_file"; then - return 0 - fi - if vulnerability_file_reports_generic_github_actions_workflow_insecurity "$vuln_file"; then - return 0 - fi - return 1 -} - -is_hallucinated_source_claim_finding() { - local latest_report_dir - if ! latest_report_dir="$(latest_strix_report_dir)"; then - return 1 - fi - - local vuln_file - for vuln_file in "$latest_report_dir"/vulnerabilities/*.md; do - if vulnerability_file_is_below_threshold "$vuln_file" && - vulnerability_file_has_hallucinated_source_claim "$vuln_file"; then - return 0 - fi - done - - return 1 -} - -is_model_retryable_error() { - local model="$1" - - if is_vertex_model "$model" && is_vertex_not_found_error; then - return 0 - fi - - if is_nvidia_nim_not_found_error; then - return 0 - fi - - if is_model_behavior_error; then - return 0 - fi - - if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then - return 0 - fi - - if is_rate_limit_error; then - return 0 - fi - - if is_llm_token_limit_error; then - return 0 - fi - - if is_timeout_error; then - # Process and provider timeouts are not clean evidence, but they are - # recoverable across distinct fallback models. Strict provider-signal - # mode still fails closed after fallback exhaustion or partial reports. - return 0 - fi - - if is_midstream_fallback_error; then - return 0 - fi - - if is_llm_api_connection_error; then - return 0 - fi - - if is_llm_service_unavailable_error; then - return 0 - fi - - # A provider failure can be recorded only in Strix's structured report log. - # run_strix_once already marks that evidence as infrastructure failure, but - # the child stdout log used by the classifiers may not contain the provider - # exception. In strict mode, let configured distinct fallbacks run instead of - # treating the report-only signal as a non-recoverable source failure. - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled && - has_strix_report_provider_failure_signal "$ACTIVE_REPORTS_DIR" "${TARGET_PATH%/}/strix_runs"; then - return 0 - fi - - if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then - return 0 - fi - - if is_pull_request_event; then - return 1 - fi - - if is_hallucinated_endpoint_finding; then - return 0 - fi - - if is_hallucinated_source_claim_finding; then - return 0 - fi - - return 1 -} - -run_current_target_scan() { - INFRA_ERROR_DETECTED=0 - ZERO_FINDINGS_REPORTED=0 - - local primary_scan_rc=0 - run_strix_with_transient_retry "$PRIMARY_MODEL" || primary_scan_rc=$? - if [ "$primary_scan_rc" -eq 0 ]; then - return 0 - fi - if [ "$primary_scan_rc" -eq 2 ]; then - return 2 - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - local strict_primary_provider_fallback=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - if is_model_retryable_error "$PRIMARY_MODEL" && has_distinct_fallback_model_for_model "$PRIMARY_MODEL"; then - strict_primary_provider_fallback=1 - else - echo "Strix scan failed after provider infrastructure or failure-signal output; failing closed." >&2 - return 1 - fi - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - - if evaluate_pull_request_findings; then - if [ "$strict_primary_provider_fallback" -eq 0 ]; then - return 0 - fi - fi - - case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - ;; - esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if [ "$strict_primary_provider_fallback" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - - if ! is_model_retryable_error "$PRIMARY_MODEL"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - - FALLBACK_MODELS_RAW="$(fallback_models_raw_for_model "$PRIMARY_MODEL")" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\r'/ }" - FALLBACK_MODELS_RAW="${FALLBACK_MODELS_RAW//$'\n'/ }" - read -r -a FALLBACK_MODELS <<<"$FALLBACK_MODELS_RAW" - - fallback_tried=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - if [ -z "$candidate" ] || [ "$candidate" = "$PRIMARY_MODEL" ]; then - if [ -n "$candidate" ]; then - echo "Skipping fallback model '$candidate' — same as primary model." >&2 - fi - continue - fi - if [ "$TOTAL_TIMEOUT_EXCEEDED" -eq 1 ]; then - return 1 - fi - - fallback_tried=1 - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Primary Vertex model unavailable; retrying with fallback '$candidate'." - else - echo "Primary model unavailable; retrying with fallback '$candidate'." - fi - local fallback_scan_rc=0 - local fallback_start_epoch - fallback_start_epoch="$(date +%s)" - run_strix_with_transient_retry "$candidate" || fallback_scan_rc=$? - local fallback_elapsed=$(( $(date +%s) - fallback_start_epoch )) - if [ "$fallback_scan_rc" -eq 0 ]; then - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan succeeded with fallback model '$candidate' in ${fallback_elapsed}s." >&2 - return 0 - fi - if [ "$fallback_scan_rc" -eq 2 ]; then - return 2 - fi - - local strict_fallback_provider_signal=0 - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled; then - strict_fallback_provider_signal=1 - fi - - if has_only_below_threshold_vulnerabilities; then - return 0 - fi - - if evaluate_pull_request_findings; then - if [ "$strict_fallback_provider_signal" -eq 0 ]; then - return 0 - fi - fi - - case "$PR_FINDINGS_DECISION" in - block_changed | block_unmapped | block_manifest_unverified) - if [ "$strict_fallback_provider_signal" -eq 1 ] && fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - ;; - esac - if fail_unmapped_threshold_report; then - return 1 - fi - - if fail_reported_vulnerabilities_before_fallback_success; then - return 1 - fi - - if [ "$strict_fallback_provider_signal" -eq 1 ]; then - if is_model_retryable_error "$candidate"; then - continue - fi - echo "Strix fallback model '$candidate' emitted provider infrastructure or failure-signal output; trying next configured fallback if available." >&2 - continue - fi - - if ! is_model_retryable_error "$candidate"; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - done - - if should_fail_pull_request_infra_zero_findings; then - return 1 - fi - - if [ "$fallback_tried" -eq 0 ]; then - local fallback_config_name - fallback_config_name="$(fallback_models_config_name_for_model "$PRIMARY_MODEL")" - local configured_fallback_count=0 - for candidate_raw in "${FALLBACK_MODELS[@]}"; do - candidate="$(normalize_model "$candidate_raw")" - [ -n "$candidate" ] && configured_fallback_count=$((configured_fallback_count + 1)) - done - if [ "$configured_fallback_count" -eq 0 ]; then - echo "ERROR: No fallback models configured ($fallback_config_name is empty). Configure distinct models." >&2 - else - echo "ERROR: All configured fallback models are the same as the primary model" >&2 - fi - return 1 - fi - - if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then - echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 - return 1 - fi - - local threshold_rank - threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" - if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then - echo "Strix quick scan failed with a non-recoverable error." >&2 - return 1 - fi - - if is_vertex_model "$PRIMARY_MODEL"; then - echo "Configured Vertex model and fallback models were unavailable." >&2 - else - echo "Configured model and fallback models were unavailable." >&2 - fi - return 1 -} - -prepare_pull_request_scan_scope -if [ "$TARGET_PATH_REQUESTS_PR_SCOPE" -eq 1 ] && - [ "$TARGET_PATH_IS_INTERNAL_PR_SCOPE" -ne 1 ]; then - echo "ERROR: STRIX_TARGET_PATH=$PR_SCOPE_TARGET_SENTINEL did not produce a PR scan scope." >&2 - exit 2 -fi - -scan_rc=0 -run_current_target_scan || scan_rc=$? -exit "$scan_rc" -; then + grep -Eq '^[^[:alnum:]]*MODEL QUALITY WARNING[^[:alnum:]]*$'; then continue fi return 0 From 09cb76e8ede836ed9e446a71f91ee43bfcb16189 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:42:07 -0700 Subject: [PATCH 6/7] test(strix): reproduce report-log quality advisory false negative --- scripts/ci/test_strix_quick_gate.sh | 41 +++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 5807a9f81..cb6abea20 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4467,7 +4467,7 @@ EOS ;; esac ;; - nim-primary-rate-limit-clean-fallback-quality-warning) + nim-primary-rate-limit-clean-fallback-quality-warning|nim-primary-rate-limit-clean-fallback-quality-report-warning) case "${STRIX_LLM:-}" in nvidia_nim/nvidia/nemotron-3-super-120b-a12b) echo "LLM CONNECTION FAILED" @@ -4481,6 +4481,13 @@ EOS echo "│ frontier model for Strix. │" echo "│ You can continue, but weaker models may miss vulnerabilities or produce │" echo "│ lower-quality findings. │" + if [ "${FAKE_STRIX_SCENARIO:?}" = "nim-primary-rate-limit-clean-fallback-quality-report-warning" ]; then + mkdir -p "$STRIX_REPORTS_DIR/fake-nim-quality-advisory" + cat >"$STRIX_REPORTS_DIR/fake-nim-quality-advisory/strix.log" <<'EOS' +│ MODEL QUALITY WARNING │ +2026-08-23 12:32:17.984 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) +EOS + fi echo "│ Vulnerabilities 0 │" echo "╰──────────────────────────────────────────────────────────────────────────────╯" exit 0 @@ -6387,7 +6394,7 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; - nim-primary-rate-limit-clean-fallback-quality-warning) + nim-primary-rate-limit-clean-fallback-quality-warning | nim-primary-rate-limit-clean-fallback-quality-report-warning) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ "" \ @@ -10415,6 +10422,36 @@ run_gate_case "nim-primary-rate-limit-clean-fallback-quality-warning" \ "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ "1" +run_gate_case "nim-primary-rate-limit-clean-fallback-quality-report-warning" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5' in [0-9]+s\\." \ + "2" \ + "nvidia_nim/nvidia/nemotron-3-super-120b-a12b|nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "https://integrate.api.nvidia.com/v1|https://integrate.api.nvidia.com/v1" \ + "nvidia_nim" \ + "https://integrate.api.nvidia.com/v1" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "nvidia_nim/nvidia/llama-3.3-nemotron-super-49b-v1.5" \ + "1" + run_gate_case "provider-fatal-success-signal" \ "vertex_ai/provider-fatal-success-signal" \ "" \ From a1fc43a13085b4dbe9367a871e33461c4eea9ae0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 05:43:27 -0700 Subject: [PATCH 7/7] fix(strix): classify quality advisory consistently across evidence files --- scripts/ci/strix_quick_gate.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index cff0f27e3..ae324f44c 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -221,7 +221,8 @@ has_strix_report_failure_signal() { report_root="$newest_report_root" fi while IFS= read -r -d '' report_log; do - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then + if has_unexpected_strix_failure_signal_file "$report_log" || + grep -Eiq '(^|[^[:alpha:]])Timeout([^[:alpha:]]|$)' "$report_log"; then return 0 fi done < <(find "$report_root" -type f -name '*.log' -print0) @@ -3150,7 +3151,8 @@ is_llm_token_limit_error() { # errors (timeout, rate-limit, transport failures) that indicate the scan # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. -has_unexpected_strix_console_failure_signal() { +has_unexpected_strix_failure_signal_file() { + local signal_file="$1" local signal_line while IFS= read -r signal_line; do # Strix emits this exact decorative banner for supported models that are @@ -3161,11 +3163,15 @@ has_unexpected_strix_console_failure_signal() { continue fi return 0 - done < <(grep -Ei '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG" || true) + done < <(grep -Ei '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$signal_file" || true) return 1 } +has_unexpected_strix_console_failure_signal() { + has_unexpected_strix_failure_signal_file "$STRIX_LOG" +} + has_detected_infrastructure_error() { if has_unexpected_strix_console_failure_signal; then return 0