diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index ed3f7b44f..0df7a17cc 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -4477,6 +4477,10 @@ jobs: # so the OpenRouter slots use cheap paid models billed against the # org's OpenRouter credits), then the full-size GPT-4.1 long-context # endpoint and provider-specific GPT/o3 fallbacks. + # The direct-OpenAI slot runs GPT-5.4: gpt-5.6-luna returns 404 on + # the OpenAI API (see a724582), so the pool keeps the newest VALID + # direct-OpenAI model instead of burning a candidate on a certain + # failure. OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.4 openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1" # One attempt per model, then fall through to the next model. Retrying # the SAME model 5x let a rate-limited/hung leader consume the whole diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 9317010e0..4bf543e3b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -882,6 +882,17 @@ jobs: export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds" export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" + # Recognized signals that the LLM backend was unavailable / starved. + # Defined before the gate loop so the bounded retry decision below + # can classify outcomes without duplicating the patterns later. + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' + model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' + # Any evidence that a vulnerability was actually reported. Its presence + # forces a hard failure so real findings are NEVER downgraded. Keep the + # severity branch anchored away from identifiers so environment lines + # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. + reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' + # Capture the gate exit code plus its console output. The gate returns # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" @@ -890,11 +901,92 @@ jobs: # could not complete a scan. Provider failure is typed infrastructure # evidence, but remains non-passing because no authoritative complete # vulnerability result exists. + # + # A typed provider outage with no reported vulnerability finding is + # retried with bounded linear backoff inside this step so transient + # provider failures do not fail the required check on the first + # attempt. Genuine findings, configuration failures, and unexpected + # exit codes never retry; the deadline keeps every path inside the + # deterministic 120-minute job budget, and all-terminal outcomes + # remain fail-closed. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" + strix_attempt_audit_log="$RUNNER_TEMP/strix_gate_attempts.log" + : > "$strix_attempt_audit_log" strix_rc=0 + strix_gate_attempt=1 + strix_gate_deadline=$(( SECONDS + 6000 )) set +e - bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log" - strix_rc="${PIPESTATUS[0]}" + while : ; do + # Rebind every invocation to the remaining outer deadline. The + # original 5700/5400 split reserves 300 seconds for gate cleanup; + # retain that internal reserve plus 300 seconds for workflow + # artifact/status cleanup even when a late retry starts. + remaining_seconds=$(( strix_gate_deadline - SECONDS )) + if [ "$remaining_seconds" -le 600 ]; then + : > "$strix_run_log" + echo "Configured model and fallback models were unavailable before another bounded retry could start: only ${remaining_seconds}s remain in the outer Strix deadline." | tee "$strix_run_log" >&2 + strix_rc=1 + { + printf "=== outer-attempt=%s rc=%s ===\\n" "$strix_gate_attempt" "$strix_rc" + cat "$strix_run_log" + } >> "$strix_attempt_audit_log" + break + fi + attempt_budget_seconds=$(( remaining_seconds - 300 )) + if [ "$attempt_budget_seconds" -gt 5700 ]; then + attempt_budget_seconds=5700 + fi + attempt_process_budget_seconds=$(( attempt_budget_seconds - 300 )) + if [ "$attempt_process_budget_seconds" -gt "$process_budget_seconds" ]; then + attempt_process_budget_seconds="$process_budget_seconds" + fi + export "STRIX_TOTAL_${budget_suffix}_SECONDS=$attempt_budget_seconds" + export "STRIX_PROCESS_${budget_suffix}_SECONDS=$attempt_process_budget_seconds" + + : > "$strix_run_log" + bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log" + strix_rc="${PIPESTATUS[0]}" + { + printf "=== outer-attempt=%s rc=%s ===\\n" "$strix_gate_attempt" "$strix_rc" + cat "$strix_run_log" + } >> "$strix_attempt_audit_log" + if [ "$strix_rc" -eq 0 ]; then + break + fi + # Only exit-code 1 scan failures can be infrastructure outcomes. + if [ "$strix_rc" -ne 1 ]; then + break + fi + # Scope this attempt's retry decision to the log tail after the + # last pipeline-continuation marker, exactly like the terminal + # classification below: an already-exempted finding before the + # marker must not mask a retryable outage after it. + strix_retry_scope_log="$strix_run_log" + if grep -Fq 'allowing pipeline continuation' "$strix_run_log"; then + strix_retry_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log" + awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \ + "$strix_run_log" > "$strix_retry_scope_log" + fi + # A reported vulnerability is authoritative evidence: never retry + # and never risk downgrading it. + if grep -Eiq "$reported_vulnerability_signal" "$strix_retry_scope_log"; then + break + fi + # Retry only recognized provider-outage / model-behavior classes. + if ! grep -Eiq "$backend_unavailable_signal" "$strix_retry_scope_log" \ + && ! grep -Eq "$model_behavior_error_signal" "$strix_retry_scope_log"; then + break + fi + remaining_seconds=$(( strix_gate_deadline - SECONDS )) + if [ "$strix_gate_attempt" -ge 3 ] || [ "$remaining_seconds" -lt 600 ]; then + echo "Provider-unavailable Strix attempt ${strix_gate_attempt} reached the bounded retry limit or the remaining job time budget (${remaining_seconds}s) is too small to retry; failing closed." >&2 + break + fi + backoff_seconds=$(( ${STRIX_GATE_RETRY_BACKOFF_SECONDS:-90} * strix_gate_attempt )) + echo "Strix provider outage on attempt ${strix_gate_attempt}; retrying after ${backoff_seconds}s backoff." >&2 + sleep "$backoff_seconds" + strix_gate_attempt=$(( strix_gate_attempt + 1 )) + done set -e if [ "$strix_rc" -eq 0 ]; then @@ -908,15 +1000,6 @@ jobs: exit "$strix_rc" fi - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' - model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - # An earlier out-of-scope/below-threshold finding may already have # been exempted by the trusted gate. Classify a later provider # outage from the tail after the last continuation marker, but keep @@ -959,6 +1042,10 @@ jobs: cp "$RUNNER_TEMP/strix_gate_console.log" "$GITHUB_WORKSPACE/strix_runs/gate-console.log" copied_reports=1 fi + if [ -f "$RUNNER_TEMP/strix_gate_attempts.log" ]; then + cp "$RUNNER_TEMP/strix_gate_attempts.log" "$GITHUB_WORKSPACE/strix_runs/gate-attempts.log" + copied_reports=1 + fi if [ -n "$(find "$GITHUB_WORKSPACE/strix_runs" -mindepth 1 -print -quit)" ]; then copied_reports=1 fi diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 16a83b935..a5d25379a 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -19,7 +19,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "ed3f7b44f9afdd6ab295426e5d0440aeca6bdfb5" +REVIEW_DISPATCH_BLOB_SHA = "0df7a17cc72a79585cec169c8299e0646f93ab02" def _workflow_text(path: Path) -> str: diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 3355a8448..17712e9a1 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -20,7 +20,7 @@ from __future__ import annotations -import re +import shlex import subprocess import tempfile import unittest @@ -63,9 +63,12 @@ def _extract_neutralization_block(workflow: str) -> str: stale logic. """ - start_marker = ( - " # Recognized signals that the LLM backend was unavailable" - ) + # The classification block starts at the neutralization-scope assignment + # and runs to the terminal failure exit. The gate-execution retry loop and + # the raw signal definitions live outside this region; the signal values + # are injected by _run_gate_tail so the extracted decision logic stays the + # single tested authority. + start_marker = ' strix_neutralization_scope_log="$strix_run_log"' terminal_failure_marker = ( ' echo "Strix reported security findings or failed for a ' 'non-backend reason; failing the required check' @@ -77,6 +80,23 @@ def _extract_neutralization_block(workflow: str) -> str: return workflow[start:end] +def _extract_signal_definitions(workflow: str) -> str: + """Return the canonical backend-outage / finding-signal definitions. + + Bounded by the same unique anchors used in production so the injected + patterns cannot drift from the ones the gate itself classifies with. + """ + + start_marker = ( + " # Recognized signals that the LLM backend was unavailable" + ) + end_marker = "reported_vulnerability_signal=" + start = workflow.index(start_marker) + end = workflow.index(end_marker, start) + end = workflow.index("\n", end) + 1 + return workflow[start:end] + + def _run_gate_tail(log_text: str) -> int: """Execute the extracted block against a synthetic log; return its exit code. @@ -85,6 +105,7 @@ def _run_gate_tail(log_text: str) -> int: """ workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + signals = _extract_signal_definitions(workflow) block = _extract_neutralization_block(workflow) with tempfile.TemporaryDirectory(prefix="strix-tail-scope-") as temp_dir: strix_run_log = Path(temp_dir) / "strix_gate_console.log" @@ -94,6 +115,7 @@ def _run_gate_tail(log_text: str) -> int: "set -uo pipefail", 'strix_run_log="$1"', "strix_rc=1", + signals, block, ) ) @@ -113,6 +135,71 @@ def _run_gate_tail(log_text: str) -> int: return completed.returncode + +def _extract_retry_loop_region(workflow: str) -> str: + """Return the bounded provider-outage retry region, verbatim from the yml. + + Spans the signal definitions through the post-loop success exit so the + retry decision, its tail-scoping, and its terminal success path are all + exercised against a scripted fake gate. + """ + + start_marker = ( + " # Recognized signals that the LLM backend was unavailable" + ) + end_marker = ( + " # Preserve configuration failures (exit 2) and any unexpected exit" + ) + start = workflow.index(start_marker) + end = workflow.index(end_marker, start) + return workflow[start:end] + + +def _run_gate_retry(gate_script: str) -> tuple[int, int, str]: + """Run the extracted retry loop; return (rc, calls, raw attempt audit). + + The fake gate appends one line to a call-counter file on every invocation + so tests can prove exactly how many attempts the loop spent. + """ + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + signals = _extract_signal_definitions(workflow) + region = _extract_retry_loop_region(workflow) + with tempfile.TemporaryDirectory(prefix="strix-retry-scope-") as temp_dir: + counter = Path(temp_dir) / "gate_calls" + counter.write_text("0\n", encoding="utf-8") + gate_path = Path(temp_dir) / "fake_gate.sh" + gate_path.write_text( + gate_script.replace("__COUNTER__", str(counter)), + encoding="utf-8", + ) + gate_path.chmod(0o755) + script = "\n".join( + ( + "set -uo pipefail", + "budget_suffix=TIMEOUT", + "process_budget_seconds=5400", + f"export TRUSTED_STRIX_GATE={shlex.quote(str(gate_path))}", + "export RUNNER_TEMP=" + shlex.quote(temp_dir), + "export STRIX_GATE_RETRY_BACKOFF_SECONDS=1", + signals, + region, + 'exit "$strix_rc"', + ) + ) + completed = subprocess.run( + ["bash", "-c", script], + check=False, + capture_output=True, + text=True, + env={"RUNNER_TEMP": temp_dir, "PATH": "/usr/bin:/bin"}, + ) + calls = int(counter.read_text().strip()) + audit_path = Path(temp_dir) / "strix_gate_attempts.log" + audit = audit_path.read_text(encoding="utf-8") if audit_path.exists() else "" + return completed.returncode, calls, audit + + class StrixBackendUnavailableAfterExemptedFindingTests(unittest.TestCase): """Protect the PR #392-shaped scenario without weakening the real gate.""" @@ -153,6 +240,73 @@ def test_bare_backend_outage_with_no_finding_is_non_passing( self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1) + def test_each_attempt_is_capped_by_the_remaining_outer_deadline(self) -> None: + """A late retry cannot inherit a fresh 5700-second gate budget.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + region = _extract_retry_loop_region(workflow) + before_gate, _ = region.split( + 'bash "$TRUSTED_STRIX_GATE"', maxsplit=1 + ) + self.assertIn( + "remaining_seconds=$(( strix_gate_deadline - SECONDS ))", + before_gate, + ) + self.assertIn( + "attempt_budget_seconds=$(( remaining_seconds - 300 ))", + before_gate, + ) + self.assertIn( + 'export "STRIX_TOTAL_${budget_suffix}_SECONDS=$attempt_budget_seconds"', + before_gate, + ) + self.assertIn( + 'export "STRIX_PROCESS_${budget_suffix}_SECONDS=$attempt_process_budget_seconds"', + before_gate, + ) + + def test_exempted_finding_then_outage_recovers_on_second_attempt(self) -> None: + """An exempt finding before continuation must not block outage retry.""" + + gate = r"""#!/usr/bin/env bash +calls=$(( $(cat __COUNTER__) + 1 )) +echo "$calls" > __COUNTER__ +if [ "$calls" -le 1 ]; then + printf '%s\n' \ + "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ + "LLM CONNECTION FAILED" \ + "Configured model and fallback models were unavailable." + exit 1 +fi +echo "scan complete" +exit 0 +""" + returncode, calls, audit = _run_gate_retry(gate) + self.assertEqual(returncode, 0) + self.assertEqual(calls, 2) + self.assertIn("outer-attempt=1 rc=1", audit) + self.assertIn("outer-attempt=2 rc=0", audit) + self.assertIn("LLM CONNECTION FAILED", audit) + self.assertIn("scan complete", audit) + + def test_real_finding_after_continuation_never_retries(self) -> None: + """A tail-scoped real finding is authoritative: zero retries, fail closed.""" + + gate = r"""#!/usr/bin/env bash +calls=$(( $(cat __COUNTER__) + 1 )) +echo "$calls" > __COUNTER__ +printf '%s\n' \ + "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \ + "LLM CONNECTION FAILED" \ + "Vulnerability Report" "Severity: CRITICAL" "Vulnerabilities 1" +exit 1 +""" + returncode, calls, audit = _run_gate_retry(gate) + self.assertEqual(returncode, 1) + self.assertEqual(calls, 1) + self.assertIn("outer-attempt=1 rc=1", audit) + self.assertIn("Vulnerabilities 1", audit) + if __name__ == "__main__": unittest.main()