fix(strix): bounded provider-outage retries

.github/workflows/opencode-review-dispatch.yml

@@ -4477,6 +4477,10 @@ jobs:
           # so the OpenRouter slots use cheap paid models billed against the
           # org's OpenRouter credits), then the full-size GPT-4.1 long-context
           # endpoint and provider-specific GPT/o3 fallbacks.
+          # The direct-OpenAI slot runs GPT-5.4: gpt-5.6-luna returns 404 on
+          # the OpenAI API (see a724582), so the pool keeps the newest VALID
+          # direct-OpenAI model instead of burning a candidate on a certain
+          # failure.
           OPENCODE_MODEL_CANDIDATES: "${{ needs.validate-pr-metadata.outputs.is_private == 'false' && 'nvidia-nim/nvidia/llama-3.3-nemotron-super-49b-v1.5 nvidia-nim/nvidia/llama-3.1-nemotron-ultra-253b-v1 nvidia-nim/nvidia/nemotron-3-super-120b-a12b nvidia-nim/nvidia/nemotron-3-ultra-550b-a55b nvidia-nim/meta/llama-3.3-70b-instruct nvidia-nim/deepseek-ai/deepseek-v4-pro nvidia-nim/mistralai/codestral-22b-instruct-v0.1 opencode-free/nemotron-3-ultra-free opencode-free/deepseek-v4-flash-free opencode-free/north-mini-code-free opencode-free/laguna-s-2.1-free opencode-free/ling-3.0-flash-free opencode-free/big-pickle opencode-free/mimo-v2.5-free opencode-free/hy3-free opencode-free/minimax-m3-free opencode-free/glm-5-free opencode-free/kimi-k2.5-free opencode-free/qwen3.6-plus-free ' || '' }}opencode/gpt-5.6-terra github-models/deepseek/deepseek-v3-0324 openai/gpt-5.4 openrouter/deepseek/deepseek-v3.2 openrouter/qwen/qwen3-coder github-models/openai/gpt-4.1 github-models/openai/gpt-5 github-models/openai/gpt-5-chat github-models/openai/o3 github-models/deepseek/deepseek-r1-0528 github-models/deepseek/deepseek-r1"
           # One attempt per model, then fall through to the next model. Retrying
           # the SAME model 5x let a rate-limited/hung leader consume the whole

.github/workflows/strix.yml

@@ -882,6 +882,17 @@ jobs:
           export "STRIX_PROCESS_${budget_suffix}_SECONDS=$process_budget_seconds"
           export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700"
 
+          # Recognized signals that the LLM backend was unavailable / starved.
+          # Defined before the gate loop so the bounded retry decision below
+          # can classify outcomes without duplicating the patterns later.
+          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080'
+          model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)'
+          # Any evidence that a vulnerability was actually reported. Its presence
+          # forces a hard failure so real findings are NEVER downgraded. Keep the
+          # severity branch anchored away from identifiers so environment lines
+          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
+          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
+
           # Capture the gate exit code plus its console output. The gate returns
           # exit 1 both for genuine blocking vulnerabilities AND for
           # LLM-backend-unavailable outcomes (GitHub Models "Too many requests"
@@ -890,11 +901,92 @@ jobs:
           # could not complete a scan. Provider failure is typed infrastructure
           # evidence, but remains non-passing because no authoritative complete
           # vulnerability result exists.
+          #
+          # A typed provider outage with no reported vulnerability finding is
+          # retried with bounded linear backoff inside this step so transient
+          # provider failures do not fail the required check on the first
+          # attempt. Genuine findings, configuration failures, and unexpected
+          # exit codes never retry; the deadline keeps every path inside the
+          # deterministic 120-minute job budget, and all-terminal outcomes
+          # remain fail-closed.
           strix_run_log="$RUNNER_TEMP/strix_gate_console.log"
+          strix_attempt_audit_log="$RUNNER_TEMP/strix_gate_attempts.log"
+          : > "$strix_attempt_audit_log"
           strix_rc=0
+          strix_gate_attempt=1
+          strix_gate_deadline=$(( SECONDS + 6000 ))
           set +e
-          bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log"
-          strix_rc="${PIPESTATUS[0]}"
+          while : ; do
+            # Rebind every invocation to the remaining outer deadline. The
+            # original 5700/5400 split reserves 300 seconds for gate cleanup;
+            # retain that internal reserve plus 300 seconds for workflow
+            # artifact/status cleanup even when a late retry starts.
+            remaining_seconds=$(( strix_gate_deadline - SECONDS ))
+            if [ "$remaining_seconds" -le 600 ]; then
+              : > "$strix_run_log"
+              echo "Configured model and fallback models were unavailable before another bounded retry could start: only ${remaining_seconds}s remain in the outer Strix deadline." | tee "$strix_run_log" >&2
+              strix_rc=1
+              {
+                printf "=== outer-attempt=%s rc=%s ===\\n" "$strix_gate_attempt" "$strix_rc"
+                cat "$strix_run_log"
+              } >> "$strix_attempt_audit_log"
+              break
+            fi
+            attempt_budget_seconds=$(( remaining_seconds - 300 ))
+            if [ "$attempt_budget_seconds" -gt 5700 ]; then
+              attempt_budget_seconds=5700
+            fi
+            attempt_process_budget_seconds=$(( attempt_budget_seconds - 300 ))
+            if [ "$attempt_process_budget_seconds" -gt "$process_budget_seconds" ]; then
+              attempt_process_budget_seconds="$process_budget_seconds"
+            fi
+            export "STRIX_TOTAL_${budget_suffix}_SECONDS=$attempt_budget_seconds"
+            export "STRIX_PROCESS_${budget_suffix}_SECONDS=$attempt_process_budget_seconds"
+
+            : > "$strix_run_log"
+            bash "$TRUSTED_STRIX_GATE" 2>&1 | tee "$strix_run_log"
+            strix_rc="${PIPESTATUS[0]}"
+            {
+              printf "=== outer-attempt=%s rc=%s ===\\n" "$strix_gate_attempt" "$strix_rc"
+              cat "$strix_run_log"
+            } >> "$strix_attempt_audit_log"
+            if [ "$strix_rc" -eq 0 ]; then
+              break
+            fi
+            # Only exit-code 1 scan failures can be infrastructure outcomes.
+            if [ "$strix_rc" -ne 1 ]; then
+              break
+            fi
+            # Scope this attempt's retry decision to the log tail after the
+            # last pipeline-continuation marker, exactly like the terminal
+            # classification below: an already-exempted finding before the
+            # marker must not mask a retryable outage after it.
+            strix_retry_scope_log="$strix_run_log"
+            if grep -Fq 'allowing pipeline continuation' "$strix_run_log"; then
+              strix_retry_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log"
+              awk '/allowing pipeline continuation/{buf=""; next} {buf=buf $0 "\n"} END{printf "%s", buf}' \
+                "$strix_run_log" > "$strix_retry_scope_log"
+            fi
+            # A reported vulnerability is authoritative evidence: never retry
+            # and never risk downgrading it.
+            if grep -Eiq "$reported_vulnerability_signal" "$strix_retry_scope_log"; then
+              break
+            fi
+            # Retry only recognized provider-outage / model-behavior classes.
+            if ! grep -Eiq "$backend_unavailable_signal" "$strix_retry_scope_log" \
+              && ! grep -Eq "$model_behavior_error_signal" "$strix_retry_scope_log"; then
+              break
+            fi
+            remaining_seconds=$(( strix_gate_deadline - SECONDS ))
+            if [ "$strix_gate_attempt" -ge 3 ] || [ "$remaining_seconds" -lt 600 ]; then
+              echo "Provider-unavailable Strix attempt ${strix_gate_attempt} reached the bounded retry limit or the remaining job time budget (${remaining_seconds}s) is too small to retry; failing closed." >&2
+              break
+            fi
+            backoff_seconds=$(( ${STRIX_GATE_RETRY_BACKOFF_SECONDS:-90} * strix_gate_attempt ))
+            echo "Strix provider outage on attempt ${strix_gate_attempt}; retrying after ${backoff_seconds}s backoff." >&2
+            sleep "$backoff_seconds"
+            strix_gate_attempt=$(( strix_gate_attempt + 1 ))
+          done
           set -e
 
           if [ "$strix_rc" -eq 0 ]; then
@@ -908,15 +1000,6 @@ jobs:
             exit "$strix_rc"
           fi
 
-          # Recognized signals that the LLM backend was unavailable / starved.
-          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080'
-          model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)'
-          # Any evidence that a vulnerability was actually reported. Its presence
-          # forces a hard failure so real findings are NEVER downgraded. Keep the
-          # severity branch anchored away from identifiers so environment lines
-          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
-          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
-
           # An earlier out-of-scope/below-threshold finding may already have
           # been exempted by the trusted gate. Classify a later provider
           # outage from the tail after the last continuation marker, but keep
@@ -959,6 +1042,10 @@ jobs:
             cp "$RUNNER_TEMP/strix_gate_console.log" "$GITHUB_WORKSPACE/strix_runs/gate-console.log"
             copied_reports=1
           fi
+          if [ -f "$RUNNER_TEMP/strix_gate_attempts.log" ]; then
+            cp "$RUNNER_TEMP/strix_gate_attempts.log" "$GITHUB_WORKSPACE/strix_runs/gate-attempts.log"
+            copied_reports=1
+          fi
           if [ -n "$(find "$GITHUB_WORKSPACE/strix_runs" -mindepth 1 -print -quit)" ]; then
             copied_reports=1
           fi

tests/test_pr_review_autofix_nvidia_nim_contract.py

@@ -19,7 +19,7 @@
 DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md")
 CHANGELOG = Path("CHANGELOG.md")
 REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml")
-REVIEW_DISPATCH_BLOB_SHA = "ed3f7b44f9afdd6ab295426e5d0440aeca6bdfb5"
+REVIEW_DISPATCH_BLOB_SHA = "0df7a17cc72a79585cec169c8299e0646f93ab02"
 
 
 def _workflow_text(path: Path) -> str:

tests/test_strix_backend_unavailable_after_exempted_finding.py

@@ -20,7 +20,7 @@
 
 from __future__ import annotations
 
-import re
+import shlex
 import subprocess
 import tempfile
 import unittest
@@ -63,9 +63,12 @@ def _extract_neutralization_block(workflow: str) -> str:
     stale logic.
     """
 
-    start_marker = (
-        "          # Recognized signals that the LLM backend was unavailable"
-    )
+    # The classification block starts at the neutralization-scope assignment
+    # and runs to the terminal failure exit. The gate-execution retry loop and
+    # the raw signal definitions live outside this region; the signal values
+    # are injected by _run_gate_tail so the extracted decision logic stays the
+    # single tested authority.
+    start_marker = '          strix_neutralization_scope_log="$strix_run_log"'
     terminal_failure_marker = (
         '          echo "Strix reported security findings or failed for a '
         'non-backend reason; failing the required check'
@@ -77,6 +80,23 @@ def _extract_neutralization_block(workflow: str) -> str:
     return workflow[start:end]
 
 
+def _extract_signal_definitions(workflow: str) -> str:
+    """Return the canonical backend-outage / finding-signal definitions.
+
+    Bounded by the same unique anchors used in production so the injected
+    patterns cannot drift from the ones the gate itself classifies with.
+    """
+
+    start_marker = (
+        "          # Recognized signals that the LLM backend was unavailable"
+    )
+    end_marker = "reported_vulnerability_signal="
+    start = workflow.index(start_marker)
+    end = workflow.index(end_marker, start)
+    end = workflow.index("\n", end) + 1
+    return workflow[start:end]
+
+
 def _run_gate_tail(log_text: str) -> int:
     """Execute the extracted block against a synthetic log; return its exit code.
 
@@ -85,6 +105,7 @@ def _run_gate_tail(log_text: str) -> int:
     """
 
     workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+    signals = _extract_signal_definitions(workflow)
     block = _extract_neutralization_block(workflow)
     with tempfile.TemporaryDirectory(prefix="strix-tail-scope-") as temp_dir:
         strix_run_log = Path(temp_dir) / "strix_gate_console.log"
@@ -94,6 +115,7 @@ def _run_gate_tail(log_text: str) -> int:
                 "set -uo pipefail",
                 'strix_run_log="$1"',
                 "strix_rc=1",
+                signals,
                 block,
             )
         )
@@ -113,6 +135,71 @@ def _run_gate_tail(log_text: str) -> int:
     return completed.returncode
 
 
+
+def _extract_retry_loop_region(workflow: str) -> str:
+    """Return the bounded provider-outage retry region, verbatim from the yml.
+
+    Spans the signal definitions through the post-loop success exit so the
+    retry decision, its tail-scoping, and its terminal success path are all
+    exercised against a scripted fake gate.
+    """
+
+    start_marker = (
+        "          # Recognized signals that the LLM backend was unavailable"
+    )
+    end_marker = (
+        "          # Preserve configuration failures (exit 2) and any unexpected exit"
+    )
+    start = workflow.index(start_marker)
+    end = workflow.index(end_marker, start)
+    return workflow[start:end]
+
+
+def _run_gate_retry(gate_script: str) -> tuple[int, int, str]:
+    """Run the extracted retry loop; return (rc, calls, raw attempt audit).
+
+    The fake gate appends one line to a call-counter file on every invocation
+    so tests can prove exactly how many attempts the loop spent.
+    """
+
+    workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+    signals = _extract_signal_definitions(workflow)
+    region = _extract_retry_loop_region(workflow)
+    with tempfile.TemporaryDirectory(prefix="strix-retry-scope-") as temp_dir:
+        counter = Path(temp_dir) / "gate_calls"
+        counter.write_text("0\n", encoding="utf-8")
+        gate_path = Path(temp_dir) / "fake_gate.sh"
+        gate_path.write_text(
+            gate_script.replace("__COUNTER__", str(counter)),
+            encoding="utf-8",
+        )
+        gate_path.chmod(0o755)
+        script = "\n".join(
+            (
+                "set -uo pipefail",
+                "budget_suffix=TIMEOUT",
+                "process_budget_seconds=5400",
+                f"export TRUSTED_STRIX_GATE={shlex.quote(str(gate_path))}",
+                "export RUNNER_TEMP=" + shlex.quote(temp_dir),
+                "export STRIX_GATE_RETRY_BACKOFF_SECONDS=1",
+                signals,
+                region,
+                'exit "$strix_rc"',
+            )
+        )
+        completed = subprocess.run(
+            ["bash", "-c", script],
+            check=False,
+            capture_output=True,
+            text=True,
+            env={"RUNNER_TEMP": temp_dir, "PATH": "/usr/bin:/bin"},
+        )
+        calls = int(counter.read_text().strip())
+        audit_path = Path(temp_dir) / "strix_gate_attempts.log"
+        audit = audit_path.read_text(encoding="utf-8") if audit_path.exists() else ""
+    return completed.returncode, calls, audit
+
+
 class StrixBackendUnavailableAfterExemptedFindingTests(unittest.TestCase):
     """Protect the PR #392-shaped scenario without weakening the real gate."""
 
@@ -153,6 +240,73 @@ def test_bare_backend_outage_with_no_finding_is_non_passing(
 
         self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1)
 
+    def test_each_attempt_is_capped_by_the_remaining_outer_deadline(self) -> None:
+        """A late retry cannot inherit a fresh 5700-second gate budget."""
+
+        workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
+        region = _extract_retry_loop_region(workflow)
+        before_gate, _ = region.split(
+            'bash "$TRUSTED_STRIX_GATE"', maxsplit=1
+        )
+        self.assertIn(
+            "remaining_seconds=$(( strix_gate_deadline - SECONDS ))",
+            before_gate,
+        )
+        self.assertIn(
+            "attempt_budget_seconds=$(( remaining_seconds - 300 ))",
+            before_gate,
+        )
+        self.assertIn(
+            'export "STRIX_TOTAL_${budget_suffix}_SECONDS=$attempt_budget_seconds"',
+            before_gate,
+        )
+        self.assertIn(
+            'export "STRIX_PROCESS_${budget_suffix}_SECONDS=$attempt_process_budget_seconds"',
+            before_gate,
+        )
+
+    def test_exempted_finding_then_outage_recovers_on_second_attempt(self) -> None:
+        """An exempt finding before continuation must not block outage retry."""
+
+        gate = r"""#!/usr/bin/env bash
+calls=$(( $(cat __COUNTER__) + 1 ))
+echo "$calls" > __COUNTER__
+if [ "$calls" -le 1 ]; then
+  printf '%s\n' \
+    "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \
+    "LLM CONNECTION FAILED" \
+    "Configured model and fallback models were unavailable."
+  exit 1
+fi
+echo "scan complete"
+exit 0
+"""
+        returncode, calls, audit = _run_gate_retry(gate)
+        self.assertEqual(returncode, 0)
+        self.assertEqual(calls, 2)
+        self.assertIn("outer-attempt=1 rc=1", audit)
+        self.assertIn("outer-attempt=2 rc=0", audit)
+        self.assertIn("LLM CONNECTION FAILED", audit)
+        self.assertIn("scan complete", audit)
+
+    def test_real_finding_after_continuation_never_retries(self) -> None:
+        """A tail-scoped real finding is authoritative: zero retries, fail closed."""
+
+        gate = r"""#!/usr/bin/env bash
+calls=$(( $(cat __COUNTER__) + 1 ))
+echo "$calls" > __COUNTER__
+printf '%s\n' \
+  "Strix findings are limited to unchanged files in this pull request; allowing pipeline continuation." \
+  "LLM CONNECTION FAILED" \
+  "Vulnerability Report" "Severity: CRITICAL" "Vulnerabilities 1"
+exit 1
+"""
+        returncode, calls, audit = _run_gate_retry(gate)
+        self.assertEqual(returncode, 1)
+        self.assertEqual(calls, 1)
+        self.assertIn("outer-attempt=1 rc=1", audit)
+        self.assertIn("Vulnerabilities 1", audit)
+
 
 if __name__ == "__main__":
     unittest.main()