fix(strix): fail closed on provider evidence gaps

.github/workflows/strix.yml

@@ -669,8 +669,8 @@ jobs:
           GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }}
         run: |
           # Direct-OpenAI scans keep GitHub Models candidates as fallbacks, so
-          # a provider quota outage degrades to a slower model instead of a
-          # neutral skip with no security evidence. github_models/* fallback
+          # a provider quota outage can retry with a slower model before the
+          # required gate fails closed without security evidence. github_models/* fallback
           # models read this token and endpoint; the primary keeps its own key.
           umask 077
           sanitized="$(printf '%s' "$GITHUB_MODELS_FALLBACK_TOKEN" | tr -d '\r\n')"
@@ -841,12 +841,11 @@ jobs:
           export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700"
 
           # Capture the gate exit code plus its console output. The gate returns
-          # exit 1 both for genuine blocking vulnerabilities AND for
-          # LLM-backend-unavailable outcomes (GitHub Models "Too many requests"
-          # rate limits, OpenAI quota starvation, 413 tokens_limit_reached
-          # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI
-          # infrastructure noise, not a security finding, so it must not fail
-          # the required check and block merges.
+          # exit 1 for genuine blocking vulnerabilities and for
+          # LLM-backend-unavailable outcomes (rate limits, quota starvation,
+          # token caps, connection/warm-up failures, or report warnings) that
+          # could not complete a clean scan. Incomplete provider evidence is a
+          # hard failure: it must never satisfy the required security check.
           strix_run_log="$RUNNER_TEMP/strix_gate_console.log"
           strix_rc=0
           set +e
@@ -859,31 +858,13 @@ jobs:
           fi
 
           # Preserve configuration failures (exit 2) and any unexpected exit
-          # code as hard failures — only the scan-failure code (1) can be an
-          # infrastructure/backend-unavailability outcome.
+          # code as hard failures. Scan failures (exit 1), including provider
+          # infrastructure and report failure signals, also fail closed.
           if [ "$strix_rc" -ne 1 ]; then
             exit "$strix_rc"
           fi
 
-          # Recognized signals that the LLM backend was unavailable / starved.
-          backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404'
-          # Any evidence that a vulnerability was actually reported. Its presence
-          # forces a hard failure so real findings are NEVER downgraded. Keep the
-          # severity branch anchored away from identifiers so environment lines
-          # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings.
-          reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:'
-
-          # Neutral skip only when ALL hold: a backend-unavailability signal is
-          # present and no vulnerability was reported anywhere. This preserves
-          # real security gating while keeping uncontrollable provider outages
-          # from blocking current-head merge progress.
-          if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \
-            && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then
-            echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log."
-            exit 0
-          fi
-
-          echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2
+          echo "Strix reported security findings or incomplete provider evidence; failing the required check (gate exit ${strix_rc})." >&2
           exit "$strix_rc"
 
       - name: Collect Strix reports for artifact upload

scripts/ci/test_strix_quick_gate.sh

@@ -297,6 +297,9 @@ assert_strix_workflow_pr_trigger_hardened() {
 	assert_file_contains "$workflow_file" "vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash)" "strix workflow accepts only exact approved organization Vertex AI models"
 	assert_file_contains "$workflow_file" 'STRIX_VERTEX_FALLBACK_MODELS: ""' "strix workflow disables silent Vertex fallbacks so timeout-class failures fail closed"
 	assert_file_contains "$workflow_file" 'STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"' "strix workflow fails closed on timeout, fatal, warning, denied, or provider failure signals"
+	assert_file_contains "$workflow_file" "Incomplete provider evidence is a" "strix workflow documents provider failure as a hard failure"
+	assert_file_contains "$workflow_file" "incomplete provider evidence; failing the required check" "strix workflow fails the required check on provider failure"
+	assert_file_not_contains "$workflow_file" "Treating as a neutral skip" "strix workflow must not downgrade incomplete provider evidence to success"
 	assert_file_contains "$workflow_file" 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables npm lifecycle scripts for untrusted PR scan data"
 	assert_file_contains "$workflow_file" 'PNPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables pnpm lifecycle scripts for untrusted PR scan data"
 	assert_file_contains "$workflow_file" 'YARN_ENABLE_SCRIPTS: "false"' "strix workflow disables yarn lifecycle scripts for untrusted PR scan data"
@@ -725,7 +728,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "not a generic model-exhaustion message" "opencode review tells models to return concrete missing-evidence findings instead of progress-only output"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow"
 	assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow"
-	assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports"
+	assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "Incomplete provider evidence is a" "strix wrapper fails closed on quota-only provider failures without vulnerability reports"
 	assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure"
 	assert_file_contains "$workflow_file" 'timeout-minutes: 325' "opencode review target contains evidence, the bounded long-review pool, publication, Noema handoff, and cleanup overhead"
 	assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue"

tests/test_required_workflow_queue_contract.py

@@ -1092,21 +1092,15 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N
     assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence
 
 
-def test_strix_provider_outage_without_findings_is_neutralized() -> None:
+def test_strix_provider_outage_without_findings_fails_closed() -> None:
     workflow = workflow_text("strix.yml")
 
-    assert "RateLimitError|Too many requests" in workflow
-    assert "exceeded your current quota" in workflow
-    assert "billing details" in workflow
-    assert "LLM warm-up failed" in workflow
-    assert "zero_vulnerabilities_signal" not in workflow
-    assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow
-    assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow
-    assert "before producing a vulnerability report" in workflow
-    assert "genuine findings still fail the check" in workflow
-    assert (
-        '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow
-    )
+    assert "STRIX_FAIL_ON_PROVIDER_SIGNAL: \"1\"" in workflow
+    assert "Incomplete provider evidence is a" in workflow
+    assert "incomplete provider evidence; failing the required check" in workflow
+    assert "backend_unavailable_signal" not in workflow
+    assert "reported_vulnerability_signal" not in workflow
+    assert "Treating as a neutral skip" not in workflow
 
 
 def test_strix_cross_repo_dispatch_uses_target_token_for_pr_scoping() -> None:

tests/test_strix_nvidia_nim_not_found_fallback.py

@@ -73,52 +73,6 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool:
     return completed.returncode == 0
 
 
-def _workflow_signal_pattern(workflow: str, variable_name: str) -> str:
-    """Extract one single-quoted POSIX ERE assigned in the Strix workflow."""
-
-    match = re.search(
-        rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$",
-        workflow,
-    )
-    if match is None:
-        raise AssertionError(f"missing workflow signal: {variable_name}")
-    return match.group(1)
-
-
-def _workflow_neutralizes(log_text: str) -> bool:
-    """Execute the outer workflow's backend-neutralization condition."""
-
-    workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
-    backend_pattern = _workflow_signal_pattern(
-        workflow,
-        "backend_unavailable_signal",
-    )
-    vulnerability_pattern = _workflow_signal_pattern(
-        workflow,
-        "reported_vulnerability_signal",
-    )
-    with tempfile.TemporaryDirectory(prefix="strix-workflow-404-") as temp_dir:
-        log_path = Path(temp_dir) / "strix.log"
-        log_path.write_text(log_text, encoding="utf-8")
-        backend = subprocess.run(
-            ["grep", "-Eiq", backend_pattern, str(log_path)],
-            check=False,
-            capture_output=True,
-            text=True,
-        )
-        vulnerability = subprocess.run(
-            ["grep", "-Eiq", vulnerability_pattern, str(log_path)],
-            check=False,
-            capture_output=True,
-            text=True,
-        )
-    if backend.returncode not in {0, 1}:
-        raise AssertionError(backend.stderr)
-    if vulnerability.returncode not in {0, 1}:
-        raise AssertionError(vulnerability.stderr)
-    return backend.returncode == 0 and vulnerability.returncode == 1
-
-
 class StrixNvidiaNotFoundFallbackTests(unittest.TestCase):
     """Protect provider-scoped 404 fallback without weakening security gates."""
 
@@ -199,62 +153,18 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None:
         )[0]
         self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate)
 
-    def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None:
-        """Reject provider-like target text in the outer neutralization gate."""
-
-        self.assertFalse(
-            _workflow_neutralizes(
-                "source literal: Nvidia_nimException Error code: 404\n"
-            )
-        )
-        self.assertTrue(
-            _workflow_neutralizes(
-                "litellm.exceptions.NotFoundError: Nvidia_nimException - "
-                "Error code: 404\nVulnerabilities 0\n"
-            )
-        )
-
-    def test_outer_workflow_rejects_cross_line_signal_assembly(self) -> None:
-        """Require exception, provider, and 404 evidence on one physical line."""
-
-        self.assertFalse(
-            _workflow_neutralizes(
-                "litellm.exceptions.NotFoundError: provider unavailable\n"
-                "Nvidia_nimException Error code: 404\n"
-            )
-        )
-
-    def test_outer_workflow_rejects_nvidia_404_without_litellm_context(self) -> None:
-        """Require LiteLLM NotFoundError context, not just NVIDIA + 404."""
-
-        self.assertFalse(
-            _workflow_neutralizes(
-                "Nvidia_nimException Error code: 404\nVulnerabilities 0\n"
-            )
-        )
-
-    def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None:
-        """Keep a real vulnerability signal blocking despite provider failure."""
-
-        self.assertFalse(
-            _workflow_neutralizes(
-                "litellm.exceptions.NotFoundError: Nvidia_nimException - "
-                "Error code: 404\nVulnerabilities 1\n"
-            )
-        )
-
-    def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None:
-        """Retain the static fail-closed vulnerability evidence contract."""
-
+    def test_outer_workflow_fails_closed_on_provider_evidence_gap(self) -> None:
+        """Provider 404s cannot become successful outer-workflow evidence."""
         workflow = STRIX_WORKFLOW.read_text(encoding="utf-8")
-        self.assertIn("Nvidia_nimException", workflow)
-        self.assertIn("Error code:[[:space:]]*404", workflow)
-        self.assertIn("reported_vulnerability_signal", workflow)
-        self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow)
+        self.assertIn('STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"', workflow)
+        self.assertIn("Incomplete provider evidence is a", workflow)
         self.assertIn(
-            '! grep -Eiq "$reported_vulnerability_signal"',
+            "incomplete provider evidence; failing the required check",
             workflow,
         )
+        self.assertNotIn("backend_unavailable_signal", workflow)
+        self.assertNotIn("reported_vulnerability_signal", workflow)
+        self.assertNotIn("Treating as a neutral skip", workflow)
 
 
 if __name__ == "__main__":