From a397820b87e5fbda1002a33e0ee13a41fab0d9bd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:03:50 +0900 Subject: [PATCH 1/2] fix(strix): fail closed on provider evidence gaps --- .github/workflows/strix.yml | 39 ++++++++--------------------- scripts/ci/test_strix_quick_gate.sh | 5 +++- 2 files changed, 14 insertions(+), 30 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index f8c361b95..c695d3472 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -669,8 +669,8 @@ jobs: GITHUB_MODELS_FALLBACK_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} run: | # Direct-OpenAI scans keep GitHub Models candidates as fallbacks, so - # a provider quota outage degrades to a slower model instead of a - # neutral skip with no security evidence. github_models/* fallback + # a provider quota outage can retry with a slower model before the + # required gate fails closed without security evidence. github_models/* fallback # models read this token and endpoint; the primary keeps its own key. umask 077 sanitized="$(printf '%s' "$GITHUB_MODELS_FALLBACK_TOKEN" | tr -d '\r\n')" @@ -841,12 +841,11 @@ jobs: export "STRIX_TOTAL_${budget_suffix}_SECONDS=5700" # Capture the gate exit code plus its console output. The gate returns - # exit 1 both for genuine blocking vulnerabilities AND for - # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI - # infrastructure noise, not a security finding, so it must not fail - # the required check and block merges. + # exit 1 for genuine blocking vulnerabilities and for + # LLM-backend-unavailable outcomes (rate limits, quota starvation, + # token caps, connection/warm-up failures, or report warnings) that + # could not complete a clean scan. Incomplete provider evidence is a + # hard failure: it must never satisfy the required security check. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" strix_rc=0 set +e @@ -859,31 +858,13 @@ jobs: fi # Preserve configuration failures (exit 2) and any unexpected exit - # code as hard failures — only the scan-failure code (1) can be an - # infrastructure/backend-unavailability outcome. + # code as hard failures. Scan failures (exit 1), including provider + # infrastructure and report failure signals, also fail closed. if [ "$strix_rc" -ne 1 ]; then exit "$strix_rc" fi - # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' - # Any evidence that a vulnerability was actually reported. Its presence - # forces a hard failure so real findings are NEVER downgraded. Keep the - # severity branch anchored away from identifiers so environment lines - # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. - reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - - # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported anywhere. This preserves - # real security gating while keeping uncontrollable provider outages - # from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ - && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." - exit 0 - fi - - echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2 + echo "Strix reported security findings or incomplete provider evidence; failing the required check (gate exit ${strix_rc})." >&2 exit "$strix_rc" - name: Collect Strix reports for artifact upload diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7343c06ac..ad500e59a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -297,6 +297,9 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$workflow_file" "vertex_ai/gemini-3.1-pro-preview-customtools | vertex_ai/gemini-2.5-flash)" "strix workflow accepts only exact approved organization Vertex AI models" assert_file_contains "$workflow_file" 'STRIX_VERTEX_FALLBACK_MODELS: ""' "strix workflow disables silent Vertex fallbacks so timeout-class failures fail closed" assert_file_contains "$workflow_file" 'STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"' "strix workflow fails closed on timeout, fatal, warning, denied, or provider failure signals" + assert_file_contains "$workflow_file" "Incomplete provider evidence is a" "strix workflow documents provider failure as a hard failure" + assert_file_contains "$workflow_file" "incomplete provider evidence; failing the required check" "strix workflow fails the required check on provider failure" + assert_file_not_contains "$workflow_file" "Treating as a neutral skip" "strix workflow must not downgrade incomplete provider evidence to success" assert_file_contains "$workflow_file" 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables npm lifecycle scripts for untrusted PR scan data" assert_file_contains "$workflow_file" 'PNPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables pnpm lifecycle scripts for untrusted PR scan data" assert_file_contains "$workflow_file" 'YARN_ENABLE_SCRIPTS: "false"' "strix workflow disables yarn lifecycle scripts for untrusted PR scan data" @@ -725,7 +728,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "not a generic model-exhaustion message" "opencode review tells models to return concrete missing-evidence findings instead of progress-only output" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "tokens_limit_reached" "opencode review detects provider context-window overflow" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" - assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" + assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "Incomplete provider evidence is a" "strix wrapper fails closed on quota-only provider failures without vulnerability reports" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$workflow_file" 'timeout-minutes: 325' "opencode review target contains evidence, the bounded long-review pool, publication, Noema handoff, and cleanup overhead" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" From 13c8e177aa9c8847a790f5696bd534d58a620467 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 14:17:05 +0900 Subject: [PATCH 2/2] test(strix): lock provider evidence to fail closed --- .../test_required_workflow_queue_contract.py | 20 ++-- ...est_strix_nvidia_nim_not_found_fallback.py | 106 ++---------------- 2 files changed, 15 insertions(+), 111 deletions(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 535fd513a..b09a740f5 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1092,21 +1092,15 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence -def test_strix_provider_outage_without_findings_is_neutralized() -> None: +def test_strix_provider_outage_without_findings_fails_closed() -> None: workflow = workflow_text("strix.yml") - assert "RateLimitError|Too many requests" in workflow - assert "exceeded your current quota" in workflow - assert "billing details" in workflow - assert "LLM warm-up failed" in workflow - assert "zero_vulnerabilities_signal" not in workflow - assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow - assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow - assert "before producing a vulnerability report" in workflow - assert "genuine findings still fail the check" in workflow - assert ( - '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow - ) + assert "STRIX_FAIL_ON_PROVIDER_SIGNAL: \"1\"" in workflow + assert "Incomplete provider evidence is a" in workflow + assert "incomplete provider evidence; failing the required check" in workflow + assert "backend_unavailable_signal" not in workflow + assert "reported_vulnerability_signal" not in workflow + assert "Treating as a neutral skip" not in workflow def test_strix_cross_repo_dispatch_uses_target_token_for_pr_scoping() -> None: diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..60aefc209 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -73,52 +73,6 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 -def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: - """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" - - match = re.search( - rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", - workflow, - ) - if match is None: - raise AssertionError(f"missing workflow signal: {variable_name}") - return match.group(1) - - -def _workflow_neutralizes(log_text: str) -> bool: - """Execute the outer workflow's backend-neutralization condition.""" - - workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - backend_pattern = _workflow_signal_pattern( - workflow, - "backend_unavailable_signal", - ) - vulnerability_pattern = _workflow_signal_pattern( - workflow, - "reported_vulnerability_signal", - ) - with tempfile.TemporaryDirectory(prefix="strix-workflow-404-") as temp_dir: - log_path = Path(temp_dir) / "strix.log" - log_path.write_text(log_text, encoding="utf-8") - backend = subprocess.run( - ["grep", "-Eiq", backend_pattern, str(log_path)], - check=False, - capture_output=True, - text=True, - ) - vulnerability = subprocess.run( - ["grep", "-Eiq", vulnerability_pattern, str(log_path)], - check=False, - capture_output=True, - text=True, - ) - if backend.returncode not in {0, 1}: - raise AssertionError(backend.stderr) - if vulnerability.returncode not in {0, 1}: - raise AssertionError(vulnerability.stderr) - return backend.returncode == 0 and vulnerability.returncode == 1 - - class StrixNvidiaNotFoundFallbackTests(unittest.TestCase): """Protect provider-scoped 404 fallback without weakening security gates.""" @@ -199,62 +153,18 @@ def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: )[0] self.assertNotIn(RETIRED_PRIMARY_MODEL, default_gate) - def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: - """Reject provider-like target text in the outer neutralization gate.""" - - self.assertFalse( - _workflow_neutralizes( - "source literal: Nvidia_nimException Error code: 404\n" - ) - ) - self.assertTrue( - _workflow_neutralizes( - "litellm.exceptions.NotFoundError: Nvidia_nimException - " - "Error code: 404\nVulnerabilities 0\n" - ) - ) - - def test_outer_workflow_rejects_cross_line_signal_assembly(self) -> None: - """Require exception, provider, and 404 evidence on one physical line.""" - - self.assertFalse( - _workflow_neutralizes( - "litellm.exceptions.NotFoundError: provider unavailable\n" - "Nvidia_nimException Error code: 404\n" - ) - ) - - def test_outer_workflow_rejects_nvidia_404_without_litellm_context(self) -> None: - """Require LiteLLM NotFoundError context, not just NVIDIA + 404.""" - - self.assertFalse( - _workflow_neutralizes( - "Nvidia_nimException Error code: 404\nVulnerabilities 0\n" - ) - ) - - def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: - """Keep a real vulnerability signal blocking despite provider failure.""" - - self.assertFalse( - _workflow_neutralizes( - "litellm.exceptions.NotFoundError: Nvidia_nimException - " - "Error code: 404\nVulnerabilities 1\n" - ) - ) - - def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: - """Retain the static fail-closed vulnerability evidence contract.""" - + def test_outer_workflow_fails_closed_on_provider_evidence_gap(self) -> None: + """Provider 404s cannot become successful outer-workflow evidence.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn("Nvidia_nimException", workflow) - self.assertIn("Error code:[[:space:]]*404", workflow) - self.assertIn("reported_vulnerability_signal", workflow) - self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn('STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"', workflow) + self.assertIn("Incomplete provider evidence is a", workflow) self.assertIn( - '! grep -Eiq "$reported_vulnerability_signal"', + "incomplete provider evidence; failing the required check", workflow, ) + self.assertNotIn("backend_unavailable_signal", workflow) + self.assertNotIn("reported_vulnerability_signal", workflow) + self.assertNotIn("Treating as a neutral skip", workflow) if __name__ == "__main__":