diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index f8c361b95..efa14958a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -844,7 +844,8 @@ jobs: # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI + # token-cap, context-window overflow, connection/warm-up failures, and + # scanner ModelBehaviorError flakes) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -866,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|ContextWindowExceededError|maximum context length|messages resulted in[[:space:]]+[0-9]+[[:space:]]+tokens|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index fd1aebf43..ba43ec7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ Semantic Versioning where the repository publishes a release. ### Added +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as a + backend-unavailable flake so a scanner-model failure does not block the + required check. `Vulnerabilities [1-9]` and a `severity:` finding still + fail closed. + - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md new file mode 100644 index 000000000..9c0233f7e --- /dev/null +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -0,0 +1,21 @@ +# Strix ModelBehaviorError classifier + +Observed required-check flake: Strix exits 1 with `ModelBehaviorError` and +`Vulnerabilities 0` after the scanner model fails to complete a turn. + +The outer `strix.yml` backend-unavailable signal now includes +`ModelBehaviorError`. Neutral skip still requires the absence of +`Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports +any numbered vulnerability stays fail-closed even when the model also +emits `ModelBehaviorError`. + +## Operator action + +If this check repeats, inspect the exact Strix log for the qualified exception +and confirm that no numbered vulnerability or `severity:` signal is present; +any finding remains fail-closed before retrying. + +## References + +OpenAI. (n.d.). *Exceptions*. OpenAI Agents SDK. Retrieved August 19, 2026, +from https://openai.github.io/openai-agents-python/ref/exceptions/ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 535fd513a..6acff89f0 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1099,7 +1099,12 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow + assert "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*" in workflow + assert "ContextWindowExceededError" in workflow + assert "maximum context length" in workflow + assert "messages resulted in[[:space:]]+[0-9]+[[:space:]]+tokens" in workflow assert "zero_vulnerabilities_signal" not in workflow + assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow assert "before producing a vulnerability report" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..c0749011c 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -251,11 +251,73 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*", + workflow, + ) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, ) + def test_outer_workflow_neutralizes_model_behavior_error_without_findings( + self, + ) -> None: + """Require the actual scanner ModelBehaviorError format before neutralizing.""" + + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) + self.assertTrue( + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( + self, + ) -> None: + """Keep Vulnerabilities [1-9] fail-closed for the actual model exception.""" + + self.assertFalse( + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 1\n" + ) + ) + self.assertFalse( + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 9\n" + ) + ) + + def test_outer_workflow_neutralizes_context_window_overflow_without_findings( + self, + ) -> None: + """Treat a provider context overflow as backend failure, not a finding.""" + + self.assertTrue( + _workflow_neutralizes( + "openai.BadRequestError: maximum context length is 1000000 " + "tokens; messages resulted in 1438805 tokens\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_context_overflow_with_findings( + self, + ) -> None: + """Keep context overflow plus a reported vulnerability fail-closed.""" + + self.assertFalse( + _workflow_neutralizes( + "openai.BadRequestError: ContextWindowExceededError\n" + "Vulnerabilities 1\n" + ) + ) + if __name__ == "__main__": unittest.main()