From 95984a6376fdb145582d9e695c81b08c45a99b66 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 04:19:22 +0000 Subject: [PATCH 1/3] fix(strix): treat ModelBehaviorError with zero findings as backend flake Keep Vulnerabilities [1-9] fail-closed. A scanner-model error without a numbered finding is infrastructure noise, not a security result. --- .github/workflows/strix.yml | 5 +++-- CHANGELOG.md | 7 ++++++ .../strix-modelbehaviorerror-classifier.md | 10 +++++++++ .../test_required_workflow_queue_contract.py | 2 ++ ...est_strix_nvidia_nim_not_found_fallback.py | 22 +++++++++++++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-modelbehaviorerror-classifier.md diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index f8c361b95..7e440b029 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -844,7 +844,8 @@ jobs: # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI + # token-cap, connection/warm-up failures, and scanner ModelBehaviorError + # flakes) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -866,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|ModelBehaviorError' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bf8ad766..991dc51b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Semantic Versioning where the repository publishes a release. ### Added +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as a + backend-unavailable flake so a scanner-model failure does not block the + required check. `Vulnerabilities [1-9]` and a `severity:` finding still + fail closed. + +### Added + - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated OriginWeave hourly caller that invokes the product-neutral central scheduler with the exact repository, protected `main` branch, one-dispatch budget, two-hour same-head retry floor, non-cancelling single-flight heartbeat, job-scoped OIDC, and only the established scheduler credentials. - Added a trusted pull-request comment router for `@cwl-noema-review` and review-only `@opencode-agent` dispatches, with an organization sweep, exact-head receipts, repository allowlisting, fixed runners, immutable checkout pins, and a permanent 100% statement/branch/docstring quality gate. diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md new file mode 100644 index 000000000..938bb1efb --- /dev/null +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -0,0 +1,10 @@ +# Strix ModelBehaviorError classifier + +Observed required-check flake: Strix exits 1 with `ModelBehaviorError` and +`Vulnerabilities 0` after the scanner model fails to complete a turn. + +The outer `strix.yml` backend-unavailable signal now includes +`ModelBehaviorError`. Neutral skip still requires the absence of +`Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports +any numbered vulnerability stays fail-closed even when the model also +emits `ModelBehaviorError`. diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 233c08584..d8fff16b6 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1086,7 +1086,9 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow + assert "ModelBehaviorError" in workflow assert "zero_vulnerabilities_signal" not in workflow + assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow assert "before producing a vulnerability report" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..dbfe7dcb5 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -251,11 +251,33 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn("ModelBehaviorError", workflow) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, ) + def test_outer_workflow_neutralizes_model_behavior_error_without_findings( + self, + ) -> None: + """Treat a scanner ModelBehaviorError with zero findings as backend flake.""" + + self.assertTrue( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) + + def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( + self, + ) -> None: + """Keep Vulnerabilities [1-9] fail-closed even when the model also errored.""" + + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 1\n") + ) + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 9\n") + ) + if __name__ == "__main__": unittest.main() From dd2eb91e43f574a7a087412485be7c4a7f245bf0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 16:35:43 +0900 Subject: [PATCH 2/3] fix(strix): qualify model behavior backend signal --- .github/workflows/strix.yml | 2 +- CHANGELOG.md | 2 -- .../strix-modelbehaviorerror-classifier.md | 11 ++++++++ .../test_required_workflow_queue_contract.py | 2 +- ...est_strix_nvidia_nim_not_found_fallback.py | 27 ++++++++++++++----- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 7e440b029..a07f7f215 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -867,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|ModelBehaviorError' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index 991dc51b8..3dbc47ffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,6 @@ Semantic Versioning where the repository publishes a release. required check. `Vulnerabilities [1-9]` and a `severity:` finding still fail closed. -### Added - - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated OriginWeave hourly caller that invokes the product-neutral central scheduler with the exact repository, protected `main` branch, one-dispatch budget, two-hour same-head retry floor, non-cancelling single-flight heartbeat, job-scoped OIDC, and only the established scheduler credentials. - Added a trusted pull-request comment router for `@cwl-noema-review` and review-only `@opencode-agent` dispatches, with an organization sweep, exact-head receipts, repository allowlisting, fixed runners, immutable checkout pins, and a permanent 100% statement/branch/docstring quality gate. diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md index 938bb1efb..9c0233f7e 100644 --- a/docs/doctoring/strix-modelbehaviorerror-classifier.md +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -8,3 +8,14 @@ The outer `strix.yml` backend-unavailable signal now includes `Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports any numbered vulnerability stays fail-closed even when the model also emits `ModelBehaviorError`. + +## Operator action + +If this check repeats, inspect the exact Strix log for the qualified exception +and confirm that no numbered vulnerability or `severity:` signal is present; +any finding remains fail-closed before retrying. + +## References + +OpenAI. (n.d.). *Exceptions*. OpenAI Agents SDK. Retrieved August 19, 2026, +from https://openai.github.io/openai-agents-python/ref/exceptions/ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index d8fff16b6..c6ffec8b4 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1086,7 +1086,7 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow - assert "ModelBehaviorError" in workflow + assert "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*" in workflow assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index dbfe7dcb5..c3e931986 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -251,7 +251,10 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) - self.assertIn("ModelBehaviorError", workflow) + self.assertIn( + "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*", + workflow, + ) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, @@ -260,22 +263,34 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: def test_outer_workflow_neutralizes_model_behavior_error_without_findings( self, ) -> None: - """Treat a scanner ModelBehaviorError with zero findings as backend flake.""" + """Require the actual scanner ModelBehaviorError format before neutralizing.""" - self.assertTrue( + self.assertFalse( _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") ) + self.assertTrue( + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 0\n" + ) + ) def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( self, ) -> None: - """Keep Vulnerabilities [1-9] fail-closed even when the model also errored.""" + """Keep Vulnerabilities [1-9] fail-closed for the actual model exception.""" self.assertFalse( - _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 1\n") + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 1\n" + ) ) self.assertFalse( - _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 9\n") + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 9\n" + ) ) From 1f4f5e0968852e453918a1c11af8e0870434739d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 19:50:50 +0900 Subject: [PATCH 3/3] fix(strix): neutralize provider context-window overflow --- .github/workflows/strix.yml | 6 ++--- .../test_required_workflow_queue_contract.py | 3 +++ ...est_strix_nvidia_nim_not_found_fallback.py | 25 +++++++++++++++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a07f7f215..efa14958a 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -844,8 +844,8 @@ jobs: # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures, and scanner ModelBehaviorError - # flakes) that could not complete a scan. A backend outage is CI + # token-cap, context-window overflow, connection/warm-up failures, and + # scanner ModelBehaviorError flakes) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -867,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|ContextWindowExceededError|maximum context length|messages resulted in[[:space:]]+[0-9]+[[:space:]]+tokens|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index c6ffec8b4..04c09299c 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1087,6 +1087,9 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "billing details" in workflow assert "LLM warm-up failed" in workflow assert "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*" in workflow + assert "ContextWindowExceededError" in workflow + assert "maximum context length" in workflow + assert "messages resulted in[[:space:]]+[0-9]+[[:space:]]+tokens" in workflow assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index c3e931986..c0749011c 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -293,6 +293,31 @@ def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( ) ) + def test_outer_workflow_neutralizes_context_window_overflow_without_findings( + self, + ) -> None: + """Treat a provider context overflow as backend failure, not a finding.""" + + self.assertTrue( + _workflow_neutralizes( + "openai.BadRequestError: maximum context length is 1000000 " + "tokens; messages resulted in 1438805 tokens\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_context_overflow_with_findings( + self, + ) -> None: + """Keep context overflow plus a reported vulnerability fail-closed.""" + + self.assertFalse( + _workflow_neutralizes( + "openai.BadRequestError: ContextWindowExceededError\n" + "Vulnerabilities 1\n" + ) + ) + if __name__ == "__main__": unittest.main()