From 1cdc02b6479c9c9487674c54692fc301a37604db Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 09:17:32 +0000 Subject: [PATCH] fix(strix): classify ModelBehaviorError as backend-unavailable Treat a trusted same-line LiteLLM or agents-SDK ModelBehaviorError with Vulnerabilities 0 as infrastructure, matching job 95148793283. Real Vulnerabilities [1-9] findings stay fail-closed. --- .github/workflows/strix.yml | 2 +- AGENTS.md | 1 + CHANGELOG.md | 5 + .../strix-model-behavior-error-fallback.md | 60 +++++ scripts/ci/strix_quick_gate.sh | 30 ++- scripts/ci/strix_required_workflow_smoke.sh | 2 + ...t_strix_model_behavior_error_is_neutral.py | 208 ++++++++++++++++++ 7 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-model-behavior-error-fallback.md create mode 100644 tests/test_strix_model_behavior_error_is_neutral.py diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 514fd8a44..52677debf 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -876,7 +876,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|Error code:[[:space:]]*410|github_models_retirement_brownout|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|Error code:[[:space:]]*410|github_models_retirement_brownout|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|litellm(\.exceptions)?\.[A-Za-z]+Error[^[:cntrl:]]*ModelBehaviorError|agents\.exceptions\.ModelBehaviorError' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/AGENTS.md b/AGENTS.md index 4e906c47c..b62cf3fd1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,6 +9,7 @@ commit and exposed without running build hooks; a lone `--require-hashes` directive is not trust evidence. See [`docs/doctoring/opencode-exact-vcs-dependency-evidence.md`](docs/doctoring/opencode-exact-vcs-dependency-evidence.md). Conflict-scope roots fail closed when the immediate parent directory is a symbolic link. +Strix classifies a trusted same-line LiteLLM or agents-SDK `ModelBehaviorError` as backend-unavailable only when the log has no `Vulnerabilities [1-9]`. See [`docs/doctoring/strix-model-behavior-error-fallback.md`](docs/doctoring/strix-model-behavior-error-fallback.md). OriginWeave hourly NVIDIA NIM repair is a thin caller at minute 10. See [`docs/doctoring/originweave-hourly-review-caller.md`](docs/doctoring/originweave-hourly-review-caller.md). nonnest2 hourly NVIDIA NIM repair is a thin caller at minute 16. See [`docs/doctoring/nonnest2-hourly-review-caller.md`](docs/doctoring/nonnest2-hourly-review-caller.md). diff --git a/CHANGELOG.md b/CHANGELOG.md index 438bc01b5..5dd02ec5b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Classify a trusted same-line LiteLLM or OpenAI Agents SDK + `ModelBehaviorError` as Strix backend-unavailable infrastructure when the + scan reported no `Vulnerabilities [1-9]`, so the job 95148793283-shaped + flake becomes a neutral skip without weakening fail-closed findings. + - Honor each trusted base project's exact, integrity-bearing pnpm `packageManager` specification in OpenCode coverage images through the pinned Node distribution's Corepack runtime, instead of admitting the specification diff --git a/docs/doctoring/strix-model-behavior-error-fallback.md b/docs/doctoring/strix-model-behavior-error-fallback.md new file mode 100644 index 000000000..a57f50369 --- /dev/null +++ b/docs/doctoring/strix-model-behavior-error-fallback.md @@ -0,0 +1,60 @@ +# Strix ModelBehaviorError classifier: evidence and design record + +## Decision + +Strix treats a trusted LiteLLM or OpenAI Agents SDK `ModelBehaviorError` as +provider/runtime abort evidence, not as a target-application vulnerability. +When the scan log also contains no `Vulnerabilities [1-9]` signal, the +required check may classify the run as a neutral skip so a malformed tool +call cannot block an otherwise empty scan. + +This classifier exists because LineageWeave PR #74 job 95148793283 printed +`Vulnerabilities 0` and then failed closed on `ModelBehaviorError`. That +abort is an agents-SDK reaction to an invalid tool call, not a finding. + +## Trust boundary + +The classifier accepts only a single bounded log line that contains both: + +1. a trusted SDK/provider marker (`litellm.exceptions.Error` or + `agents.exceptions.ModelBehaviorError`); and +2. the `ModelBehaviorError` exception name. + +It does not assemble those signals from different lines. A source literal +that merely mentions `ModelBehaviorError`, an application traceback without +the SDK marker, or any `Vulnerabilities [1-9]` / `severity:` finding remains +blocking. Real findings are never downgraded. + +The gate also routes the same classifier into infrastructure detection, +cross-model fallback, and same-model retry. A catalog 404 is not retried on +the same model; a `ModelBehaviorError` may be, because the next sample from +the same model can emit a valid tool call. + +## Verification contract + +Regression evidence proves that: + +1. the agents-SDK abort observed in required CI is recognized; +2. a LiteLLM-wrapped `APIError: ModelBehaviorError` is recognized; +3. a source literal without the SDK marker is not recognized; +4. provider context and `ModelBehaviorError` on different lines are not + recognized; +5. `Vulnerabilities [1-9]` prevents neutralization; +6. the classifier is wired into infrastructure, retry, and same-model retry; + and +7. the required-workflow smoke contract pins the workflow and gate strings. + +## Limitations + +This change does not treat arbitrary model errors as success. It does not +weaken Strix severity, changed-file attribution, incomplete-scan fail-closed +behavior, or independent approval requirements. NVIDIA NIM catalog 404s stay +on their own same-line classifier. + +## References + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC +9110). Internet Engineering Task Force. https://doi.org/10.17487/RFC9110 + +OpenAI. (2026). *Agents SDK exceptions*. OpenAI Agents SDK. +https://openai.github.io/openai-agents-python/ref/exceptions/ diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index ed434ff91..8f545c977 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2664,14 +2664,31 @@ is_nvidia_nim_not_found_error() { return 1 } +is_model_behavior_error() { + # Classify only one bounded LiteLLM or OpenAI Agents SDK error line + # that names ModelBehaviorError. Observed in LineageWeave PR #74 + # (job 95148793283): the scan printed Vulnerabilities 0, then the + # agents SDK aborted on a malformed tool call. Target-source text + # that merely mentions ModelBehaviorError, or splits the SDK marker + # and the exception name across lines, stays non-retryable so a + # real finding cannot spoof infrastructure fallback. + if grep -Ei '(litellm(\.exceptions)?\.[A-Za-z]+Error|agents\.exceptions\.ModelBehaviorError)' "$STRIX_LOG" | + grep -Eiq 'ModelBehaviorError'; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). -## Five error families qualify: +## Six error families qualify: ## - RateLimit / RESOURCE_EXHAUSTED / HTTP 429 ## - litellm API connection failures with LLM-provider evidence ## - litellm service-unavailable / high-demand provider failures ## - MidStreamFallbackError (litellm mid-stream provider switch) ## - Caido bootstrap timing failures (guest login before the local proxy is up) +## - ModelBehaviorError (agents SDK / LiteLLM malformed-tool-call abort) ## Timeouts are infrastructure failures. In strict CI mode they fail closed; ## otherwise the caller may still move to fallback model evaluation. is_transient_same_model_retry_error() { @@ -2694,6 +2711,9 @@ is_transient_same_model_retry_error() { if is_caido_bootstrap_timing_error; then return 0 fi + if is_model_behavior_error; then + return 0 + fi return 1 } @@ -3003,6 +3023,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_caido_bootstrap_timing_error; then return 0 fi @@ -3857,6 +3881,10 @@ is_model_retryable_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index d56de5a02..8a92937c0 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -160,6 +160,8 @@ assert_file_contains "$workflow_file" "nvidia_nim/nvidia/llama-3.3-nemotron-supe assert_file_not_contains "$workflow_file" "github_models/openai/o3" "Strix fallback list must not depend on GitHub Models, which is in platform-wide retirement" assert_file_contains "$workflow_file" "Nvidia_nimException" "Strix workflow recognizes provider-scoped NVIDIA NIM failures" assert_file_contains "$gate_script" "is_nvidia_nim_not_found_error" "Strix gate classifies NVIDIA NIM model-catalog 404s" +assert_file_contains "$workflow_file" "ModelBehaviorError" "Strix workflow recognizes ModelBehaviorError as backend-unavailable" +assert_file_contains "$gate_script" "is_model_behavior_error" "Strix gate classifies ModelBehaviorError infrastructure flakes" if [ "$failures" -ne 0 ]; then echo "Strix required workflow smoke test failed with $failures failure(s)." >&2 diff --git a/tests/test_strix_model_behavior_error_is_neutral.py b/tests/test_strix_model_behavior_error_is_neutral.py new file mode 100644 index 000000000..8ed6d7c89 --- /dev/null +++ b/tests/test_strix_model_behavior_error_is_neutral.py @@ -0,0 +1,208 @@ +"""Regression contract for Strix ModelBehaviorError infrastructure flakes. + +Strix runs through the OpenAI Agents SDK. When a model emits a malformed +tool call the SDK raises ``agents.exceptions.ModelBehaviorError`` and the +scan aborts. LiteLLM wraps the same abort as +``litellm.exceptions.APIError: ... ModelBehaviorError``. Job 95148793283 +on LineageWeave PR #74 printed ``Vulnerabilities 0`` then failed the +required check because the outer workflow did not classify that abort as +backend-unavailable. + +This contract keeps that flake as a neutral skip when -- and only when -- +a trusted LiteLLM or agents SDK marker shares a physical log line with +``ModelBehaviorError`` and no ``Vulnerabilities [1-9]`` signal is present. +Target-source text, cross-line assembly, and real findings stay fail-closed. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" + +AGENTS_SDK_MODEL_BEHAVIOR_ERROR = ( + "Model nvidia_nim/nvidia/nemotron-3-super-120b-a12b\n" + "Vulnerabilities 0\n" + "agents.exceptions.ModelBehaviorError: Tool create_vulnerability_report " + "not found in agent strix\n" + "Strix run failed for model 'nvidia_nim/nvidia/nemotron-3-super-120b-a12b' " + "after 41s (exit code 1).\n" + "Strix scan failed after provider infrastructure or failure-signal " + "output; failing closed.\n" +) + +LITELLM_WRAPPED_MODEL_BEHAVIOR_ERROR = ( + "litellm.exceptions.APIError: ModelBehaviorError - invalid tool call\n" + "Vulnerabilities 0\n" +) + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_model_behavior_error(log_text: str) -> bool: + """Execute the production classifier against a bounded synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_behavior_error") + with tempfile.TemporaryDirectory(prefix="strix-model-behavior-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_model_behavior_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern(workflow, "backend_unavailable_signal") + vulnerability_pattern = _workflow_signal_pattern( + workflow, "reported_vulnerability_signal" + ) + with tempfile.TemporaryDirectory(prefix="strix-model-behavior-wf-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixModelBehaviorErrorIsNeutralTests(unittest.TestCase): + """Protect the PR #74-shaped ModelBehaviorError flake without weakening the gate.""" + + def test_agents_sdk_model_behavior_error_is_classified(self) -> None: + self.assertTrue(_classifies_as_model_behavior_error(AGENTS_SDK_MODEL_BEHAVIOR_ERROR)) + + def test_litellm_wrapped_model_behavior_error_is_classified(self) -> None: + self.assertTrue( + _classifies_as_model_behavior_error(LITELLM_WRAPPED_MODEL_BEHAVIOR_ERROR) + ) + + def test_source_literal_without_trusted_sdk_context_is_not_classified(self) -> None: + log = "source literal: ModelBehaviorError\nVulnerabilities 0\n" + self.assertFalse(_classifies_as_model_behavior_error(log)) + + def test_provider_and_exception_must_share_one_log_line(self) -> None: + log = ( + "litellm.exceptions.APIError: provider unavailable\n" + "ModelBehaviorError\n" + "Vulnerabilities 0\n" + ) + self.assertFalse(_classifies_as_model_behavior_error(log)) + + def test_classifier_is_wired_into_infrastructure_retry_and_same_model_retry(self) -> None: + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block(gate_source, "has_detected_infrastructure_error") + retryable = _function_block(gate_source, "is_model_retryable_error") + same_model_retry = _function_block( + gate_source, "is_transient_same_model_retry_error" + ) + + self.assertIn("is_model_behavior_error", infrastructure) + self.assertIn("is_model_behavior_error", retryable) + self.assertIn("is_model_behavior_error", same_model_retry) + + def test_workflow_neutralizes_agents_sdk_abort_with_zero_findings(self) -> None: + self.assertTrue(_workflow_neutralizes(AGENTS_SDK_MODEL_BEHAVIOR_ERROR)) + + def test_workflow_neutralizes_litellm_wrapped_abort_with_zero_findings(self) -> None: + self.assertTrue(_workflow_neutralizes(LITELLM_WRAPPED_MODEL_BEHAVIOR_ERROR)) + + def test_workflow_rejects_source_literal_without_trusted_context(self) -> None: + self.assertFalse( + _workflow_neutralizes("source literal: ModelBehaviorError\nVulnerabilities 0\n") + ) + + def test_workflow_rejects_cross_line_signal_assembly(self) -> None: + self.assertFalse( + _workflow_neutralizes( + "litellm.exceptions.APIError: provider unavailable\n" + "ModelBehaviorError\n" + "Vulnerabilities 0\n" + ) + ) + + def test_still_fails_closed_when_a_real_vulnerability_is_also_reported(self) -> None: + log = AGENTS_SDK_MODEL_BEHAVIOR_ERROR + ( + "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" + ) + self.assertFalse(_workflow_neutralizes(log)) + self.assertFalse( + _workflow_neutralizes( + "litellm.exceptions.APIError: ModelBehaviorError - invalid tool call\n" + "Vulnerabilities 1\n" + ) + ) + + def test_workflow_keeps_fail_closed_vulnerability_evidence_contract(self) -> None: + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("ModelBehaviorError", workflow) + self.assertIn(r"agents\.exceptions\.ModelBehaviorError", workflow) + self.assertIn("reported_vulnerability_signal", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + '! grep -Eiq "$reported_vulnerability_signal"', + workflow, + ) + + +if __name__ == "__main__": + unittest.main()