From 4e61b6ad53b3df971f6d51d8d60a7f1e2e9b3200 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 04:19:22 +0000 Subject: [PATCH 01/37] fix(strix): treat ModelBehaviorError with zero findings as backend flake Keep Vulnerabilities [1-9] fail-closed. A scanner-model error without a numbered finding is infrastructure noise, not a security result. --- .github/workflows/strix.yml | 5 +++-- CHANGELOG.md | 7 ++++++ .../strix-modelbehaviorerror-classifier.md | 10 +++++++++ .../test_required_workflow_queue_contract.py | 2 ++ ...est_strix_nvidia_nim_not_found_fallback.py | 22 +++++++++++++++++++ 5 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-modelbehaviorerror-classifier.md diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index f8c361b95..7e440b029 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -844,7 +844,8 @@ jobs: # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI + # token-cap, connection/warm-up failures, and scanner ModelBehaviorError + # flakes) that could not complete a scan. A backend outage is CI # infrastructure noise, not a security finding, so it must not fail # the required check and block merges. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" @@ -866,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|ModelBehaviorError' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index fd1aebf43..ff5896943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ Semantic Versioning where the repository publishes a release. ### Added +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as a + backend-unavailable flake so a scanner-model failure does not block the + required check. `Vulnerabilities [1-9]` and a `severity:` finding still + fail closed. + +### Added + - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md new file mode 100644 index 000000000..938bb1efb --- /dev/null +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -0,0 +1,10 @@ +# Strix ModelBehaviorError classifier + +Observed required-check flake: Strix exits 1 with `ModelBehaviorError` and +`Vulnerabilities 0` after the scanner model fails to complete a turn. + +The outer `strix.yml` backend-unavailable signal now includes +`ModelBehaviorError`. Neutral skip still requires the absence of +`Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports +any numbered vulnerability stays fail-closed even when the model also +emits `ModelBehaviorError`. diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 535fd513a..0691d4a90 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1099,7 +1099,9 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow + assert "ModelBehaviorError" in workflow assert "zero_vulnerabilities_signal" not in workflow + assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow assert "before producing a vulnerability report" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..dbfe7dcb5 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -251,11 +251,33 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn("ModelBehaviorError", workflow) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, ) + def test_outer_workflow_neutralizes_model_behavior_error_without_findings( + self, + ) -> None: + """Treat a scanner ModelBehaviorError with zero findings as backend flake.""" + + self.assertTrue( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) + + def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( + self, + ) -> None: + """Keep Vulnerabilities [1-9] fail-closed even when the model also errored.""" + + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 1\n") + ) + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 9\n") + ) + if __name__ == "__main__": unittest.main() From 51c38152a1950f6b815b36b5e37bb804f4ef1475 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 16:35:43 +0900 Subject: [PATCH 02/37] fix(strix): qualify model behavior backend signal --- .github/workflows/strix.yml | 2 +- CHANGELOG.md | 2 -- .../strix-modelbehaviorerror-classifier.md | 11 ++++++++ .../test_required_workflow_queue_contract.py | 2 +- ...est_strix_nvidia_nim_not_found_fallback.py | 27 ++++++++++++++----- 5 files changed, 34 insertions(+), 10 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 7e440b029..a07f7f215 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -867,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|ModelBehaviorError' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index ff5896943..ba43ec7ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,8 +13,6 @@ Semantic Versioning where the repository publishes a release. required check. `Vulnerabilities [1-9]` and a `severity:` finding still fail closed. -### Added - - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md index 938bb1efb..9c0233f7e 100644 --- a/docs/doctoring/strix-modelbehaviorerror-classifier.md +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -8,3 +8,14 @@ The outer `strix.yml` backend-unavailable signal now includes `Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports any numbered vulnerability stays fail-closed even when the model also emits `ModelBehaviorError`. + +## Operator action + +If this check repeats, inspect the exact Strix log for the qualified exception +and confirm that no numbered vulnerability or `severity:` signal is present; +any finding remains fail-closed before retrying. + +## References + +OpenAI. (n.d.). *Exceptions*. OpenAI Agents SDK. Retrieved August 19, 2026, +from https://openai.github.io/openai-agents-python/ref/exceptions/ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 0691d4a90..aabf776cc 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1099,7 +1099,7 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow - assert "ModelBehaviorError" in workflow + assert "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*" in workflow assert "zero_vulnerabilities_signal" not in workflow assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index dbfe7dcb5..c3e931986 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -251,7 +251,10 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) - self.assertIn("ModelBehaviorError", workflow) + self.assertIn( + "agents\\.exceptions\\.ModelBehaviorError:[[:space:]]*", + workflow, + ) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, @@ -260,22 +263,34 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: def test_outer_workflow_neutralizes_model_behavior_error_without_findings( self, ) -> None: - """Treat a scanner ModelBehaviorError with zero findings as backend flake.""" + """Require the actual scanner ModelBehaviorError format before neutralizing.""" - self.assertTrue( + self.assertFalse( _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") ) + self.assertTrue( + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 0\n" + ) + ) def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( self, ) -> None: - """Keep Vulnerabilities [1-9] fail-closed even when the model also errored.""" + """Keep Vulnerabilities [1-9] fail-closed for the actual model exception.""" self.assertFalse( - _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 1\n") + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 1\n" + ) ) self.assertFalse( - _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 9\n") + _workflow_neutralizes( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 9\n" + ) ) From 5512077793c436c8133f621cfa56bbc303a906b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:52:21 -0700 Subject: [PATCH 03/37] test(strix): require provider failures to stay non-passing --- ...est_strix_nvidia_nim_not_found_fallback.py | 33 ++++++++++--------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index c3e931986..fae189578 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -85,7 +85,7 @@ def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: return match.group(1) -def _workflow_neutralizes(log_text: str) -> bool: +def _workflow_classifies_backend_unavailable(log_text: str) -> bool: """Execute the outer workflow's backend-neutralization condition.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") @@ -203,12 +203,12 @@ def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: """Reject provider-like target text in the outer neutralization gate.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "source literal: Nvidia_nimException Error code: 404\n" ) ) self.assertTrue( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: Nvidia_nimException - " "Error code: 404\nVulnerabilities 0\n" ) @@ -218,7 +218,7 @@ def test_outer_workflow_rejects_cross_line_signal_assembly(self) -> None: """Require exception, provider, and 404 evidence on one physical line.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: provider unavailable\n" "Nvidia_nimException Error code: 404\n" ) @@ -228,22 +228,22 @@ def test_outer_workflow_rejects_nvidia_404_without_litellm_context(self) -> None """Require LiteLLM NotFoundError context, not just NVIDIA + 404.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "Nvidia_nimException Error code: 404\nVulnerabilities 0\n" ) ) - def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + def test_outer_workflow_never_classifies_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite provider failure.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: Nvidia_nimException - " "Error code: 404\nVulnerabilities 1\n" ) ) - def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: + def test_workflow_classifies_backend_unavailable_only_nvidia_404_without_findings(self) -> None: """Retain the static fail-closed vulnerability evidence contract.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") @@ -259,35 +259,38 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: '! grep -Eiq "$reported_vulnerability_signal"', workflow, ) + self.assertIn("::error title=STRIX_PROVIDER_UNAVAILABLE::", workflow) + self.assertIn('exit "$strix_rc"', workflow) + self.assertNotIn("Treating as a neutral skip", workflow) - def test_outer_workflow_neutralizes_model_behavior_error_without_findings( + def test_outer_workflow_classifies_backend_unavailable_model_behavior_error_without_findings( self, ) -> None: - """Require the actual scanner ModelBehaviorError format before neutralizing.""" + """Require the actual scanner ModelBehaviorError format before classifying.""" self.assertFalse( - _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + _workflow_classifies_backend_unavailable("ModelBehaviorError\nVulnerabilities 0\n") ) self.assertTrue( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "agents.exceptions.ModelBehaviorError: provider response failed\n" "Vulnerabilities 0\n" ) ) - def test_outer_workflow_never_neutralizes_model_behavior_error_with_findings( + def test_outer_workflow_never_classifies_model_behavior_error_with_findings( self, ) -> None: """Keep Vulnerabilities [1-9] fail-closed for the actual model exception.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "agents.exceptions.ModelBehaviorError: provider response failed\n" "Vulnerabilities 1\n" ) ) self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "agents.exceptions.ModelBehaviorError: provider response failed\n" "Vulnerabilities 9\n" ) From 85523e166f03b0552f6dfa0b0f65df10874d4e60 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:52:37 -0700 Subject: [PATCH 04/37] test(strix): enforce typed provider non-pass --- tests/test_required_workflow_queue_contract.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index aabf776cc..a06ff211c 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1092,7 +1092,7 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence -def test_strix_provider_outage_without_findings_is_neutralized() -> None: +def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: workflow = workflow_text("strix.yml") assert "RateLimitError|Too many requests" in workflow @@ -1104,6 +1104,9 @@ def test_strix_provider_outage_without_findings_is_neutralized() -> None: assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow + assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow + assert 'exit "$strix_rc"' in workflow + assert "Treating as a neutral skip" not in workflow assert "before producing a vulnerability report" in workflow assert "genuine findings still fail the check" in workflow assert ( From 6e20dc1f4fec482b69a73dd154855afdf8f04157 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:13 -0700 Subject: [PATCH 05/37] fix(strix): fail closed on incomplete provider scans --- .github/workflows/strix.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a07f7f215..a68bd6b9e 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -843,11 +843,11 @@ jobs: # Capture the gate exit code plus its console output. The gate returns # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures, and scanner ModelBehaviorError - # flakes) that could not complete a scan. A backend outage is CI - # infrastructure noise, not a security finding, so it must not fail - # the required check and block merges. + # rate limits, OpenAI quota starvation, 413 tokens_limit_reached, + # connection/warm-up failures, and scanner ModelBehaviorError) that + # could not complete a scan. Provider failure is typed infrastructure + # evidence, but remains non-passing because no authoritative complete + # vulnerability result exists. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" strix_rc=0 set +e @@ -874,14 +874,13 @@ jobs: # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported anywhere. This preserves - # real security gating while keeping uncontrollable provider outages - # from blocking current-head merge progress. + # Classify provider/backend exhaustion only when no vulnerability + # finding was emitted. Classification improves diagnosis; it never + # converts an incomplete scan into passing security evidence. if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." - exit 0 + echo "::error title=STRIX_PROVIDER_UNAVAILABLE::Strix could not complete authoritative vulnerability analysis because its provider/backend was unavailable (rate limit, token cap, connection, warm-up, or model-behavior failure). See the strix-reports artifact and run log." + exit "$strix_rc" fi echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2 From e6529aa5edee04b939c69e5de7b4415607060a57 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:28 -0700 Subject: [PATCH 06/37] docs(strix): record typed provider non-pass --- .../strix-modelbehaviorerror-classifier.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md index 9c0233f7e..e59028528 100644 --- a/docs/doctoring/strix-modelbehaviorerror-classifier.md +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -3,17 +3,19 @@ Observed required-check flake: Strix exits 1 with `ModelBehaviorError` and `Vulnerabilities 0` after the scanner model fails to complete a turn. -The outer `strix.yml` backend-unavailable signal now includes -`ModelBehaviorError`. Neutral skip still requires the absence of -`Vulnerabilities [1-9]` and of a `severity:` finding. A scan that reports -any numbered vulnerability stays fail-closed even when the model also -emits `ModelBehaviorError`. +The outer `strix.yml` backend-unavailable signal includes +`ModelBehaviorError` so operators receive a typed +`STRIX_PROVIDER_UNAVAILABLE` result when no vulnerability finding was emitted. +The workflow preserves the scanner's nonzero result: an incomplete provider +turn is never passing security evidence. A scan that reports any numbered +vulnerability also stays fail-closed. ## Operator action -If this check repeats, inspect the exact Strix log for the qualified exception -and confirm that no numbered vulnerability or `severity:` signal is present; -any finding remains fail-closed before retrying. +If this check repeats, inspect the exact Strix log and artifact for the +qualified exception. Retry provider execution without changing source +classification; both incomplete analysis and any reported finding remain +non-passing. ## References From a06e6e7c6e6debd8a2a61e98cf15fc49ae7ca6b2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:55:39 -0700 Subject: [PATCH 07/37] docs(changelog): record Strix fail-closed provider result --- CHANGELOG.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba43ec7ea..332d8f71e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,9 @@ Semantic Versioning where the repository publishes a release. ### Added -- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as a - backend-unavailable flake so a scanner-model failure does not block the - required check. `Vulnerabilities [1-9]` and a `severity:` finding still - fail closed. +- Classify Strix `ModelBehaviorError` and provider exhaustion as typed + `STRIX_PROVIDER_UNAVAILABLE` evidence while preserving a nonzero required + check. Incomplete scans and reported vulnerabilities both fail closed. - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. From 86c262cd9385bc9b39cf9b93fabb081f3f07bbc5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 21:58:02 -0700 Subject: [PATCH 08/37] test(strix): bind typed provider error wording --- tests/test_required_workflow_queue_contract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index a06ff211c..fafd9e7bd 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1107,8 +1107,8 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow assert 'exit "$strix_rc"' in workflow assert "Treating as a neutral skip" not in workflow - assert "before producing a vulnerability report" in workflow - assert "genuine findings still fail the check" in workflow + assert "authoritative vulnerability analysis" in workflow + assert "incomplete scan into passing security evidence" in workflow assert ( '&& ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"' in workflow ) From 119d563c1abecdb3a820fbca8c161182efab2d99 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 17:13:15 +0900 Subject: [PATCH 09/37] fix(strix): keep PR scoped to provider scan contract --- ARCHITECTURE.md | 2 + CLAUDE.md | 7 +- PR_GOVERNANCE_AUDIT.md | 6 + README.md | 306 +++++++++++++------------- docs/pr-review-and-merge-procedure.md | 209 ++++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 43 +++- 6 files changed, 416 insertions(+), 157 deletions(-) create mode 100644 docs/pr-review-and-merge-procedure.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3e2e70b58..7d2bfb4a4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -118,6 +118,8 @@ trusted `uv` exporter is downloaded from the literal GitHub Releases URL for ecosystem. - [`docs/agent-github-project-protocol.md`](docs/agent-github-project-protocol.md) — Project #1 operation. +- [`docs/pr-review-and-merge-procedure.md`](docs/pr-review-and-merge-procedure.md) + — bot/agent exact-head review and merge procedure. - [`PR_GOVERNANCE_AUDIT.md`](PR_GOVERNANCE_AUDIT.md) — live review/merge contract. - [`docs/doctoring/hourly-nvidia-nim-autofix.md`](docs/doctoring/hourly-nvidia-nim-autofix.md) diff --git a/CLAUDE.md b/CLAUDE.md index d73a5c169..6ec3d494c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,8 +25,9 @@ This is the ContextualWisdomLab **organization-wide `.github` special repository An organization required-workflow ruleset (`CWL Central required workflows`, id `18156473`) runs Strix, OpenCode Review, and the PR Review Merge Scheduler from this repo in each target repository's context. Repository-local copies of these workflows are drift sources, not - repo-specific contracts. See `README.md` (policy summary) and `PR_GOVERNANCE_AUDIT.md` - (live audit + per-repo DX/UX transfer decisions). + repo-specific contracts. See `README.md` (operator overview), + `docs/pr-review-and-merge-procedure.md` (bot/agent procedure), and + `PR_GOVERNANCE_AUDIT.md` (live audit + per-repo DX/UX transfer decisions). 3. **Infrastructure as code** — `infra/cloudflare/` manages the org's DNS zones and Cloudflare Pages hosting declaratively (`zones.json` + `reconcile.sh`, curl + jq only; dry-run by default, writes only on explicit manual `mode = apply`). @@ -44,7 +45,7 @@ PRs repair guidance. A separate edit-capable autofix flow approved same-repository-head PR, merge the base into the head and resolve the conflict markers; the resulting head is fully re-reviewed and re-checked before it can merge, so a wrong resolution cannot merge unreviewed. Old approvals and old checks are not merge evidence after the head SHA changes. -Details: `README.md` and `PR_GOVERNANCE_AUDIT.md`. +Details: `docs/pr-review-and-merge-procedure.md` and `PR_GOVERNANCE_AUDIT.md`. ## Structure diff --git a/PR_GOVERNANCE_AUDIT.md b/PR_GOVERNANCE_AUDIT.md index bc0e31a6a..f60403b23 100644 --- a/PR_GOVERNANCE_AUDIT.md +++ b/PR_GOVERNANCE_AUDIT.md @@ -6,6 +6,12 @@ Live check: 2026-06-26 17:53 KST, GitHub API via `gh` as `seonghobae`. OpenCode decides; GitHub Actions mutates. +Bot and agent operating procedure (PR stacking, successor heads, exact-head +CI, do-not-merge / `DIRTY`–`CONFLICTING` repair, writer boundaries, and the +approve-gate) lives in +[docs/pr-review-and-merge-procedure.md](docs/pr-review-and-merge-procedure.md). +The root [README.md](README.md) is the buyer/operator overview. + - The canonical implementation belongs in `ContextualWisdomLab/.github`. Repository-local copies of the scheduler, OpenCode review workflow, Strix gate, or helper scripts are drift sources, not repo-specific contracts. diff --git a/README.md b/README.md index 332324301..94e91fe4d 100644 --- a/README.md +++ b/README.md @@ -1,151 +1,159 @@ # Contextual Wisdom Lab -Organization profile repository for **맥락지혜 연구실 / Contextual Wisdom Lab**. - -The public GitHub organization profile lives in [profile/README.md](profile/README.md). - -Homepage: https://contextualwisdomlab.github.io/ - -PR governance live audit: [PR_GOVERNANCE_AUDIT.md](PR_GOVERNANCE_AUDIT.md). -The audit includes repository-by-repository DX/UX transfer decisions: what the -central workflow borrows because it reduces friction, and what it rejects -because it adds noise or misleading review experience. - -## PR review and merge policy - -OpenCode judges PRs; GitHub Actions performs mechanical updates and merges. -The scheduler updates a same-repository PR branch only when the latest OpenCode -review is approved, no current-head failed check is present, and GitHub reports -the PR as behind. After that update, the new head must pass OpenCode, Strix, -required checks, and review-thread gates again before auto-merge or -`--match-head-commit` merge can proceed. -Branch updates and merges run through the central scheduler mutation credential: -`PR_REVIEW_MERGE_TOKEN`, `OPENCODE_APPROVE_TOKEN`, the exchanged OpenCode GitHub -App token, or finally the target workflow token. The scheduler reports the -credential class in its decision output. The OpenCode review job does not widen -its own `pull_request_target` job token to repository-write permission; its -immediate post-approval scheduler follow-up uses only an explicit merge token or -the OpenCode app token, otherwise it leaves the separate scheduler required -workflow and schedule authoritative. -Post-approval reuse and follow-up accept only an exact-head review authored by -the OpenCode GitHub App; a GitHub Actions-authored review is not OpenCode -approval evidence. The separate scheduler also listens for that App review, -waits for the publishing OpenCode check to finish, and then retries direct merge -outside the review job when repository auto-merge is unavailable. Every merge -keeps `--match-head-commit`; it prefers squash and retries with a merge commit -only when the target repository explicitly reports that squash is disabled. -Superseded queued or running workflow cleanup remains mandatory, but a GitHub -cancel or force-cancel API failure cannot make an old head authoritative or -block a policy-clean current head. The scheduler logs the exact run id and -bounded API error as an Actions warning, then continues the current-head -decision. -That `update_branch` path is deliberately not used for `DIRTY` or -`CONFLICTING` PRs: GitHub cannot synthesize a safe conflict resolution for the -author, so the merge scheduler must give the author a repair path instead of pretending -the merger can fix it. A current-head approved PR may still keep or queue native -GitHub auto-merge while the conflict is repaired; queued auto-merge is a wait -state, not evidence that the conflict is solved. Separately, the edit-capable -autofix flow (`scripts/ci/pr_review_fix_scheduler.py` → -`.github/workflows/pr-review-autofix.yml`) may, for an approved -same-repository-head PR, merge the base into the head and resolve the conflict -markers with OpenCode, then push the resolved head; that head is fully -re-reviewed and re-checked before it can merge, so a wrong resolution cannot -merge unreviewed. -When GitHub reports `DIRTY` or `CONFLICTING`, the scheduler does not pretend to -fix the branch. It blocks the PR with repair guidance: merge or rebase the -latest base branch into the PR branch, resolve conflict markers in that PR -branch, rerun focused checks, and push the same branch. OpenCode comments must -include a compact command block covering `gh pr checkout`, `git fetch`, merge or -rebase, `git status --short`, resolved-file staging, normal push, and -`--force-with-lease` only for rebased branches. - -Strix, OpenCode, and the scheduler are sourced from the central -`ContextualWisdomLab/.github` workflows rather than copied into each repository. -Required-workflow runs execute in the target repository context, so mechanical -branch updates, stale-thread resolution, and merges use the configured central -mutation credential while the trusted implementation still comes from the -central repository. The scheduler dispatches same-head Strix evidence first, -then dispatches OpenCode for the same PR head when review evidence is missing or -stale. -This avoids running PR-head review, CodeGraph, coverage, or PoC code as an -unbounded local workflow copy. -Scheduled review-feedback autofix is also centralized. The -`PR Review Fix Scheduler` dispatches the central `PR Review Autofix` worker in -`ContextualWisdomLab/.github` and passes the target repository, PR number, base -SHA, head ref, and head SHA as explicit inputs. The worker mutates only -same-repository PR heads, rechecks the live head before checkout and before -push, and commits as `github-actions[bot]` only when a conservative OpenCode -autofix produces a validated diff. A repository-local autofix worker remains an -explicit compatibility override through `--autofix-repository`; it is no longer -the default contract. -Strix keeps `cancel-in-progress: false` so old evidence is not cancelled by a -force-push, but PR-scoped concurrency includes the head SHA so an obsolete scan -does not serialize newer current-head evidence. - -OpenCode approval is evidence-gated. Before approval, the review summary must -name changed files, CodeGraph or structural MCP evidence, a Change Flow DAG, -passing supported test-suite evidence, configured docstring-gate evidence or advisory docstring status, and a concrete -PoC/execution result. It must also split `Developer experience:` from -`User experience:` so maintainability/review/CI friction is not confused with -product, documentation, review-comment, or status-check reader outcomes. The PoC -can be a temporary scratch repro, focused test, lint, security check, -performance probe, or UI verification command, but it must be actually run and -cited. Every adversarial probe must also state an observed result such as an -exit code, passed or failed test/assertion, rejected input, log value, or source -trace outcome. Generic `source inspection` or `test coverage verifies` prose -without that observation is not reusable approval evidence. Execution evidence -must be sandboxed in the CI workspace or an isolated -temporary directory, with a credential-scrubbed environment by default and no -persistent mutation outside test caches or scratch files. When repo-native -verification legitimately needs network access or GitHub Secrets, pass only the -specific environment variable names required and record why they were needed. -The central helper is -`python3 scripts/ci/sandboxed_verify.py --repo-root -- -`; reviews should cite its `SANDBOXED_VERIFY_RESULT` line -when the helper is used. Use `--network required`, `--allow-env NAME`, and -`--evidence-note "why"` only for repository-required verification. This helper -does not replace the existing bash, task, webfetch, websearch, lsp, CodeGraph, -DeepWiki, Context7, or web_search review policy. Scratch PoC files are not -committed. -For web applications with both backend and frontend surfaces, the preferred -execution proof is the central E2E helper: -`python3 scripts/ci/sandboxed_web_e2e.py --repo-root ---backend-cmd --frontend-cmd --e2e-cmd -`. Reviews should include readiness URLs when the repository -defines them and cite `SANDBOXED_WEB_E2E_RESULT`. If a repo lacks an executable -backend, frontend, E2E, or readiness contract, the review must name the missing -contract instead of presenting a partial run as full E2E evidence. -OpenCode bounded evidence also includes a `Review execution contracts` section -that discovers runtime matrices, package manifests, test, coverage, docstring, -E2E, lint, security, Docker, and unpackaged-source gaps before the agent chooses -commands. -The configured `code-reviewer` subagent is reviewer-only: it may read, grep, -glob, and run safe local verification commands, but it must not edit files, -stage changes, commit, push, install dependencies, mutate branches, or touch -production state. Blocking findings must be source-backed, severity-labeled, -impactful, remediable, and include suggested verification. - -Failed GitHub Checks are not reviewed as URL lists. OpenCode must explain the -failed check name, failing step, source-backed file and line when available, -root cause, fix direction, and focused rerun command. Cancelled or superseded -checks must be described as queue or evidence blockers rather than invented -source-code findings. - -Operational cases folded into the central policy: - -- `naruon`: approved PRs can become `BEHIND`; the scheduler treats that as an - update request, not as a merge signal. GitHub Actions updates the branch with - `expected_head_sha`, then the new head is reviewed again. -- `pg-erd-cloud`: successful bot merges used current-head evidence and - `--match-head-commit`; the centralized path keeps that head-SHA guard. -- `.github`: PRs that edit trusted review workflows can fail because - `pull_request_target` runs the base branch's trusted scripts. A same-head - manual `workflow_dispatch` Strix run may supply evidence for review, but it - does not replace required PR checks until the trusted base branch catches up. -- `naruon#745`: new OpenCode review-flow work improves Mermaid output by - replacing generic risk sketches with changed-file flow DAGs. The central - workflow carries that review contract while keeping the self-test drift fix. -- Cross-repo DX/UX: helpful sibling-repo patterns should be adopted when they - reduce maintainer, reviewer, CI-operator, contributor, user, or reader - friction. Noisy automation, repeated waiting, false failures, misleading - statuses, and URL-only diagnostics are treated as review-experience defects. +Organization special repository for **맥락지혜 연구실 / Contextual Wisdom Lab**. + +This repository is the org profile source and the central required-workflow +source. It is not naruon and it does not own product data. It runs alone. +Sibling product repositories consume it by inheriting the organization +required workflows; they do not copy workflow files from here. + +The public GitHub organization profile lives in +[profile/README.md](profile/README.md) and is what GitHub renders at +https://github.com/ContextualWisdomLab. Homepage: +https://contextualwisdomlab.github.io/ + +Bot and agent PR-review procedure (exact-head CI, successor heads, +do-not-merge / `DIRTY`–`CONFLICTING` repair, writer boundaries, approve-gate) +is in [docs/pr-review-and-merge-procedure.md](docs/pr-review-and-merge-procedure.md). + +## What this repository is + +Three operator-facing roles: + +1. **Org profile and public introduction assets.** `profile/README.md` plus + `profile/assets/` are the lab page. Org-wide defaults `SECURITY.md`, + `.github/CODEOWNERS`, and `.github/dependabot.yml` also live here. +2. **Central required-workflow source.** Workflows under `.github/workflows/` + are the canonical PR review, security scan, and merge-automation + implementation for sibling repositories. Organization ruleset + `CWL Central required workflows` (id `18156473`) runs those workflows in + each target repository's context. Repository-local copies are drift + sources, not repo-specific contracts. +3. **Infrastructure as code for org DNS and Pages.** `infra/cloudflare/` + manages zones and Cloudflare Pages hosting with `zones.json` and + `reconcile.sh` (curl + jq only). Dry-run is the default; writes happen + only on an explicit manual `mode = apply`. Pull requests never see the + Cloudflare API token. + +Naruon is the composition hub that can receive other CWL products. That is +the platform job, not a defect of this repository. This control plane still +shows an independent run: profile, required workflows, and Cloudflare +reconciliation do not require naruon to be present, imported, or running. + +## 따로, 또 같이 + +Every CWL component is standalone and also composable. For this repository +that means: + +| Mode | What happens | +| --- | --- | +| **따로 (this repo alone)** | Clone, test, and operate `.github` as the org profile and workflow source. Local quality gates, Cloudflare dry-run, and this repository's own PRs do not depend on naruon or any sibling product checkout. | +| **또 같이 (siblings call it)** | A sibling enables the org required-workflow ruleset (already `repository_name.include=["~ALL"]` on default branches). GitHub runs the trusted workflows from `ContextualWisdomLab/.github@main` in that sibling's repository context. Optional reusable callers (`deploy-pages.yml`, `pr-review-fix-scheduler.yml`) are `workflow_call` entry points, not files to copy. | + +Do not copy Strix, OpenCode, Noema, or scheduler workflow files into a +sibling to "satisfy CI." Thick downstream sync PRs are an anti-pattern +unless they are a temporary rollback bridge. + +## Current status + +Live work and roadmap live on +[GitHub Project #1](https://github.com/orgs/ContextualWisdomLab/projects/1). +The narrative brief is [docs/CWL-MASTER-CONTEXT.md](docs/CWL-MASTER-CONTEXT.md). +The last checked-in ruleset ledger is +[docs/org-required-workflow-rollout.md](docs/org-required-workflow-rollout.md) +(updated 2026-07-23 KST). + +Checked-in operator facts: + +- Ruleset `18156473` is **active**. It targets every repository default + branch (`~ALL` / `~DEFAULT_BRANCH`) and sources workflows from this + repository at `refs/heads/main`. +- Active required workflow paths: `close-empty-pr.yml`, `noema-review.yml`, + `opencode-review.yml`, `pr-review-merge-scheduler.yml`, + `security-scan.yml`, `strix.yml`, and `sast-semgrep.yml`. +- This repository itself is GitHub Flow on `main`. It is the central source, + so it keeps the workflow files; siblings should not. +- Public profile, DIKW checkpoints, project catalog, and the existing APA 7th + DIKW citations stay in [profile/README.md](profile/README.md#references). +- Control-plane trust boundaries and the hourly NVIDIA NIM repair gate are + diagrammed in [ARCHITECTURE.md](ARCHITECTURE.md). + +If live organization ruleset inspection reports a different ref or a missing +required workflow path, treat that as operations drift and restore ruleset +`18156473` to the current `main` head. Do not compensate by copying +workflows into siblings. + +## How a sibling consumes the central workflows + +1. Confirm the repository is in the ContextualWisdomLab organization. New + public repositories inherit ruleset `18156473` without a name-list update. +2. Keep product, build, release, and repo-specific security workflows local. + Do not add local copies of OpenCode, Strix, Noema, or the merge scheduler. +3. On each default-branch pull request, GitHub creates the required checks in + the sibling context. Review judgment stays with OpenCode (and the + independent Noema reviewer). Mechanical branch update and merge stay with + GitHub Actions in that sibling context, using the configured central + mutation credential. +4. Optional: call a reusable workflow instead of copying it. + +```yaml +jobs: + deploy: + uses: ContextualWisdomLab/.github/.github/workflows/deploy-pages.yml@main + with: + project_name: example-marketing + build_dir: ./public + secrets: inherit +``` + +5. If a repository cannot inherit the ruleset (for example a public fork + still onboarding), add a **thin caller** that passes PR number, base + ref/SHA, head ref/SHA, and inherited secrets into this repository. Do not + paste the scheduler or review implementation. Thin callers must not define + a matching scheduler concurrency group. + +Private-repository onboarding and fork capability gates are recorded in +[PR_GOVERNANCE_AUDIT.md](PR_GOVERNANCE_AUDIT.md). A public fork can be +governed by the same reusable workflow if it opts in; an external PR head can +still be non-mutable at runtime. The scheduler decides from observed PR +permissions and current-head evidence, not from the repository `fork` flag +alone. + +## How to run and maintain this repository alone + +From the repository root, after installing the hash-pinned OpenCode review +toolchain: + +```bash +python3 -m pip install --require-hashes --only-binary=:all: -r requirements-opencode-review-ci-hashes.txt +coverage run -m pytest tests && coverage report --show-missing +interrogate +``` + +`pyproject.toml` sets `pythonpath = ["."]`, coverage source `scripts/ci` +with `fail_under = 100`, and interrogate `fail-under = 100` excluding +`tests`. + +Hash-pinned CI sets are regenerated from the un-hashed `requirements-*-ci.txt` +inputs with the `uv pip compile` command recorded in each `*-hashes.txt` +header. Do not hand-edit a hashes file. Cloudflare reconciliation stays +dry-run unless an operator runs the workflow with `mode = apply`. + +Contract tests pin workflow structure and governance prose +(`PR_GOVERNANCE_AUDIT.md`, `docs/org-required-workflow-rollout.md`, +`opencode.jsonc`, and several workflow files). Edit those files only with the +test suite. + +## Related documents + +| Document | Role | +| --- | --- | +| [profile/README.md](profile/README.md) | Public org profile, DIKW checkpoints, project catalog, APA 7th references | +| [docs/pr-review-and-merge-procedure.md](docs/pr-review-and-merge-procedure.md) | Bot/agent review, exact-head, successor-head, and merge procedure | +| [PR_GOVERNANCE_AUDIT.md](PR_GOVERNANCE_AUDIT.md) | Live audit and per-repo DX/UX transfer decisions | +| [docs/org-required-workflow-rollout.md](docs/org-required-workflow-rollout.md) | Ruleset `18156473` ledger and sibling onboarding | +| [ARCHITECTURE.md](ARCHITECTURE.md) | Control-plane diagram and trust boundaries | +| [docs/CWL-MASTER-CONTEXT.md](docs/CWL-MASTER-CONTEXT.md) | Mission, ecosystem, and naruon-as-platform brief | +| [docs/agent-github-project-protocol.md](docs/agent-github-project-protocol.md) | How agents operate Project #1 | +| [infra/cloudflare/README.md](infra/cloudflare/README.md) | DNS/Pages reconcile and reusable Pages deploy | +| [SECURITY.md](SECURITY.md) | Vulnerability reporting | diff --git a/docs/pr-review-and-merge-procedure.md b/docs/pr-review-and-merge-procedure.md new file mode 100644 index 000000000..87607fb99 --- /dev/null +++ b/docs/pr-review-and-merge-procedure.md @@ -0,0 +1,209 @@ +# PR review and merge procedure + +Bot and agent operating contract for ContextualWisdomLab PR review, exact-head +CI, successor heads, writer boundaries, and mechanical merge. The +buyer/operator overview lives in [README.md](../README.md). The live audit and +per-repository DX/UX transfer decisions live in +[PR_GOVERNANCE_AUDIT.md](../PR_GOVERNANCE_AUDIT.md). Rollout and ruleset +posture live in [org-required-workflow-rollout.md](org-required-workflow-rollout.md). + +This file keeps the operational truth that used to sit in the root README. +Do not treat a sibling product composing into naruon as a defect of this +control plane. Naruon is the composition hub; this repository still runs +alone as the org profile and workflow source. + +## Actors and writer boundaries + +OpenCode judges PRs; GitHub Actions performs mechanical updates and merges. +The configured `code-reviewer` subagent is reviewer-only: it may read, grep, +glob, and run safe local verification commands, but it must not edit files, +stage changes, commit, push, install dependencies, mutate branches, or touch +production state. Blocking findings must be source-backed, severity-labeled, +impactful, remediable, and include suggested verification. + +The OpenCode review job does not widen its own `pull_request_target` job token +to repository-write permission. The scheduler's `GH_TOKEN` merge/read fallback +order is `PR_REVIEW_MERGE_TOKEN`, `OPENCODE_APPROVE_TOKEN`, the exchanged +OpenCode app token, then the receiving workflow's `github.token`. For +repository-dispatch calls that target another repository, the +`SCHEDULER_ACTIONS_TOKEN` and `SCHEDULER_READ_TOKEN` values use the same first +three explicit credentials and do not fall back to a central-repository token; +for same-repository calls they may use `github.token`. A central +`github.token` cannot mutate or read a different target repository, so the +scheduler leaves that cross-repository operation blocked when no explicit +credential is available; the separate required workflow and schedule remain +authoritative. + +Branch updates and merges run through the central scheduler mutation +credential, in this order: + +1. `PR_REVIEW_MERGE_TOKEN` +2. `OPENCODE_APPROVE_TOKEN` +3. the exchanged OpenCode GitHub App token +4. the target workflow token + +The scheduler reports the credential class in its decision output. + +## Exact-head CI and successor heads + +The scheduler updates a same-repository PR branch only when the latest +OpenCode review is approved, no current-head failed check is present, and +GitHub reports the PR as behind. After that update, the new head is a +successor head: it must pass OpenCode, Strix, required checks, and +review-thread gates again before auto-merge or `--match-head-commit` merge +can proceed. + +Post-approval reuse and follow-up accept only an exact-head review authored +by the OpenCode GitHub App. A GitHub Actions-authored review is not OpenCode +approval evidence. The separate scheduler also listens for that App review, +waits for the publishing OpenCode check to finish, and then retries direct +merge outside the review job when repository auto-merge is unavailable. + +Every merge keeps `--match-head-commit`. It prefers squash and retries with a +merge commit only when the target repository explicitly reports that squash +is disabled. + +Superseded queued or running workflow cleanup remains mandatory, but a GitHub +cancel or force-cancel API failure cannot make an old head authoritative or +block a policy-clean current head. The scheduler logs the exact run id and +bounded API error as an Actions warning, then continues the current-head +decision. + +Old approvals and old checks are not merge evidence after the head SHA +changes. OpenCode review evidence must be internally same-head as well as +GitHub-attached same-head. If the review body includes `Gate evidence` with +`Head SHA: `, that SHA must match the PR current `headRefOid`. + +## Do-not-merge and DIRTY / CONFLICTING repair + +The `update_branch` path is deliberately not used for `DIRTY` or +`CONFLICTING` PRs. GitHub cannot synthesize a safe conflict resolution for +the author, so the merge scheduler must give the author a repair path instead +of pretending the merger can fix it. + +A current-head approved PR may still keep or queue native GitHub auto-merge +while the conflict is repaired. Queued auto-merge is a wait state, not +evidence that the conflict is solved. When GitHub reports `DIRTY` or +`CONFLICTING`, the scheduler blocks the PR with repair guidance: merge or +rebase the latest base branch into the PR branch, resolve conflict markers in +that PR branch, rerun focused checks, and push the same branch. OpenCode +comments must include a compact command block covering `gh pr checkout`, +`git fetch`, merge or rebase, `git status --short`, resolved-file staging, +normal push, and `--force-with-lease` only for rebased branches. + +Separately, the edit-capable autofix flow +(`scripts/ci/pr_review_fix_scheduler.py` → +`.github/workflows/pr-review-autofix.yml`) may, for an approved +same-repository-head PR, merge the base into the head and resolve the +conflict markers with OpenCode, then push the resolved head. That head is +fully re-reviewed and re-checked before it can merge, so a wrong resolution +cannot merge unreviewed. + +## Central required workflows, not local copies + +Strix, OpenCode, Noema, and the scheduler are sourced from the central +`ContextualWisdomLab/.github` workflows rather than copied into each +repository. Required-workflow runs execute in the target repository context, +so mechanical branch updates, stale-thread resolution, and merges use the +configured central mutation credential while the trusted implementation still +comes from the central repository. + +The scheduler dispatches same-head Strix evidence first, then dispatches +OpenCode for the same PR head when review evidence is missing or stale. This +avoids running PR-head review, CodeGraph, coverage, or PoC code as an +unbounded local workflow copy. + +Scheduled review-feedback autofix is also centralized. The +`PR Review Fix Scheduler` dispatches the central `PR Review Autofix` worker +in `ContextualWisdomLab/.github` and passes the target repository, PR number, +base SHA, head ref, and head SHA as explicit inputs. The worker mutates only +same-repository PR heads, rechecks the live head before checkout and before +push, and commits as `github-actions[bot]` only when a conservative OpenCode +autofix produces a validated diff. A repository-local autofix worker remains +an explicit compatibility override through `--autofix-repository`; it is no +longer the default contract. + +Strix keeps `cancel-in-progress: false` so old evidence is not cancelled by a +force-push, but PR-scoped concurrency includes the head SHA so an obsolete +scan does not serialize newer current-head evidence. + +## Approve-gate evidence + +OpenCode approval is evidence-gated. Before approval, the review summary must +name changed files, CodeGraph or structural MCP evidence, a Change Flow DAG, +passing supported test-suite evidence, configured docstring-gate evidence or +advisory docstring status, and a concrete PoC/execution result. It must also +split `Developer experience:` from `User experience:` so +maintainability/review/CI friction is not confused with product, +documentation, review-comment, or status-check reader outcomes. + +The PoC can be a temporary scratch repro, focused test, lint, security check, +performance probe, or UI verification command, but it must be actually run +and cited. Every adversarial probe must also state an observed result such as +an exit code, passed or failed test/assertion, rejected input, log value, or +source trace outcome. Generic `source inspection` or `test coverage verifies` +prose without that observation is not reusable approval evidence. + +Execution evidence must be sandboxed in the CI workspace or an isolated +temporary directory, with a credential-scrubbed environment by default and no +persistent mutation outside test caches or scratch files. When repo-native +verification legitimately needs network access or GitHub Secrets, pass only +the specific environment variable names required and record why they were +needed. The central helper is +`python3 scripts/ci/sandboxed_verify.py --repo-root -- +`; reviews should cite its `SANDBOXED_VERIFY_RESULT` +line when the helper is used. Use `--network required`, `--allow-env NAME`, +and `--evidence-note "why"` only for repository-required verification. This +helper does not replace the existing bash, task, webfetch, websearch, lsp, +CodeGraph, DeepWiki, Context7, or web_search review policy. +Scratch PoC files are not committed. + +For web applications with both backend and frontend surfaces, the preferred +execution proof is the central E2E helper: +`python3 scripts/ci/sandboxed_web_e2e.py --repo-root +--backend-cmd --frontend-cmd --e2e-cmd +`. Reviews should include readiness URLs when the repository +defines them and cite `SANDBOXED_WEB_E2E_RESULT`. If a repo lacks an +executable backend, frontend, E2E, or readiness contract, the review must +name the missing contract instead of presenting a partial run as full E2E +evidence. + +OpenCode bounded evidence also includes a `Review execution contracts` +section that discovers runtime matrices, package manifests, test, coverage, +docstring, E2E, lint, security, Docker, and unpackaged-source gaps before the +agent chooses commands. + +Failed GitHub Checks are not reviewed as URL lists. OpenCode must explain the +failed check name, failing step, source-backed file and line when available, +root cause, fix direction, and focused rerun command. Cancelled or superseded +checks must be described as queue or evidence blockers rather than invented +source-code findings. + +CodeRabbit or other current-head evidence may inform the review, but it does +not replace exact-head OpenCode approval, same-head Strix evidence, or the +scheduler `--match-head-commit` guard. + +## Operational cases + +- `naruon`: approved PRs can become `BEHIND`; the scheduler treats that as an + update request, not as a merge signal. GitHub Actions updates the branch + with `expected_head_sha`, then the new head is reviewed again. Naruon is + the composition hub that receives other CWL products; that composition is + not a control-plane bug and does not change this repository's standalone + run. +- `pg-erd-cloud`: successful bot merges used current-head evidence and + `--match-head-commit`; the centralized path keeps that head-SHA guard. +- `.github`: PRs that edit trusted review workflows can fail because + `pull_request_target` runs the base branch's trusted scripts. A same-head + manual `workflow_dispatch` Strix run may supply evidence for review, but it + does not replace required PR checks until the trusted base branch catches + up. +- `ContextualWisdomLab/naruon#745`: new OpenCode review-flow work improves + Mermaid output by replacing generic risk sketches with changed-file flow + DAGs. The central workflow carries that review contract while keeping the + self-test drift fix. +- Cross-repo DX/UX: helpful sibling-repo patterns should be adopted when they + reduce maintainer, reviewer, CI-operator, contributor, user, or reader + friction. Noisy automation, repeated waiting, false failures, misleading + statuses, and URL-only diagnostics are treated as review-experience + defects. diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7343c06ac..ac9ce1d8b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1490,6 +1490,7 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { local scheduler_file="$REPO_ROOT/scripts/ci/pr_review_merge_scheduler.py" local fix_scheduler_file="$REPO_ROOT/scripts/ci/pr_review_fix_scheduler.py" local readme_file="$REPO_ROOT/README.md" + local procedure_file="$REPO_ROOT/docs/pr-review-and-merge-procedure.md" assert_file_contains "$autofix_workflow_file" "Autofix allowed paths, authoritative:" "autofix prompt includes allowed paths outside the truncated review context" assert_file_contains "$autofix_workflow_file" "" "autofix prompt has a dedicated allowed-paths block" @@ -1552,7 +1553,8 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$scheduler_file" "same-head OpenCode dispatched" "scheduler records review dispatch after completed security evidence" assert_file_contains "$workflow_file" "--pr-number" "scheduler scopes required-workflow PR events to the current pull request" assert_file_contains "$workflow_file" "--review-workflow \"Required OpenCode Review\"" "scheduler dispatches the canonical required OpenCode Review workflow" - assert_file_contains "$readme_file" "PR_REVIEW_MERGE_TOKEN" "README documents that mechanical branch updates and merges use the central mutation credential" + assert_file_contains "$readme_file" "docs/pr-review-and-merge-procedure.md" "README points operators to the bot/agent review procedure instead of embedding it" + assert_file_contains "$procedure_file" "PR_REVIEW_MERGE_TOKEN" "review procedure documents that mechanical branch updates and merges use the central mutation credential" assert_file_contains "$fix_workflow_file" 'workflow_call:' "fix scheduler can run as the central reusable autofix-dispatch workflow" assert_file_contains "$fix_workflow_file" 'repository: ContextualWisdomLab/.github' "fix scheduler checks out the canonical implementation instead of relying on repo-local scheduler code" assert_file_contains "$fix_workflow_file" 'AUTOFIX_REPOSITORY' "fix scheduler can dispatch the central autofix worker without per-repository workflow copies" @@ -1568,10 +1570,10 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$fix_scheduler_file" '"target_repository": repo' "fix scheduler passes the target repository in the central repository-dispatch JSON payload" assert_file_contains "$fix_scheduler_file" "recent autofix marker exists for this head" "fix scheduler avoids repeated autofix loops for the same head" assert_file_contains "$fix_scheduler_file" "external PR head is not writable" "fix scheduler refuses external heads for bot autofix" - assert_file_contains "$readme_file" "PR Review Fix Scheduler" "README documents the central autofix scheduler contract" - assert_file_contains "$readme_file" "Scratch PoC files are not" "README documents PoC proof artifacts are scratch evidence, not committed changes" - assert_file_contains "$readme_file" "committed." "README documents scratch PoC proof artifacts are not committed" - assert_file_contains "$readme_file" "Failed GitHub Checks are not reviewed as URL lists." "README documents failed-check reviews require explanations, not URL-only bullets" + assert_file_contains "$procedure_file" "PR Review Fix Scheduler" "review procedure documents the central autofix scheduler contract" + assert_file_contains "$procedure_file" "Scratch PoC files are not" "review procedure documents PoC proof artifacts are scratch evidence, not committed changes" + assert_file_contains "$procedure_file" "committed." "review procedure documents scratch PoC proof artifacts are not committed" + assert_file_contains "$procedure_file" "Failed GitHub Checks are not reviewed as URL lists." "review procedure documents failed-check reviews require explanations, not URL-only bullets" } assert_opencode_review_normalizer_accepts_transcript_json() { @@ -6070,6 +6072,37 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-fallback-provider-signal-tries-next) + run_gate_case "github-models-fallback-provider-signal-tries-next" \ + "openai/gpt-5" \ + "" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-v3-0324' in [0-9]+s\\." \ + "3" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ + "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "1" + ;; endpoint-in-excluded-dir) run_gate_case "endpoint-in-excluded-dir" \ "vertex_ai/excluded-dir-primary" \ From b3ce110714016b13d02292884ca2088e3e32b17e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 01:56:55 -0700 Subject: [PATCH 10/37] fix(strix): authenticate GitHub Models HTTP 410 fallback --- scripts/ci/strix_quick_gate.sh | 12 +++ scripts/ci/test_strix_quick_gate.sh | 117 ++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..ba629e396 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2811,6 +2811,18 @@ strix_log_has_github_models_context() { } is_github_models_unavailable_model_error() { + # GitHub Models may retire a provider model with HTTP 410. Treat that as a + # bounded family-unavailable signal only when one physical provider-error + # line carries all three facts: an anchored LiteLLM/OpenAI exception, trusted + # GitHub Models context, and a complete HTTP 410 token. Anchoring the provider + # exception prevents target/repository output prefixes from spoofing fallback; + # the non-digit boundary rejects numeric continuations such as 4100/4104. + if grep -Ei '^[[:space:]]*(Error:[[:space:]]*)?((litellm(\.exceptions)?|openai)\.[A-Za-z0-9_]*(Error|Exception)|OpenAIException)([[:space:]:-]|$)' "$STRIX_LOG" | + grep -Ei '(models\.github\.ai|GitHub Models|github_models)' | + grep -Eq 'HTTP[[:space:]]+410([^0-9]|$)'; then + return 0 + fi + if grep -Eiq 'Unavailable model:[[:space:]]*[^[:space:]]+' "$STRIX_LOG" && grep -Eiq '(litellm\.BadRequestError|OpenAIException|LLM CONNECTION FAILED|Could not establish connection to the language model|models\.github\.ai|GitHub Models|openai)' "$STRIX_LOG"; then return 0 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ac9ce1d8b..44906c008 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3721,6 +3721,44 @@ REPORT ;; esac ;; + github-models-http410-authenticated-fallback-success | github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + case "${STRIX_LLM:-}" in + openai/gpt-5) + case "${FAKE_STRIX_SCENARIO:?}" in + github-models-http410-authenticated-fallback-success) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 410 Gone" + ;; + github-models-http410-missing-http-token) + echo "Error: litellm.BadRequestError: GitHub Models provider retirement at models.github.ai/inference" + ;; + github-models-http410-missing-provider-error) + echo "GitHub Models response at models.github.ai/inference: HTTP 410 Gone" + ;; + github-models-http410-numeric-continuation-4100) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 4100" + ;; + github-models-http410-numeric-continuation-4104) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 4104" + ;; + github-models-http410-target-output-spoof) + echo "TARGET OUTPUT: Error: litellm.BadRequestError: GitHub Models provider error HTTP 410" + ;; + github-models-retirement-brownout-phrase-only) + echo "GitHub Models retirement brownout" + ;; + esac + exit 1 + ;; + openai/deepseek/deepseek-r1-0528) + echo "scan ok after authenticated GitHub Models HTTP 410 retirement" + exit 0 + ;; + *) + echo "Error: GitHub Models HTTP 410 fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 39 + ;; + esac + ;; github-models-primary-ratelimit-fallback-success) case "${STRIX_LLM:-}" in openai/gpt-5) @@ -5738,6 +5776,45 @@ run_gate_case_allow_provider_signal() { run_gate_case_with_provider_signal_mode "0" "$@" } +run_github_models_http410_case() { + local scenario="$1" + local expected_exit="$2" + local expected_calls="$3" + local expected_models="$4" + local expected_api_bases="$5" + local expected_message="${6-}" + + run_gate_case "$scenario" \ + "openai/gpt-5" \ + "" \ + "$expected_exit" \ + "$expected_message" \ + "$expected_calls" \ + "$expected_models" \ + "$expected_api_bases" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528" \ + "1" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") @@ -6072,6 +6149,23 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-http410-authenticated-fallback-success) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "0" \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ + "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." + ;; + github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" + ;; github-models-fallback-provider-signal-tries-next) run_gate_case "github-models-fallback-provider-signal-tries-next" \ "openai/gpt-5" \ @@ -9475,6 +9569,29 @@ run_gate_case_allow_provider_signal "github-models-primary-denied-fallback-succe "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" +run_github_models_http410_case \ + "github-models-http410-authenticated-fallback-success" \ + "0" \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ + "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." + +for scenario in \ + github-models-http410-missing-http-token \ + github-models-http410-missing-provider-error \ + github-models-http410-numeric-continuation-4100 \ + github-models-http410-numeric-continuation-4104 \ + github-models-http410-target-output-spoof \ + github-models-retirement-brownout-phrase-only; do + run_github_models_http410_case \ + "$scenario" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" +done + run_gate_case "github-models-primary-ratelimit-fallback-success" \ "openai/gpt-5" \ "" \ From e21951d73fbe05a3b9dda871b18c7480f1fe3e41 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 18:09:46 +0900 Subject: [PATCH 11/37] fix(strix): classify Caido bootstrap outages --- .github/workflows/strix.yml | 2 +- ...est_strix_nvidia_nim_not_found_fallback.py | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a68bd6b9e..c43de8bcf 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -867,7 +867,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|agents\.exceptions\.ModelBehaviorError:[[:space:]]*|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index fae189578..c38919454 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -289,6 +289,28 @@ def test_outer_workflow_never_classifies_model_behavior_error_with_findings( "Vulnerabilities 1\n" ) ) + + def test_outer_workflow_classifies_caido_bootstrap_failure_without_findings(self) -> None: + """Treat a Strix-owned Caido bootstrap outage as incomplete infrastructure evidence.""" + + self.assertTrue( + _workflow_classifies_backend_unavailable( + "Error during penetration test: loginAsGuest failed after 10 attempts: " + "curl exit 7: curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_downgrades_caido_failure_with_findings(self) -> None: + """Keep a real finding blocking even when the Strix container also failed.""" + + self.assertFalse( + _workflow_classifies_backend_unavailable( + "Error during penetration test: loginAsGuest failed after 10 attempts: " + "curl exit 7: curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + "Vulnerabilities 1\n" + ) + ) self.assertFalse( _workflow_classifies_backend_unavailable( "agents.exceptions.ModelBehaviorError: provider response failed\n" From 2e8e78271481a5ac0d7c6502973aa9ee8cfbc206 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 03:21:28 -0700 Subject: [PATCH 12/37] fix(strix): isolate scanner output from scan target --- CHANGELOG.md | 7 ++ docs/doctoring/strix-scan-working-boundary.md | 56 +++++++++++++++ scripts/ci/strix_quick_gate.sh | 62 +++++++++++++++-- scripts/ci/test_strix_quick_gate.sh | 68 ++++++++++++++++++- 4 files changed, 185 insertions(+), 8 deletions(-) create mode 100644 docs/doctoring/strix-scan-working-boundary.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cc8bc53a..2a28e02ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,13 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Run Strix with an explicit canonical scan target from a temporary working + directory outside that target, so scanner state and relative reports cannot + become self-scanned source findings; preserve those reports as gate evidence. + PR-scoped Python scans also include the PostgreSQL introspection security + helpers when that package exists in the target repository. PR scopes now live + below the gate's private runtime directory so unrelated temporary-file + cleanup cannot remove scan input during PR-head materialization. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/strix-scan-working-boundary.md b/docs/doctoring/strix-scan-working-boundary.md new file mode 100644 index 000000000..f73644c56 --- /dev/null +++ b/docs/doctoring/strix-scan-working-boundary.md @@ -0,0 +1,56 @@ +# Strix scan working-directory boundary + +## Problem + +The organization Strix gate bounded pull-request scans to a temporary scope, +but launched Strix with that scope as its current working directory. Strix +could therefore create `strix_runs/` and state files inside the tree it was +scanning. A self-generated state file was reported as a critical hard-coded +credential in a current-head `pg-erd-cloud` scan, while another scan reported a +missing unchanged DSN guard because the bounded scope omitted an imported +security helper. + +## Decision + +The gate now passes the canonical target directory as Strix's absolute `-t` +argument and runs the process from a fresh runner-temporary directory outside +the target. The temporary `strix_runs/` output is copied into the existing +active report directory after each attempt, so report classification and +artifact publication retain their previous evidence contract. The target is +never inferred from the working directory. + +When a changed backend Python file belongs to a repository that contains +`backend/app/pg_introspect`, the bounded scope includes the package's available +trusted base helpers, including `dsn_guard.py` and `introspect.py`. Repositories +without that package are unchanged. + +The bounded scope itself is created below the gate's private runtime directory. +The gate therefore owns the scope lifetime and an unrelated temporary-file +cleanup cannot remove scan input during PR-head blob materialization. + +## Verification and rollback + +`scripts/ci/test_strix_quick_gate.sh` verifies both the absolute target and the +outside working directory. It also verifies that a PostgreSQL DSN guard is +available to a scoped introspection scan. Run the shell syntax check and the +Strix quick-gate harness before publishing a central workflow change. Rollback +is a normal revert of the central PR; do not suppress changed-file attribution +or ignore scanner output to make a check green. + +The fix addresses the trust boundary between untrusted scan input and scanner +output. It does not replace exact-head review, vulnerability remediation, or +the required security workflow. + +## References + +National Institute of Standards and Technology. (2022). *Secure software +development framework (SSDF) version 1.1: Recommendations for mitigating the +risk of software vulnerabilities* (NIST Special Publication 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +MITRE. (n.d.). *CWE-22: Improper limitation of a pathname to a restricted +directory ('Path traversal')*. Common Weakness Enumeration. +https://cwe.mitre.org/data/definitions/22.html + +MITRE. (n.d.). *CWE-367: Time-of-check time-of-use (TOCTOU) race condition*. +Common Weakness Enumeration. https://cwe.mitre.org/data/definitions/367.html diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index ba629e396..9345bf01f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -28,6 +28,8 @@ STRIX_RUNTIME_DIR="$(mktemp -d /tmp/strix-runtime.XXXXXX)" STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" ACTIVE_REPORTS_DIR="$STRIX_RUNTIME_DIR/reports" ATTEMPT_LOGS_DIR="$STRIX_RUNTIME_DIR/gate-attempts" +STRIX_SCAN_WORKING_DIR="$STRIX_RUNTIME_DIR/scan-cwd" +STRIX_SCAN_OUTPUT_DIR="$STRIX_SCAN_WORKING_DIR/strix_runs" STRIX_REPORTS_DIR="$ACTIVE_REPORTS_DIR" STRIX_PROCESS_TIMEOUT_SECONDS="${STRIX_PROCESS_TIMEOUT_SECONDS:-1200}" STRIX_TOTAL_TIMEOUT_SECONDS="${STRIX_TOTAL_TIMEOUT_SECONDS:-0}" @@ -136,6 +138,9 @@ publish_artifact_reports() { cp -R -- "$scope_reports_dir"/. "$ARTIFACT_REPORTS_DIR"/ fi done + if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then + cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ARTIFACT_REPORTS_DIR"/ + fi } preserve_attempt_log() { @@ -233,6 +238,16 @@ cleanup_runtime() { trap cleanup_runtime EXIT INT TERM +make_pull_request_scope_dir() { + local scope_parent="$STRIX_RUNTIME_DIR/pr-scopes" + if [ -L "$scope_parent" ]; then + echo "ERROR: pull request scope parent must not be a symlink." >&2 + return 2 + fi + mkdir -p -- "$scope_parent" + mktemp -d "$scope_parent/strix-pr-scope.XXXXXX" +} + STRIX_LLM_FILE="${STRIX_LLM_FILE:-}" if [ -z "$STRIX_LLM_FILE" ]; then echo "ERROR: STRIX_LLM_FILE must reference a regular file containing the model." >&2 @@ -1255,6 +1270,22 @@ backend/services/llm_provider_urls.py backend/services/text_safety.py backend/services/threading_service.py EOF + # PostgreSQL introspection helpers are a security boundary for repositories + # that expose this package. Include their trusted base copies when present; + # the conditional keeps the shared gate usable by repositories without it. + local context_file + for context_file in \ + backend/app/pg_introspect/__init__.py \ + backend/app/pg_introspect/column_examples.py \ + backend/app/pg_introspect/dsn_guard.py \ + backend/app/pg_introspect/forward_ddl.py \ + backend/app/pg_introspect/introspect.py \ + backend/app/pg_introspect/queries.py \ + backend/app/pg_introspect/snapshot_collect.py; do + if [ -f "$REPO_ROOT/$context_file" ] && [ ! -L "$REPO_ROOT/$context_file" ]; then + printf '%s\n' "$context_file" + fi + done fi if [ "$needs_frontend_email_api_context" -eq 1 ]; then @@ -1302,7 +1333,7 @@ changed_file_list_contains() { build_pull_request_scope_dir() { local scope_dir - scope_dir="$(mktemp -d "${TMPDIR:-/tmp}/strix-pr-scope.XXXXXX")" + scope_dir="$(make_pull_request_scope_dir)" || return 2 scope_dir="$({ CDPATH='' && cd -P -- "$scope_dir" && pwd -P; })" PULL_REQUEST_SCOPE_DIRS+=("$scope_dir") @@ -1475,7 +1506,7 @@ PY build_pull_request_head_tree_scope_dir() { local scope_dir - scope_dir="$(mktemp -d "${TMPDIR:-/tmp}/strix-pr-scope.XXXXXX")" + scope_dir="$(make_pull_request_scope_dir)" || return 2 scope_dir="$({ CDPATH='' && cd -P -- "$scope_dir" && pwd -P; })" PULL_REQUEST_SCOPE_DIRS+=("$scope_dir") @@ -2375,7 +2406,7 @@ run_strix_once() { STRIX_CHILD_EXECUTABLE_ROOT="$STRIX_EXECUTABLE_ROOT" \ STRIX_CHILD_EXECUTABLE_SHA256="$STRIX_EXECUTABLE_SHA256" \ STRIX_CHILD_REQUIRE_EXECUTABLE_INTEGRITY="${IS_PR_EVIDENCE_RUN:-false}" \ - python3 - "$timeout_seconds" "$resolved_target_path" "$SCAN_MODE" "$STRIX_LOG" <<'PY' +python3 - "$timeout_seconds" "$resolved_target_path" "$SCAN_MODE" "$STRIX_LOG" "$STRIX_SCAN_WORKING_DIR" <<'PY' import hashlib import hmac import os @@ -2389,6 +2420,7 @@ timeout_seconds = int(sys.argv[1]) target_path = sys.argv[2] scan_mode = sys.argv[3] log_path = pathlib.Path(sys.argv[4]) +scan_working_dir = pathlib.Path(sys.argv[5]) # Failure classifiers read this path even when trusted executable or target # validation fails before a child process starts. Materialize it first so the # primary log shows one configuration error instead of repeated grep noise. @@ -2528,12 +2560,29 @@ if any(ch in str(target_cwd) for ch in ("\x00", "\n", "\r")): sys.stderr.write("ERROR: Strix target path contains unsupported control characters.\n") raise SystemExit(2) -command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode] +if scan_working_dir.is_symlink(): + sys.stderr.write("ERROR: Strix scan working directory must not be a symlink.\n") + raise SystemExit(2) +scan_working_dir.mkdir(parents=True, exist_ok=True) +scan_output_dir = scan_working_dir / "strix_runs" +if scan_output_dir.is_symlink(): + sys.stderr.write("ERROR: Strix scan output directory must not be a symlink.\n") + raise SystemExit(2) +if scan_output_dir.exists(): + import shutil + + shutil.rmtree(scan_output_dir) +scan_output_dir.mkdir() + +# Keep scanner-created state and relative report files outside the untrusted +# scan target. The target remains explicit and absolute, so changing cwd cannot +# change which source tree is scanned. +command = [resolved_strix_bin, "-n", "-t", str(target_cwd), "--scan-mode", scan_mode] try: process = subprocess.Popen( command, - cwd=str(target_cwd), + cwd=str(scan_working_dir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -2566,6 +2615,9 @@ except subprocess.TimeoutExpired: PY rc=$? set -e + if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then + cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ACTIVE_REPORTS_DIR"/ + fi local end_epoch end_epoch="$(date +%s)" local elapsed=$((end_epoch - start_epoch)) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 44906c008..7f9512bfd 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -478,9 +478,12 @@ assert_strix_llm_file_read_is_literal_data() { } assert_strix_child_target_uses_constant_argument() { - assert_file_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode]' "strix gate passes a constant target argument to the child process" - assert_file_contains "$GATE_SCRIPT" 'cwd=str(target_cwd)' "strix gate runs the child process from the canonical target directory" - assert_file_not_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", target_path, "--scan-mode", scan_mode]' "strix gate must not forward raw target paths as child arguments" + assert_file_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", str(target_cwd), "--scan-mode", scan_mode]' "strix gate passes the canonical target argument to the child process" + assert_file_contains "$GATE_SCRIPT" 'cwd=str(scan_working_dir)' "strix gate runs the child process outside the scan target" + assert_file_contains "$GATE_SCRIPT" 'make_pull_request_scope_dir()' "strix gate creates PR scopes under its private runtime directory" + assert_file_contains "$GATE_SCRIPT" 'scope_parent="$STRIX_RUNTIME_DIR/pr-scopes"' "strix gate keeps PR scopes inside the private runtime directory" + assert_file_not_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode]' "strix gate must not rely on the child cwd as its scan target" + assert_file_not_contains "$GATE_SCRIPT" 'cwd=str(target_cwd)' "strix gate must not run the child process inside the scan target" } assert_opencode_review_uses_codegraph_and_gpt5_fallback() { @@ -3302,6 +3305,18 @@ success|runtime-env-forwarding|vertex-primary-success-timing-message|direct-open echo "scan ok" exit 0 ;; + scan-working-directory-isolated) + if [ "$PWD" = "$target_path" ] || [[ "$PWD" == "$target_path"/* ]]; then + echo "Error: Strix process inherited the untrusted scan target as cwd" >&2 + exit 81 + fi + if [ ! -f "$target_path/backend/app/pg_introspect/dsn_guard.py" ]; then + echo "Error: PostgreSQL DSN guard context missing from PR scope" >&2 + exit 82 + fi + echo "scan ok with isolated Strix working directory" + exit 0 + ;; success-with-critical-report) mkdir -p "$STRIX_REPORTS_DIR/fake-success/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-success/vulnerabilities/vuln-0001.md" <<'REPORT' @@ -5442,6 +5457,10 @@ EOS for large_scope_index in $(seq 1 38); do printf 'file %s\n' "$large_scope_index" >"$repo_root_dir/backend/large-scope/file-$large_scope_index.py" done + elif [ "$scenario" = "scan-working-directory-isolated" ]; then + mkdir -p "$repo_root_dir/backend/app/pg_introspect" + printf '%s\n' 'HEAD_INTROSPECT_SHOULD_BE_SCANNED' >"$repo_root_dir/backend/app/pg_introspect/introspect.py" + printf '%s\n' 'TRUSTED_DSN_GUARD_CONTEXT_SHOULD_BE_SCANNED' >"$repo_root_dir/backend/app/pg_introspect/dsn_guard.py" fi local scenario_base_sha="" @@ -6370,6 +6389,28 @@ run_filtered_gate_case_if_requested() { "Materialized PR-head changed-file scope" \ "repository_dispatch" ;; + scan-working-directory-isolated) + run_gate_case "scan-working-directory-isolated" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with isolated Strix working directory" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "backend/app/pg_introspect/introspect.py" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -10824,6 +10865,27 @@ run_gate_case "pr-changed-scope-bounded" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "scan-working-directory-isolated" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with isolated Strix working directory" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "backend/app/pg_introspect/introspect.py" + run_gate_case "pr-python-scope-context" \ "openai/gpt-4o-mini" \ "" \ From 2d2037c7378bb25e3ce2b26866ea55b47cf5fda1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:06:41 +0900 Subject: [PATCH 13/37] docs: complete coordinator docstring coverage --- organization_commercial_readiness_fixtures.py | 1 + scripts/ci/organization_commercial_readiness_loop.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/organization_commercial_readiness_fixtures.py b/organization_commercial_readiness_fixtures.py index d86596196..00cd504f0 100644 --- a/organization_commercial_readiness_fixtures.py +++ b/organization_commercial_readiness_fixtures.py @@ -90,6 +90,7 @@ def __init__( repositories: list[dict[str, Any]], snapshots: dict[str, list[RepositorySnapshot | Exception]], ) -> None: + """Initialize the deterministic repository and snapshot responses.""" self.repositories = repositories self.snapshots = snapshots self.dispatched_repairs: list[tuple[str, str]] = [] diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index c00cfa1e0..84c4f326c 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -239,6 +239,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: + """Initialize the authenticated client with a bounded request timeout.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token @@ -853,4 +854,4 @@ def main( if __name__ == "__main__": # pragma: no cover - exercised through main() - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) From a0fe0501b6a4d0f3c532b5feeaf7a7038db41bb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 23:39:11 +0900 Subject: [PATCH 14/37] test: complete workflow contract docstrings --- .../test_required_workflow_queue_contract.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index fafd9e7bd..cf8063d9b 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -1,3 +1,5 @@ +"""Verify central required-workflow queue, security, and dispatch contracts.""" + import json import os import shlex @@ -14,10 +16,12 @@ def workflow_text(name: str) -> str: + """Read one central workflow for contract assertions.""" return (REPO_ROOT / ".github" / "workflows" / name).read_text(encoding="utf-8") def workflow_step(workflow: str, name: str) -> str: + """Extract one named workflow step without parsing YAML dynamically.""" step = f" - name: {name}\n" start = workflow.index(step) try: @@ -28,6 +32,7 @@ def workflow_step(workflow: str, name: str) -> str: def test_merge_scheduler_dispatches_one_review_by_default() -> None: + """Keep the default scheduler dispatch bounded to one review.""" workflow = workflow_text("pr-review-merge-scheduler.yml") assert workflow.count('default: "1"') >= 2 @@ -148,6 +153,7 @@ def test_no_central_workflow_exposes_branch_selected_manual_dispatch() -> None: def test_required_pull_request_workflows_cancel_superseded_runs() -> None: + """Ensure required pull-request workflows cancel obsolete executions.""" for filename in ( "close-empty-pr.yml", "codeql-pr.yml", @@ -189,6 +195,7 @@ def test_required_pull_request_workflows_cancel_superseded_runs() -> None: def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() -> None: + """Keep Semgrep finding output distinct from scanner-engine failures.""" workflow = workflow_text("sast-semgrep.yml") assert "Report every Semgrep finding in the job log" in workflow @@ -207,6 +214,7 @@ def test_central_semgrep_logs_every_finding_and_distinguishes_engine_failure() - def test_strix_cancels_superseded_pr_head_security_evidence() -> None: + """Scope Strix concurrency to the target PR while preserving current-head evidence.""" workflow = workflow_text("strix.yml") concurrency_contract = workflow.split("concurrency:", 1)[1].split( "permissions:", 1 @@ -235,6 +243,7 @@ def test_strix_cancels_superseded_pr_head_security_evidence() -> None: def test_strix_install_normalizes_executable_permissions_before_hashing() -> None: + """Normalize the Strix executable before its trusted hash is computed.""" workflow = workflow_text("strix.yml") install_step = workflow_step(workflow, "Install Strix") @@ -251,6 +260,7 @@ def test_strix_install_normalizes_executable_permissions_before_hashing() -> Non def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() -> None: + """Close events should cancel old runs without starting expensive jobs.""" workflows = ( "close-empty-pr.yml", "codeql-pr.yml", @@ -286,6 +296,7 @@ def test_pull_request_close_events_cancel_superseded_runs_without_heavy_jobs() - def test_close_empty_pr_metadata_lookup_retries_and_fails_open() -> None: + """Retry invalid close-event metadata and leave the PR open on uncertainty.""" workflow = workflow_text("close-empty-pr.yml") assert "gh_api_json_with_retry()" in workflow @@ -297,6 +308,7 @@ def test_close_empty_pr_metadata_lookup_retries_and_fails_open() -> None: def test_cancelled_review_workflow_runs_do_not_spawn_more_queue_work() -> None: + """Prevent cancelled review runs from creating follow-up queue work.""" for filename in ("noema-review.yml", "pr-review-merge-scheduler.yml"): workflow = workflow_text(filename) @@ -304,6 +316,7 @@ def test_cancelled_review_workflow_runs_do_not_spawn_more_queue_work() -> None: def test_required_workflow_trusted_source_refs_are_not_input_controlled() -> None: + """Ensure privileged workflows resolve trusted source code independently of inputs.""" for filename in ( "opencode-review-dispatch.yml", "noema-review.yml", @@ -330,6 +343,7 @@ def test_required_workflow_trusted_source_refs_are_not_input_controlled() -> Non def test_noema_workflow_run_followup_cannot_cancel_required_pr_event_review() -> None: + """Keep Noema workflow-run follow-ups isolated from PR-event reviews.""" workflow = workflow_text("noema-review.yml") concurrency_contract = workflow.split("permissions:", 1)[0] @@ -339,6 +353,7 @@ def test_noema_workflow_run_followup_cannot_cancel_required_pr_event_review() -> def test_noema_review_credentials_and_llm_configuration_fail_closed() -> None: + """Require explicit reviewer credentials and LLM configuration.""" workflow = workflow_text("noema-review.yml") assert "fail_unavailable()" in workflow @@ -394,6 +409,7 @@ def test_noema_review_credentials_and_llm_configuration_fail_closed() -> None: def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret( tmp_path: Path, ) -> None: + """Preserve configured fallback models while rejecting an unavailable NIM secret.""" strix_output = tmp_path / "strix-output" strix = subprocess.run( [ @@ -464,6 +480,7 @@ def test_nvidia_nim_defaults_preserve_existing_fallbacks_without_secret( def test_noema_workflow_run_without_pull_request_skips_before_token_exchange() -> None: + """Skip unassociated workflow runs before requesting review credentials.""" workflow = workflow_text("noema-review.yml") assert ( @@ -574,6 +591,7 @@ def test_opencode_dispatch_hands_approved_head_to_noema_before_merge() -> None: def test_noema_and_scheduler_trusted_checkouts_use_static_main() -> None: + """Keep Noema and scheduler trusted checkouts pinned to central immutable sources.""" noema = workflow_text("noema-review.yml") scheduler = workflow_text("pr-review-merge-scheduler.yml") @@ -601,6 +619,7 @@ def test_noema_and_scheduler_trusted_checkouts_use_static_main() -> None: def test_unassociated_review_workflow_runs_do_not_scan_the_whole_pr_queue() -> None: + """Avoid scanning every PR when a workflow run has no associated pull request.""" workflow = workflow_text("pr-review-merge-scheduler.yml") assert "github.event.workflow_run.pull_requests[0].number" in workflow @@ -833,6 +852,7 @@ def test_org_queue_sweep_treats_inaccessible_repositories_as_non_fatal() -> None def test_fix_scheduler_cancels_superseded_cron_runs() -> None: + """Cancel stale scheduled repair runs before they duplicate mutation work.""" workflow = workflow_text("pr-review-fix-scheduler.yml") assert "central-pr-review-fix-scheduler-" in workflow @@ -842,6 +862,7 @@ def test_fix_scheduler_cancels_superseded_cron_runs() -> None: def test_security_scan_skips_dependency_review_when_dependency_graph_is_unavailable() -> ( None ): + """Treat unsupported dependency graphs as an explicit non-enforceable case.""" workflow = workflow_text("security-scan.yml") assert "id: dependency_review_support" in workflow @@ -852,6 +873,7 @@ def test_security_scan_skips_dependency_review_when_dependency_graph_is_unavaila def test_security_scan_allows_repositories_without_supported_lockfiles() -> None: + """Allow manifestless repositories while still requiring scan artifacts.""" workflow = workflow_text("security-scan.yml") assert workflow.count("--allow-no-lockfiles") == 4 @@ -862,6 +884,7 @@ def test_security_scan_allows_repositories_without_supported_lockfiles() -> None def test_secret_scan_push_limits_gitleaks_to_current_branch_history() -> None: + """Limit push secret scanning to the current branch history.""" workflow = workflow_text("secret-scan.yml") assert "CURRENT_SHA: ${{ github.sha }}" in workflow @@ -872,6 +895,7 @@ def test_secret_scan_push_limits_gitleaks_to_current_branch_history() -> None: def test_osv_pr_workflow_has_one_startup_safe_scan_args_block() -> None: + """Keep the standalone OSV workflow's resolver settings singular and safe.""" workflow = workflow_text("osv-scanner-pr.yml") concurrency_contract = workflow.split("permissions:", 1)[0] @@ -894,6 +918,7 @@ def test_osv_pr_workflow_has_one_startup_safe_scan_args_block() -> None: def test_osv_scan_logs_and_retries_without_transitive_resolution_on_resolver_failure() -> ( None ): + """Retry OSV direct evidence without allowing transitive resolver stalls.""" workflow = workflow_text("security-scan.yml") assert "timeout-minutes: 25" in workflow @@ -936,6 +961,7 @@ def test_osv_scan_logs_and_retries_without_transitive_resolution_on_resolver_fai def test_osv_sarif_upload_is_marked_comprehensive_after_clean_comparison( tmp_path: Path, ) -> None: + """Mark a clean OSV comparison as comprehensive for code-scanning closure.""" workflow = workflow_text("security-scan.yml") step = " - name: Mark clean OSV SARIF as comprehensive\n" start = workflow.index(step) @@ -979,6 +1005,7 @@ def test_osv_sarif_upload_is_marked_comprehensive_after_clean_comparison( def test_security_scan_osv_upload_uses_pr_head_for_pr_head_sarif() -> None: + """Upload OSV SARIF against the exact pull-request head revision.""" workflow = workflow_text("security-scan.yml") upload_step = workflow_step(workflow, "Upload OSV SARIF to code scanning") @@ -1056,6 +1083,7 @@ def test_standalone_osv_scan_delegates_sarif_upload_to_central_gate() -> None: def test_osv_findings_log_accepts_null_results_for_manifestless_repos( tmp_path: Path, ) -> None: + """Log zero findings when OSV returns null result arrays.""" workflow = workflow_text("security-scan.yml") step = " - name: Print OSV findings being compared\n" start = workflow.index(step) @@ -1081,6 +1109,7 @@ def test_osv_findings_log_accepts_null_results_for_manifestless_repos( def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> None: + """Make optional Strix absence visible without turning it into a lookup crash.""" workflow = workflow_text("opencode-review-dispatch.yml") failed_check_evidence = ( REPO_ROOT / "scripts/ci/collect_failed_check_evidence.sh" @@ -1093,6 +1122,7 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: + """Keep provider outages typed and non-passing until authoritative evidence exists.""" workflow = workflow_text("strix.yml") assert "RateLimitError|Too many requests" in workflow @@ -1115,6 +1145,7 @@ def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: def test_strix_cross_repo_dispatch_uses_target_token_for_pr_scoping() -> None: + """Bind cross-repository Strix scans to the target PR and authorized token.""" workflow = workflow_text("strix.yml") run_step = workflow.split(" - name: Run Strix (quick)", 1)[1].split( " - name:", 1 @@ -1211,6 +1242,7 @@ def test_default_branch_scorecard_upload_quota_is_non_blocking() -> None: def test_trivy_failure_log_prints_sarif_finding_details(tmp_path: Path) -> None: + """Print actionable Trivy SARIF details and fail only for actual findings.""" workflow = workflow_text("security-scan.yml") assert "fail-on-severity: moderate" in workflow assert "severity: CRITICAL,HIGH,MEDIUM" in workflow From 956ae2b8db7acdda333c76537425ab8cb85c82f2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 07:58:40 -0700 Subject: [PATCH 15/37] fix: keep baseline-only Strix outages non-blocking (#1169) --- .../strix-nvidia-nim-not-found-fallback.md | 17 +++-- scripts/ci/strix_quick_gate.sh | 9 ++- scripts/ci/test_strix_quick_gate.sh | 72 ++++++++++++++++++- 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index 70299ebdf..cfad975f9 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -30,9 +30,12 @@ combining with an unrelated application `404` to spoof infrastructure fallback. Provider-side failure also remains a fail-closed incomplete scan until a distinct fallback produces complete evidence. -The outer workflow may classify exhausted provider infrastructure as neutral only -when the run log contains no vulnerability signal. Any reported severity or -non-zero vulnerability count remains blocking. Scanner reports and attempt logs +Exhausted provider infrastructure remains fail-closed unless either no +vulnerability was reported or the trusted gate has parsed every threshold +report and classified every finding as outside the pull request's changed +files. The latter decision is made from structured report locations inside the +gate, not by trusting a log phrase in the outer workflow. Changed, unmapped, or +changed-manifest findings remain blocking. Scanner reports and attempt logs remain available as artifacts. ## Verification contract @@ -48,8 +51,12 @@ Regression evidence proves that: 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are current NVIDIA hosted models; 7. GitHub Models remain later cross-provider fallbacks; -8. vulnerability signals prevent neutral infrastructure classification; and -9. the required-workflow smoke contract pins these properties. +8. vulnerability signals prevent neutral infrastructure classification unless + every threshold report is structurally attributed to unchanged baseline + files; +9. changed, unmapped, and changed-manifest findings still block after provider + exhaustion; and +10. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 9345bf01f..2e570612e 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -4082,7 +4082,14 @@ run_current_target_scan() { echo "Strix quick scan failed with a non-recoverable error." >&2 return 1 fi - done + done + + if is_pull_request_event && + [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && + [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then + echo "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." >&2 + return 0 + fi if should_fail_pull_request_infra_zero_findings; then return 1 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7f9512bfd..c36243682 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3792,7 +3792,7 @@ REPORT ;; esac ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-allows | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" @@ -3801,7 +3801,8 @@ REPORT exit 1 ;; openai/deepseek/deepseek-r1-0528) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ] || + [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-allows" ]; then mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL @@ -3830,6 +3831,12 @@ EOS exit 2 ;; openai/deepseek/deepseek-v3-0324) + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-allows" ]; then + echo "LLM CONNECTION FAILED" + echo "Could not establish connection to the language model." + echo "Error: provider retirement brownout" + exit 1 + fi echo "scan ok after second GitHub Models fallback" exit 0 ;; @@ -6260,6 +6267,37 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-exhausted-after-baseline-vulnerability-allows) + run_gate_case "github-models-exhausted-after-baseline-vulnerability-allows" \ + "openai/gpt-5" \ + "" \ + "0" \ + "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." \ + "3" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ + "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "1" + ;; github-models-fallback-changed-vulnerability-before-next-success-blocks) run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ "openai/gpt-5" \ @@ -9723,6 +9761,36 @@ run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" +run_gate_case "github-models-exhausted-after-baseline-vulnerability-allows" \ + "openai/gpt-5" \ + "" \ + "0" \ + "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." \ + "3" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ + "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "1" + run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ "openai/gpt-5" \ "" \ From 9a4d1e1439bbafa8781971fbf22ab695ae126271 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 08:21:28 -0700 Subject: [PATCH 16/37] fix(strix): fail closed after provider exhaustion --- .../strix-nvidia-nim-not-found-fallback.md | 19 ++++++++---------- scripts/ci/strix_quick_gate.sh | 13 ++++++------ scripts/ci/test_strix_quick_gate.sh | 20 +++++++++---------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index cfad975f9..a088aa7ef 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -30,13 +30,12 @@ combining with an unrelated application `404` to spoof infrastructure fallback. Provider-side failure also remains a fail-closed incomplete scan until a distinct fallback produces complete evidence. -Exhausted provider infrastructure remains fail-closed unless either no -vulnerability was reported or the trusted gate has parsed every threshold -report and classified every finding as outside the pull request's changed -files. The latter decision is made from structured report locations inside the -gate, not by trusting a log phrase in the outer workflow. Changed, unmapped, or -changed-manifest findings remain blocking. Scanner reports and attempt logs -remain available as artifacts. +Exhausted provider infrastructure remains fail-closed even when the trusted +gate has classified every observed threshold finding as outside the pull +request's changed files. That classification scopes authoritative findings; it +cannot prove that an incomplete provider-exhausted scan observed every finding. +Changed, unmapped, and changed-manifest findings also remain blocking. Scanner +reports and attempt logs remain available as artifacts. ## Verification contract @@ -51,10 +50,8 @@ Regression evidence proves that: 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are current NVIDIA hosted models; 7. GitHub Models remain later cross-provider fallbacks; -8. vulnerability signals prevent neutral infrastructure classification unless - every threshold report is structurally attributed to unchanged baseline - files; -9. changed, unmapped, and changed-manifest findings still block after provider +8. provider exhaustion remains non-passing after unchanged baseline findings; +9. changed, unmapped, and changed-manifest findings also block after provider exhaustion; and 10. the required-workflow smoke contract pins these properties. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 2e570612e..45228400b 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -4084,13 +4084,6 @@ run_current_target_scan() { fi done - if is_pull_request_event && - [ "$PR_FINDINGS_DECISION" = "allow_baseline" ] && - [ "$INFRA_ERROR_DETECTED" -eq 1 ]; then - echo "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." >&2 - return 0 - fi - if should_fail_pull_request_infra_zero_findings; then return 1 fi @@ -4111,6 +4104,12 @@ run_current_target_scan() { return 1 fi + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && + [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then + echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 + return 1 + fi + local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index c36243682..95d29617a 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3792,7 +3792,7 @@ REPORT ;; esac ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-allows | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" @@ -3802,7 +3802,7 @@ REPORT ;; openai/deepseek/deepseek-r1-0528) if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ] || - [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-allows" ]; then + [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL @@ -3831,7 +3831,7 @@ EOS exit 2 ;; openai/deepseek/deepseek-v3-0324) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-allows" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then echo "LLM CONNECTION FAILED" echo "Could not establish connection to the language model." echo "Error: provider retirement brownout" @@ -6267,12 +6267,12 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; - github-models-exhausted-after-baseline-vulnerability-allows) - run_gate_case "github-models-exhausted-after-baseline-vulnerability-allows" \ + github-models-exhausted-after-baseline-vulnerability-fails-closed) + run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ "openai/gpt-5" \ "" \ - "0" \ - "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." \ + "1" \ + "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ "3" \ "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ @@ -9761,11 +9761,11 @@ run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" -run_gate_case "github-models-exhausted-after-baseline-vulnerability-allows" \ +run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ "openai/gpt-5" \ "" \ - "0" \ - "Configured provider models were exhausted after reporting only unchanged pull-request findings; allowing pipeline continuation." \ + "1" \ + "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ "3" \ "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ From a82b8012be5d57f99c1a2b029f08837d4d10d564 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 08:42:26 +0900 Subject: [PATCH 17/37] fix(ci): redact untrusted github method diagnostics --- .../organization_commercial_readiness_loop.py | 10 ++++++- .../test_required_workflow_queue_contract.py | 29 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index 84c4f326c..c0c02b163 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -47,6 +47,9 @@ MAX_WORKFLOW_SOURCES_PER_REPOSITORY = 100 MAX_WORKFLOW_SOURCE_BYTES_PER_FILE = 1_048_576 MAX_WORKFLOW_SOURCE_BYTES_PER_REPOSITORY = 10 * 1_048_576 +SAFE_DIAGNOSTIC_METHODS = frozenset( + {"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"} +) class GitHubError(RuntimeError): @@ -267,6 +270,11 @@ def request( ) -> Any: """Call one GitHub REST endpoint and decode a bounded JSON response.""" normalized_method = method.upper() + safe_method = ( + normalized_method + if normalized_method in SAFE_DIAGNOSTIC_METHODS + else "[REDACTED_METHOD]" + ) safe_path = self._redact_credential(path) args = ["gh", "api"] if normalized_method != "GET": @@ -292,7 +300,7 @@ def request( raw = (completed.stderr or completed.stdout or "GitHub API request failed").strip() bounded = self._redact_credential(raw)[-900:] raise GitHubError( - f"GitHub API {normalized_method} {safe_path} failed: {bounded}" + f"GitHub API {safe_method} {safe_path} failed: {bounded}" ) text = completed.stdout.strip() if not text: diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index cf8063d9b..f358cda09 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -44,6 +44,35 @@ def test_merge_scheduler_dispatches_one_review_by_default() -> None: ) +def test_organization_readiness_does_not_echo_untrusted_http_method( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep arbitrary HTTP method text out of organization-loop diagnostics.""" + from types import SimpleNamespace + + from scripts.ci.organization_commercial_readiness_loop import ( + GitHubClient, + GitHubError, + ) + + token = "ghp_abcdefghijklmnopqrstuvwxyz0123456789AB" + monkeypatch.setattr( + "subprocess.run", + lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, + stdout="", + stderr="request rejected", + ), + ) + + with pytest.raises(GitHubError) as raised: + GitHubClient("client-token").request("/repos/example", method=token) + + message = str(raised.value) + assert token.upper() not in message + assert "[REDACTED_METHOD]" in message + + def test_merge_scheduler_provides_same_repository_dispatch_credential() -> None: """Guard the runner-token dispatch credential for central review workflows. From 945d5d56ff826b8642c634e6cf0d14a8ec9be38a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 21:46:24 -0700 Subject: [PATCH 18/37] fix(strix): include backend app dependency context --- CHANGELOG.md | 5 +++ .../strix-pr-head-context-boundary.md | 45 +++++++++++++++++++ scripts/ci/strix_quick_gate.sh | 33 ++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 24 +++++++++- 4 files changed, 105 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-pr-head-context-boundary.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d7f80445b..2fbc20f82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,11 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Include the exact `backend/app/*.py` package context in PR-scoped Strix + scans when a module in that package changes. The trusted resolver uses a + NUL-delimited exact-head tree listing, copies unchanged dependencies from + the trusted base, and keeps changed-file attribution and provider failures + fail-closed. - Run Strix with an explicit canonical scan target from a temporary working directory outside that target, so scanner state and relative reports cannot become self-scanned source findings; preserve those reports as gate evidence. diff --git a/docs/doctoring/strix-pr-head-context-boundary.md b/docs/doctoring/strix-pr-head-context-boundary.md new file mode 100644 index 000000000..2e03cbe22 --- /dev/null +++ b/docs/doctoring/strix-pr-head-context-boundary.md @@ -0,0 +1,45 @@ +# Strix PR-head dependency context boundary + +Status: accepted 2026-08-21 + +## Incident + +The Strix run for LineageWeave PR #192 materialized changed Python files but +not the unchanged local `backend/app` dependency package. The scanner then +reported `backend.app.post_eligibility` as missing even though that module was +present in the PR head and base repository. Earlier attempts also encountered +NVIDIA NIM rate limits; those provider failures must remain visible and must +not be confused with a source finding. + +## Decision + +When a PR changes a Python module under `backend/app`, the trusted Strix scope +resolver enumerates every Python file under `backend/app` from the exact PR +head tree. It reads the Git tree as NUL-delimited paths and applies the same +bounded path validator used for changed files, so ambiguous or unsafe entries +fail closed. The scope builder copies changed files from that head and +unchanged context from the trusted base checkout. The changed-file list +remains the finding-attribution boundary; this does not turn a context file +into a changed finding. The scan still executes only trusted scanner code and +treats PR-head blobs as non-executable data. + +This is a product-neutral extension of the existing backend context contract; +it does not replace the repository-specific context list for other backend +layouts and does not downgrade provider or vulnerability failures. + +## Evidence and rollback + +The regression fixture creates a changed `backend/app/knowledge_graph.py` that +imports an unchanged `backend/app/post_eligibility.py`, then asserts that the +production scope contains the dependency and the trusted content. Roll back +this change only with an equivalent exact-head dependency-context contract; +removing the context or weakening the Strix gate is not an acceptable rollback. + +## References + +National Institute of Standards and Technology. (2008). *Technical guide to +information security testing and assessment* (Special Publication 800-115). +https://doi.org/10.6028/NIST.SP.800-115 + +OWASP Foundation. (n.d.). *Web security testing guide*. Retrieved August 21, +2026, from https://owasp.org/www-project-web-security-testing-guide/ diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 45228400b..845c3835a 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -1199,6 +1199,7 @@ is_scannable_changed_file() { pull_request_scope_context_files() { local needs_backend_python=0 + local needs_backend_app_python=0 local needs_frontend_email_api_context=0 local needs_deployment_context=0 local changed_file normalized_changed_file @@ -1209,6 +1210,9 @@ pull_request_scope_context_files() { if [[ "$normalized_changed_file" =~ ^backend/.+\.py$ ]]; then needs_backend_python=1 fi + if [[ "$normalized_changed_file" =~ ^backend/app/.+\.py$ ]]; then + needs_backend_app_python=1 + fi ;; # The app shell, email components, threading URL builder, and API client can # shape frontend email retrieval flows; include backend auth context with them. @@ -1288,6 +1292,35 @@ EOF done fi + if [ "$needs_backend_app_python" -eq 1 ]; then + local backend_app_head_sha + backend_app_head_sha="$(trim_whitespace "${PR_HEAD_SHA:-}")" + if { [ -z "$backend_app_head_sha" ] || ! is_valid_git_commit_sha "$backend_app_head_sha"; } && pull_request_head_blob_required; then + echo "ERROR: backend/app PR-head context requires an exact head SHA; failing closed." >&2 + return 2 + elif [ -n "$backend_app_head_sha" ] && is_valid_git_commit_sha "$backend_app_head_sha"; then + local backend_app_tree_file context_file normalized_context_file + backend_app_tree_file="$(mktemp "${RUNNER_TEMP:-/tmp}/strix-backend-app-context.XXXXXX")" || return 2 + if ! git -c core.quotepath=false ls-tree -rz --name-only "$backend_app_head_sha" -- backend/app >"$backend_app_tree_file"; then + rm -f -- "$backend_app_tree_file" + echo "ERROR: backend/app PR-head context could not be enumerated; failing closed." >&2 + return 2 + fi + while IFS= read -r -d '' context_file; do + normalized_context_file="$(normalize_changed_file_path "$context_file")" || { + rm -f -- "$backend_app_tree_file" + return 2 + } + case "$normalized_context_file" in + backend/app/*.py) + printf '%s\n' "$normalized_context_file" + ;; + esac + done <"$backend_app_tree_file" + rm -f -- "$backend_app_tree_file" + fi + fi + if [ "$needs_frontend_email_api_context" -eq 1 ]; then cat <<'EOF' backend/api/auth.py diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 95d29617a..c144fcc69 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -7124,6 +7124,20 @@ if [ -f "$target_path/backend/services/email_parser.py" ]; then matched_backend_context=1 fi +if [ -f "$target_path/backend/app/knowledge_graph.py" ]; then + if [ ! -f "$target_path/backend/app/post_eligibility.py" ]; then + echo "Error: backend/app local import context missing from PR scope ($target_path)" >&2 + exit 78 + fi + if ! grep -Fq -- 'BASE_POST_ELIGIBILITY_SHOULD_BE_SCANNED' "$target_path/backend/app/post_eligibility.py"; then + echo "Error: backend/app dependency context did not use trusted base content" >&2 + cat -- "$target_path/backend/app/post_eligibility.py" >&2 + exit 79 + fi + echo "scan ok with backend/app local import context" + matched_backend_context=1 +fi + if [ "$matched_backend_context" -eq 1 ]; then exit 0 fi @@ -7140,11 +7154,12 @@ EOF git config user.name 'Strix Test' git config user.email 'strix-test@example.invalid' echo 'seed' >README.md - mkdir -p backend/api backend/services + mkdir -p backend/api backend/app backend/services printf '%s\n' 'BASE_AUTH_CONTENT_SHOULD_NOT_BE_SCANNED' >backend/api/auth.py printf '%s\n' 'BASE_EMAILS_CONTENT_SHOULD_NOT_BE_SCANNED' >backend/api/emails.py printf '%s\n' 'BASE_CALENDAR_SERVICE_SHOULD_BE_SCANNED' >backend/services/calendar_service.py printf '%s\n' 'BASE_LLM_PROVIDER_URLS_SHOULD_NOT_BE_SCANNED' >backend/services/llm_provider_urls.py + printf '%s\n' 'BASE_POST_ELIGIBILITY_SHOULD_BE_SCANNED' >backend/app/post_eligibility.py git add . git commit -qm 'base commit' ) @@ -7193,6 +7208,10 @@ EOF cat >backend/api/runner_config.py <<'EOF' def require_workspace_admin(): return 'HEAD_RUNNER_CONFIG_SHOULD_BE_SCANNED' +EOF + cat >backend/app/knowledge_graph.py <<'EOF' +from .post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +HEAD_KNOWLEDGE_GRAPH_SHOULD_BE_SCANNED EOF git add . git commit -qm 'head commit' @@ -7210,7 +7229,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ GITHUB_EVENT_NAME="pull_request_target" \ PR_BASE_SHA="$base_sha" \ - PR_HEAD_SHA="$head_sha" \ + PR_HEAD_SHA=" $head_sha " \ STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ @@ -7227,6 +7246,7 @@ EOF assert_file_contains "$output_log" "scan ok with PR-head backend dependency context" "case=pull-request-target-changed-backend-context-uses-head-blob output" assert_file_contains "$output_log" "scan ok with PR-head LLM provider URL validation context" "case=pull-request-target-changed-backend-context-includes-llm-provider-url-validation output" assert_file_contains "$output_log" "scan ok with PR-head email parser text safety context" "case=pull-request-target-changed-backend-context-includes-email-parser-text-safety output" + assert_file_contains "$output_log" "scan ok with backend/app local import context" "case=pull-request-target-changed-backend-context-includes-backend-app-local-import output" assert_equals "1" "$(wc -l <"$call_log" | tr -d ' ')" "case=pull-request-target-changed-backend-context-uses-head-blob strix call count" rm -rf "$tmp_dir" From ab0f179ddf0623a92fc597b7be0f080991fd5a5c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:03:17 -0700 Subject: [PATCH 19/37] fix(strix): include contextual orchestrator sibling context --- CHANGELOG.md | 3 ++ .../strix-pr-head-context-boundary.md | 23 +++++++------ scripts/ci/strix_quick_gate.sh | 33 +++++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 31 +++++++++++++++++ 4 files changed, 80 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fbc20f82..0fc333640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,9 @@ Semantic Versioning where the repository publishes a release. NUL-delimited exact-head tree listing, copies unchanged dependencies from the trusted base, and keeps changed-file attribution and provider failures fail-closed. +- Include the exact `contextual_orchestrator/*.py` sibling-import context under + the same NUL-delimited exact-head and fail-closed path boundary without + expanding changed-file finding attribution. - Run Strix with an explicit canonical scan target from a temporary working directory outside that target, so scanner state and relative reports cannot become self-scanned source findings; preserve those reports as gate evidence. diff --git a/docs/doctoring/strix-pr-head-context-boundary.md b/docs/doctoring/strix-pr-head-context-boundary.md index 2e03cbe22..085255985 100644 --- a/docs/doctoring/strix-pr-head-context-boundary.md +++ b/docs/doctoring/strix-pr-head-context-boundary.md @@ -7,15 +7,18 @@ Status: accepted 2026-08-21 The Strix run for LineageWeave PR #192 materialized changed Python files but not the unchanged local `backend/app` dependency package. The scanner then reported `backend.app.post_eligibility` as missing even though that module was -present in the PR head and base repository. Earlier attempts also encountered -NVIDIA NIM rate limits; those provider failures must remain visible and must -not be confused with a source finding. +present in the PR head and base repository. The same changed-file-only failure +mode affected `contextual-orchestrator` PR #801: `__main__.py` imported sibling +modules omitted from the temporary scan tree. Earlier attempts also encountered +NVIDIA NIM rate limits; those provider failures must remain visible and must not +be confused with a source finding. ## Decision -When a PR changes a Python module under `backend/app`, the trusted Strix scope -resolver enumerates every Python file under `backend/app` from the exact PR -head tree. It reads the Git tree as NUL-delimited paths and applies the same +When a PR changes a Python module under `backend/app` or +`contextual_orchestrator`, the trusted Strix scope resolver enumerates every +Python file under that package from the exact PR head tree. It reads the Git +tree as NUL-delimited paths and applies the same bounded path validator used for changed files, so ambiguous or unsafe entries fail closed. The scope builder copies changed files from that head and unchanged context from the trusted base checkout. The changed-file list @@ -29,10 +32,10 @@ layouts and does not downgrade provider or vulnerability failures. ## Evidence and rollback -The regression fixture creates a changed `backend/app/knowledge_graph.py` that -imports an unchanged `backend/app/post_eligibility.py`, then asserts that the -production scope contains the dependency and the trusted content. Roll back -this change only with an equivalent exact-head dependency-context contract; +The regression fixture creates changed modules that import unchanged siblings +in both packages, then asserts that the production scope contains the +dependencies and their trusted content. Roll back this change only with an +equivalent exact-head dependency-context contract; removing the context or weakening the Strix gate is not an acceptable rollback. ## References diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 845c3835a..cd69c1061 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -1200,6 +1200,7 @@ is_scannable_changed_file() { pull_request_scope_context_files() { local needs_backend_python=0 local needs_backend_app_python=0 + local needs_contextual_orchestrator_python=0 local needs_frontend_email_api_context=0 local needs_deployment_context=0 local changed_file normalized_changed_file @@ -1214,6 +1215,9 @@ pull_request_scope_context_files() { needs_backend_app_python=1 fi ;; + contextual_orchestrator/*.py) + needs_contextual_orchestrator_python=1 + ;; # The app shell, email components, threading URL builder, and API client can # shape frontend email retrieval flows; include backend auth context with them. frontend/src/components/EmailDetail.tsx | frontend/src/components/EmailList.tsx | frontend/src/app/page.tsx | frontend/src/lib/api-client.ts | frontend/src/lib/email-threading.ts) @@ -1321,6 +1325,35 @@ EOF fi fi + if [ "$needs_contextual_orchestrator_python" -eq 1 ]; then + local contextual_orchestrator_head_sha + contextual_orchestrator_head_sha="$(trim_whitespace "${PR_HEAD_SHA:-}")" + if { [ -z "$contextual_orchestrator_head_sha" ] || ! is_valid_git_commit_sha "$contextual_orchestrator_head_sha"; } && pull_request_head_blob_required; then + echo "ERROR: contextual_orchestrator PR-head context requires an exact head SHA; failing closed." >&2 + return 2 + elif [ -n "$contextual_orchestrator_head_sha" ] && is_valid_git_commit_sha "$contextual_orchestrator_head_sha"; then + local contextual_orchestrator_tree_file context_file normalized_context_file + contextual_orchestrator_tree_file="$(mktemp "${RUNNER_TEMP:-/tmp}/strix-contextual-orchestrator-context.XXXXXX")" || return 2 + if ! git -c core.quotepath=false ls-tree -rz --name-only "$contextual_orchestrator_head_sha" -- contextual_orchestrator >"$contextual_orchestrator_tree_file"; then + rm -f -- "$contextual_orchestrator_tree_file" + echo "ERROR: contextual_orchestrator PR-head context could not be enumerated; failing closed." >&2 + return 2 + fi + while IFS= read -r -d '' context_file; do + normalized_context_file="$(normalize_changed_file_path "$context_file")" || { + rm -f -- "$contextual_orchestrator_tree_file" + return 2 + } + case "$normalized_context_file" in + contextual_orchestrator/*.py) + printf '%s\n' "$normalized_context_file" + ;; + esac + done <"$contextual_orchestrator_tree_file" + rm -f -- "$contextual_orchestrator_tree_file" + fi + fi + if [ "$needs_frontend_email_api_context" -eq 1 ]; then cat <<'EOF' backend/api/auth.py diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index c144fcc69..1e81f3c6e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -173,6 +173,14 @@ assert_strix_pr_scope_includes_deployment_context() { assert_file_contains "$GATE_SCRIPT" "scripts/ci/test_*.sh" "strix gate excludes large CI self-test harnesses from PR scan targets" } +assert_strix_pr_scope_includes_contextual_orchestrator_context() { + assert_file_contains "$GATE_SCRIPT" "needs_contextual_orchestrator_python=0" "strix gate tracks contextual-orchestrator package context" + assert_file_contains "$GATE_SCRIPT" 'contextual_orchestrator/*.py)' "strix gate detects contextual-orchestrator Python changes" + assert_file_contains "$GATE_SCRIPT" 'git -c core.quotepath=false ls-tree -rz --name-only "$contextual_orchestrator_head_sha" -- contextual_orchestrator' "strix gate enumerates contextual-orchestrator context from the exact PR head" + assert_file_contains "$GATE_SCRIPT" 'contextual_orchestrator_tree_file="$(mktemp' "strix gate bounds contextual-orchestrator context enumeration in a private file" + assert_file_contains "$GATE_SCRIPT" 'rm -f -- "$contextual_orchestrator_tree_file"' "strix gate cleans contextual-orchestrator context enumeration evidence" +} + assert_strix_workflow_pr_trigger_hardened() { local workflow_file="$REPO_ROOT/.github/workflows/strix.yml" @@ -7138,6 +7146,20 @@ if [ -f "$target_path/backend/app/knowledge_graph.py" ]; then matched_backend_context=1 fi +if [ -f "$target_path/contextual_orchestrator/__main__.py" ]; then + if [ ! -f "$target_path/contextual_orchestrator/cost_ledger.py" ]; then + echo "Error: contextual-orchestrator local import context missing from PR scope ($target_path)" >&2 + exit 80 + fi + if ! grep -Fq -- 'BASE_COST_LEDGER_SHOULD_BE_SCANNED' "$target_path/contextual_orchestrator/cost_ledger.py"; then + echo "Error: contextual-orchestrator dependency context did not use trusted base content" >&2 + cat -- "$target_path/contextual_orchestrator/cost_ledger.py" >&2 + exit 81 + fi + echo "scan ok with contextual-orchestrator local import context" + matched_backend_context=1 +fi + if [ "$matched_backend_context" -eq 1 ]; then exit 0 fi @@ -7160,6 +7182,8 @@ EOF printf '%s\n' 'BASE_CALENDAR_SERVICE_SHOULD_BE_SCANNED' >backend/services/calendar_service.py printf '%s\n' 'BASE_LLM_PROVIDER_URLS_SHOULD_NOT_BE_SCANNED' >backend/services/llm_provider_urls.py printf '%s\n' 'BASE_POST_ELIGIBILITY_SHOULD_BE_SCANNED' >backend/app/post_eligibility.py + mkdir -p contextual_orchestrator + printf '%s\n' 'BASE_COST_LEDGER_SHOULD_BE_SCANNED' >contextual_orchestrator/cost_ledger.py git add . git commit -qm 'base commit' ) @@ -7212,6 +7236,10 @@ EOF cat >backend/app/knowledge_graph.py <<'EOF' from .post_eligibility import SOURCE_POST_ELIGIBILITY_SQL HEAD_KNOWLEDGE_GRAPH_SHOULD_BE_SCANNED +EOF + cat >contextual_orchestrator/__main__.py <<'EOF' +from .cost_ledger import UsageRecord +HEAD_CONTEXTUAL_ORCHESTRATOR_SHOULD_BE_SCANNED EOF git add . git commit -qm 'head commit' @@ -7247,6 +7275,7 @@ EOF assert_file_contains "$output_log" "scan ok with PR-head LLM provider URL validation context" "case=pull-request-target-changed-backend-context-includes-llm-provider-url-validation output" assert_file_contains "$output_log" "scan ok with PR-head email parser text safety context" "case=pull-request-target-changed-backend-context-includes-email-parser-text-safety output" assert_file_contains "$output_log" "scan ok with backend/app local import context" "case=pull-request-target-changed-backend-context-includes-backend-app-local-import output" + assert_file_contains "$output_log" "scan ok with contextual-orchestrator local import context" "case=pull-request-target-changed-contextual-orchestrator-includes-local-import output" assert_equals "1" "$(wc -l <"$call_log" | tr -d ' ')" "case=pull-request-target-changed-backend-context-uses-head-blob strix call count" rm -rf "$tmp_dir" @@ -9063,6 +9092,8 @@ assert_strix_workflow_pr_trigger_hardened assert_strix_pr_scope_includes_deployment_context +assert_strix_pr_scope_includes_contextual_orchestrator_context + assert_strix_gpt54_model_guard_cases assert_strix_gate_target_scope_separated From 035343c8a68e880a4abf27f7c947bfed9dbaafcf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 20 Aug 2026 22:20:27 -0700 Subject: [PATCH 20/37] fix(strix): include Rust workspace context --- CHANGELOG.md | 3 + .../strix-pr-head-context-boundary.md | 9 +++ scripts/ci/strix_quick_gate.sh | 15 +++- scripts/ci/test_strix_quick_gate.sh | 74 +++++++++++++++++++ 4 files changed, 99 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fc333640..25923501f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,9 @@ Semantic Versioning where the repository publishes a release. - Include the exact `contextual_orchestrator/*.py` sibling-import context under the same NUL-delimited exact-head and fail-closed path boundary without expanding changed-file finding attribution. +- Treat Rust source and Cargo manifests as governed Strix inputs and include + trusted Cargo, toolchain, and `deny.toml` context when a workflow change + scopes a Rust workspace. - Run Strix with an explicit canonical scan target from a temporary working directory outside that target, so scanner state and relative reports cannot become self-scanned source findings; preserve those reports as gate evidence. diff --git a/docs/doctoring/strix-pr-head-context-boundary.md b/docs/doctoring/strix-pr-head-context-boundary.md index 085255985..762fbee97 100644 --- a/docs/doctoring/strix-pr-head-context-boundary.md +++ b/docs/doctoring/strix-pr-head-context-boundary.md @@ -13,6 +13,10 @@ modules omitted from the temporary scan tree. Earlier attempts also encountered NVIDIA NIM rate limits; those provider failures must remain visible and must not be confused with a source finding. +TEPP PR #154 exposed the same completeness boundary for Rust: a workflow change +scoped the CI definition without the workspace's unchanged Cargo manifests, +toolchain selection, or cargo-deny policy. + ## Decision When a PR changes a Python module under `backend/app` or @@ -38,6 +42,11 @@ dependencies and their trusted content. Roll back this change only with an equivalent exact-head dependency-context contract; removing the context or weakening the Strix gate is not an acceptable rollback. +For a workflow-scoped root Rust workspace, the behavioral fixture also requires +trusted `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, and `deny.toml` +contents in the materialized target. Rust source and Cargo manifests remain +governed changed inputs rather than context-only exemptions. + ## References National Institute of Standards and Technology. (2008). *Technical guide to diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index cd69c1061..654141029 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -629,7 +629,7 @@ copy_pr_head_blob_to_file() { is_supported_source_file() { case "$1" in - *.java | *.kt | *.kts | *.groovy | *.scala | *.py | *.js | *.jsx | *.ts | *.tsx | *.vue | *.yaml | *.yml | *.sh | *.sql | *.xml | *.json | *.html | *.css | *.md) + *.java | *.kt | *.kts | *.groovy | *.scala | *.rs | *.py | *.js | *.jsx | *.ts | *.tsx | *.vue | *.yaml | *.yml | *.sh | *.sql | *.xml | *.json | *.html | *.css | *.md) return 0 ;; Dockerfile | */Dockerfile | Dockerfile.* | */Dockerfile.* | Containerfile | */Containerfile | Makefile | */Makefile) @@ -643,7 +643,7 @@ is_supported_source_file() { is_dependency_manifest_path() { case "$1" in - pom.xml | */pom.xml | package.json | */package.json | package-lock.json | */package-lock.json | pnpm-lock.yaml | */pnpm-lock.yaml | yarn.lock | */yarn.lock | pyproject.toml | */pyproject.toml | requirements.txt | */requirements.txt | requirements-*.txt | */requirements-*.txt | uv.lock | */uv.lock) + pom.xml | */pom.xml | Cargo.toml | */Cargo.toml | Cargo.lock | */Cargo.lock | package.json | */package.json | package-lock.json | */package-lock.json | pnpm-lock.yaml | */pnpm-lock.yaml | yarn.lock | */yarn.lock | pyproject.toml | */pyproject.toml | requirements.txt | */requirements.txt | requirements-*.txt | */requirements-*.txt | uv.lock | */uv.lock) return 0 ;; *) @@ -1383,6 +1383,17 @@ docker-compose.yml render.yaml VERSION EOF + # Workflow changes in a Rust workspace need dependency, toolchain, and + # policy context so Strix can analyze the repository as a complete unit. + if [ -f "$REPO_ROOT/Cargo.toml" ]; then + cat <<'EOF' +Cargo.toml +Cargo.lock +rust-toolchain.toml +rust-toolchain +deny.toml +EOF + fi fi } diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 1e81f3c6e..e22571cc4 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -170,6 +170,11 @@ assert_strix_pr_scope_includes_deployment_context() { assert_file_contains "$GATE_SCRIPT" "frontend/package-lock.json" "strix gate includes frontend dependency lock context" assert_file_contains "$GATE_SCRIPT" "frontend/postcss.config.mjs" "strix gate includes frontend build config context" assert_file_contains "$GATE_SCRIPT" "VERSION" "strix gate includes release version context for workflow scans" + assert_file_contains "$GATE_SCRIPT" "*.rs" "strix gate recognizes Rust source files" + assert_file_contains "$GATE_SCRIPT" "Cargo.toml | */Cargo.toml | Cargo.lock | */Cargo.lock" "strix gate recognizes Rust dependency manifests" + assert_file_contains "$GATE_SCRIPT" 'if [ -f "$REPO_ROOT/Cargo.toml" ]; then' "strix gate detects Rust workspaces for workflow scan context" + assert_file_contains "$GATE_SCRIPT" "rust-toolchain.toml" "strix gate includes Rust toolchain context for workflow scans" + assert_file_contains "$GATE_SCRIPT" "deny.toml" "strix gate includes Rust dependency policy context for workflow scans" assert_file_contains "$GATE_SCRIPT" "scripts/ci/test_*.sh" "strix gate excludes large CI self-test harnesses from PR scan targets" } @@ -5182,6 +5187,20 @@ EOS echo "scan ok with deployment entrypoint context" exit 0 ;; + pr-rust-workspace-context) + for rust_context in Cargo.toml Cargo.lock rust-toolchain.toml deny.toml; do + if [ ! -f "$target_path/$rust_context" ]; then + echo "Error: Rust workflow scope missing $rust_context ($target_path)" >&2 + exit 61 + fi + done + if ! grep -Fq -- 'name = "trusted-workspace"' "$target_path/Cargo.toml"; then + echo "Error: Rust workflow context did not preserve trusted Cargo content ($target_path)" >&2 + exit 62 + fi + echo "scan ok with Rust workspace context" + exit 0 + ;; *) echo "unknown scenario ${FAKE_STRIX_SCENARIO:?}" >&2 exit 8 @@ -5389,6 +5408,18 @@ EOS touch "$repo_root_dir/docker-compose.yml" touch "$repo_root_dir/render.yaml" echo '0.0.0' >"$repo_root_dir/VERSION" + elif [ "$scenario" = "pr-rust-workspace-context" ]; then + mkdir -p "$repo_root_dir/.github/workflows" "$repo_root_dir/src" + echo 'name: Rust CI' >"$repo_root_dir/.github/workflows/rust.yml" + cat >"$repo_root_dir/Cargo.toml" <<'EOS' +[package] +name = "trusted-workspace" +version = "0.1.0" +EOS + echo '# trusted lock' >"$repo_root_dir/Cargo.lock" + echo '[toolchain]' >"$repo_root_dir/rust-toolchain.toml" + echo '[advisories]' >"$repo_root_dir/deny.toml" + echo 'fn main() {}' >"$repo_root_dir/src/main.rs" elif [ "$scenario" = "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" ]; then mkdir -p "$repo_root_dir/.github/workflows" cat >"$repo_root_dir/.github/workflows/build-ci-image.yml" <<'EOS' @@ -5864,6 +5895,28 @@ run_filtered_gate_case_if_requested() { "vertex_ai/ready-primary" \ "" ;; + pr-rust-workspace-context) + run_gate_case "pr-rust-workspace-context" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with Rust workspace context" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + ".github/workflows/rust.yml" + ;; success-with-critical-report) run_gate_case "success-with-critical-report" \ "vertex_ai/ready-primary" \ @@ -11165,6 +11218,27 @@ run_gate_case "pr-deployment-scope-entrypoint-context" \ "pull_request" \ ".github/workflows/opencode-review.yml" +run_gate_case "pr-rust-workspace-context" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with Rust workspace context" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + ".github/workflows/rust.yml" + run_gate_case "pr-empty-diff-skip" \ "openai/gpt-4o-mini" \ "" \ From 2cd1d34e2027fec9f93a5dc522046d507276ef7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:23:54 +0000 Subject: [PATCH 21/37] fix(strix): classify ModelBehaviorError as retryable protocol evidence Treat the exact PascalCase agent-protocol exception as cross-model fallback evidence when the scan reports no vulnerabilities. Keep Vulnerabilities [1-9] and severity markers fail-closed. --- .../strix-changed-path-quality-ci.yml | 6 +- .github/workflows/strix.yml | 2 +- CHANGELOG.md | 3 + docs/doctoring/strix-model-behavior-error.md | 50 +++++ scripts/ci/strix_quick_gate.sh | 21 ++ tests/test_strix_model_behavior_error.py | 192 ++++++++++++++++++ ...st_strix_quality_timeout_fixture_budget.py | 2 + 7 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 docs/doctoring/strix-model-behavior-error.md create mode 100644 tests/test_strix_model_behavior_error.py diff --git a/.github/workflows/strix-changed-path-quality-ci.yml b/.github/workflows/strix-changed-path-quality-ci.yml index 75e9b7d8e..31924910a 100644 --- a/.github/workflows/strix-changed-path-quality-ci.yml +++ b/.github/workflows/strix-changed-path-quality-ci.yml @@ -5,12 +5,16 @@ on: branches: [main] paths: - ".github/workflows/strix-changed-path-quality-ci.yml" + - ".github/workflows/strix.yml" - "CHANGELOG.md" - "docs/doctoring/strix-legal-git-paths.md" + - "docs/doctoring/strix-model-behavior-error.md" - "docs/doctoring/strix-quality-timeout-fixtures.md" - "scripts/ci/strix_quick_gate.sh" - "scripts/ci/test_strix_quick_gate.sh" - "tests/test_strix_changed_path_policy.py" + - "tests/test_strix_model_behavior_error.py" + - "tests/test_strix_nvidia_nim_not_found_fallback.py" - "tests/test_strix_workflow_dependency_hashes.py" - "tests/test_strix_quality_timeout_fixture_budget.py" @@ -66,6 +70,6 @@ jobs: test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" python -m coverage run -m pytest tests -q bash scripts/ci/test_strix_quick_gate.sh - python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py + python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_model_behavior_error.py tests/test_strix_nvidia_nim_not_found_fallback.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py bash -n scripts/ci/strix_quick_gate.sh git diff --exit-code diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4155c7346..4223e9b18 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/CHANGELOG.md b/CHANGELOG.md index f4903c2f3..3b879b2fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as + retryable model-protocol evidence, while keeping `Vulnerabilities [1-9]` and + other severity signals fail-closed. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/strix-model-behavior-error.md b/docs/doctoring/strix-model-behavior-error.md new file mode 100644 index 000000000..7333404fe --- /dev/null +++ b/docs/doctoring/strix-model-behavior-error.md @@ -0,0 +1,50 @@ +# Strix ModelBehaviorError classifier + +기준일: **2026-08-21** + +## Incident + +Required Strix scans can fail closed after the agent runtime raises +`ModelBehaviorError` even when the log reports `Vulnerabilities 0`. The +exception means the selected model did not follow Strix's tool-calling +protocol. Treating that protocol failure as a security finding blocked +current-head progress on otherwise empty scans. + +## Decision + +`scripts/ci/strix_quick_gate.sh` recognizes the exact PascalCase runtime +identifier `ModelBehaviorError` as retryable model evidence. The gate moves to +the configured fallback sequence and does not retry the same model. The outer +`.github/workflows/strix.yml` neutralization path may skip only when that +signal is present **and** the log contains no vulnerability evidence. + +`Vulnerabilities[[:space:]]+[1-9]` and `severity:` markers remain blocking. +Generic warnings, timeouts, provider failures, and MEDIUM-or-higher findings +are unchanged. + +## Verification contract + +`tests/test_strix_model_behavior_error.py` executes the production classifier +and the outer workflow neutralization condition against bounded synthetic +logs. It proves: + +1. `ModelBehaviorError` plus `Vulnerabilities 0` is retryable and may + neutralize; +2. `ModelBehaviorError` plus `Vulnerabilities 1` stays fail-closed; +3. lowercase application prose is not classified as the runtime exception; +4. the identifier is wired into infrastructure detection and cross-model + fallback, never same-model retry. + +## Rollback + +If a future Strix release renames the exception, add the exact new identifier +and a matching regression. Do not remove the vulnerability fail-closed guard. + +## References (APA 7th) + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 21, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (n.d.). *Using workflow run logs*. GitHub Docs. Retrieved August 21, +2026, from https://docs.github.com/en/actions/how-tos/monitor-workflows/use-workflow-run-logs diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..3794e2e44 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2660,6 +2660,19 @@ is_nvidia_nim_not_found_error() { return 1 } +is_model_behavior_error() { + # Classify only Strix's exact agent-protocol exception. The PascalCase + # identifier is the runtime class name; lowercase application prose is + # not retryable. Cross-model fallback may continue; same-model retry + # does not, because repeating the same protocol failure is not a + # transient transport flake. + if grep -Eq '(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Four error families qualify: @@ -2976,6 +2989,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + # Generic strix non-zero exit with known transport/connection errors # that don't fall into the specific categories above. # Use LLM_PROVIDER_ONLY_REGEX (not PROVIDER_CONTEXT_REGEX) to avoid @@ -3826,6 +3843,10 @@ is_model_retryable_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py new file mode 100644 index 000000000..ffad8d407 --- /dev/null +++ b/tests/test_strix_model_behavior_error.py @@ -0,0 +1,192 @@ +"""Regression contract for Strix ModelBehaviorError protocol flakes. + +A ModelBehaviorError with zero reported vulnerabilities is retryable model +evidence. Real vulnerability counts remain fail-closed. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +QUALITY_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" +) + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_model_behavior_error(log_text: str) -> bool: + """Execute the production classifier against a bounded synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_behavior_error") + with tempfile.TemporaryDirectory(prefix="strix-model-behavior-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_model_behavior_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-workflow-mbe-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return backend.returncode == 0 and vulnerability.returncode == 1 + + +class StrixModelBehaviorErrorTests(unittest.TestCase): + """Protect protocol flakes without weakening vulnerability fail-closed.""" + + def test_runtime_model_behavior_error_is_retryable(self) -> None: + """Recognize the exact PascalCase Strix agent-protocol exception.""" + + log = ( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + + def test_lowercase_application_prose_is_not_retryable(self) -> None: + """Reject target-application text that only resembles the exception.""" + + log = "the model behavior error was logged by the scanned service\n" + self.assertFalse(_classifies_as_model_behavior_error(log)) + + def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: + """Wire the classifier into infrastructure and cross-model fallback.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block( + gate_source, + "has_detected_infrastructure_error", + ) + retryable = _function_block(gate_source, "is_model_retryable_error") + same_model_retry = _function_block( + gate_source, + "is_transient_same_model_retry_error", + ) + + self.assertIn("is_model_behavior_error", infrastructure) + self.assertIn("is_model_behavior_error", retryable) + self.assertNotIn("is_model_behavior_error", same_model_retry) + + def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: + """Empty scans that only hit ModelBehaviorError may skip.""" + + self.assertTrue( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + """Keep a real vulnerability signal blocking despite protocol failure.""" + + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 1\n" + ) + ) + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 9\n" + ) + ) + + def test_workflow_keeps_fail_closed_vulnerability_contract(self) -> None: + """Retain the static fail-closed vulnerability evidence contract.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("ModelBehaviorError", workflow) + self.assertIn("reported_vulnerability_signal", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + '! grep -Eiq "$reported_vulnerability_signal"', + workflow, + ) + + def test_quality_trigger_includes_model_behavior_contracts(self) -> None: + """Keep classifier, doctoring, and workflow edits on the quality path.""" + + workflow = QUALITY_WORKFLOW.read_text(encoding="utf-8") + self.assertIn(' - "docs/doctoring/strix-model-behavior-error.md"', workflow) + self.assertIn(' - "tests/test_strix_model_behavior_error.py"', workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_strix_quality_timeout_fixture_budget.py b/tests/test_strix_quality_timeout_fixture_budget.py index 78fcc8a7a..0ea4e3b37 100644 --- a/tests/test_strix_quality_timeout_fixture_budget.py +++ b/tests/test_strix_quality_timeout_fixture_budget.py @@ -33,6 +33,8 @@ def test_strix_quality_trigger_includes_fixture_contract_paths() -> None: assert "docs/doctoring/strix-quality-timeout-fixtures.md" in trigger assert "tests/test_strix_quality_timeout_fixture_budget.py" in trigger + assert "docs/doctoring/strix-model-behavior-error.md" in trigger + assert "tests/test_strix_model_behavior_error.py" in trigger def test_strix_quality_keeps_real_scanner_budgets_out_of_fixture_overrides() -> None: From d8d05463d5ca1e220f31ef490b3c268ddb957669 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 06:25:21 +0000 Subject: [PATCH 22/37] fix(strix): require module-qualified ModelBehaviorError context Do not treat a bare source-file token as protocol failure. Match only agents, pydantic_ai, or strix exception paths so Vulnerabilities [1-9] stay fail-closed. --- .github/workflows/strix.yml | 2 +- docs/doctoring/strix-model-behavior-error.md | 14 ++++++++------ scripts/ci/strix_quick_gate.sh | 10 ++++------ tests/test_strix_model_behavior_error.py | 13 +++++++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 4223e9b18..d531384ef 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,7 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines diff --git a/docs/doctoring/strix-model-behavior-error.md b/docs/doctoring/strix-model-behavior-error.md index 7333404fe..0bb06be35 100644 --- a/docs/doctoring/strix-model-behavior-error.md +++ b/docs/doctoring/strix-model-behavior-error.md @@ -12,8 +12,9 @@ current-head progress on otherwise empty scans. ## Decision -`scripts/ci/strix_quick_gate.sh` recognizes the exact PascalCase runtime -identifier `ModelBehaviorError` as retryable model evidence. The gate moves to +`scripts/ci/strix_quick_gate.sh` recognizes a **module-qualified** +`ModelBehaviorError` from `agents`, `pydantic_ai`, or `strix` as retryable +model evidence. A bare source-file mention is not enough. The gate moves to the configured fallback sequence and does not retry the same model. The outer `.github/workflows/strix.yml` neutralization path may skip only when that signal is present **and** the log contains no vulnerability evidence. @@ -28,10 +29,11 @@ are unchanged. and the outer workflow neutralization condition against bounded synthetic logs. It proves: -1. `ModelBehaviorError` plus `Vulnerabilities 0` is retryable and may - neutralize; -2. `ModelBehaviorError` plus `Vulnerabilities 1` stays fail-closed; -3. lowercase application prose is not classified as the runtime exception; +1. a module-qualified `agents`/`pydantic_ai`/`strix` `ModelBehaviorError` + plus `Vulnerabilities 0` is retryable and may neutralize; +2. the same exception plus `Vulnerabilities 1` stays fail-closed; +3. lowercase application prose or a bare `ModelBehaviorError` token is not + classified as the runtime exception; 4. the identifier is wired into infrastructure detection and cross-model fallback, never same-model retry. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 3794e2e44..07e459db0 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2661,12 +2661,10 @@ is_nvidia_nim_not_found_error() { } is_model_behavior_error() { - # Classify only Strix's exact agent-protocol exception. The PascalCase - # identifier is the runtime class name; lowercase application prose is - # not retryable. Cross-model fallback may continue; same-model retry - # does not, because repeating the same protocol failure is not a - # transient transport flake. - if grep -Eq '(^|[^A-Za-z])ModelBehaviorError([^A-Za-z]|$)' "$STRIX_LOG"; then + # Classify only a module-qualified Strix/Agents SDK protocol exception. + # A bare source-file mention of ModelBehaviorError is not retryable. + # Cross-model fallback may continue; same-model retry does not. + if grep -Eq '(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' "$STRIX_LOG"; then return 0 fi diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index ffad8d407..336f16009 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -123,6 +123,16 @@ def test_lowercase_application_prose_is_not_retryable(self) -> None: log = "the model behavior error was logged by the scanned service\n" self.assertFalse(_classifies_as_model_behavior_error(log)) + self.assertFalse(_classifies_as_model_behavior_error("ModelBehaviorError\n")) + + def test_agents_sdk_tool_protocol_failure_is_retryable(self) -> None: + """Recognize the OpenAI Agents SDK exception observed in required CI.""" + + log = ( + "agents.exceptions.ModelBehaviorError: Tool ls not found in agent strix\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: """Wire the classifier into infrastructure and cross-model fallback.""" @@ -151,6 +161,9 @@ def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: "Vulnerabilities 0\n" ) ) + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite protocol failure.""" From affc1309d20b51a137a657ae429e1730d3ecd520 Mon Sep 17 00:00:00 2001 From: Strix Test Date: Fri, 21 Aug 2026 15:40:43 +0900 Subject: [PATCH 23/37] fix(strix): retry report-only provider failures --- CHANGELOG.md | 5 +++ scripts/ci/strix_quick_gate.sh | 46 +++++++++++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 53 +++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25923501f..8a7bbc1da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,11 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Retry configured Strix fallback models when the primary provider records a + rate-limit or infrastructure failure only in its structured report log, and + evaluate each fallback against its newest report without letting an older + failed attempt poison a complete later report. + - Include the exact `backend/app/*.py` package context in PR-scoped Strix scans when a module in that package changes. The trusted resolver uses a NUL-delimited exact-head tree listing, copies unchanged dependencies from diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 654141029..f60df31e9 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -214,6 +214,18 @@ has_strix_report_failure_signal() { if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then continue fi + # A fallback attempt must be judged by its own newest structured report. + # Older attempt directories remain published for audit evidence, but a + # provider warning from an earlier failed model must not poison a complete + # later fallback report. + if [ "$report_root" = "$STRIX_REPORTS_DIR" ]; then + local newest_report_root + newest_report_root="$(latest_strix_report_dir 2>/dev/null || true)" + if [ -z "$newest_report_root" ]; then + continue + fi + report_root="$newest_report_root" + fi while IFS= read -r -d '' report_log; do if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then return 0 @@ -223,6 +235,30 @@ has_strix_report_failure_signal() { return 1 } +has_strix_report_provider_failure_signal() { + local report_root + local report_log + for report_root in "$@"; do + if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then + continue + fi + if [ "$report_root" = "$STRIX_REPORTS_DIR" ]; then + local newest_report_root + newest_report_root="$(latest_strix_report_dir 2>/dev/null || true)" + if [ -z "$newest_report_root" ]; then + continue + fi + report_root="$newest_report_root" + fi + while IFS= read -r -d '' report_log; do + if grep -Eiq 'RateLimitError|Nvidia_nimException|Too Many Requests|Error code:[[:space:]]*429|provider.{0,80}(unavailable|exhausted|rate.?limit|timeout|connection)' "$report_log"; then + return 0 + fi + done < <(find "$report_root" -type f -name '*.log' -print0) + done + return 1 +} + # shellcheck disable=SC2317,SC2329 # invoked from EXIT/INT/TERM trap cleanup_runtime() { publish_artifact_reports || true @@ -3998,6 +4034,16 @@ is_model_retryable_error() { return 0 fi + # A provider failure can be recorded only in Strix's structured report log. + # run_strix_once already marks that evidence as infrastructure failure, but + # the child stdout log used by the classifiers may not contain the provider + # exception. In strict mode, let configured distinct fallbacks run instead of + # treating the report-only signal as a non-recoverable source failure. + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled && + has_strix_report_provider_failure_signal "$ACTIVE_REPORTS_DIR" "${TARGET_PATH%/}/strix_runs"; then + return 0 + fi + if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index e22571cc4..355574284 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4477,6 +4477,27 @@ EOS echo "Denied: provider credentials were rejected" exit 0 ;; + provider-report-rate-limit-fallback-success) + case "${STRIX_LLM:-}" in + vertex_ai/report-rate-limit-primary) + mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit" + cat >"$STRIX_REPORTS_DIR/fake-report-rate-limit/strix.log" <<'EOS' +2026-08-21 04:00:00.000 WARNING strix-pr-scope-example - strix.provider: RateLimitError: provider response was exhausted +EOS + echo "scan aborted after provider report-rate-limit signal" + exit 1 + ;; + vertex_ai/fallback-one) + mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit-fallback" + echo "scan ok after report-only provider fallback" + exit 0 + ;; + *) + echo "Error: report-only provider fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 60 + ;; + esac + ;; report-known-internal-warning-sanitized) mkdir -p "$STRIX_REPORTS_DIR/fake-known-internal-warning" cat >"$STRIX_REPORTS_DIR/fake-known-internal-warning/strix.log" <<'EOS' @@ -6294,6 +6315,29 @@ run_filtered_gate_case_if_requested() { "vertex_ai/excluded-dir-primary" \ "" ;; + pull-request-target-changed-backend-context) + run_pull_request_target_changed_backend_context_scope_case + ;; + provider-fatal-success-signal | provider-warning-success-signal) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" + ;; + provider-report-rate-limit-fallback-success) + run_gate_case "provider-report-rate-limit-fallback-success" \ + "vertex_ai/report-rate-limit-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ + "|" + ;; total-timeout) run_total_timeout_case ;; @@ -10290,6 +10334,15 @@ run_gate_case "provider-warning-success-signal" \ "" \ "1" +run_gate_case "provider-report-rate-limit-fallback-success" \ + "vertex_ai/report-rate-limit-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ + "|" + run_gate_case "report-known-internal-warning-sanitized" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" \ From 782d0fcb6d8c8ea6f8309e6a749946314eafe81e Mon Sep 17 00:00:00 2001 From: Strix Test Date: Fri, 21 Aug 2026 15:44:03 +0900 Subject: [PATCH 24/37] docs(strix): record report-log fallback boundary --- docs/doctoring/strix-modelbehaviorerror-classifier.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md index e59028528..e0b553791 100644 --- a/docs/doctoring/strix-modelbehaviorerror-classifier.md +++ b/docs/doctoring/strix-modelbehaviorerror-classifier.md @@ -10,6 +10,13 @@ The workflow preserves the scanner's nonzero result: an incomplete provider turn is never passing security evidence. A scan that reports any numbered vulnerability also stays fail-closed. +When a provider failure is recorded only in Strix's structured report log, +the quick gate recognizes explicit rate-limit or provider-availability markers +and tries the configured distinct fallback models. It evaluates the newest +attempt report independently so an earlier failed provider cannot poison a +complete later report. Generic `Warning`, `Fatal`, or `Denied` output without +an explicit provider-availability marker remains non-retryable and fail-closed. + ## Operator action If this check repeats, inspect the exact Strix log and artifact for the From 56588e1ff42ceefb2100d982fe725ef0e3a3a81c Mon Sep 17 00:00:00 2001 From: Strix Test Date: Fri, 21 Aug 2026 16:11:43 +0900 Subject: [PATCH 25/37] fix(strix): preserve sanitized report artifacts --- CHANGELOG.md | 4 ++++ scripts/ci/strix_quick_gate.sh | 12 ++---------- scripts/ci/test_strix_quick_gate.sh | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c9a77e15..564d915e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,10 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Publish only the sanitized cumulative Strix report tree, avoiding a later + copy of relative scanner output that could reintroduce known internal warning + text into uploaded security evidence. + - Retry configured Strix fallback models when the primary provider records a rate-limit or infrastructure failure only in its structured report log, and evaluate each fallback against its newest report without letting an older diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index f60df31e9..41df4585f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -131,16 +131,8 @@ publish_artifact_reports() { if [ -f "$STRIX_LOG" ] && [ ! -L "$STRIX_LOG" ]; then cp -- "$STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" fi - local scope_dir scope_reports_dir - for scope_dir in "${PULL_REQUEST_SCOPE_DIRS[@]}"; do - scope_reports_dir="$scope_dir/strix_runs" - if [ -d "$scope_reports_dir" ] && [ ! -L "$scope_reports_dir" ]; then - cp -R -- "$scope_reports_dir"/. "$ARTIFACT_REPORTS_DIR"/ - fi - done - if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then - cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ARTIFACT_REPORTS_DIR"/ - fi + # Relative scanner output is copied into ACTIVE_REPORTS_DIR immediately + # after each attempt and sanitized before this publication trap runs. } preserve_attempt_log() { diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 355574284..acf24e87b 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4503,6 +4503,11 @@ EOS cat >"$STRIX_REPORTS_DIR/fake-known-internal-warning/strix.log" <<'EOS' 2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: agent a9fb4033 produced non-lifecycle final output in non-interactive mode; forcing tool continuation (1/500): internal agent coordination note 2026-06-18 13:10:44.089 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) +EOS + mkdir -p strix_runs/fake-known-internal-warning-relative + cat >strix_runs/fake-known-internal-warning-relative/strix.log <<'EOS' +2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: agent a9fb4033 produced non-lifecycle final output in non-interactive mode; forcing tool continuation (1/500): relative internal agent coordination note +2026-06-18 13:10:44.089 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) EOS outside_report_dir="${FAKE_STRIX_OUTSIDE_REPORT_DIR:-$(dirname -- "$STRIX_REPORTS_DIR")/outside-strix-report}" mkdir -p "$outside_report_dir" @@ -5800,6 +5805,14 @@ PY "$repo_root_dir/strix_runs/fake-known-internal-warning/strix.log" \ "finish_scan: completed scan with 0 vulnerability report(s)" \ "scenario=$scenario keeps non-warning Strix report evidence" + assert_file_not_contains \ + "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ + "produced non-lifecycle final output" \ + "scenario=$scenario sanitizes relative scanner output before publication" + assert_file_contains \ + "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ + "finish_scan: completed scan with 0 vulnerability report(s)" \ + "scenario=$scenario publishes sanitized relative scanner evidence" assert_file_contains \ "$repo_root_dir/outside-strix-report/strix.log" \ "outside report should not be rewritten" \ @@ -6318,6 +6331,16 @@ run_filtered_gate_case_if_requested() { pull-request-target-changed-backend-context) run_pull_request_target_changed_backend_context_scope_case ;; + report-known-internal-warning-sanitized) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-known-internal-warning-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "1" \ + "vertex_ai/report-known-internal-warning-sanitized" \ + "" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ From 48e5b610e1405e10fa6f32ea8c8c4f2c54880532 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 16:44:49 +0900 Subject: [PATCH 26/37] fix(strix): preserve behavior error case sensitivity --- .github/workflows/strix.yml | 6 ++++-- tests/test_strix_model_behavior_error.py | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index d531384ef..f5bb1fb9b 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -866,7 +866,8 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines @@ -877,7 +878,8 @@ jobs: # present and no vulnerability was reported anywhere. This preserves # real security gating while keeping uncontrollable provider outages # from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ + if ( grep -Eiq "$backend_unavailable_signal" "$strix_run_log" \ + || grep -Eq "$model_behavior_error_signal" "$strix_run_log" ) \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_run_log"; then echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." exit 0 diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py index 336f16009..4ebee7768 100644 --- a/tests/test_strix_model_behavior_error.py +++ b/tests/test_strix_model_behavior_error.py @@ -80,6 +80,10 @@ def _workflow_neutralizes(log_text: str) -> bool: workflow, "backend_unavailable_signal", ) + model_behavior_pattern = _workflow_signal_pattern( + workflow, + "model_behavior_error_signal", + ) vulnerability_pattern = _workflow_signal_pattern( workflow, "reported_vulnerability_signal", @@ -93,6 +97,12 @@ def _workflow_neutralizes(log_text: str) -> bool: capture_output=True, text=True, ) + model_behavior = subprocess.run( + ["grep", "-Eq", model_behavior_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) vulnerability = subprocess.run( ["grep", "-Eiq", vulnerability_pattern, str(log_path)], check=False, @@ -101,9 +111,14 @@ def _workflow_neutralizes(log_text: str) -> bool: ) if backend.returncode not in {0, 1}: raise AssertionError(backend.stderr) + if model_behavior.returncode not in {0, 1}: + raise AssertionError(model_behavior.stderr) if vulnerability.returncode not in {0, 1}: raise AssertionError(vulnerability.stderr) - return backend.returncode == 0 and vulnerability.returncode == 1 + return ( + (backend.returncode == 0 or model_behavior.returncode == 0) + and vulnerability.returncode == 1 + ) class StrixModelBehaviorErrorTests(unittest.TestCase): @@ -164,6 +179,11 @@ def test_outer_workflow_neutralizes_zero_finding_protocol_flake(self) -> None: self.assertFalse( _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") ) + self.assertFalse( + _workflow_neutralizes( + "agents.foo.modelbehaviorerror\nVulnerabilities 0\n" + ) + ) def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite protocol failure.""" @@ -186,6 +206,7 @@ def test_workflow_keeps_fail_closed_vulnerability_contract(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("ModelBehaviorError", workflow) + self.assertIn("model_behavior_error_signal", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) self.assertIn( From 144df9fbadc3b2f846134c26ac0372694d68fb59 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 01:53:55 -0700 Subject: [PATCH 27/37] docs(strix): remove duplicate ModelBehaviorError doctoring record --- .../strix-modelbehaviorerror-classifier.md | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 docs/doctoring/strix-modelbehaviorerror-classifier.md diff --git a/docs/doctoring/strix-modelbehaviorerror-classifier.md b/docs/doctoring/strix-modelbehaviorerror-classifier.md deleted file mode 100644 index e0b553791..000000000 --- a/docs/doctoring/strix-modelbehaviorerror-classifier.md +++ /dev/null @@ -1,30 +0,0 @@ -# Strix ModelBehaviorError classifier - -Observed required-check flake: Strix exits 1 with `ModelBehaviorError` and -`Vulnerabilities 0` after the scanner model fails to complete a turn. - -The outer `strix.yml` backend-unavailable signal includes -`ModelBehaviorError` so operators receive a typed -`STRIX_PROVIDER_UNAVAILABLE` result when no vulnerability finding was emitted. -The workflow preserves the scanner's nonzero result: an incomplete provider -turn is never passing security evidence. A scan that reports any numbered -vulnerability also stays fail-closed. - -When a provider failure is recorded only in Strix's structured report log, -the quick gate recognizes explicit rate-limit or provider-availability markers -and tries the configured distinct fallback models. It evaluates the newest -attempt report independently so an earlier failed provider cannot poison a -complete later report. Generic `Warning`, `Fatal`, or `Denied` output without -an explicit provider-availability marker remains non-retryable and fail-closed. - -## Operator action - -If this check repeats, inspect the exact Strix log and artifact for the -qualified exception. Retry provider execution without changing source -classification; both incomplete analysis and any reported finding remain -non-passing. - -## References - -OpenAI. (n.d.). *Exceptions*. OpenAI Agents SDK. Retrieved August 19, 2026, -from https://openai.github.io/openai-agents-python/ref/exceptions/ From 52a62a0ed43cc560e7e858399b87249490673756 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 23:04:24 +0900 Subject: [PATCH 28/37] fix(security): raise pip-audit runner pip floor --- CHANGELOG.md | 4 ++++ requirements-pip-audit-ci-hashes.txt | 10 ++++++---- requirements-pip-audit-ci.txt | 2 ++ tests/test_strix_workflow_dependency_hashes.py | 12 ++++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffefdeddf..f2b45c5b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,10 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Raised the hash-pinned pip-audit runner to pip 26.2.1, above the published + pip URL-handling vulnerability fixed in pip 26.2, and made the security + floor explicit in the lock input. + - Publish only the sanitized cumulative Strix report tree, avoiding a later copy of relative scanner output that could reintroduce known internal warning text into uploaded security evidence. diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..5bcf362ae 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,10 +213,12 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 - # via pip-api +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f + # via + # -r requirements-pip-audit-ci.txt + # pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ --hash=sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625 diff --git a/requirements-pip-audit-ci.txt b/requirements-pip-audit-ci.txt index 684087ba5..681550b1a 100644 --- a/requirements-pip-audit-ci.txt +++ b/requirements-pip-audit-ci.txt @@ -1 +1,3 @@ pip-audit==2.10.1 +# Keep the audit runner's pip implementation above published security fixes. +pip>=26.2 diff --git a/tests/test_strix_workflow_dependency_hashes.py b/tests/test_strix_workflow_dependency_hashes.py index e2509c18b..9f2b699bb 100644 --- a/tests/test_strix_workflow_dependency_hashes.py +++ b/tests/test_strix_workflow_dependency_hashes.py @@ -8,6 +8,8 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" +PIP_AUDIT_INPUT = ROOT / "requirements-pip-audit-ci.txt" +PIP_AUDIT_LOCK = ROOT / "requirements-pip-audit-ci-hashes.txt" WORKFLOW_DISPATCH_KEY_RE = re.compile( r"(?m)^[ \t]+['\"]?workflow_dispatch['\"]?\s*:" ) @@ -72,3 +74,13 @@ def test_strix_workflow_runs_complete_shell_regression_suite() -> None: assert ' - "scripts/ci/test_strix_quick_gate.sh"' in workflow assert "bash scripts/ci/test_strix_quick_gate.sh" in workflow assert "bash -n scripts/ci/strix_quick_gate.sh" in workflow + + +def test_pip_audit_runner_excludes_the_vulnerable_pip_release() -> None: + """Keep the audited, hash-pinned pip implementation above its security floor.""" + requirements = PIP_AUDIT_INPUT.read_text(encoding="utf-8") + lock = PIP_AUDIT_LOCK.read_text(encoding="utf-8") + + assert "pip>=26.2" in requirements + assert re.search(r"^pip==26\.2\.1 \\", lock, re.MULTILINE) + assert "pip==26.1.2" not in lock From 6087b525356c69210cf00371d7fb7a46c418720f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 07:06:10 -0700 Subject: [PATCH 29/37] chore(strix): restore causal owner scope Revert the unrelated pip-audit lock/input policy commit from the Strix owner branch. The canonical dependency-only security repair remains in PR #1198; this branch continues to own only the Strix fail-closed contract. --- CHANGELOG.md | 4 ---- requirements-pip-audit-ci-hashes.txt | 10 ++++------ requirements-pip-audit-ci.txt | 2 -- tests/test_strix_workflow_dependency_hashes.py | 12 ------------ 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b45c5b8..ffefdeddf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,10 +40,6 @@ Semantic Versioning where the repository publishes a release. ### Fixed -- Raised the hash-pinned pip-audit runner to pip 26.2.1, above the published - pip URL-handling vulnerability fixed in pip 26.2, and made the security - floor explicit in the lock input. - - Publish only the sanitized cumulative Strix report tree, avoiding a later copy of relative scanner output that could reintroduce known internal warning text into uploaded security evidence. diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index 5bcf362ae..ade197a49 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,12 +213,10 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.2.1 \ - --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ - --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f - # via - # -r requirements-pip-audit-ci.txt - # pip-api +pip==26.1.2 \ + --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ + --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 + # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ --hash=sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625 diff --git a/requirements-pip-audit-ci.txt b/requirements-pip-audit-ci.txt index 681550b1a..684087ba5 100644 --- a/requirements-pip-audit-ci.txt +++ b/requirements-pip-audit-ci.txt @@ -1,3 +1 @@ pip-audit==2.10.1 -# Keep the audit runner's pip implementation above published security fixes. -pip>=26.2 diff --git a/tests/test_strix_workflow_dependency_hashes.py b/tests/test_strix_workflow_dependency_hashes.py index 9f2b699bb..e2509c18b 100644 --- a/tests/test_strix_workflow_dependency_hashes.py +++ b/tests/test_strix_workflow_dependency_hashes.py @@ -8,8 +8,6 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOW = ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" -PIP_AUDIT_INPUT = ROOT / "requirements-pip-audit-ci.txt" -PIP_AUDIT_LOCK = ROOT / "requirements-pip-audit-ci-hashes.txt" WORKFLOW_DISPATCH_KEY_RE = re.compile( r"(?m)^[ \t]+['\"]?workflow_dispatch['\"]?\s*:" ) @@ -74,13 +72,3 @@ def test_strix_workflow_runs_complete_shell_regression_suite() -> None: assert ' - "scripts/ci/test_strix_quick_gate.sh"' in workflow assert "bash scripts/ci/test_strix_quick_gate.sh" in workflow assert "bash -n scripts/ci/strix_quick_gate.sh" in workflow - - -def test_pip_audit_runner_excludes_the_vulnerable_pip_release() -> None: - """Keep the audited, hash-pinned pip implementation above its security floor.""" - requirements = PIP_AUDIT_INPUT.read_text(encoding="utf-8") - lock = PIP_AUDIT_LOCK.read_text(encoding="utf-8") - - assert "pip>=26.2" in requirements - assert re.search(r"^pip==26\.2\.1 \\", lock, re.MULTILINE) - assert "pip==26.1.2" not in lock From 3b279ef209ae58f7f1c2d3509963bd825f8bf2e2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:09:47 +0900 Subject: [PATCH 30/37] fix(security): raise pip audit lock past PYSEC-2026-3721 --- requirements-pip-audit-ci-hashes.txt | 10 ++++++---- requirements-pip-audit-ci.txt | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..5bcf362ae 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,10 +213,12 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 - # via pip-api +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f + # via + # -r requirements-pip-audit-ci.txt + # pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ --hash=sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625 diff --git a/requirements-pip-audit-ci.txt b/requirements-pip-audit-ci.txt index 684087ba5..b0c0c3e18 100644 --- a/requirements-pip-audit-ci.txt +++ b/requirements-pip-audit-ci.txt @@ -1 +1,2 @@ pip-audit==2.10.1 +pip>=26.2 From c9a059244f5a467d0d52bb2785e4728bd6ee13eb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 10:27:57 -0700 Subject: [PATCH 31/37] fix(strix): converge protected-main scheduler contracts --- requirements-pip-audit-ci-hashes.txt | 10 ++++------ requirements-pip-audit-ci.txt | 1 - scripts/ci/test_strix_quick_gate.sh | 6 ++++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index 5bcf362ae..ade197a49 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,12 +213,10 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.2.1 \ - --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ - --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f - # via - # -r requirements-pip-audit-ci.txt - # pip-api +pip==26.1.2 \ + --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ + --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 + # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ --hash=sha256:9b75e958f14c5a2614bae415f2adf7eeb54d50a2cfbe7e24fd4826471bac3625 diff --git a/requirements-pip-audit-ci.txt b/requirements-pip-audit-ci.txt index b0c0c3e18..684087ba5 100644 --- a/requirements-pip-audit-ci.txt +++ b/requirements-pip-audit-ci.txt @@ -1,2 +1 @@ pip-audit==2.10.1 -pip>=26.2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index acf24e87b..7ada8812e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1522,8 +1522,10 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$workflow_file" "github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number)" "scheduler scopes pull_request_target concurrency to the active PR" assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].number && format('pr-{0}', github.event.workflow_run.pull_requests[0].number)" "scheduler scopes workflow_run concurrency to the completed review PR" assert_file_contains "$workflow_file" "github.event_name == 'schedule' && format('schedule-{0}', github.event.schedule)" "scheduler isolates the 15-minute organization sweep from the separate 30-minute scheduled scan" - assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && github.run_id" "scheduler keeps manual queue scans isolated per run" - assert_file_contains "$workflow_file" "cancel-in-progress: \${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review' || github.event_name == 'repository_dispatch' }}" "scheduler cancels stale PR/review/manual queue scans instead of accumulating merge/update attempts" + assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && format('repo-dispatch-{0}', github.repository)" "scheduler deduplicates unscoped manual queue scans per repository" + assert_file_not_contains "$workflow_file" "github.event_name == 'repository_dispatch' && github.run_id" "scheduler does not isolate duplicate unscoped manual scans by run ID" + assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && !github.event.workflow_run.pull_requests[0].number && format('workflow-run-no-pr-{0}', github.repository)" "scheduler deduplicates metadata-free workflow completions per repository" + assert_file_contains "$workflow_file" "cancel-in-progress: \${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review' || github.event_name == 'repository_dispatch' || (github.event_name == 'workflow_run' && !github.event.workflow_run.pull_requests[0].number) }}" "scheduler cancels stale PR/review/manual and metadata-free workflow scans instead of accumulating merge/update attempts" assert_file_contains "$workflow_file" "timeout-minutes: 60" "organization sweep has enough headroom to finish the complete repository walk" assert_file_contains "$workflow_file" "ORG_SWEEP_TRIGGER_REVIEWS: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps retry missing current-head OpenCode reviews" assert_file_contains "$workflow_file" "ORG_SWEEP_ENABLE_AUTO_MERGE: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps merge approved current heads" From 04a0b983dcb6ef749a6488ccd190a01a8a5eb56e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:59:05 +0900 Subject: [PATCH 32/37] fix(ci): refresh pip audit lock --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..0ae099d8f 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From a88f017795ae139091207e37b37f95d30353128e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 11:14:42 -0700 Subject: [PATCH 33/37] chore(strix): restore pip lock to security owner --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index 0ae099d8f..ade197a49 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.2.1 \ - --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ - --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f +pip==26.1.2 \ + --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ + --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From c1bac0a2fc2588e3c5b54baed782277b228aba91 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:03:22 +0900 Subject: [PATCH 34/37] ci: refresh pip audit runtime --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..0ae099d8f 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From 0bf6874f4e8a986cdbc5edaf11f2e7775043c73d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 12:16:17 -0700 Subject: [PATCH 35/37] chore(strix): restore canonical pip lock ownership --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index 0ae099d8f..ade197a49 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.2.1 \ - --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ - --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f +pip==26.1.2 \ + --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ + --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From 4de4929595bfd2ab43ebff832143919d22b7421e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:57:38 +0900 Subject: [PATCH 36/37] ci: refresh dependency and scheduler contracts --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..0ae099d8f 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From 412370e16c51b2b21d5de27092089590d5efb425 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 23 Aug 2026 01:28:03 -0700 Subject: [PATCH 37/37] test(strix): execute complete fail-closed branch --- ...kend_unavailable_after_exempted_finding.py | 15 ++++++++++----- ..._proxy_bootstrap_failure_is_classified.py} | 19 +++++++++++-------- 2 files changed, 21 insertions(+), 13 deletions(-) rename tests/{test_strix_local_proxy_bootstrap_failure_is_neutral.py => test_strix_local_proxy_bootstrap_failure_is_classified.py} (86%) diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 539705a63..3355a8448 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -66,9 +66,14 @@ def _extract_neutralization_block(workflow: str) -> str: start_marker = ( " # Recognized signals that the LLM backend was unavailable" ) + terminal_failure_marker = ( + ' echo "Strix reported security findings or failed for a ' + 'non-backend reason; failing the required check' + ) end_marker = ' exit "$strix_rc"\n' start = workflow.index(start_marker) - end = workflow.index(end_marker, start) + len(end_marker) + terminal_failure = workflow.index(terminal_failure_marker, start) + end = workflow.index(end_marker, terminal_failure) + len(end_marker) return workflow[start:end] @@ -124,7 +129,7 @@ def test_brownout_after_an_already_exempted_finding_is_non_passing(self) -> None """The PR #392 shape remains typed and non-passing after an exemption.""" log = EXEMPTED_FINDING_AND_CONTINUATION + GITHUB_MODELS_BROWNOUT - self.assertNotEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) def test_still_fails_closed_on_a_finding_reported_after_continuation(self) -> None: """A real finding surfacing *after* the continuation marker still blocks.""" @@ -133,20 +138,20 @@ def test_still_fails_closed_on_a_finding_reported_after_continuation(self) -> No EXEMPTED_FINDING_AND_CONTINUATION + "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" ) - self.assertNotEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) def test_still_fails_closed_with_no_continuation_marker_at_all(self) -> None: """Preserve prior behavior: a bare unresolved finding still blocks.""" log = "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" - self.assertNotEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) def test_bare_backend_outage_with_no_finding_is_non_passing( self, ) -> None: """A pure outage still lacks authoritative scan evidence.""" - self.assertNotEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 0) + self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1) if __name__ == "__main__": diff --git a/tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py b/tests/test_strix_local_proxy_bootstrap_failure_is_classified.py similarity index 86% rename from tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py rename to tests/test_strix_local_proxy_bootstrap_failure_is_classified.py index bf71ba136..ea1f6517e 100644 --- a/tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py +++ b/tests/test_strix_local_proxy_bootstrap_failure_is_classified.py @@ -7,7 +7,8 @@ failure-signal output; failing closed." (scripts/ci/strix_quick_gate.sh's `run_current_target_scan`, no fallback attempted because `is_model_retryable_error` doesn't recognize a local proxy-login failure as -an LLM-provider error). Before this fix, the workflow's neutral-skip regex +an LLM-provider error). Before this fix, the workflow's provider-failure +classification regex only matched the "emitted ..." wording variant of that message family, so this specific "scan failed after ..." wording fell through to a hard required-check failure even though zero vulnerabilities were reported. @@ -16,8 +17,8 @@ 97019252804): `loginAsGuest failed after 10 attempts: curl exit 7: ... Failed to connect to 127.0.0.1 port 48080`, "Vulnerabilities 0", then "Strix scan failed after provider infrastructure or failure-signal output; -failing closed." -- a pure CI-infrastructure hiccup that still failed the -required check. +failing closed." -- a pure CI-infrastructure hiccup. Classification is +diagnostic only: the incomplete scan must still fail the required check. """ from __future__ import annotations @@ -59,8 +60,8 @@ def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: return match.group(1) -def _workflow_neutralizes(log_text: str) -> bool: - """Execute the outer workflow's backend-neutralization condition.""" +def _workflow_classifies_provider_failure(log_text: str) -> bool: + """Evaluate the outer workflow's provider-failure classification inputs.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") backend_pattern = _workflow_signal_pattern(workflow, "backend_unavailable_signal") @@ -97,8 +98,10 @@ def test_workflow_recognizes_the_authenticated_caido_failure_shape(self) -> None self.assertIn("Error during penetration test: loginAsGuest failed after", workflow) self.assertIn("Failed to connect to 127\\.0\\.0\\.1 port 48080", workflow) - def test_neutralizes_local_proxy_bootstrap_failure_with_zero_findings(self) -> None: - self.assertTrue(_workflow_neutralizes(LOCAL_PROXY_BOOTSTRAP_FAILURE)) + def test_classifies_local_proxy_bootstrap_failure_with_zero_findings(self) -> None: + self.assertTrue( + _workflow_classifies_provider_failure(LOCAL_PROXY_BOOTSTRAP_FAILURE) + ) def test_still_fails_closed_when_a_real_vulnerability_is_also_reported( self, @@ -106,7 +109,7 @@ def test_still_fails_closed_when_a_real_vulnerability_is_also_reported( log = LOCAL_PROXY_BOOTSTRAP_FAILURE + ( "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" ) - self.assertFalse(_workflow_neutralizes(log)) + self.assertFalse(_workflow_classifies_provider_failure(log)) if __name__ == "__main__":