From a6ab3074490409a2c1677af3650bc0428d9a5fc6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 03:39:33 +0900 Subject: [PATCH 1/6] fix: retry Strix model quality warnings --- CHANGELOG.md | 3 ++ .../strix-quality-warning-fallback.md | 49 +++++++++++++++++++ scripts/ci/strix_quick_gate.sh | 12 +++++ ...est_strix_nvidia_nim_not_found_fallback.py | 35 +++++++++++++ 4 files changed, 99 insertions(+) create mode 100644 docs/doctoring/strix-quality-warning-fallback.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d2f9f24d..87ae3447b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Retry a Strix provider model that emits its exact model-quality warning via + the configured fallback sequence, while retaining fail-closed handling for + all other warning, timeout, provider, and vulnerability signals. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/strix-quality-warning-fallback.md b/docs/doctoring/strix-quality-warning-fallback.md new file mode 100644 index 000000000..d39dc4f8c --- /dev/null +++ b/docs/doctoring/strix-quality-warning-fallback.md @@ -0,0 +1,49 @@ +# Strix model-quality warning fallback + +검토 기준일: **2026-08-21** + +## Incident + +The central Strix workflow selected the public-repository NVIDIA NIM model +`nvidia/nemotron-3-super-120b-a12b`. Strix completed a scan and produced only a +`LOW` finding, but it also emitted `MODEL QUALITY WARNING` because the selected +model was not a recommended frontier model. The gate correctly classified the +warning as non-clean evidence, then stopped before trying the configured +fallback models. This left an actionable low-severity report indistinguishable +from an unrecoverable provider failure and blocked the target pull request. + +## Decision + +`scripts/ci/strix_quick_gate.sh` recognizes only Strix's exact model-quality +warning (`MODEL QUALITY WARNING` / `is not a recommended frontier model for +Strix`) as retryable model evidence. It remains an infrastructure/failure +signal, so the scan never passes merely because the warning was seen. The +existing fallback sequence must obtain a clean result or the gate fails closed. + +All other `Warn`, `Warning`, `Fatal`, `Denied`, and `Timeout` output remains a +hard failure. A `MEDIUM` or higher vulnerability remains blocking even when a +fallback succeeds; below-threshold findings are handled by the existing +`STRIX_FAIL_ON_MIN_SEVERITY` policy. + +## Verification contract + +`tests/test_strix_nvidia_nim_not_found_fallback.py` executes the production +classifier against a bounded quality-warning log and asserts that the warning +is wired into `is_model_retryable_error`. The existing shell harness continues +to cover generic warning signals and fallback failure paths. + +## Rollback + +If a future Strix release changes the warning wording, add the exact new +provider-produced wording to the narrow classifier and its regression test. +Do not remove generic warning failure handling or neutralize the entire warning +class. + +## References (APA 7th) + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 21, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (n.d.). *Using workflow run logs*. GitHub Docs. Retrieved August 21, +2026, from https://docs.github.com/en/actions/how-tos/monitor-workflows/use-workflow-run-logs diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..4a8d66c42 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2944,6 +2944,10 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { + if is_model_quality_warning; then + return 0 + fi + if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then return 0 fi @@ -2990,6 +2994,10 @@ has_detected_infrastructure_error() { return 1 } +is_model_quality_warning() { + grep -Eiq 'MODEL QUALITY WARNING|is not a recommended[[:space:]]+frontier model for Strix' "$STRIX_LOG" +} + latest_strix_report_dir() { local latest="" local run_dir @@ -3818,6 +3826,10 @@ is_hallucinated_source_claim_finding() { is_model_retryable_error() { local model="$1" + if is_model_quality_warning; then + return 0 + fi + if is_vertex_model "$model" && is_vertex_not_found_error; then return 0 fi diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..cb1296be8 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -171,6 +171,41 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) + def test_model_quality_warning_enters_configured_fallback(self) -> None: + """Retry a weaker provider model instead of treating its warning as clean evidence.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + quality_warning = _function_block( + gate_source, + "is_model_quality_warning", + ) + retryable = _function_block(gate_source, "is_model_retryable_error") + self.assertIn("is_model_quality_warning", retryable) + + with tempfile.TemporaryDirectory(prefix="strix-quality-warning-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text( + "MODEL QUALITY WARNING\n" + "'nvidia_nim/example' is not a recommended frontier model for Strix.\n", + encoding="utf-8", + ) + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + quality_warning, + "is_model_quality_warning", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-quality-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(completed.returncode, 0, completed.stderr) + def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: """Prefer a documented hosted NIM and another NIM before GitHub.""" From 4d397eaba66331daea6b530c9381e1fa0110c957 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:11:50 +0900 Subject: [PATCH 2/6] fix(strix): bind model warning to provider detail --- scripts/ci/strix_quick_gate.sh | 5 +- ...est_strix_nvidia_nim_not_found_fallback.py | 63 ++++++++++++------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 4a8d66c42..1340f78a8 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2995,7 +2995,10 @@ has_detected_infrastructure_error() { } is_model_quality_warning() { - grep -Eiq 'MODEL QUALITY WARNING|is not a recommended[[:space:]]+frontier model for Strix' "$STRIX_LOG" + local quality_detail_regex + quality_detail_regex="(^|[^[:alnum:]_.-])[[:alnum:]_.-]+(/[[:alnum:]_.:-]+)+['\"]?[[:space:]]+is not a recommended[[:space:]]+frontier model for Strix([.!]|$)" + grep -Eiq 'MODEL QUALITY WARNING' "$STRIX_LOG" && + grep -Eiq "$quality_detail_regex" "$STRIX_LOG" } latest_strix_report_dir() { diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index cb1296be8..062d53e13 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -73,6 +73,33 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 +def _classifies_as_model_quality_warning(log_text: str) -> bool: + """Execute the production model-quality classifier against a bounded log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_quality_warning") + with tempfile.TemporaryDirectory(prefix="strix-quality-warning-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + completed = subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail; STRIX_LOG=\"$1\"; " + f"{function_source}\n" + "is_model_quality_warning", + "strix-quality-classifier", + str(log_path), + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" @@ -175,36 +202,26 @@ def test_model_quality_warning_enters_configured_fallback(self) -> None: """Retry a weaker provider model instead of treating its warning as clean evidence.""" gate_source = STRIX_GATE.read_text(encoding="utf-8") - quality_warning = _function_block( - gate_source, - "is_model_quality_warning", - ) retryable = _function_block(gate_source, "is_model_retryable_error") self.assertIn("is_model_quality_warning", retryable) - with tempfile.TemporaryDirectory(prefix="strix-quality-warning-") as temp_dir: - log_path = Path(temp_dir) / "strix.log" - log_path.write_text( + self.assertTrue( + _classifies_as_model_quality_warning( "MODEL QUALITY WARNING\n" - "'nvidia_nim/example' is not a recommended frontier model for Strix.\n", - encoding="utf-8", + "'nvidia_nim/example' is not a recommended frontier model for Strix.\n" ) - script = "\n".join( - ( - "set -euo pipefail", - 'STRIX_LOG="$1"', - quality_warning, - "is_model_quality_warning", - ) + ) + self.assertFalse( + _classifies_as_model_quality_warning( + "MODEL QUALITY WARNING\n" + "target output: the model warning is ordinary application text.\n" ) - completed = subprocess.run( - ["bash", "-c", script, "strix-quality-classifier", str(log_path)], - check=False, - capture_output=True, - text=True, + ) + self.assertFalse( + _classifies_as_model_quality_warning( + "target output: 'nvidia_nim/example' is not a recommended frontier model for Strix.\n" ) - - self.assertEqual(completed.returncode, 0, completed.stderr) + ) def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: """Prefer a documented hosted NIM and another NIM before GitHub.""" From 07fa2a69c2820d7eff348fcacc017154e85748f6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 17:56:03 +0900 Subject: [PATCH 3/6] refactor(strix): remove redundant warning branch --- scripts/ci/strix_quick_gate.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 1340f78a8..41853689f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2944,10 +2944,6 @@ is_llm_token_limit_error() { # was interrupted or incomplete. Used as a guard to prevent the # below-threshold override from silently passing an aborted scan. has_detected_infrastructure_error() { - if is_model_quality_warning; then - return 0 - fi - if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning)([^[:alpha:]]|$)' "$STRIX_LOG"; then return 0 fi From b2cdb96a86a116533605fa6180abd7873acd524a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 02:34:05 +0900 Subject: [PATCH 4/6] test(strix): align scheduler contract with current concurrency --- scripts/ci/test_strix_quick_gate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ac9ce1d8b..38ab060d9 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1506,8 +1506,8 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$workflow_file" "github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number)" "scheduler scopes pull_request_target concurrency to the active PR" assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].number && format('pr-{0}', github.event.workflow_run.pull_requests[0].number)" "scheduler scopes workflow_run concurrency to the completed review PR" assert_file_contains "$workflow_file" "github.event_name == 'schedule' && format('schedule-{0}', github.event.schedule)" "scheduler isolates the 15-minute organization sweep from the separate 30-minute scheduled scan" - assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && github.run_id" "scheduler keeps manual queue scans isolated per run" - assert_file_contains "$workflow_file" "cancel-in-progress: \${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review' || github.event_name == 'repository_dispatch' }}" "scheduler cancels stale PR/review/manual queue scans instead of accumulating merge/update attempts" + assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && format('repo-dispatch-{0}', github.repository)" "scheduler keeps unscoped repository dispatch scans isolated per repository" + assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && !github.event.workflow_run.pull_requests[0].number" "scheduler cancels only metadata-free workflow-run scans in their isolated fallback group" assert_file_contains "$workflow_file" "timeout-minutes: 60" "organization sweep has enough headroom to finish the complete repository walk" assert_file_contains "$workflow_file" "ORG_SWEEP_TRIGGER_REVIEWS: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps retry missing current-head OpenCode reviews" assert_file_contains "$workflow_file" "ORG_SWEEP_ENABLE_AUTO_MERGE: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps merge approved current heads" From 77877831999213965e925fc7418b25d77d19c660 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 03:10:04 +0900 Subject: [PATCH 5/6] fix(security): refresh pip audit dependency --- requirements-pip-audit-ci-hashes.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..0ae099d8f 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ From f6c2404b1f32d98a97c10305bc651f2a989d6a4a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 04:07:09 +0900 Subject: [PATCH 6/6] test(coverage): document coordinator client initializer --- scripts/ci/organization_commercial_readiness_loop.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index c00cfa1e0..a4d7fa983 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -239,6 +239,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: + """Initialize the client with one bounded GitHub credential.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token @@ -853,4 +854,4 @@ def main( if __name__ == "__main__": # pragma: no cover - exercised through main() - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main())