diff --git a/CHANGELOG.md b/CHANGELOG.md index e42afe76a..e38804b30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Retry a Strix provider model that emits its exact model-quality warning via + the configured fallback sequence, while retaining fail-closed handling for + all other warning, timeout, provider, and vulnerability signals. - Refused PR Review Merge Scheduler head mutations, `update-branch` and the last-push approval head restamp, whenever the resolved mutation credential is the workflow `GITHUB_TOKEN`. GitHub starts no workflow run for events created with that credential, so the moved head collected no current-head required checks and the PR stayed permanently `BLOCKED` with a `github-actions[bot]` merge commit that no later scheduler run could repair, because the branch was no longer behind. The scheduler now waits with `head_mutation_credential_upgrade` guidance naming `PR_REVIEW_MERGE_TOKEN`, `OPENCODE_APPROVE_TOKEN`, and the OpenCode app token exchange. - Parsed `opencode.jsonc` as JSONC (stripping `//` and `/* */` comments outside string literals) in the reasoning-effort guard and its contract tests, instead of raw `json.loads`, which rejected the file the moment it carried its first explanatory comment (added for the `contextual-orchestrator` provider block) with `Expecting property name enclosed in double quotes`. Comment markers inside string values, such as the `$schema` URL, are left untouched. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). diff --git a/docs/doctoring/strix-quality-warning-fallback.md b/docs/doctoring/strix-quality-warning-fallback.md new file mode 100644 index 000000000..d39dc4f8c --- /dev/null +++ b/docs/doctoring/strix-quality-warning-fallback.md @@ -0,0 +1,49 @@ +# Strix model-quality warning fallback + +검토 기준일: **2026-08-21** + +## Incident + +The central Strix workflow selected the public-repository NVIDIA NIM model +`nvidia/nemotron-3-super-120b-a12b`. Strix completed a scan and produced only a +`LOW` finding, but it also emitted `MODEL QUALITY WARNING` because the selected +model was not a recommended frontier model. The gate correctly classified the +warning as non-clean evidence, then stopped before trying the configured +fallback models. This left an actionable low-severity report indistinguishable +from an unrecoverable provider failure and blocked the target pull request. + +## Decision + +`scripts/ci/strix_quick_gate.sh` recognizes only Strix's exact model-quality +warning (`MODEL QUALITY WARNING` / `is not a recommended frontier model for +Strix`) as retryable model evidence. It remains an infrastructure/failure +signal, so the scan never passes merely because the warning was seen. The +existing fallback sequence must obtain a clean result or the gate fails closed. + +All other `Warn`, `Warning`, `Fatal`, `Denied`, and `Timeout` output remains a +hard failure. A `MEDIUM` or higher vulnerability remains blocking even when a +fallback succeeds; below-threshold findings are handled by the existing +`STRIX_FAIL_ON_MIN_SEVERITY` policy. + +## Verification contract + +`tests/test_strix_nvidia_nim_not_found_fallback.py` executes the production +classifier against a bounded quality-warning log and asserts that the warning +is wired into `is_model_retryable_error`. The existing shell harness continues +to cover generic warning signals and fallback failure paths. + +## Rollback + +If a future Strix release changes the warning wording, add the exact new +provider-produced wording to the narrow classifier and its regression test. +Do not remove generic warning failure handling or neutralize the entire warning +class. + +## References (APA 7th) + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 21, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (n.d.). *Using workflow run logs*. GitHub Docs. Retrieved August 21, +2026, from https://docs.github.com/en/actions/how-tos/monitor-workflows/use-workflow-run-logs diff --git a/requirements-pip-audit-ci-hashes.txt b/requirements-pip-audit-ci-hashes.txt index ade197a49..0ae099d8f 100644 --- a/requirements-pip-audit-ci-hashes.txt +++ b/requirements-pip-audit-ci-hashes.txt @@ -213,9 +213,9 @@ packaging==26.2 \ # via # pip-audit # pip-requirements-parser -pip==26.1.2 \ - --hash=sha256:382ff9f685ee3bc25864f820aa50505825f10f5458ffff07e30a6d96e5715cab \ - --hash=sha256:f49cd134c61cf2fd75e0ce2676db03e4054504a5a4986d00f8299ae632dc4605 +pip==26.2.1 \ + --hash=sha256:71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e \ + --hash=sha256:f6ad667e89a1fe78046c8f13232b247200f5258d7828f3f7883d660878e0813f # via pip-api pip-api==0.0.34 \ --hash=sha256:8b2d7d7c37f2447373aa2cf8b1f60a2f2b27a84e1e9e0294a3f6ef10eb3ba6bb \ diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index c00cfa1e0..a4d7fa983 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -239,6 +239,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: + """Initialize the client with one bounded GitHub credential.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token @@ -853,4 +854,4 @@ def main( if __name__ == "__main__": # pragma: no cover - exercised through main() - raise SystemExit(main()) \ No newline at end of file + raise SystemExit(main()) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..41853689f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2990,6 +2990,13 @@ has_detected_infrastructure_error() { return 1 } +is_model_quality_warning() { + local quality_detail_regex + quality_detail_regex="(^|[^[:alnum:]_.-])[[:alnum:]_.-]+(/[[:alnum:]_.:-]+)+['\"]?[[:space:]]+is not a recommended[[:space:]]+frontier model for Strix([.!]|$)" + grep -Eiq 'MODEL QUALITY WARNING' "$STRIX_LOG" && + grep -Eiq "$quality_detail_regex" "$STRIX_LOG" +} + latest_strix_report_dir() { local latest="" local run_dir @@ -3818,6 +3825,10 @@ is_hallucinated_source_claim_finding() { is_model_retryable_error() { local model="$1" + if is_model_quality_warning; then + return 0 + fi + if is_vertex_model "$model" && is_vertex_not_found_error; then return 0 fi diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index ac9ce1d8b..38ab060d9 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -1506,8 +1506,8 @@ assert_pr_review_merge_scheduler_uses_github_actions_bot_token() { assert_file_contains "$workflow_file" "github.event_name == 'pull_request_target' && format('pr-{0}', github.event.pull_request.number)" "scheduler scopes pull_request_target concurrency to the active PR" assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && github.event.workflow_run.pull_requests[0].number && format('pr-{0}', github.event.workflow_run.pull_requests[0].number)" "scheduler scopes workflow_run concurrency to the completed review PR" assert_file_contains "$workflow_file" "github.event_name == 'schedule' && format('schedule-{0}', github.event.schedule)" "scheduler isolates the 15-minute organization sweep from the separate 30-minute scheduled scan" - assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && github.run_id" "scheduler keeps manual queue scans isolated per run" - assert_file_contains "$workflow_file" "cancel-in-progress: \${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request_review' || github.event_name == 'repository_dispatch' }}" "scheduler cancels stale PR/review/manual queue scans instead of accumulating merge/update attempts" + assert_file_contains "$workflow_file" "github.event_name == 'repository_dispatch' && format('repo-dispatch-{0}', github.repository)" "scheduler keeps unscoped repository dispatch scans isolated per repository" + assert_file_contains "$workflow_file" "github.event_name == 'workflow_run' && !github.event.workflow_run.pull_requests[0].number" "scheduler cancels only metadata-free workflow-run scans in their isolated fallback group" assert_file_contains "$workflow_file" "timeout-minutes: 60" "organization sweep has enough headroom to finish the complete repository walk" assert_file_contains "$workflow_file" "ORG_SWEEP_TRIGGER_REVIEWS: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps retry missing current-head OpenCode reviews" assert_file_contains "$workflow_file" "ORG_SWEEP_ENABLE_AUTO_MERGE: \${{ github.event_name == 'schedule' ||" "scheduled organization sweeps merge approved current heads" diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index a48f3092d..062d53e13 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -73,6 +73,33 @@ def _classifies_as_nvidia_not_found(log_text: str) -> bool: return completed.returncode == 0 +def _classifies_as_model_quality_warning(log_text: str) -> bool: + """Execute the production model-quality classifier against a bounded log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_quality_warning") + with tempfile.TemporaryDirectory(prefix="strix-quality-warning-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + completed = subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail; STRIX_LOG=\"$1\"; " + f"{function_source}\n" + "is_model_quality_warning", + "strix-quality-classifier", + str(log_path), + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" @@ -171,6 +198,31 @@ def test_not_found_skips_same_model_and_enters_cross_model_fallback(self) -> Non self.assertIn("is_nvidia_nim_not_found_error", retryable) self.assertNotIn("is_nvidia_nim_not_found_error", same_model_retry) + def test_model_quality_warning_enters_configured_fallback(self) -> None: + """Retry a weaker provider model instead of treating its warning as clean evidence.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + retryable = _function_block(gate_source, "is_model_retryable_error") + self.assertIn("is_model_quality_warning", retryable) + + self.assertTrue( + _classifies_as_model_quality_warning( + "MODEL QUALITY WARNING\n" + "'nvidia_nim/example' is not a recommended frontier model for Strix.\n" + ) + ) + self.assertFalse( + _classifies_as_model_quality_warning( + "MODEL QUALITY WARNING\n" + "target output: the model warning is ordinary application text.\n" + ) + ) + self.assertFalse( + _classifies_as_model_quality_warning( + "target output: 'nvidia_nim/example' is not a recommended frontier model for Strix.\n" + ) + ) + def test_workflow_uses_available_free_first_nvidia_plan(self) -> None: """Prefer a documented hosted NIM and another NIM before GitHub."""