diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 553111b54..2aa245e7f 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -5605,12 +5605,12 @@ jobs: } emit_known_missing_string_finding \ - "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" \ - "Strix PR scans must use contextual-orchestrator/orchestrator/auto" \ + "STRIX_MODEL: contextual-orchestrator/orchestrator/free" \ + "Strix PR scans must use contextual-orchestrator/orchestrator/free" \ ".github/workflows/strix.yml" \ "scripts/ci/strix_required_workflow_smoke.sh" emit_known_missing_string_finding \ - "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto." \ + "Strix model overrides are limited to contextual-orchestrator/orchestrator/free." \ "Strix model overrides must remain gateway-only" \ ".github/workflows/strix.yml" \ "scripts/ci/strix_required_workflow_smoke.sh" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index a0d95e5af..93e493aae 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -549,14 +549,14 @@ jobs: - name: Gate Strix secrets id: gate env: - STRIX_MODEL: contextual-orchestrator/orchestrator/auto + STRIX_MODEL: contextual-orchestrator/orchestrator/free STRIX_MODEL_REQUESTED: ${{ github.event.client_payload.strix_llm || '' }} run: | requested_model="$(printf '%s' "$STRIX_MODEL_REQUESTED" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" case "$requested_model" in - ""|orchestrator/auto|contextual-orchestrator/orchestrator/auto) ;; + ""|orchestrator/free|contextual-orchestrator/orchestrator/free) ;; *) - echo '::error::Strix model overrides are limited to contextual-orchestrator/orchestrator/auto.' + echo '::error::Strix model overrides are limited to contextual-orchestrator/orchestrator/free.' exit 1 ;; esac @@ -574,7 +574,7 @@ jobs: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.is_private }} - CONTEXTUAL_ORCHESTRATOR_POOL: auto + CONTEXTUAL_ORCHESTRATOR_POOL: free run: | set -euo pipefail bash "$TRUSTED_STRIX_SOURCE/scripts/ci/contextual_orchestrator_review_sidecar.sh" @@ -713,11 +713,11 @@ jobs: strix_llm_file="$RUNNER_TEMP/strix_llm.txt" strix_model="$(printf '%s' "$STRIX_MODEL" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" case "$strix_model" in - orchestrator/auto | contextual-orchestrator/orchestrator/auto) - printf '%s' 'orchestrator/auto' > "$strix_llm_file" + orchestrator/free | contextual-orchestrator/orchestrator/free) + printf '%s' 'orchestrator/free' > "$strix_llm_file" ;; *) - echo '::error::STRIX_LLM must select contextual-orchestrator/orchestrator/auto.' + echo '::error::STRIX_LLM must select contextual-orchestrator/orchestrator/free.' exit 1 ;; esac diff --git a/AGENTS.md b/AGENTS.md index d84840833..6e598cfe1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,11 +21,12 @@ sidecar (`scripts/ci/contextual_orchestrator_review_sidecar.sh`). The five provider secrets (`BYTEZ_API_KEY`, `NVIDIA_NIM_API_KEY`, `NVIDIA_NIM_API_KEY_SUB`, `OPENROUTER_API_KEY`, `OPENAI_API_KEY`) enter its KV as bootstrap transport in the same process that discovers models and serves; -OpenCode and Noema use the fail-closed zero-cost pool `orchestrator/free`. -Authoritative Strix analysis uses the correctness-first `orchestrator/auto` -provider-diverse pool. Non-free Strix routes -are admitted only with complete published prompt/completion price and currency -evidence, and private targets still require ZDR-compliant routes under +OpenCode, Noema, and Strix all use the fail-closed zero-cost pool +`orchestrator/free`. Strix uses the zero-cost `orchestrator/free` pool by +explicit 2026-08-30 owner decision, superseding the prior `orchestrator/auto` +(provider-diverse, non-free-admitting) default; private targets still require +ZDR-compliant routes under [`scripts/ci/zdr_policy.py`](scripts/ci/zdr_policy.py). -See [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md). +See [`docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`](docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md) +and its 2026-08-30 amendment. The materialization contract is also covered by [`docs/doctoring/exact-artifact-sbom-attestation.md`](docs/doctoring/exact-artifact-sbom-attestation.md). diff --git a/CHANGELOG.md b/CHANGELOG.md index 41d02f6ee..fc84661ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,40 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Raise `contextual_orchestrator_review_sidecar.sh`'s + `ORCHESTRATOR_CATALOG_FAMILY_CAP` default from 4 to 8: root-caused the + live "no provider route passed the Strix plain-chat preflight" outage + blocking `noema-review`/`opencode-review`/`strix` org-wide to + `contextual_orchestrator_review_policy.py`'s family-cap candidate + selection deterministically admitting the same 4 alphabetically-first + `nvidia_nim`/`nvidia_nim_sub` free-model candidates on every run — 2 of + which are confirmed NVIDIA-retired model ids returning HTTP 404 forever — + while ~19 other healthy free candidates in the same discovery report + never got a chance. See the 2026-08-30 sidecar-preflight gap-baseline + entry for the full evidence trail, the exact trade-off reasoned through + (not live-verified, since this session lacks provider credentials), and + the more complete fix if this proves insufficient. +- Switch Strix from `orchestrator/auto` to `orchestrator/free`, matching + OpenCode and Noema: `strix.yml`'s `STRIX_MODEL`/`CONTEXTUAL_ORCHESTRATOR_POOL` + default and both model-override allowlists, and + `scripts/ci/strix_quick_gate.sh`'s `is_contextual_orchestrator_model`, now + accept only `orchestrator/free`. This is an explicit, informed owner + override of `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md`'s + original `orchestrator/auto` decision (see that ADR's 2026-08-30 + amendment and the matching gap-baseline entry for the full trade-off and + evidence trail): Strix no longer has a paid-model fallback and can go + fully dark during the class of single-provider-family-collapse incident + the original decision was written to survive, until the free-catalog's + stale-model and provider-diversity gaps are separately closed. +- Strengthen `scripts/ci/zdr_policy.py`'s `nvidia_nim`/`nvidia_nim_sub` ZDR + attestation with a direct primary-source citation: NVIDIA's own current + *NVIDIA API Trial Terms of Service* (v. September 19, 2025), Section + 3.3(iv), states User Content and Generated Content are collected "to + improve NVIDIA products and services, including AI models" — affirmative + evidence against zero data retention, not just an absence of attestation. + `zero_data_retention` stays `False` as it already was; only the citation + and note change. See the 2026-08-30 ZDR/NIM-routing gap-baseline entry for + the full architecture review this citation was part of. - Bump the vendored `contextual-orchestrator` review-sidecar pin from `5f2753a` (the #1422 pin) to current `main` `30c6d716`, picking up `ContextualWisdomLab/contextual-orchestrator#919`: generalizes the diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index c1700aaf5..ee5706e03 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -1,9 +1,11 @@ # ADR-0003: Vendored contextual-orchestrator review sidecar with governed gateway pools -- Status: accepted +- Status: accepted, amended 2026-08-30 (see "2026-08-30 amendment" below — Strix + now uses `orchestrator/free`, not the `orchestrator/auto` this header + originally recorded) - Date: 2026-08-27 - Scope: ContextualWisdomLab/.github central review pipelines (OpenCode autofix/dispatch + shared `opencode.jsonc` default + required Noema + Strix review) -- Decision: Route every central CI review write/model execution that touches contracts in this repository through the **vendored** `contextual-orchestrator` gateway, served as a per-runner sidecar. OpenCode and Noema retain the fail-closed zero-cost virtual model id `orchestrator/free`; authoritative Strix security analysis uses the provider-diverse `orchestrator/auto` pool. Strix is intentionally correctness-first rather than zero-cost. **Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets.** +- Decision: Route every central CI review write/model execution that touches contracts in this repository through the **vendored** `contextual-orchestrator` gateway, served as a per-runner sidecar. OpenCode, Noema, and (as of the 2026-08-30 amendment) Strix all use the fail-closed zero-cost virtual model id `orchestrator/free`. **Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets.** - Ownership: `.github` owns control-plane evidence; `ContextualWisdomLab/contextual-orchestrator` owns the gateway. The 2026-08-18 org decision (recorded in `ContextualWisdomLab/contextual-orchestrator` AGENTS.md) already migrated OpenCode/Noema/Strix to the orchestrator backend; this ADR is the org-repo (provider-config) half of that decision. - Figma File ID: N/A (no customer UI). @@ -142,3 +144,35 @@ all five, and auto-optimize routing by cost. set `CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR=true`; the catalog then excludes every non-ZDR route and fails closed when no attested ZDR route exists in the selected workflow pool. + +- **2026-08-30 amendment: Strix uses `orchestrator/free`, superseding this + ADR's original `orchestrator/auto` decision.** The org owner explicitly + directed Strix off the paid-inclusive `orchestrator/auto` pool and onto the + same zero-cost `orchestrator/free` pool OpenCode and Noema already use, so + no central review path executes a paid model. This is a deliberate, + informed override of the original decision above, not an oversight of it: + the trade-off the original decision recorded — "the 2026-08-29 exact-head + DiskSage scan proved that four discovered free routes all shared the + OpenRouter outage domain, which the gateway correctly collapsed to one + provider attempt... Strix has no external fallback" — was surfaced to the + owner explicitly, including a live 2026-08-30 reproduction of that same + single-family-collapse pattern (a `strix` run's `orchestrator/auto` + primary/free stage rejected 4/4 candidates — 2 timeouts, 2 HTTP 404s from + retired NVIDIA-hosted models — and only the `auto` pool's paid fallback + kept that run alive; see `docs/product-technical-gap-baseline.md`'s + 2026-08-30 sidecar-preflight entries for the full evidence trail). The + owner's response, verbatim in substance: implement the free-only directive + as originally instructed. **Accepted consequence**: Strix has no external + fallback and can go fully dark (rather than degraded-but-running) during + the exact class of incident this ADR originally used `orchestrator/auto` + to survive, until the free-catalog's stale-model and provider-diversity + gaps documented alongside this amendment are separately closed. This is + the owner's accepted risk, not an unnoticed regression. + `scripts/ci/strix_quick_gate.sh`'s `is_contextual_orchestrator_model` no + longer accepts `orchestrator/auto`; `strix.yml`'s `STRIX_MODEL`/ + `CONTEXTUAL_ORCHESTRATOR_POOL` default to `orchestrator/free`; and + `scripts/ci/strix_required_workflow_smoke.sh`/`AGENTS.md` were updated to + match. The `orchestrator/auto` pool mode itself is unchanged and still + exists in `contextual_orchestrator_review_policy.py`/the sidecar for any + other caller that opts into it explicitly — this amendment only removes it + as Strix's default and as an accepted Strix override value. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 0933f55e8..794dc9de9 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -792,6 +792,506 @@ recurrence" section below out of the file entirely; both are restored here.) Following up on that hosted-run confirmation is the concrete next check for this entry, not a new code change. +## 2026-08-30 hosted-run confirmation of #1430 fails at a new stage: live preflight, not discovery + +- This is exactly the follow-up hosted-run confirmation the entry above asked + for, and it does **not** come back clean. Three independent fresh + `noema-review` runs were forced against current `main` + (`755fe8e1`/`30c6d716`, i.e. with #1430's fix already in effect, since + `pull_request_target` always executes the *base* branch's copy of + `scripts/ci/contextual_orchestrator_review_sidecar.sh` regardless of the + PR's own content): #1432 twice (`61de349f`, jobs `33303869223` then + `33304289755` after a second forced re-run) and #1418 once (`7b4161fd`, + job containing check id `99238526905`). All three reproduce the identical + new failure, verbatim: `vendoring contextual-orchestrator @ + 30c6d71680e659f25a0a433d4726ad0d437f9757` → discovery completes with + **zero** `provider_discovery_failed` lines (the sentinel + `discovery_diagnostics_complete` is reached cleanly, so `orchestrator/free` + is genuinely populated this time, unlike the pre-#1430 empty-pool + signature) → `review sidecar preflight failed` (the launcher's + `_preflight_review_agents` in `scripts/ci/contextual_orchestrator_review_launcher.py` + raises `ReviewPreflightError("no provider route passed the Strix + plain-chat preflight", report)`) → `sidecar exited before healthz (status + 1)`. Every run also logs `omitted_unstructured_lines=4`: the redacting + stream sanitizer (`scripts/ci/sanitize_contextual_orchestrator_sidecar_stream.py`) + is, by design, dropping the four lines that would explain *which* routes + were rejected and why (provider response bodies/exception text are + intentionally never allowlisted into CI logs) — so the exact per-route + `error_type`/`http_status` only exists in the `preflight_report` JSON + (`$STRIX_EVIDENCE_DIR/contextual-orchestrator-preflight.json`), which only + `strix.yml` uploads as an artifact; `noema-review.yml` and + `opencode-review-dispatch.yml` run the identical sidecar script but do not + upload it, so this pass could not retrieve the artifact (a same-cycle + `strix` run on unrelated PR #1176 was still queued behind the + per-repository concurrency group after 15+ minutes and was not waited + out). +- This is a **different** defect from the one #1430 fixed, not a recurrence + of it: the pool is not empty and discovery is not failing. Something + downstream — plausibly (not yet confirmed) shared-provider-key rate/burst + pressure from the large number of PRs' `noema-review`/`opencode-review`/ + `strix` jobs re-triggered by #1430 landing, or a genuine defect newly + exposed by #919's provider-family generalization (`nvidia_nim`/ + `nvidia_nim_sub`/`openai` routes that previously never reached live + discovery) — is rejecting every one of the (up to 12) selected zero-cost + candidates at `ModelClient.proxy_send_once`. Two observations argue + against pure rate-limiting: the failure is 3-for-3 reproducible with no + intervening success, and the two #1432 runs were ~9 minutes apart (well + outside a typical burst window) yet failed identically. This needs a + `preflight_report` artifact (or direct provider-side log access this + session does not have) to root-cause conclusively — not assumed to be one + cause or the other here. +- **Scope of impact**: essentially every non-draft open PR's + `noema-review`/`opencode-review`/`strix` required checks are currently + blocked on this, independent of anything in the PR's own diff or how + stale its branch is — confirmed by sampling ~45 open PRs' latest check + runs and finding the `noema-review`/`opencode-review`/`strix` failures + either stale (pre-dating one of today's earlier fixes: #1413, #1414, + #1422, or #1430) or, on the three forced fresh re-runs above, this new + signature. No PR sampled this pass showed a `noema-review` failure + distinct from this signature or from the three already-diagnosed + pre-#1430 systemic causes recorded in the 2026-08-30 hourly-recheck entry + above. +- **Not bypassed.** The owner's standing bypass authorization for this repo + covers two verified structural signatures only: a PR whose own diff edits + `.github/workflows/`/`scripts/ci/` review-pipeline files (the + `pull_request_target` trust-boundary case #1430 itself hit) or the + pre-#1430 empty-pool chicken-and-egg. Neither applies here: discovery is + not empty, and none of the PRs sampled this pass (including #1176, which + edits `.github/workflows/audit-central-ruleset.yml` and + `scripts/ci/audit_central_required_workflows.py` — real workflow/CI files, + but not the review-pipeline ones, and not the cause of its own + `noema-review` failure) edit the review-pipeline files themselves. Per the + owner's explicit conservative instruction, an unclear or newly-surfaced + failure reason is not bypass-eligible, so nothing was bypass-merged this + pass. +- Given the above, this pass deliberately did **not** mass-retry + `update_pull_request_branch`/re-runs across the ~45 affected open PRs: + three independent forced reproductions already established the failure is + systemic and deterministic, not per-PR or transient, so repeating the same + forced re-run dozens more times would only burn shared runner/provider + quota for the same evidence already in hand. +- Next concrete step (not attempted this pass, given the time budget): get + one `strix` run's `contextual-orchestrator-preflight.json` artifact on a + current-`main`-based head (wait out or avoid the concurrency queue) to + read the real per-route `error_type`/`http_status`, then decide whether + the fix belongs in `contextual_orchestrator_review_launcher.py` (e.g. + lower `REVIEW_PREFLIGHT_MAX_TOTAL_ROUTES`/serialize discovery to avoid a + self-inflicted burst) or in `contextual-orchestrator` itself (e.g. a + credential-resolution or request-shape regression for the newly-widened + `nvidia_nim`/`nvidia_nim_sub`/`openai` routes from #919). + +## 2026-08-30 sidecar-preflight outage: consolidated evidence and why it is not one deterministic bug + +**Supersedes the framing (not the evidence) of the entry above** — same incident, +now with the actual per-route rejection data and a third independent run +sequence, from three converging sources this pass: this session's own three +forced reproductions on `.github` (#1432 x2, #1418 x1, all `SystemExit` +before `healthz`), the `contextual-orchestrator-preflight.json`/ +`contextual-orchestrator-discovery.json` artifact recovered from PR #1176's +`strix` run (queued behind #1418's, completed ~09:45), and a fourth +independently-reported run on PR #1433's `noema-review` (`healthz` reached, +then a 502 on the actual gateway request). + +- **PR #1176's `strix` artifact is the first look at the real per-route + reasons**, previously invisible because the sanitizer intentionally + redacts them from job logs. That run used `orchestrator/auto` (pre-dating + this pass's now-reverted Strix free/auto edit — see below), so it exercised + both stages `_preflight_with_fallback` runs: + - **Primary (free) stage, 4/4 candidates rejected, zero ready**: two + `nvidia_nim` `deepseek-ai/deepseek-v4-*` candidates timed out + (`TimeoutError`); two `nvidia_nim` `google/gemma-3-*b-it` candidates got + `HTTPError` **404** — i.e. NVIDIA has retired those hosted model ids + (the exact failure class `scripts/ci/select_nvidia_nim_model.py`'s own + docstring already describes for a *different*, currently-unwired + caller: "NVIDIA retires hosted models on published end-of-life dates, + and the endpoint then answers every request with HTTP 410/404"). The + discovery report shows 46 free-priced rows existed, all `nvidia_nim`/ + `nvidia_nim_sub` duplicates of the same ~23 model ids — so this was not + a bad selection out of a large pool; it is the **entire** free-tier + catalog for this run, and 2 of ~23 distinct ids are already dead. + - **Fallback (priced/auto) stage, 2/8 ready**: `nvidia_nim` and + `nvidia_nim_sub` `nvidia/nemotron-3-super-120b-a12b` both succeeded; + `nemotron-3-ultra-550b-a55b` timed out on both keys; all four `openai` + candidates (`gpt-3.5-turbo`, `gpt-4`, `gpt-4-turbo`, `gpt-4.1`) were + rejected with **HTTPError 429** (rate-limited) on every single attempt. + The run only survived because `auto`'s fallback tier existed at all. +- **PR #1433's `noema-review` (pool is always `free` there, no fallback tier) + reached `healthz` successfully after 23s** — its own internal + `_preflight_review_agents` found a viable route this time — but the + shell script's separate, subsequent real `/v1/chat/completions` gateway + smoke request against the now-serving `orchestrator/free` virtual model + came back **HTTP 502**. This is a different code path than the launcher's + own preflight (`ModelClient.proxy_send_once` against explicit candidate + agents) — it is the running server's own virtual-model routing under a + real request — so a route that passed the launcher's own preflight + moments earlier still failed when the server tried to actually serve it. + A `provider_discovery_failed provider=bytez code=http_status_500` warning + in the same run is flagged non-fatal by the sidecar itself; not confirmed + either way as related. +- **Reading all four data points together**, this is not one deterministic + code defect to patch: it is a **mix of (a) a stale/retired-model gap in + the free-tier catalog** (the 404s — a real, fixable bug: nothing in + `contextual_orchestrator_review_launcher.py`'s selection path + cross-checks a discovered "free" model id against the provider's live + `/v1/models` catalog before adding it as a preflight candidate, unlike + `select_nvidia_nim_model.py`'s already-solved pattern for its own, + currently-unwired caller) **and (b) load-sensitive provider instability** + (timeouts, the 429s across every OpenAI candidate in one run, the 502 on + an already-healthy server in another) most consistent with the shared + five org provider keys being hit by concurrent review-check volume across + many simultaneously re-triggered PRs org-wide, though this pass could not + instrument request volume to confirm that mechanism directly. Two runs on + the same PR #1432 nine minutes apart failing identically (both times + `omitted_unstructured_lines=4`, same overall shape) argues the *retired- + model* component is deterministic and load-independent; PR #1176/#1433's + more varied outcomes (partial success, a different failure stage + entirely) argue the *timeout/429/502* component is not. +- **Root-caused precisely (code-verified, not just log-pattern-matched) and + a first mitigation implemented, though not confirmed on a live hosted + run** — this session lacks the five provider credentials the sidecar + registers into its KV, so nothing here could be locally reproduced end to + end; the fix below was reasoned from reading + `scripts/ci/contextual_orchestrator_review_policy.py`'s actual selection + code against the PR #1176 artifact's exact discovery/preflight data, not + from guessing at the log-pattern level: + - `contextual_orchestrator_review_policy.py`'s + `build_zdr_prioritized_catalog` groups `nvidia_nim`/`nvidia_nim_sub` + into one outage-domain "family" (`PROVIDER_FAMILIES`) and caps how many + candidates from one family it will ever select + (`family_cap`, default 4) — a guard originally meant to stop one + provider family from crowding out others. But eligible rows are sorted + purely alphabetically by `(cost_rank, zdr_rank, provider, model)`, with + **no reliability signal at all**, and per the PR #1176 discovery report, + 100% of `orchestrator/free`'s 46 rows (23 distinct model ids, mirrored + across the two NVIDIA keys) currently belong to this one family. The + combination is deterministic, not merely load-sensitive: every run + admits the exact same alphabetically-first 4 candidates — + `deepseek-ai/deepseek-v4-flash-0731`, `deepseek-ai/deepseek-v4-pro-0813`, + `google/gemma-3-12b-it`, `google/gemma-3-4b-it` — and the PR #1176 + artifact shows two of those four (the `gemma-3` pair) are NVIDIA-retired + model ids returning HTTP 404, forever, on every future run, regardless + of load or timing, while the other ~19 free `nvidia_nim`/`nvidia_nim_sub` + model ids in the same discovery report (`nemotron`, `llama`, `mistral`, + `minimax`, `moonshot`, `openai/gpt-oss-*`, `poolside`) never get a + chance to preflight at all. This fully explains the earlier finding that + two runs on PR #1432 nine minutes apart failed identically + (`omitted_unstructured_lines=4` both times, same shape): it was never + going to vary run to run. + - **Implemented**: raised `contextual_orchestrator_review_sidecar.sh`'s + `ORCHESTRATOR_CATALOG_FAMILY_CAP` default from 4 to 8 (see the dated + comment left at that line for the full reasoning and numbers). This is a + deliberately moderate, bounded change, not a full fix: it roughly + doubles how many of the ~23 distinct free `nvidia_nim`/`nvidia_nim_sub` + model ids get a chance per run, which — assuming the retired/slow + candidates observed in the one artifact available are a minority of that + set, not the majority — meaningfully improves the odds of finding a + working route without needing new retry/exclude logic in + `contextual_orchestrator_review_launcher.py` or touching + `contextual_orchestrator_review_policy.py`'s tested, shared + `family_cap` contract (its own default and tests are untouched; only + this one deployment-level env-var default changed). It does **not** + remove the two permanently-dead `gemma-3` candidates from the pool — + they will still be tried and still fail, just alongside more real + chances rather than crowding out all of them. The trade-off made + explicitly, not silently. The picking loop also stops at the overall + `CATALOG_LIMIT` (12) regardless of `family_cap`, so the absolute + worst case across any number of distinct families was already + `REVIEW_PREFLIGHT_TIMEOUT_SECONDS=10` × 12 = 120s before this change + (reached once `family_cap` × distinct families ≥ 12, i.e. ≥3 families + at the old cap of 4) and stays 120s after it — this raise does not move + that pre-existing ceiling. What changes is *when* that ceiling is + reached and the typical case today: with the single family + (`nvidia_nim`) currently filling 100% of `orchestrator/free`, + worst-case preflight time rises from ~40s (4 candidates) to ~80s (8 + candidates); with exactly two distinct families it would now also + reach the 120s ceiling (previously ~80s at `family_cap=4`). Both + figures stay within the sidecar's existing 180s readiness-wait + ceiling in the common case but not verified against real provider + latency, since this session cannot exercise that path live. + - **Not implemented, and the more complete fix if 8 turns out + insufficient or the added latency itself becomes the new bottleneck**: + cross-check discovered "free" model ids against the provider's live + `/v1/models` catalog before admitting them to the candidate pool at all, + dropping retired ids at discovery time rather than paying their + preflight cost every single run. `scripts/ci/select_nvidia_nim_model.py` + already implements exactly this pattern (see its docstring) — for a + different, currently-unwired caller (this same pass's ZDR/NIM-routing + entry above). Wiring that same live-catalog-freshness check into + `contextual_orchestrator_review_launcher.py`'s own selection path was + not attempted this pass: it requires new network-call error handling in + a security-relevant path this session cannot exercise against real + NVIDIA endpoints, which is a materially different risk profile than the + bounded, config-only change above. + - The separate timeout/429/502 half of the four-source evidence above + (real transient provider-side load, not a catalog-freshness issue) is + unaffected by this change and remains unconfirmed either way; a + properly-diverse candidate set (which this change moves toward) is the + best available mitigation for it without direct provider-side + observability this session does not have. + - **Next concrete step for whoever has runner access next**: watch the + next real hosted `noema-review`/`opencode-review`/`strix` run's + artifact/logs against this change. If it still fails with "no provider + route passed" and `omitted_unstructured_lines` stays non-zero, pull the + `contextual-orchestrator-preflight.json` artifact (`strix` only uploads + it; a targeted `strix` run may be needed) and check whether the newly + admitted 4 candidates (ranks 5-8 alphabetically) are also all rejected, + which would mean the dead/slow fraction of this provider's free catalog + is larger than assumed and the live-catalog cross-check above is the + real fix, not a further family_cap increase. + - **A second, independent, complementary fix landed on `main` mid-pass**: + PR #1436 ("give the gateway preflight probe a real reasoning budget"), + authored elsewhere in parallel, fixes `contextual_orchestrator_review_ + sidecar.sh`'s own post-`healthz` gateway smoke request — it previously + used a `max_tokens` value desynchronized from + `REVIEW_MAX_OUTPUT_TOKENS`, so a reasoning-capable free-tier route (e.g. + a DeepSeek NIM model) that the launcher's own internal preflight had + already proved "ready" could still spend its whole budget on internal + reasoning before any visible answer, making the shell script's separate + end-to-end smoke request see empty assistant content and fail closed + with `502 invalid_structured_output`. This is the precise mechanism + behind the PR #1433 "healthz reached, then 502" signature this entry's + earlier revision (see the superseded framing note above) described + without yet knowing the cause — it is a genuinely different bug from + this entry's own family-cap/stale-model finding (that one is about + *which* candidates ever reach a preflight attempt; #1436's is about the + *separate*, later smoke-test step that re-checks whichever candidate + the server ends up actually routing to), not a duplicate or a + correction of it. Both fixes are now in this branch's ancestry + (merged `main` into `fix/zdr-nim-nvidia-citation-20260830` mid-pass); + a hosted run against the combined state is the next real test of + whether the outage is now closed or whether further work (the + live-catalog cross-check above, or something neither fix covers) is + still needed. +- **Strix `orchestrator/auto` → `orchestrator/free`: implemented, per the + owner's explicit, informed decision.** This pass first drafted the switch, + then reverted it unpushed on discovering `docs/adr/0003-contextual- + orchestrator-vendored-free-zdr.md`'s original, evidence-based rationale for + `orchestrator/auto` ("the 2026-08-29 exact-head DiskSage scan proved that + four discovered free routes all shared the OpenRouter outage domain... + Strix has no external fallback") and today's own PR #1176 artifact showing + that exact single-family-collapse pattern reproducing live (free-only + primary stage: 4/4 candidates rejected — 2 timeouts, 2 HTTP 404s on retired + NVIDIA models; only `auto`'s paid fallback kept that run alive). That + conflict — a fresh verbal directive versus a documented prior decision with + a specific, currently-reproducing technical rationale — was surfaced to the + owner rather than resolved unilaterally. The owner's response, having seen + both: "아니 일단 내가 지시한대로 해봐" ("no, do what I originally instructed + first") — an explicit, informed override, accepting that Strix can now go + fully dark rather than degraded-but-running during the exact incident class + ADR-0003 originally used `orchestrator/auto` to survive, until the + free-catalog's stale-model and provider-diversity gaps (documented in the + entries above and below) are separately closed. + **Implemented this pass**: `strix.yml`'s `STRIX_MODEL`/ + `CONTEXTUAL_ORCHESTRATOR_POOL` and both model-selection-step allowlists now + default to and accept only `orchestrator/free`; + `scripts/ci/strix_quick_gate.sh`'s `is_contextual_orchestrator_model` no + longer accepts `orchestrator/auto`; `scripts/ci/ + strix_required_workflow_smoke.sh`, `AGENTS.md`, and the diagnostic-string + lookups in `opencode-review-dispatch.yml`'s failed-check diagnosis were + updated to match; `docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md` + carries a dated amendment recording this as a superseding decision (not a + silent contradiction) with the owner's accepted risk spelled out + explicitly. All 6 previously-`auto`-pinning test files plus one + reviewed-workflow blob-SHA pin (`opencode-review-dispatch.yml` changed + content, so its independently-reviewed-blob contract in + `tests/test_pr_review_autofix_nvidia_nim_contract.py` was re-pinned to the + new blob SHA) were updated; full local suite: 1880 passed, 1 skipped, 100% + interrogate, `pingora_edge_policy.py`'s single pre-existing coverage miss + unrelated to this change. **Not yet confirmed on a real hosted run**: this + makes Strix subject to the same currently-open sidecar-preflight outage + documented above — a real `strix` run against this change will very likely + fail (or go dark) until that outage's stale-model/provider-diversity gaps + are fixed, which is the accepted, expected, and now-explicitly-owner-chosen + state, not a new defect. +- **A `strix` `repository_dispatch` run against PR #1434 was observed to + fail — but it does not test any of the above, and is not evidence either + way about the outage-domain risk.** Run + `ContextualWisdomLab/.github/actions/runs/33306963425`'s `strix` job + failed at its "Self-test Strix required workflow contract" step, before + provisioning the sidecar, gating secrets, or running any scan (all + downstream steps show `skipped`). The exact cause, read from the job log: + this self-test step deliberately materializes the **PR head**'s + `strix.yml` (`"Materialized PR-head Strix workflow for self-test."`) and + checks it with the **trusted-base** (i.e. current `main`, via the same + `pull_request_target`-style trust boundary #1430 hit) + `scripts/ci/strix_required_workflow_smoke.sh`. `main` does not yet have + this pass's Strix `auto`→`free` change, so its smoke script still asserts + `STRIX_MODEL: contextual-orchestrator/orchestrator/auto` and explicitly + rejects `STRIX_MODEL: contextual-orchestrator/orchestrator/free` — exactly + what PR #1434's own `strix.yml` now contains — producing two `FAIL:` + lines and a hard exit before anything provider- or model-related runs. + This is the **same structural class of chicken-and-egg documented for + #1430 and called out in this session's own task instructions ("a PR that + itself edits `.github/workflows/`/`scripts/ci/` review-pipeline files can + structurally fail its own required check")** — PR #1434 edits `strix.yml` + and `strix_required_workflow_smoke.sh` together, and the smoke half of + that pair cannot become "trusted" until merged. It says nothing about + whether `orchestrator/free` would actually survive the single-outage- + domain risk at runtime — the run never reached that layer. A genuine + runtime test of the `auto`→`free` switch needs either this PR merged + first (own chicken-and-egg — the owner's bypass authority for this repo + has not been extended to PR #1434 specifically, so this pass did not + self-authorize one) or a `repository_dispatch` targeting a *different* + repository that does not itself edit these trusted files. +- **Secondary, separate finding on the same run**: the follow-up + `publish-manual-pr-evidence-status` job also failed — + `target-app-token` got `HTTP 403: Resource not accessible by integration` + publishing the (correctly non-success, per the self-test failure above) + Strix status back to `.github`'s own PR #1434. The publisher's own logic + only tolerates a publish failure silently when `STRIX_RESULT=success`; a + non-success result that also cannot be published hard-fails by design, so + this is arguably correct fail-closed behavior surfacing a real, + previously-unobserved token-scoping gap, not a logic bug. Plausibly an + edge case specific to `.github` being the `target_repository` of its own + `repository_dispatch` Strix run (this central repo normally dispatches + Strix *to* sibling repos, not to itself) rather than a gap sibling repos + would hit; not investigated further or fixed this pass given it is + downstream of, and only surfaced by, the self-test failure above. + +## 2026-08-30 ZDR/NIM-routing architecture review (owner-directed) + +Investigated the owner's stated goal that Noema/OpenCode/Strix review route +through `contextual-orchestrator`'s `orchestrator/free` specifically, and that +direct-NVIDIA-NIM communication is a removal target. + +- **Repo visibility, checked directly rather than assumed**: `.github`, + `noema`, `contextual-orchestrator`, `naruon`, `fast-mlsirm`, `TEPP`, + `scopeweave`, `pg-llm-batch`, and `keyverse` are all confirmed **public** + (this session's git proxy serves them as anonymous public reads with no + attachment needed). `gyeot` required a genuine authenticated attachment + (the proxy's "added"/`push`-capable response, not the "already public" + response the others got) — strong evidence it is **private**, making it + (or any other private sibling repo not checked here) the concrete case + where `CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR` actually evaluates `true` and + the free+ZDR intersection below matters. For `.github`/`noema`/ + `contextual-orchestrator` themselves, confirmed directly in job env + (`CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: false` in every log pulled this + pass) that ZDR is not gating their own reviews — the sidecar-preflight + outage above is a separate, ZDR-independent problem for those three. +- **`scripts/ci/zdr_policy.py`'s conservative `nvidia_nim`/`nvidia_nim_sub` + = not-ZDR classification is correct, and now has a direct primary-source + citation rather than an indirect one.** Fetched NVIDIA's own current + *NVIDIA API Trial Terms of Service* (the terms actually governing this + org's free/trial `integrate.api.nvidia.com` key; PDF, v. September 19, + 2025, confirmed still the live document as of 2026-08-30) directly from + `assets.ngc.nvidia.com` rather than relying on third-party summaries. + Section 3.3(iv) states NVIDIA collects "User Content and Generated + Content to improve NVIDIA products and services, including AI models" — + i.e., prompts/completions from this API **are** used for training; this + is not merely "unattested," it is affirmative evidence against ZDR. + Updated both `PROVIDER_ZDR_SCOPE` entries' `source`/`note`/`as_of` fields + to cite this document and quote the operative clause (code change only, + `zero_data_retention` stays `False` as it already was); `scripts/ci/` + interrogate coverage stays 100% and `tests/test_zdr_policy.py`/ + `tests/test_contextual_orchestrator_review_policy.py` (67 tests) still + pass unchanged, since neither pins the old source URL. **Did not + reclassify `opencode_zen`** (present in + `contextual_orchestrator/model_discovery.py`'s five... six provider + sources but absent from `PROVIDER_ZDR_SCOPE`'s five entries — a real, + pre-existing gap: `provider_zdr_scope()` would `KeyError` on it if it + were ever ZDR-checked) because this org's CI sidecar never registers an + `opencode_zen` credential (only the five `BYTEZ_/NVIDIA_NIM_/ + NVIDIA_NIM_SUB_/OPENROUTER_/OPENAI_API_KEY` secrets exist), so the + dormant `KeyError` risk is not live here; flagged rather than silently + left, since it would surface the moment any caller registers that + credential and requires ZDR. +- **The "free + ZDR is structurally near-empty for private targets" premise + is confirmed, and is not fixable by reclassifying NVIDIA** — the Section + 3.3(iv) evidence above forecloses that specific path. The only + theoretical non-empty free+ZDR route left is an OpenRouter model that is + simultaneously free-priced and present in the live + `/api/v1/endpoints/zdr` feed; not verified live this pass (would need a + fresh discovery run against real credentials, which circles back to the + same access gap as the sidecar-outage investigation above). This remains + a real, unresolved architecture question for private-repo reviews + specifically (public repos are unaffected, per the visibility check + above) and is a policy/product decision, not a code bug this pass can + close. +- **Direct-NIM-communication audit — narrower than the initial description, + most of it already resolved or dormant, nothing changed this pass:** + - `scripts/ci/select_nvidia_nim_model.py` (the "ask NVIDIA's live + `/v1/models` catalog which model is actually still served" resolver, + written specifically to survive NVIDIA's own model end-of-life + rotations) has **zero callers** anywhere in `.github/workflows/` or + `scripts/`; only its own test (`tests/test_select_nvidia_nim_model.py`) + exercises it. It is not wired into `pr_review_fix_scheduler.py` or any + hourly-repair workflow despite its docstring's framing ("the scheduled + autofix worker"). Dead code today, not a live direct-NIM path — and, + notably, it already implements the exact live-catalog cross-check that + would fix this entry's 404-retired-model finding above, just for a + different, currently-unwired caller. + - `scripts/ci/run_opencode_review_model_pool.sh`'s `is_nvidia_nim_candidate`/ + `NVIDIA_API_KEY` handling is real, wired code, but its candidate list + comes entirely from `OPENCODE_MODEL_CANDIDATES`, which + `.github/workflows/opencode-review-dispatch.yml` (contract-pinned by + `tests/test_opencode_agent_contract.py`) currently sets to the single + value `"contextual-orchestrator/orchestrator/free"` — already + gateway-only, no direct-NIM entries active. `docs/nvidia-nim-opencode-hotfix.md` + documents that a six-model NIM-prefix hotfix existed for exactly this + script during a past GitHub-Models outage and was already rolled back + per its own "Rollback" section; that doc is now stale (describes a + reverted state as current) and its own instructions say to delete it + once catalog reliability is restored — worth a follow-up doc cleanup, + not attempted this pass. The dormant `nvidia-nim` provider block still + present in root `opencode.jsonc` (lines ~289-294) is inert for the CI + dispatch path (which generates its own `enabled_providers: + ["contextual-orchestrator"]` config) but was left as-is since it may + still serve local/interactive OpenCode use outside CI, which is outside + the owner's stated CI-routing goal. + - `scripts/ci/strix_quick_gate.sh`'s `is_contextual_orchestrator_model` + was narrowed to `orchestrator/free` only, per the owner's explicit + override decision recorded above — see the "Strix `orchestrator/auto` → + `orchestrator/free`" entry above for the full sequencing conflict, how + it was surfaced, and the owner's decision. +- **Net effect on the owner's goal**: the OpenCode review-dispatch path was + already fully gateway-only (`orchestrator/free`, no direct-NIM) before + this pass. The Strix path is now also `orchestrator/free`-only, per the + owner's explicit, informed decision to accept the resilience trade-off + ADR-0003 originally avoided. The private-repo free+ZDR gap is real, + unresolved, and not a code bug. No dead NIM-direct code was removed this + pass because none of the + three flagged call sites turned out to be a live, unconditional + direct-NIM path that could be safely deleted without either doing nothing + (already dead) or removing the one resilience mechanism keeping a + required check alive during a live outage. + +## 2026-08-30 pingora_edge_policy.py binary-evidence gap: two competing open fixes + +A live failure on `ContextualWisdomLab/contextual-orchestrator#906`'s `required-workflow-bootstrap` +job (`GitHub content evidence for docs/papers/helm-holistic-evaluation-2211.09110.pdf +is not a regular base64 file`) traces to `scripts/ci/pingora_edge_policy.py`'s +`_load_file_content`: GitHub's Contents API stops returning inline +`encoding: "base64"` once a file crosses roughly 1 MB (returning +`encoding: "none"` + a `download_url` instead), and this policy scanner's +`_needs_content_scan` has no exemption for genuinely binary evidence files in +general — any added/modified file without a `patch` (i.e. any binary file, +regardless of size) reaches `_load_file_content`, which always fails once it +tries `raw.decode("utf-8")`. Two **already-open, independent, partially +conflicting** PRs address pieces of this: + +- **#1420** adds real, structural validation (`_is_recognized_documentation_image`: + PNG magic header, chunk order, CRC, zlib-stream, dimension, and scanline + checks) so an image *suffix* alone cannot exempt a file — consistent with + this policy's own stated principle. Covers `.png` only; does not touch + `.pdf`, so it would not by itself fix `ContextualWisdomLab/contextual-orchestrator#906`. +- **#1427** adds a flat `NON_RUNTIME_BINARY_SUFFIXES` allowlist (`.avif`, + `.gif`, `.ico`, `.jpeg`, `.jpg`, `.pdf`, `.png`, `.webp`) that skips + content-scanning by **extension alone**, no byte-level verification. This + does fix `ContextualWisdomLab/contextual-orchestrator#906`, but for every + suffix in that list (not just `.pdf`) it + reintroduces the exact "extension alone is not an exception" gap #1420 + exists to close for PNG — a shell/config file renamed to `evidence.pdf` + (or `.png`, `.jpg`, ...) would now bypass the Nginx-runtime-artifact scan + entirely. +- Left substantive comments on both PRs (this pass) recommending #1420's + structural-validation pattern be extended to `.pdf` (a bounded magic- + header/`%%EOF`-trailer check, short of full parsing) rather than merging + #1427's blanket suffix-trust list, and that the two PRs coordinate so the + org does not land two divergent implementations of the same policy + surface. Not resolved in code this pass — both PRs are themselves + currently blocked by the sidecar-preflight outage above, so neither could + be re-reviewed to a genuine pass yet regardless of which approach wins. + ## 5. 실행 루프와 고객의 다음 행동 각 hourly pass는 아래 순서를 유지한다. diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index 28b36e8dd..9e35db4bc 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -36,7 +36,46 @@ SIDECAR_LOG_SANITIZER="$ORG_REPO_ROOT/scripts/ci/sanitize_contextual_orchestrato # of guessing whether the async sanitizer has caught up. SIDECAR_DISCOVERY_DIAGNOSTICS_SENTINEL="discovery_diagnostics_complete" CATALOG_LIMIT="${ORCHESTRATOR_CATALOG_LIMIT:-12}" -CATALOG_FAMILY_CAP="${ORCHESTRATOR_CATALOG_FAMILY_CAP:-4}" +# 2026-08-30: raised from 4. contextual_orchestrator_review_policy.py's +# family_cap groups nvidia_nim and nvidia_nim_sub as one outage-domain family +# and, per an exact-head evidence trail, currently that single family is the +# *only* one populating orchestrator/free (46 free rows, 100% nvidia_nim* -- +# 23 distinct model ids shared by both keys). Candidate selection sorts +# eligible rows alphabetically by (provider, model) with no reliability +# awareness, so a family_cap of 4 deterministically admitted the same four +# alphabetically-first candidates on every run -- always including two +# NVIDIA-retired model ids (google/gemma-3-12b-it, google/gemma-3-4b-it; +# confirmed HTTP 404 on live preflight) plus two others that timed out in the +# same recovered run -- while never giving the other ~19 healthy free +# nvidia_nim* models in the same run's own discovery report a chance. This is +# not throughput tuning: it is the confirmed, reproducible root cause of +# orchestrator/free's "no provider route passed the Strix plain-chat +# preflight" failures (see docs/product-technical-gap-baseline.md's +# 2026-08-30 sidecar-preflight entries for the full evidence, including the +# exact discovery/preflight artifact this comment is based on). +# 8 is a deliberately moderate raise, not a wholesale removal of the cap. The +# picking loop below also stops at CATALOG_LIMIT (12) total regardless of +# family_cap, so the absolute worst case across any number of families was +# already REVIEW_PREFLIGHT_TIMEOUT_SECONDS (10s) x 12 = 120s before this +# change (reached once family_cap x distinct-families >= 12, i.e. >=3 +# families at the old cap of 4) and stays 120s after it -- this raise does +# not move that pre-existing ceiling. What it does change is when that +# ceiling is reached and the typical case today: with the single family +# (nvidia_nim) that currently fills 100% of orchestrator/free, worst-case +# preflight time rises from ~40s (4 candidates) to ~80s (8 candidates); with +# exactly two distinct families it would now also reach the 120s ceiling +# (previously ~80s at family_cap=4). Both figures stay within the sidecar's +# existing 180s readiness-wait budget in the common case; this was reasoned +# from, not verified against, live provider timing, since this session has +# no access to the five provider credentials the sidecar's KV requires. If +# real hosted +# runs show this is still insufficient (all 8 still failing) or the added +# latency itself becomes the bottleneck, the more complete fix is a live +# provider /v1/models cross-check at discovery time to drop retired model ids +# before they ever reach preflight (scripts/ci/select_nvidia_nim_model.py +# already implements that exact pattern for a different, currently-unwired +# caller) rather than raising this further. +CATALOG_FAMILY_CAP="${ORCHESTRATOR_CATALOG_FAMILY_CAP:-8}" ORCHESTRATOR_GITHUB_ENV="${GITHUB_ENV:-}" sidecar_python="$(command -v python3)" diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index c4cd33cfa..66be927b5 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -322,8 +322,7 @@ is_vertex_model() { is_contextual_orchestrator_model() { case "$1" in - orchestrator/free | contextual-orchestrator/orchestrator/free | \ - orchestrator/auto | contextual-orchestrator/orchestrator/auto) + orchestrator/free | contextual-orchestrator/orchestrator/free) return 0 ;; *) @@ -2563,8 +2562,8 @@ child_model_for_api_base() { # LiteLLM requires an explicit provider prefix even when the gateway is an # OpenAI-compatible local endpoint. Strip only the connector-facing alias so - # the selected orchestrator/free or orchestrator/auto virtual pool reaches - # contextual-orchestrator unchanged. + # the selected orchestrator/free virtual pool reaches contextual-orchestrator + # unchanged. if is_contextual_orchestrator_model "$model" && is_contextual_orchestrator_api_base "$llm_api_base_value"; then local contextual_orchestrator_model diff --git a/scripts/ci/strix_required_workflow_smoke.sh b/scripts/ci/strix_required_workflow_smoke.sh index 1243911d2..e2f1fda40 100755 --- a/scripts/ci/strix_required_workflow_smoke.sh +++ b/scripts/ci/strix_required_workflow_smoke.sh @@ -181,12 +181,11 @@ assert_file_contains "$full_gate_test" "assert_strix_workflow_pr_trigger_hardene assert_file_contains "$workflow_file" "Provision contextual-orchestrator Strix sidecar" "Strix workflow provisions the trusted contextual-orchestrator gateway" assert_file_contains "$workflow_file" "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" "Strix workflow binds target visibility to the gateway ZDR policy" active_strix_models="$(sed -n -E 's/^[[:space:]]*STRIX_MODEL:[[:space:]]*([^#[:space:]]+)[[:space:]]*$/\1/p' "$workflow_file")" -[ "$active_strix_models" = "contextual-orchestrator/orchestrator/auto" ] || record_failure "Strix must define exactly one active provider-diverse auto default model" -assert_file_not_contains "$workflow_file" "STRIX_MODEL: contextual-orchestrator/orchestrator/free" "Strix must not retain the free default route" -assert_file_contains "$decision_record" "authoritative Strix security analysis uses the provider-diverse \`orchestrator/auto\` pool" "The binding ADR authorizes the Strix auto route" +[ "$active_strix_models" = "contextual-orchestrator/orchestrator/free" ] || record_failure "Strix must define exactly one active zero-cost free default model" +assert_file_not_contains "$workflow_file" "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" "Strix must not retain the paid-inclusive auto default route" +assert_file_contains "$decision_record" "2026-08-30 amendment: Strix uses \`orchestrator/free\`" "The binding ADR amendment records the owner's explicit free-only override" assert_file_contains "$decision_record" "Zero Data Retention (ZDR)-compliant routes remain mandatory for private targets" "The binding ADR preserves private-target privacy" -assert_file_contains "$decision_record" "Strix is intentionally correctness-first rather than zero-cost" "The binding ADR records the Strix cost boundary" -assert_file_contains "$agent_policy" "Authoritative Strix analysis uses the correctness-first \`orchestrator/auto\`" "Repository guidance agrees with the binding Strix route" +assert_file_contains "$agent_policy" "Strix uses the zero-cost \`orchestrator/free\`" "Repository guidance agrees with the binding Strix route" assert_file_contains "$workflow_file" "provider_mode=contextual_orchestrator" "Strix workflow selects the contextual-orchestrator provider mode" assert_file_contains "$workflow_file" "STRIX_FALLBACK_MODELS: \"\"" "Strix delegates provider discovery and failover to the gateway" assert_file_not_contains "$workflow_file" "Resolve live NVIDIA NIM Strix models" "Strix does not resolve a direct provider outside the gateway" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index c44e82c5a..b528e8baf 100644 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -313,8 +313,8 @@ assert_strix_workflow_pr_trigger_hardened() { assert_file_contains "$REPO_ROOT/tests/test_strix_repository_visibility_contract.py" "test_dispatch_api_visibility_preserves_internal_privacy" "strix visibility contract executes public, private, and internal dispatch fixtures" assert_file_contains "$workflow_file" 'STRIX_MODEL: ${{ steps.gate.outputs.strix_model }}' "strix workflow propagates the gate-selected fallback model to the scanner" assert_file_not_contains "$workflow_file" "secrets.STRIX_LLM ||" "strix workflow must not let the legacy STRIX_LLM secret override PR defaults" - assert_file_contains "$workflow_file" "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto" "strix workflow rejects non-gateway model overrides" - assert_file_contains "$workflow_file" "STRIX_LLM must select contextual-orchestrator/orchestrator/auto" "strix workflow accepts only the gateway model" + assert_file_contains "$workflow_file" "Strix model overrides are limited to contextual-orchestrator/orchestrator/free" "strix workflow rejects non-gateway model overrides" + assert_file_contains "$workflow_file" "STRIX_LLM must select contextual-orchestrator/orchestrator/free" "strix workflow accepts only the gateway model" assert_file_contains "$workflow_file" 'STRIX_FALLBACK_MODELS: ""' "strix workflow disables external fallback models" assert_file_contains "$workflow_file" 'STRIX_FAIL_ON_PROVIDER_SIGNAL: "1"' "strix workflow fails closed on timeout, fatal, warning, denied, or provider failure signals" assert_file_contains "$workflow_file" 'NPM_CONFIG_IGNORE_SCRIPTS: "true"' "strix workflow disables npm lifecycle scripts for untrusted PR scan data" diff --git a/scripts/ci/zdr_policy.py b/scripts/ci/zdr_policy.py index 848bcb232..eb327c8ca 100644 --- a/scripts/ci/zdr_policy.py +++ b/scripts/ci/zdr_policy.py @@ -69,22 +69,38 @@ class ProviderZdrScope: "authoritative per-endpoint membership source.", openrouter_endpoints_feed=True, ), + # Full citation for the two NVIDIA entries below: NVIDIA's own current + # *NVIDIA API Trial Terms of Service* (v. September 19, 2025 -- the terms + # governing this org's free/trial integrate.api.nvidia.com key; confirmed + # still the live document as of the as_of date on these entries), Section + # 3.3(iv), states NVIDIA collects "User Content and Generated Content to + # improve NVIDIA products and services, including AI models" -- i.e. + # prompts and completions ARE used for training. This is not merely an + # absence of attestation; it is an affirmative not-ZDR fact. Section 2.3's + # "will not store or use User Content or Generated Content at the end of + # each API Service session" does not override this: 3.3 is the operative + # carve-out. Do not reclassify either provider as ZDR without a + # superseding, dated NVIDIA document that repeals or narrows Section + # 3.3(iv) for this specific API Service. "nvidia_nim": ProviderZdrScope( provider_name="nvidia_nim", zero_data_retention=False, - source="https://openrouter.ai/docs/guides/privacy/provider-logging", - as_of="2026-08-27", - note="Direct NVIDIA NIM hosted API (integrate.api.nvidia.com) is not " - "attested as zero-retention; treat as retained unless a dated provider " - "attestation is added.", + source="https://assets.ngc.nvidia.com/products/api-catalog/legal/" + "NVIDIA%20API%20Trial%20Terms%20of%20Service.pdf", + as_of="2026-08-30", + note="NVIDIA API Trial Terms of Service Section 3.3(iv) states User " + "Content and Generated Content are used to improve NVIDIA products " + "and services, including AI models -- affirmatively not ZDR.", ), "nvidia_nim_sub": ProviderZdrScope( provider_name="nvidia_nim_sub", zero_data_retention=False, - source="https://openrouter.ai/docs/guides/privacy/provider-logging", - as_of="2026-08-27", - note="Secondary NVIDIA NIM key shares the nvidia_nim scope and is not " - "attested as zero-retention.", + source="https://assets.ngc.nvidia.com/products/api-catalog/legal/" + "NVIDIA%20API%20Trial%20Terms%20of%20Service.pdf", + as_of="2026-08-30", + note="Secondary NVIDIA NIM key is the same integrate.api.nvidia.com " + "trial API and shares the nvidia_nim entry's Section 3.3(iv) " + "training-use scope verbatim.", ), "openai": ProviderZdrScope( provider_name="openai", diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 60b3ade48..ce5f3cc13 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -348,7 +348,7 @@ def test_strix_gateway_uses_provider_neutral_reasoning_effort() -> None: """Gateway free-pool scans must not force unsupported provider controls.""" text = _read(STRIX_WORKFLOW) assert "STRIX_REASONING_EFFORT: none" in text - assert "CONTEXTUAL_ORCHESTRATOR_POOL: auto" in text + assert "CONTEXTUAL_ORCHESTRATOR_POOL: free" in text def test_sidecar_probes_the_pinned_server_body_limit_at_http_boundary() -> None: @@ -497,7 +497,7 @@ def test_required_strix_uses_the_gateway_and_zdr_visibility_contract() -> None: workflow = _read(STRIX_WORKFLOW) assert "Provision contextual-orchestrator Strix sidecar" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow - assert 'STRIX_MODEL: contextual-orchestrator/orchestrator/auto' in workflow + assert 'STRIX_MODEL: contextual-orchestrator/orchestrator/free' in workflow assert "provider_mode=contextual_orchestrator" in workflow assert "STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator" in workflow assert workflow.index("Resolve target repository visibility") < workflow.index( diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index dfa9aa2c8..481b3356a 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -87,7 +87,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "contextual-orchestrator/orchestrator/auto", + "STRIX_MODEL": "contextual-orchestrator/orchestrator/free", "STRIX_MODEL_REQUESTED": "", }, capture_output=True, @@ -96,12 +96,12 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> ) assert strix.returncode == 0, strix.stderr assert { - "strix_model=contextual-orchestrator/orchestrator/auto", + "strix_model=contextual-orchestrator/orchestrator/free", "enabled=true", "provider_mode=contextual_orchestrator", } <= set(strix_output.read_text().splitlines()) assert ( - "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" + "STRIX_MODEL: contextual-orchestrator/orchestrator/free" in workflow_text("strix.yml") ) assert ( diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 986dd24ff..3dcfe2cdd 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -19,7 +19,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "553111b54287c56a78833f8463b2a75e646ae208" +REVIEW_DISPATCH_BLOB_SHA = "2aa245e7f2a053a4c0b7a9cc8bac0d5d44d38092" def _workflow_text(path: Path) -> str: diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index a00b0c426..77594cc1f 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -576,7 +576,7 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( env={ **os.environ, "GITHUB_OUTPUT": str(strix_output), - "STRIX_MODEL": "contextual-orchestrator/orchestrator/auto", + "STRIX_MODEL": "contextual-orchestrator/orchestrator/free", "STRIX_MODEL_REQUESTED": "", }, capture_output=True, @@ -585,12 +585,12 @@ def test_strix_gateway_default_and_noema_sidecar_fail_closed( ) assert strix.returncode == 0, strix.stderr assert { - "strix_model=contextual-orchestrator/orchestrator/auto", + "strix_model=contextual-orchestrator/orchestrator/free", "enabled=true", "provider_mode=contextual_orchestrator", } <= set(strix_output.read_text().splitlines()) assert ( - "STRIX_MODEL: contextual-orchestrator/orchestrator/auto" + "STRIX_MODEL: contextual-orchestrator/orchestrator/free" in workflow_text("strix.yml") ) assert ( diff --git a/tests/test_strix_contextual_orchestrator_contract.py b/tests/test_strix_contextual_orchestrator_contract.py index 0db9f6b4e..52763ecc8 100644 --- a/tests/test_strix_contextual_orchestrator_contract.py +++ b/tests/test_strix_contextual_orchestrator_contract.py @@ -26,7 +26,7 @@ def setUp(self) -> None: def test_default_scan_provisions_the_existing_gateway_sidecar(self) -> None: """Every scan uses the five-provider gateway, never a direct pool.""" self.assertIn("Provision contextual-orchestrator Strix sidecar", self.workflow) - self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/auto", self.workflow) + self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/free", self.workflow) self.assertIn("provider_mode=contextual_orchestrator", self.workflow) self.assertIn("STRIX_FALLBACK_MODELS: \"\"", self.workflow) self.assertNotIn( @@ -48,7 +48,7 @@ def test_model_override_cannot_escape_the_gateway(self) -> None: """A dispatch payload cannot select a direct provider route.""" self.assertIn("github.event.client_payload.strix_llm", self.workflow) self.assertIn( - "Strix model overrides are limited to contextual-orchestrator/orchestrator/auto", + "Strix model overrides are limited to contextual-orchestrator/orchestrator/free", self.workflow, ) for direct_route in ("nvidia_nim/*)", "openrouter/free", "openai-direct/gpt-5.4"): @@ -75,7 +75,7 @@ def test_required_smoke_pins_the_gateway_default(self) -> None: self.assertIn("contextual-orchestrator Strix sidecar", self.smoke) self.assertIn("active_strix_models=", self.smoke) self.assertIn( - '"$active_strix_models" = "contextual-orchestrator/orchestrator/auto"', + '"$active_strix_models" = "contextual-orchestrator/orchestrator/free"', self.smoke, ) self.assertIn("Strix does not resolve a direct provider outside the gateway", self.smoke) diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index 07ab21bad..ba8344455 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -190,7 +190,7 @@ def test_workflow_routes_all_scans_through_contextual_orchestrator(self) -> None workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("Provision contextual-orchestrator Strix sidecar", workflow) - self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/auto", workflow) + self.assertIn("STRIX_MODEL: contextual-orchestrator/orchestrator/free", workflow) self.assertIn("provider_mode=contextual_orchestrator", workflow) self.assertIn("STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator", workflow) self.assertNotIn("Resolve live NVIDIA NIM Strix models", workflow) @@ -201,7 +201,7 @@ def test_workflow_rejects_non_gateway_model_overrides(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("STRIX_MODEL_REQUESTED", workflow) - self.assertIn("Strix model overrides are limited to contextual-orchestrator/orchestrator/auto.", workflow) + self.assertIn("Strix model overrides are limited to contextual-orchestrator/orchestrator/free.", workflow) self.assertIn("STRIX_FALLBACK_MODELS: \"\"", workflow) def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: diff --git a/tests/test_strix_openai_fallback_api_base.py b/tests/test_strix_openai_fallback_api_base.py index 4b20d3580..7919a7468 100644 --- a/tests/test_strix_openai_fallback_api_base.py +++ b/tests/test_strix_openai_fallback_api_base.py @@ -306,13 +306,11 @@ def test_workflow_does_not_configure_an_external_fallback(self) -> None: self.assertIn("Provision contextual-orchestrator Strix sidecar", workflow) def test_workflow_gateway_base_is_the_only_http_exception(self) -> None: - """Both gateway pools accept only the pinned process-local HTTP base.""" + """The free gateway pool accepts only the pinned process-local HTTP base.""" for model in ( "orchestrator/free", "contextual-orchestrator/orchestrator/free", - "orchestrator/auto", - "contextual-orchestrator/orchestrator/auto", ): with self.subTest(model=model): rc, api_base = _resolve_api_base( @@ -324,6 +322,16 @@ def test_workflow_gateway_base_is_the_only_http_exception(self) -> None: rc, _ = _resolve_api_base( {"LLM_API_BASE_FILE": "http://127.0.0.1:18081/v1"}, + "orchestrator/free", + ) + self.assertEqual(rc, 2) + + # 2026-08-30: orchestrator/auto is no longer a recognized Strix gateway + # model (owner decision superseding ADR-0003's auto default) -- the + # gate must now reject it rather than resolve it, the same as any + # other unrecognized virtual pool. + rc, _ = _resolve_api_base( + {"LLM_API_BASE_FILE": "http://127.0.0.1:18080/v1"}, "orchestrator/auto", ) self.assertEqual(rc, 2) @@ -335,13 +343,11 @@ def test_workflow_gateway_base_is_the_only_http_exception(self) -> None: self.assertEqual(rc, 2) def test_gateway_child_model_preserves_selected_virtual_pool(self) -> None: - """LiteLLM qualification must not rewrite auto back to free.""" + """LiteLLM qualification must not rewrite the selected free pool.""" expected_child_models = { "orchestrator/free": "openai/orchestrator/free", "contextual-orchestrator/orchestrator/free": "openai/orchestrator/free", - "orchestrator/auto": "openai/orchestrator/auto", - "contextual-orchestrator/orchestrator/auto": "openai/orchestrator/auto", } for model, expected_child_model in expected_child_models.items(): with self.subTest(model=model):