diff --git a/CHANGELOG.md b/CHANGELOG.md index 58e53f8..09793f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 persists the complete `CouncilResult` JSON with user-private temporary-file semantics while preserving stdout and exit-code behavior. Persistence remains opt-in and is rejected with streaming; write failures retain normal stdout before exiting 1. +- **Degraded-run exit code (DSE-901).** `CouncilResult.degraded` is a new computed + field, `True` whenever `synthesis_error` or `adversarial.verdict_error` is set -- + i.e. at least one member answered but the judge/synthesizer step failed. The CLI + now exits a distinct code `3` in that case (`--json` still emits the full payload, + now including `"degraded"`), so a verification gate reading only the exit code can + no longer mistake a partial run for a clean pass. This closes the gap found + 2026-07-25: an Anthropic credit failure took out the `claude` judge while 4/5 + members still answered, `adversarial.verdict` came back `null`, and the CLI still + exited `0`. Judge/synthesizer failover to another provider remains explicitly out + of scope for this fix (tracked separately). + +### Fixed + +- **Adversarial mode's `--json` gate contract (DSE-901).** Previously a judge/ + synthesizer failure in `adversarial` (or `synthesize`/`debate`) mode was only + observable by inspecting `synthesis_error`/`adversarial.verdict_error` in the + payload; the exit code stayed `0`. See "Degraded-run exit code" above. ## [1.2.0] - 2026-07-18 diff --git a/DOCUMENTATION_INDEX.md b/DOCUMENTATION_INDEX.md index 7d870fb..d120be2 100644 --- a/DOCUMENTATION_INDEX.md +++ b/DOCUMENTATION_INDEX.md @@ -82,7 +82,7 @@ Package root: `src/conclave/` (installed as the `conclave` package; console scri | Adapter tests | [`tests/test_adapters.py`](tests/test_adapters.py) | Per-adapter `build_request` + `parse_response` for openai-compat/anthropic/gemini: system-hoist, max_tokens, role mapping, usage parsing, empty/malformed/error-status raises. | | Provider highway tests | [`tests/test_providers.py`](tests/test_providers.py) | `resolve_adapter` (built-in prefixes, per-provider URLs, custom endpoints, unknown-prefix raise), end-to-end `call_model`, and `redact()` (bearer/`sk-`/env-var-value/`x-api-key` scrubbing; pre-redacted provider errors). | | Registry/config tests | [`tests/test_registry_config.py`](tests/test_registry_config.py) | Name resolution, key-presence logic, config merge. | -| CLI tests | [`tests/test_cli.py`](tests/test_cli.py) | Typer `CliRunner`: exit-code contract (0 usable result and ready Elite / 1 zero-usable or non-ready Elite / 2 usage error), `--json` payload + exit code, human renderers per mode, `providers` table never prints secrets, aclose lifecycle. | +| CLI tests | [`tests/test_cli.py`](tests/test_cli.py) | Typer `CliRunner`: exit-code contract (0 usable result and ready Elite / 1 zero-usable or non-ready Elite / 2 usage error / 3 degraded — usable answers but judge/synthesizer failed, DSE-901), `--json` payload (including `degraded`) + exit code, human renderers per mode, `providers` table never prints secrets, aclose lifecycle. | | Eval tests | [`tests/evals/`](tests/evals/) | Frozen matrix, offline replay, live reservation/checkpoint/resume gates, blinding, scoring, reporting, and CLI assertions. | | Transport tests | [`tests/test_transport.py`](tests/test_transport.py) | `post_json` via httpx `MockTransport`: success/error-status/non-JSON fallback, timeout & connect/HTTP errors → `TransportError` (key never leaks), client reuse/pooling, aclose idempotency. | | Streaming tests | [`tests/test_streaming.py`](tests/test_streaming.py) | Per-adapter SSE via `MockTransport` (openai-compat/anthropic/gemini): incremental chunks + assembled answer == concatenation == buffered result; mid-stream malformed-frame/connection-drop/non-2xx → error set with partial text preserved (never raises); key redaction in stream errors; buffered `ask()` never opens a stream; `Council.ask_stream` interleaving + terminal `done` shape; CLI `--stream` smoke + exit-code contract + debate rejection; `--stream` + cache one-shot replay. | diff --git a/src/conclave/cli.py b/src/conclave/cli.py index 2f5b88e..f76ca22 100644 --- a/src/conclave/cli.py +++ b/src/conclave/cli.py @@ -436,6 +436,15 @@ def on_event(event: StreamEvent) -> None: # round-over-round, so the default is conservative and rarely fires spuriously. _DEFAULT_CONVERGE_THRESHOLD = 0.95 +# Distinct exit code for a "degraded" run (DSE-901): at least one member +# answered, but the judge/synthesizer step failed (``CouncilResult.degraded``). +# Deliberately NOT 2 -- that code already means "usage/config error" (unknown +# mode, unresolved council) below, and colliding the two would reintroduce the +# same ambiguity this change fixes. Kept below the existing 0/1/2 meanings so a +# caller doing a bare ``echo $?`` can add one new branch without reinterpreting +# any exit code it already handles. +_DEGRADED_EXIT_CODE = 3 + def _resolve_converge_threshold( converge: bool | None, @@ -555,7 +564,8 @@ def ask( Exit codes: - * 0 -- the run produced at least one usable member answer and, for Elite, + * 0 -- the run produced at least one usable member answer, the + judge/synthesizer step (when attempted) succeeded, and for Elite, ``decision_readiness`` is ``ready``. * 1 -- the run produced zero usable member answers (e.g. no council member had an API key, or every member failed), or an Elite result is missing or @@ -563,6 +573,18 @@ def ask( the full JSON result is still emitted to stdout first, so a script can both parse the payload and detect the failure via the non-zero exit code. * 2 -- a usage/config error (unknown mode, or no members resolved). + * 3 -- degraded (DSE-901): at least one member answered, but the + judge/synthesizer step failed -- ``CouncilResult.degraded`` is ``True`` + (``synthesis_error`` or, in ``adversarial`` mode, ``adversarial.verdict_error`` + is set). Under ``--json`` the full JSON result -- including the top-level + ``"degraded": true`` field -- is still emitted to stdout first. Kept + distinct from both 0 and 1 so a caller that checks only the exit code (e.g. + a verification gate) cannot mistake a partial run for a clean pass: this was + exactly the 2026-07-25 incident that produced this code (an Anthropic + credit failure took out the judge while 4/5 members still answered, and the + run exited 0 with ``adversarial.verdict`` silently ``null``). Not raised for + Elite (its own readiness gate above already covers a failed + synthesis/verdict step with exit code 1). """ mode_lower = mode.lower() if mode_lower not in _VALID_MODES: @@ -609,6 +631,10 @@ def ask( "[red]No usable council answers. Run 'conclave providers' to check keys.[/red]" ) raise typer.Exit(code=1) + if result.degraded: + # _stream_to_terminal already printed the "No synthesis: ..." warning + # (from synthesis_error) to stderr; only the exit code is new here. + raise typer.Exit(code=_DEGRADED_EXIT_CODE) return if mode_lower == "debate": @@ -628,7 +654,11 @@ def ask( # A run that produced no usable member answers is a failure for scripting # purposes regardless of output format. We compute this once and apply the - # same exit-code contract to both the JSON and human paths. + # same exit-code contract to both the JSON and human paths. A run that DID + # get usable member answers but whose judge/synthesizer step failed is a + # distinct, less severe failure (DSE-901): ``result.degraded`` (checked + # below, after the hard-failure/usage-error exits) drives exit code + # ``_DEGRADED_EXIT_CODE`` instead of silently returning 0. no_usable_answers = not result.successful_answers elite_not_ready = mode_lower == "elite" and ( result.elite is None or result.elite.decision_readiness != "ready" @@ -651,6 +681,8 @@ def ask( console.print_json(json.dumps(payload)) if json_output_failed or no_usable_answers or elite_not_ready: raise typer.Exit(code=1) + if result.degraded: + raise typer.Exit(code=_DEGRADED_EXIT_CODE) return if mode_lower == "elite": @@ -682,6 +714,11 @@ def ask( _RENDERERS[result.mode](result) if json_output_failed: raise typer.Exit(code=1) + if result.degraded: + # The mode-specific renderer above already printed the "No synthesis: ..." + # / "No verdict: ..." warning (from synthesis_error / verdict_error) to + # stderr; only the exit code is new here (DSE-901). + raise typer.Exit(code=_DEGRADED_EXIT_CODE) @app.command() diff --git a/src/conclave/models.py b/src/conclave/models.py index 826a039..fbcdaeb 100644 --- a/src/conclave/models.py +++ b/src/conclave/models.py @@ -11,7 +11,7 @@ from collections.abc import Iterable from typing import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, computed_field from .verdict import ( CouncilConflict, @@ -355,6 +355,11 @@ class CouncilResult(BaseModel): member_answers: Read-only alias for ``answers`` (the per-member raw responses), exposed under the contract-v2 name. Returns the same list object as ``answers``; there is one underlying field. + degraded: Computed field (DSE-901) -- ``True`` when the run produced at + least one usable member answer but the judge/synthesizer step failed, + so a caller cannot mistake a partial run for a clean pass. See the + property docstring below for the exact rule and the CLI's exit-code + contract (:func:`conclave.cli.ask`) that keys off it. """ prompt: str @@ -402,6 +407,46 @@ def member_answers(self) -> list[ModelAnswer]: """Contract-v2 alias for :attr:`answers` (the per-member raw responses).""" return self.answers + @computed_field # type: ignore[prop-decorator] + @property + def degraded(self) -> bool: + """True when members answered but the judge/synthesizer step failed (DSE-901). + + Distinguishes "partial run: judge/synthesis failed" from a clean pass, so + a verification gate that only checks ``CouncilResult`` fields (or the + CLI's exit code -- see :func:`conclave.cli.ask`) cannot read a degraded + run as a full success. This closed a real gap: on 2026-07-25 an + Anthropic credit failure meant 4/5 members answered but the ``claude`` + judge/synthesizer failed, ``adversarial.verdict`` was ``null``, and the + CLI still exited 0. + + ``True`` whenever either of the two fields the judge/synthesizer path + writes on failure is set: + + * ``synthesis_error`` is non-``None`` (``synthesize``/``debate``/``elite`` + modes, and mirrored into ``adversarial`` mode's top-level fields too); or + * ``adversarial.verdict_error`` is non-``None`` (checked directly as a + defense-in-depth fallback in case a future code path sets it without + mirroring to ``synthesis_error``). + + Always ``False`` for ``raw``/``vote`` runs, where the judge/synthesizer is + never invoked, and for any run where it ran and succeeded. Because + ``synthesis_error`` is also set when a run has zero usable member answers + (nothing to synthesize), that harder failure is *also* ``degraded=True`` + here; the CLI's exit-code contract gives that case its own, more severe + exit code, so this flag alone does not distinguish "nothing answered" + from "some members answered, judge/synthesis failed" -- callers wanting + that distinction should also check ``successful_answers``. Included as a + top-level key in ``model_dump(mode="json")`` output (a Pydantic + ``computed_field``), so a scripted consumer can check it directly instead + of reaching into ``synthesis_error``/``adversarial``. + """ + if self.synthesis_error: + return True + if self.adversarial is not None and self.adversarial.verdict_error: + return True + return False + # Late import (see the note near the top): ``manifest`` imports ``TokenUsage`` # from this module, so it can only be imported once the leaf types above exist. diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a431a3..f78fb02 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,11 +3,16 @@ These exercise ``conclave.cli.ask`` through Typer's ``CliRunner`` (no real keys, no network). Three concerns are pinned here: -* **Exit-code contract (#17).** A run that produces zero *usable* member answers, - or an Elite run whose ``decision_readiness`` is not ``ready``, exits non-zero - (code 1) on both the human and ``--json`` paths. Under ``--json`` the full JSON - payload is still emitted to stdout so a script can parse the result *and* - detect the failure via the exit code. Other runs with a usable answer exit 0. +* **Exit-code contract (#17, extended by DSE-901).** A run that produces zero + *usable* member answers, or an Elite run whose ``decision_readiness`` is not + ``ready``, exits non-zero (code 1) on both the human and ``--json`` paths. + Under ``--json`` the full JSON payload is still emitted to stdout so a script + can parse the result *and* detect the failure via the exit code. A run with a + usable answer whose judge/synthesizer step nonetheless failed + (``CouncilResult.degraded``) exits a distinct code 3 rather than 0 -- see + ``tests/test_modes.py`` / ``tests/test_synthesizer.py`` for the mode-level + ``degraded`` behavior this exit code surfaces. Every other run with a usable + answer exits 0. * **Durable result output (#58).** Buffered runs can atomically persist the same complete JSON payload with user-private permissions, including before a failure exit. Invalid destinations and streaming combinations fail before council construction. @@ -28,6 +33,7 @@ from conclave.config import ConclaveConfig from conclave.models import CouncilResult, EliteResult, ModelAnswer from conclave.verdict import CouncilVerdict +from tests.conftest import make_response runner = CliRunner() @@ -52,6 +58,14 @@ def patch_cli_config(monkeypatch): monkeypatch.setattr(cli, "load_config", _config) +def _system_text(messages) -> str: + """Return the system-role content of a message list, or '' if none.""" + for m in messages: + if m.get("role") == "system": + return m.get("content", "") + return "" + + def test_no_members_human_exits_one(clear_keys, patch_cli_config): """Plain (human) path: zero usable answers -> exit code 1.""" result = runner.invoke(cli.app, ["ask", "hello"]) @@ -285,6 +299,125 @@ def test_unknown_mode_exits_two(patch_cli_config): assert result.exit_code == 2 +# --------------------------------------------------------------------------- # +# Degraded-run exit code (DSE-901): members answered, judge/synthesizer failed. +# --------------------------------------------------------------------------- # + + +def test_adversarial_judge_failure_json_is_degraded_not_clean( + monkeypatch, patch_cli_config, patch_call_model +): + """DSE-901: members answer but the judge/synthesizer call fails -> degraded. + + Reproduces the 2026-07-25 incident that filed this ticket: the Anthropic key + had no credit, so the ``claude`` judge's call returned an HTTP 400 while + ``grok``/``gemini`` still answered normally, and the CLI exited 0 with + ``adversarial.verdict`` silently ``null``. The fix must surface + ``degraded: true`` in the JSON payload and exit the new degraded code + (``cli._DEGRADED_EXIT_CODE`` == 3), not the clean-pass code 0. + """ + for var in ("XAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.setenv(var, "dummy-key") + + def handler(model, messages, **kwargs): + if "judge of an adversarial review" in _system_text(messages): + raise RuntimeError("HTTP 400: your credit balance is too low") + return make_response(f"answer from {model}") + + patch_call_model(handler) + result = runner.invoke( + cli.app, + ["ask", "hello", "--council", "grok,gemini", "--mode", "adversarial", "--json"], + ) + + assert result.exit_code == cli._DEGRADED_EXIT_CODE + assert result.exit_code not in (0, 1, 2) + payload = json.loads(result.stdout) + assert payload["degraded"] is True + assert payload["adversarial"]["verdict"] is None + assert "credit balance" in payload["adversarial"]["verdict_error"] + assert payload["synthesis_error"] == payload["adversarial"]["verdict_error"] + # This is a PARTIAL run, not the hard "zero usable answers" failure (code 1): + # the proposer/critic that didn't hit the judge path still answered. + assert len(payload["answers"]) >= 1 + assert any(a["error"] is None and a["answer"] is not None for a in payload["answers"]) + + +def test_adversarial_judge_failure_human_exits_degraded_code( + monkeypatch, patch_cli_config, patch_call_model +): + """Same DSE-901 scenario on the human (non-``--json``) render path.""" + for var in ("XAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.setenv(var, "dummy-key") + + def handler(model, messages, **kwargs): + if "judge of an adversarial review" in _system_text(messages): + raise RuntimeError("HTTP 400: your credit balance is too low") + return make_response(f"answer from {model}") + + patch_call_model(handler) + result = runner.invoke( + cli.app, ["ask", "hello", "--council", "grok,gemini", "--mode", "adversarial"] + ) + + assert result.exit_code == cli._DEGRADED_EXIT_CODE + assert "No verdict" in result.output + assert "credit balance" in result.output + + +def test_adversarial_clean_pass_still_exits_zero(monkeypatch, patch_cli_config, patch_call_model): + """Regression: a fully successful adversarial run (proposer/critic/judge all + succeed) is NOT degraded and keeps exiting 0 -- the new check must not fire + on the existing clean-pass path.""" + for var in ("XAI_API_KEY", "GEMINI_API_KEY", "ANTHROPIC_API_KEY"): + monkeypatch.setenv(var, "dummy-key") + + def handler(model, messages, **kwargs): + system = _system_text(messages) + if "judge of an adversarial review" in system: + return make_response("verdict text") + if "critic on an adversarial review" in system: + return make_response(f"critique from {model}") + return make_response(f"proposal from {model}") + + patch_call_model(handler) + result = runner.invoke( + cli.app, + ["ask", "hello", "--council", "grok,gemini", "--mode", "adversarial", "--json"], + ) + + assert result.exit_code == 0 + payload = json.loads(result.stdout) + assert payload["degraded"] is False + assert payload["adversarial"]["verdict"] == "verdict text" + + +def test_all_members_failed_synthesize_mode_exits_one_not_degraded( + monkeypatch, patch_cli_config, patch_call_model +): + """Regression: zero usable answers keeps exit 1, never the new degraded code. + + In default (synthesize) mode, every member erroring also sets + ``synthesis_error`` ("no successful member answers to synthesize"), so + ``result.degraded`` reads ``True`` here too -- the CLI's priority order must + still pick the more severe hard-failure code (1) over the new degraded code. + """ + for var in ("XAI_API_KEY", "GEMINI_API_KEY"): + monkeypatch.setenv(var, "dummy-key") + + def handler(model, messages, **kwargs): + raise RuntimeError("provider down") + + patch_call_model(handler) + result = runner.invoke(cli.app, ["ask", "hello", "--council", "grok,gemini", "--json"]) + + assert result.exit_code == 1 + payload = json.loads(result.stdout) + assert payload["degraded"] is True + assert len(payload["answers"]) == 2 + assert all(a["error"] is not None for a in payload["answers"]) + + def _elite_cli_result( *, completed: bool = True, diff --git a/tests/test_synthesizer.py b/tests/test_synthesizer.py index a44579a..72c1eba 100644 --- a/tests/test_synthesizer.py +++ b/tests/test_synthesizer.py @@ -186,6 +186,8 @@ def handler(model, messages, **kwargs): assert payload["synthesizer"] == "openai" assert payload["synthesizer_model_id"] == "openai/gpt-4.1" assert payload["synthesis"] == "CLI OPENAI MERGE" + # DSE-901: a clean pass is explicitly NOT degraded (regression guard). + assert payload["degraded"] is False # --------------------------------------------------------------------------- # @@ -242,6 +244,9 @@ def handler(model, messages, **kwargs): assert result.synthesis is None assert result.synthesis_error is not None assert "synthesizer 503 from provider" in result.synthesis_error + # DSE-901: members answered but synthesis failed -> a distinct "degraded" + # run, not a clean pass. + assert result.degraded is True async def test_no_usable_answers_is_signaled(monkeypatch, patch_call_model): @@ -299,6 +304,8 @@ def handler(model, messages, **kwargs): assert adv.judge == "claude" assert adv.judge_model_id == "anthropic/claude-sonnet-4-6" assert result.synthesis_error == adv.verdict_error + # DSE-901: a usable proposal/critique but no verdict is a degraded run. + assert result.degraded is True async def test_adversarial_judge_call_failure_is_signaled(monkeypatch, patch_call_model): @@ -323,6 +330,10 @@ def handler(model, messages, **kwargs): assert adv.verdict is None assert adv.verdict_error is not None assert "judge 500 from provider" in adv.verdict_error + # DSE-901 reference scenario: this is exactly the 2026-07-25 incident shape + # (members/critics answered, the judge call itself errored) -- must read as + # a degraded run so a verification gate can't mistake it for a clean pass. + assert result.degraded is True async def test_debate_synthesizer_unkeyed_is_signaled(monkeypatch, patch_call_model, clear_keys): @@ -422,3 +433,65 @@ def handler(model, messages, **kwargs): result = council.adversarial_sync("q") assert result.prompt_version == SYNTHESIS_PROMPT_VERSION + + +# --------------------------------------------------------------------------- # +# (f) ``degraded`` computed field (DSE-901): members answered, judge/synthesis +# failed. Unit-level pins on the ``CouncilResult`` model itself; the CLI's +# exit-code contract built on top of this field is pinned in ``tests/test_cli.py`` +# (``test_adversarial_judge_failure_*``, ``test_adversarial_clean_pass_still_exits_zero``). +# --------------------------------------------------------------------------- # + + +def test_degraded_false_by_default(): + """A bare result (no synthesis attempted at all) is not degraded.""" + result = CouncilResult(prompt="x") + assert result.degraded is False + + +def test_degraded_true_when_synthesis_error_set(): + """``synthesis_error`` alone (synthesize/debate/elite shape) marks degraded.""" + result = CouncilResult(prompt="x", synthesis_error="synthesizer 503 from provider") + assert result.degraded is True + + +def test_degraded_true_when_only_adversarial_verdict_error_set(): + """``adversarial.verdict_error`` alone also marks degraded (defense-in-depth). + + In the real adversarial code path ``synthesis_error`` is always mirrored + from ``adversarial.verdict_error`` (see ``conclave.modes.run_adversarial``), + so this exercises the fallback branch directly in case a future code path + ever sets one without the other. + """ + from conclave.models import AdversarialResult, ModelAnswer + + adv = AdversarialResult( + proposer="grok", + proposal=ModelAnswer(name="grok", model_id="xai/grok-4.3", answer="proposal text"), + verdict_error="judge 500 from provider", + ) + result = CouncilResult(prompt="x", adversarial=adv) + assert result.synthesis_error is None + assert result.degraded is True + + +def test_degraded_false_on_clean_synthesis(): + """A successful synthesis (no error anywhere) is not degraded.""" + result = CouncilResult(prompt="x", synthesis="the merged answer") + assert result.degraded is False + + +def test_degraded_is_a_top_level_json_key(): + """The computed field serializes as a top-level ``"degraded"`` JSON key. + + This is the exact shape a verification-gate consumer (e.g. ``/conclave-verify``) + reads: no need to reach into ``synthesis_error``/``adversarial`` -- ``degraded`` + is present and boolean directly on the JSON payload. + """ + clean = CouncilResult(prompt="x", synthesis="ok") + dumped = clean.model_dump(mode="json") + assert "degraded" in dumped + assert dumped["degraded"] is False + + failed = CouncilResult(prompt="x", synthesis_error="synthesizer 503 from provider") + assert failed.model_dump(mode="json")["degraded"] is True