diff --git a/.env.example b/.env.example index 50d77e1..6bf37d1 100644 --- a/.env.example +++ b/.env.example @@ -149,6 +149,59 @@ CLK_ROBUSTNESS_MAX_QA_DEPTH=3 CLK_ROBUSTNESS_PLATEAU_WINDOW=3 CLK_ROBUSTNESS_PLATEAU_ACTION=escalate_then_reframe +# Adversarial debate panel: instead of a single critic, spawn N critics +# with distinct lenses that try to break the work and debate each other +# across rounds before the worker revises. +# off | careful_only (default) | all +CLK_ROBUSTNESS_DEBATE=careful_only +# Comma-separated adversarial lenses (one critic per lens, run in parallel). +CLK_ROBUSTNESS_DEBATE_LENSES=correctness,security,simplicity +# Cap on debate rounds (each round = panel critique + worker revision). +CLK_ROBUSTNESS_DEBATE_MAX_ROUNDS=2 + +# ---------------------------------------------------------------------- +# Autonomous missions, done-gate, no-op guard, deliberation +# (clk run drives a full mission to a code-gated done; see README +# "Autonomous missions"). Every check has a kill switch here. +# ---------------------------------------------------------------------- +# Mission loop bounds + behavior. +CLK_MISSION_MAX_PHASES=12 +CLK_MISSION_MAX_ITERATIONS_PER_PHASE=3 +CLK_MISSION_MAX_TOTAL_CYCLES=60 +CLK_MISSION_PHASE_GATE=true +CLK_MISSION_REFINE_REQUIRED=true +CLK_MISSION_AUTO_CONSENSUS_ON_STALL=true +CLK_MISSION_CHARTER_FIRST=true +CLK_MISSION_COMMIT_TRACE=true +CLK_MISSION_COMMIT_GRANULARITY=batch +CLK_MISSION_MIN_CYCLES_BEFORE_DONE=1 +CLK_MISSION_TELEMETRY_STDOUT=true +CLK_MISSION_ON_BUDGET_EXHAUSTED=advance +# Comma-separated default lifecycle phases (used when planning can't parse). +CLK_MISSION_DEFAULT_PHASES=discovery,product,engineering,validation,deployment +# Machine-checkable done-gate: ACTION:done is only granted when these pass. +CLK_DONE_GATE_ENABLED=true +CLK_DONE_GATE_REQUIRE_TESTS_GREEN=true +CLK_DONE_GATE_REQUIRE_DELIVERABLES=true +CLK_DONE_GATE_MIN_DELIVERABLE_FILES=1 +CLK_DONE_GATE_REQUIRE_QA_PASS=true +CLK_DONE_GATE_REQUIRE_RALPH_PASS=true +CLK_DONE_GATE_FORBID_TODO_MARKERS=false +CLK_DONE_GATE_MAX_FINISH_ATTEMPTS=5 +# No-op guard: re-dispatch a producing stage that changed no files. +CLK_NOOP_GUARD_ENABLED=true +CLK_NOOP_GUARD_MAX_REDISPATCH=2 +CLK_NOOP_GUARD_PRODUCING_AGENTS=engineer,ralph +CLK_NOOP_GUARD_TREAT_OUTPUTS_STAGE_AS_PRODUCING=true +# Deliberation: self-reflection + blocking peer Q&A before a phase passes. +CLK_DELIBERATION_ENABLED=true +CLK_DELIBERATION_ENCOURAGE_QUESTIONS=true +CLK_DELIBERATION_REQUIRE_OPEN_QUESTIONS_RESOLVED=true +CLK_DELIBERATION_SELF_REFLECT_PREAMBLE=true +CLK_DELIBERATION_MIN_DEBATE_ROUNDS=1 +# Auto-derive a real validation command when a producing stage has none. +CLK_VALIDATION_AUTO_DERIVE=true + # ---------------------------------------------------------------------- # Prior-knob reference (already supported; documented here for parity) # ---------------------------------------------------------------------- diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e619b7..30a0689 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,11 +32,20 @@ jobs: pip install -e ".[api,telegram]" pip install pytest pytest-asyncio httpx - - name: Run tests + - name: Run unit tests run: pytest tests/ -v env: CLK_WORKSPACES_DIR: /tmp/clk-workspaces + - name: Run offline autonomous-mission e2e (shell provider, no network) + # Exercises the single-prompt mission, the done-gate / no-op guard, the + # charter-first ordering, and the commit trace end to end using the + # shell provider — no API keys or network access required. + run: pytest user_tests/test_mission_e2e.py -v + env: + CLK_PROVIDER: shell + CLK_WORKSPACES_DIR: /tmp/clk-workspaces + webui: name: Web UI (build + unit tests) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 44a97d2..b48410a 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,19 @@ committed automatically. If you've used CLK before, the highlights of this release: +- **Autonomous missions — one prompt to done.** `clk run` (and the TUI's first + message) now drive the whole lifecycle autonomously: the chief writes a + **charter**, authors a **living plan**, and walks discovery → … → + deployment through chief-evaluated phase gates to a **code-gated** done. + Reliability is enforced, not hoped for: a machine-checkable **done-gate** + (tests/qa/ralph/deliverables, adaptive for test-less projects) makes + `ACTION:done` a *request*; a **no-op guard** re-dispatches stages that changed + no files; evaluation **auto-derives** a real command instead of vacuously + passing; refinement is on for every producing stage by default; agents + **deliberate** (blocking Q&A + self-reflection); and every boundary leaves a + structured **git commit trace** with a per-cycle telemetry line. Use + `clk mission ""` for the explicit form or `clk run --once` for a single + cycle. See [Autonomous missions](#autonomous-missions). - **Web dashboard (`clk web`).** A beautiful browser UI that mirrors the TUI: configure every feature and `.env` setting, kick off workflows, and watch the agents work in real time with live cards, a colour-coded @@ -1247,9 +1260,12 @@ quality. Use this table to pick a regime: | Knob | Worst-case multiplier per affected dispatch | Recommended starting point | |----------------------------------------|--------------------------------------------------------------------------|----------------------------| | `robustness.auto_consensus` | `off` → ×1; `on_careful` → ×(N+1) on careful stages only; `always` → ×(N+1) on every dispatch (where N = `consensus.max_samples`, default 6) | `on_careful` (default) | -| `robustness.auto_refine` | `off` → ×1; `careful_only` → ×(1 + 1 worker revision + 1 critic) on careful stages; `all` → that on every stage | `careful_only` (default) | -| `robustness.max_quality_retries` | At most this many extra dispatches when a response fails the quality check; 0 disables | 2 (default) | -| `robustness.refine_max_rounds` | Cap on critic↔worker round-trips inside a refine loop | 3 (default) | +| `robustness.auto_refine` | `off` → ×1; `careful_only` → ×(1 + 1 worker revision + 1 critic) on careful stages; `all` → that on every producing stage | `all` (default — every producing stage refines) | +| `robustness.max_quality_retries` | At most this many extra dispatches when a response fails the quality check; 0 disables | 4 (default) | +| `robustness.refine_max_rounds` | Cap on critic↔worker round-trips inside a refine loop | 10 (default) | +| `robustness.debate` | `off` → ×1; `careful_only` → an adversarial panel (one critic per lens) on careful stages; `all` → on every producing stage | `careful_only` (default) | +| `robustness.debate_lenses` | Adversarial lenses (one parallel critic each) — more lenses = more critic dispatches per round | `[correctness, security, simplicity]` | +| `robustness.debate_max_rounds` | Cap on debate rounds (panel critique + worker revision) per stage | 2 (default) | | `robustness.max_qa_depth` | Cap on inter-agent Q&A chain depth (each peer answer can ask one peer) | 3 (default) | | `robustness.plateau_window` | How many no-improvement Ralph/autoresearch iterations before escalation | 3 (default) | | `robustness.plateau_action` | `off` disables adaptive loop termination entirely | `escalate_then_reframe` | @@ -1660,6 +1676,98 @@ blackboard) so `git log` in the kickoff dir tells the project's story without harness chatter. Deleting `.clk/` resets the harness without touching the project tree. +## Autonomous missions + +Give CLK one objective and it drives the **whole lifecycle to a code-gated +"done" on its own** — no babysitting with separate `plan` / `run` / `loop` +commands. This is the plan→execute→evaluate→refine→iterate loop made +*reliable*: the guarantees that used to live only in prompt text are now +enforced in the harness. + +**Autonomy is the default.** `clk run` (and the TUI's first message, and the +web/REST single-prompt flow) drive the full mission. `clk mission ""` +(alias `clk auto`) is the explicit form; `clk run --once` restores the legacy +single-workflow pass. + +```bash +clk mission "a local-first journaling app that summarizes my week" +# charter → plan → discovery → product → engineering(+ralph) → validation +# → deployment → done-gate → done_granted.md +``` + +### Charter first, then a living plan + +1. **Charter** — before any work, the chief authors `.clk/state/charter.md` + (+ `charter.json`): mission statement, scope, explicit non-goals, success + criteria, constraints. The plan and the done-gate are *derived from it*, so + "done" is judged against an up-front commitment instead of drifting. +2. **Living plan** — the chief emits a `PROPOSE_PLAN` block; the harness + persists an ordered phase plan to `.clk/state/mission.json` (human mirror + `MISSION.md`). After each phase a chief **phase-gate** returns + `pass | repeat | revise | done`; `revise` re-plans the remaining phases, so + the chief can reorder, insert, or skip phases as it learns. + +Three nested loops: the per-stage **supervise/ralph** loops (unchanged) inside a +per-phase **repeat** loop inside the **phase-sequencing** loop — bounded by +`mission.max_phases` and `mission.max_total_cycles`. + +### Reliability reinforcements (enforced in code, not prose) + +- **Machine-checkable done-gate.** `ACTION:done` / `done.md` is now a *request*. + The loop only stops when `done_gate.evaluate_done_gate` passes and writes + `.clk/state/done_granted.md`. Default checks: tests green, deliverables on + disk, a `POST: qa` PASS, a ralph refinement pass, plus any file-named charter + success criteria. **Adaptive:** the tests-green check is auto-relaxed when no + real test command can be derived (docs / research / content projects), so the + gate is strict where it's meaningful but never deadlocks. Each `require_*` is + an independent switch; `done_gate.enabled: false` restores legacy behavior. +- **No-op guard.** A producing stage (engineer/ralph, or any stage with an + `outputs` contract) whose response changed **no files** is re-dispatched with + an escalating "emit ACTION blocks now" preamble — descriptions stop counting + as work. Kill: `noop_guard.enabled: false`. +- **Evaluation never silently skips.** A producing stage with no `validation:` + no longer auto-passes — the harness derives a real command from the project + shape (`pytest` / `npm test` / a `compileall` smoke). Kill: + `validation.auto_derive: false`. +- **Ungated refinement.** `robustness.auto_refine` defaults to `all`, so every + producing stage gets a critic pass without the chief having to mark it + `careful`. +- **Deliberation — the team thinks.** Producing dispatches get a + self-reflect preamble and an invitation to ask peers + `POST: question TO: URGENCY: blocking`; a phase gate cannot `pass` + while a blocking question is unanswered. Kill: `deliberation.enabled: false`. +- **Execution trace.** Structured `[clk:charter|plan|phase-start|phase-gate:*|done]` + commits land at every boundary, so the git log *is* the audit trail. Kill: + `mission.commit_trace: false`. +- **Observability.** Each cycle prints a one-line summary and writes a + `loop_cycle_summary` event to `.clk/logs/activity.jsonl`, e.g. + `cycle 3/60 | phase engineering | stages 5 (4 ok) | actions 7 | refine 1r | eval FAIL(2) | done-gate REJECT(no_qa_pass)`. + +### Mission config (`clk.config.json`) + +```jsonc +"mission": { "max_phases": 12, "max_iterations_per_phase": 3, + "max_total_cycles": 60, "phase_gate": true, + "refine_required": true, "charter_first": true, + "commit_trace": true, "telemetry_stdout": true, + "default_phases": ["discovery","product","engineering","validation","deployment"] }, +"done_gate": { "enabled": true, "require_tests_green": true, + "require_deliverables": true, "min_deliverable_files": 1, + "require_qa_pass": true, "require_ralph_pass": true, + "forbid_todo_markers": false }, +"noop_guard": { "enabled": true, "max_redispatch": 2, + "producing_agents": ["engineer","ralph"] }, +"deliberation": { "enabled": true, "encourage_questions": true, + "require_open_questions_resolved": true, + "self_reflect_preamble": true, "min_debate_rounds": 1 } +``` + +Every knob above is also overridable from the environment via the matching +`CLK_MISSION_*`, `CLK_DONE_GATE_*`, `CLK_NOOP_GUARD_*`, and +`CLK_DELIBERATION_*` variables (see `.env.example`; `kickoff.sh` maps them into +`clk.config.json`). CLI overrides: `clk run --max-phases N --max-cycles M`, +`clk run --once`, `clk mission "" --resume` (continue a persisted plan). + ## Chief supervisor loop The default `engineering` workflow ends with a `supervise` stage where @@ -1992,8 +2100,8 @@ decides whether one round runs anyway: | `auto_refine` | Behavior | |----------------------------|---------------------------------------------------------| | `off` | Only stages with `refine:` use the inner loop. | -| `careful_only` *(default)* | Stages marked `careful: true` get one critic pass. | -| `all` | Every non-chief, non-qa, non-critic stage gets one pass.| +| `careful_only` | Stages marked `careful: true` get one critic pass. | +| `all` *(default)* | Every non-chief, non-qa, non-critic stage gets a critic pass — so refinement fires without relying on the chief marking stages careful. | The critic's last two lines must be: @@ -2048,6 +2156,44 @@ iteration is recorded with `improved=False`. `autoresearch_step_skipped_low_quality`, `autoresearch_revert` - Kill switch: `robustness.plateau_action: "off"` +### 11. Adversarial debate panel (new) + +Instead of a single critic, a stage can be refined by a **panel of +adversarial critics** that each take a distinct lens, try to *break* the work, +and engage with each other across rounds before the worker revises. This +catches failure modes a single reviewer misses (a correctness reviewer won't +think like a security reviewer). + +Each round fans out one critic per lens **in parallel** (reusing the +`critic` agent with a lens-specific adversarial prompt); the worker output is +kept only when a **majority of lenses accept** and the mean score clears +`refine_accept_threshold`. Otherwise the combined critiques drive a revision, +and the next round's critics see the prior panel transcript (posted to the +blackboard as `post_type: debate`) so they can reinforce, refute, or concede +each other's points. Bounded by `debate_max_rounds`. + +```yaml +- id: implement + agent: engineer + refine: + mode: debate + critics: [correctness, security, performance] + max_rounds: 2 + objective: Implement the slice. +``` + +When a stage has no explicit `refine: {mode: debate}`, `robustness.debate` +decides whether the panel runs anyway (`off` | `careful_only` *(default)* | +`all`). The debate panel takes precedence over the single-critic loop +(layer 9) when both would apply. Built-in lenses: `correctness`, `security`, +`simplicity`, `performance`, `robustness`, `tests`, `ux` (configure via +`robustness.debate_lenses`). + +- Code: `workflow.py::WorkflowRunner._debate_loop` / `_dispatch_lens_critic` +- Config: `robustness.{debate, debate_lenses, debate_max_rounds}` +- Logged events: `debate_round` +- Kill switch: `robustness.debate: "off"` AND remove any `refine: {mode: debate}` blocks. + ### Putting it together A typical "careful" engineering stage now runs: @@ -2055,7 +2201,7 @@ A typical "careful" engineering stage now runs: 1. Stage dispatched with `careful: true`. 2. `auto_consensus=on_careful` → N samples fan out in parallel. 3. Chief coalesces the samples. -4. `auto_refine=careful_only` → critic scores the coalesced output; +4. `auto_refine=all` (default) → critic scores the coalesced output; the worker is revised until critic accepts or `max_rounds`. 5. Stage validation runs. 6. Checkpoint (if enabled) — chief CONTINUE / REDIRECT / ABORT diff --git a/clk_harness/api.py b/clk_harness/api.py index b4c9f29..e4c81b6 100644 --- a/clk_harness/api.py +++ b/clk_harness/api.py @@ -72,7 +72,7 @@ def get_bind_port() -> int: return DEFAULT_PORT -COMMANDS = ["init", "idea", "plan", "run", "loop", "status"] +COMMANDS = ["init", "idea", "plan", "run", "mission", "auto", "loop", "status"] MAX_TASK_LINES = 10_000 diff --git a/clk_harness/cli.py b/clk_harness/cli.py index f2c6428..c0d09d4 100644 --- a/clk_harness/cli.py +++ b/clk_harness/cli.py @@ -4,7 +4,8 @@ init - bootstrap .clk/, configs, prompts, workflows, git repo idea - capture an idea plan - run the discovery + product workflows - run - run a single development cycle (engineering workflow by default) + run - drive the autonomous mission to a code-gated done (--once for one cycle) + mission - autonomous single-prompt mission (alias: auto): objective -> done loop - repeat the Ralph (or autoresearch) loop status - print harness status and recent activity providers - list providers and availability @@ -56,6 +57,7 @@ AgentRunner, AutoresearchLoop, Evaluator, + MissionRunner, RalphLoop, RoleProposal, WorkflowRunner, @@ -206,12 +208,19 @@ def _make_runner(paths: Paths) -> AgentRunner: def _make_evaluator(paths: Paths) -> Evaluator: cfg = load_clk_config(paths) checks = cfg.get("validation_checks") or [] - if not checks: - # Default sanity check: the project is still initialized. Users should - # override `validation_checks` in clk.config.json with a project-specific - # gate (e.g. `pytest -q` or `npm test`) once the project has real code. - checks = ["test -f .clk/config/clk.config.json"] - return Evaluator(root=paths.root, default_checks=checks) + # FM4: when the user has not configured an explicit validation gate, do NOT + # fall back to a vacuous "is the project still initialized" check (which + # always passes and makes "evaluate" a no-op). Instead let the Evaluator + # auto-derive a real command from the project shape (pytest / npm / smoke). + val_cfg = cfg.get("validation") or {} + auto_derive = bool(val_cfg.get("auto_derive", True)) + derived_command = val_cfg.get("derived_command") + return Evaluator( + root=paths.root, + default_checks=checks, + auto_derive=auto_derive, + derived_command=derived_command, + ) # --------------------------------------------------------------------------- @@ -272,6 +281,28 @@ def cmd_init(args: argparse.Namespace) -> int: return 0 +def _capture_idea(paths: Paths, statement: str, title: Optional[str] = None) -> str: + """Persist an idea (idea.json + system_brief.md). Returns the title.""" + title = title or (statement.split(".")[0][:80] if statement else "Untitled idea") + payload = { + "title": title, + "statement": statement, + "captured_at": datetime.now().isoformat(timespec="seconds"), + "tags": [], + } + save_json(paths.state / "idea.json", payload) + brief_path = paths.state / "system_brief.md" + try: + brief_path.write_text( + f"# System brief\n\n**Title:** {title}\n\n## Idea\n{statement}\n\n" + f"## Captured at\n{payload['captured_at']}\n", + encoding="utf-8", + ) + except Exception as exc: + log_exception("cli._capture_idea.brief", exc) + return title + + def cmd_idea(args: argparse.Namespace) -> int: paths = project_paths() if not _ensure_initialized(paths): @@ -401,7 +432,59 @@ def cmd_plan(args: argparse.Namespace) -> int: return 0 if overall_ok else 1 +def _apply_mission_overrides(paths: Paths, args: argparse.Namespace) -> None: + """Apply CLI --max-phases / --max-cycles overrides into the live config.""" + cfg = load_clk_config(paths) + mission_cfg = dict(cfg.get("mission") or {}) + changed = False + if getattr(args, "max_phases", None): + mission_cfg["max_phases"] = int(args.max_phases) + changed = True + if getattr(args, "max_cycles", None): + mission_cfg["max_total_cycles"] = int(args.max_cycles) + changed = True + if changed: + cfg["mission"] = mission_cfg + save_json(paths.config / "clk.config.json", cfg) + + +def _drive_mission(args: argparse.Namespace, *, log_label: str, objective: Optional[str]) -> int: + """Run the autonomous mission to a code-gated done. Shared by run/mission.""" + paths = project_paths() + if not _ensure_initialized(paths): + return 2 + init_log_file(paths.logs, log_label) + _apply_mission_overrides(paths, args) + runner = _make_runner(paths) + evaluator = _make_evaluator(paths) + mr = MissionRunner(paths, runner, evaluator) + plan = mr.run( + objective, + resume=bool(getattr(args, "resume", False)), + dry_run=bool(getattr(args, "dry_run", False)), + ) + done = sum(1 for p in plan.phases if p.status == "done") + print( + f"mission {plan.status}: {done}/{len(plan.phases)} phases done, " + f"{plan.total_cycles_used} cycles used" + ) + if plan.status == "done": + print("done-gate granted — see .clk/state/done_granted.md") + else: + last = plan.done_gate_last or {} + if last.get("failures"): + print("done-gate unmet: " + ", ".join(last.get("failures") or [])) + close_log() + # A dry run is a plan/preview, not a real completion — treat as success. + return 0 if (getattr(args, "dry_run", False) or plan.status == "done") else 1 + + def cmd_run(args: argparse.Namespace) -> int: + # Autonomy by default: `clk run` drives the full mission to a code-gated + # done. `--once` restores the legacy single-workflow-pass behavior. + if not getattr(args, "once", False): + return _drive_mission(args, log_label="run", objective=None) + paths = project_paths() if not _ensure_initialized(paths): return 2 @@ -427,6 +510,21 @@ def cmd_run(args: argparse.Namespace) -> int: return 0 if fail_count == 0 else 1 +def cmd_mission(args: argparse.Namespace) -> int: + """Autonomous single-prompt mission: one objective -> code-gated done.""" + paths = project_paths() + if not _ensure_initialized(paths): + return 2 + objective = getattr(args, "idea", None) + # Capture the idea first so resumes / context have it on disk. + if objective: + try: + _capture_idea(paths, objective) + except Exception as exc: + log_exception("cli.cmd_mission.capture_idea", exc) + return _drive_mission(args, log_label="mission", objective=objective) + + def cmd_loop(args: argparse.Namespace) -> int: paths = project_paths() if not _ensure_initialized(paths): @@ -834,11 +932,31 @@ def build_parser() -> argparse.ArgumentParser: p_plan.add_argument("--dry-run", action="store_true") p_plan.set_defaults(func=cmd_plan) - p_run = sub.add_parser("run", help="Run a single development cycle.") - p_run.add_argument("--workflow", help="Workflow name (default: engineering).") + p_run = sub.add_parser( + "run", + help="Drive the autonomous mission to a code-gated done (use --once for a single cycle).", + ) + p_run.add_argument("--workflow", help="Workflow name for --once (default: engineering).") + p_run.add_argument("--once", action="store_true", + help="Run a single workflow pass instead of the full mission (legacy).") + p_run.add_argument("--resume", action="store_true", help="Resume an existing mission plan.") + p_run.add_argument("--max-phases", type=int, help="Override mission.max_phases.") + p_run.add_argument("--max-cycles", type=int, help="Override mission.max_total_cycles.") p_run.add_argument("--dry-run", action="store_true") p_run.set_defaults(func=cmd_run) + p_mission = sub.add_parser( + "mission", + help="Autonomous single-prompt mission: one objective -> code-gated done.", + aliases=["auto"], + ) + p_mission.add_argument("idea", nargs="?", help="The mission objective (optional; uses captured idea).") + p_mission.add_argument("--resume", action="store_true", help="Resume an existing mission plan.") + p_mission.add_argument("--max-phases", type=int, help="Override mission.max_phases.") + p_mission.add_argument("--max-cycles", type=int, help="Override mission.max_total_cycles.") + p_mission.add_argument("--dry-run", action="store_true") + p_mission.set_defaults(func=cmd_mission) + p_loop = sub.add_parser("loop", help="Run a Ralph or autoresearch loop.") p_loop.add_argument("--mode", choices=["ralph", "autoresearch"], default="ralph") p_loop.add_argument("--max-iterations", type=int) diff --git a/clk_harness/config.py b/clk_harness/config.py index c21ee00..52967b0 100644 --- a/clk_harness/config.py +++ b/clk_harness/config.py @@ -220,6 +220,13 @@ def save_json(path: Path, data: Dict[str, Any], *, backup: bool = True) -> None: # Keeping work means batch commits survive, the Files tab shows the # latest state, and the supervise loop fixes problems forward. "rollback_on_failure": "careful", + # FM4 — when a producing stage declares no validation command, derive + # a real one from the project shape (pytest / npm test / compileall + # smoke) instead of vacuously passing. Set false to restore the legacy + # "no validation = pass" behavior. + "auto_derive": True, + # Explicit override for the derived command (null = auto-detect). + "derived_command": None, }, "casting": { "max_dynamic_roles": 12, @@ -230,15 +237,28 @@ def save_json(path: Path, data: Dict[str, Any], *, backup: bool = True) -> None: # off | on_careful (fan out only stages marked careful=true) | always. "auto_consensus": "on_careful", # Critic-judge inner refinement loop. - # off | careful_only (default) | all - "auto_refine": "careful_only", + # off | careful_only | all + # FM3 — default is now "all": every producing stage gets at least one + # critic pass so refinement actually fires without relying on the chief + # remembering to mark a stage careful. Set back to "careful_only" to + # reduce token cost. + "auto_refine": "all", # Cap on automatic re-dispatch attempts after a quality failure. "max_quality_retries": 4, # Below this, responses are treated as suspect and may be re-run. "min_response_chars": 40, # Critic-judge loop bounds. - "refine_max_rounds": 6, + "refine_max_rounds": 10, "refine_accept_threshold": 0.8, + # Adversarial debate panel: instead of a single critic, spawn N + # critics with distinct lenses that try to break the work and engage + # with each other's critiques across rounds, then the worker revises. + # off | careful_only (default) | all + "debate": "careful_only", + # The adversarial lenses (one critic per lens, fanned out in parallel). + "debate_lenses": ["correctness", "security", "simplicity"], + # Cap on debate rounds (each round = panel critique + worker revision). + "debate_max_rounds": 2, # Inter-agent Q&A bounds. "qa_parallel_judges": 1, "max_qa_depth": 6, @@ -249,6 +269,53 @@ def save_json(path: Path, data: Dict[str, Any], *, backup: bool = True) -> None: # escalate_then_reframe | escalate_only | reframe_only | off "plateau_action": "escalate_then_reframe", }, + # Autonomous mission driver: one objective -> full lifecycle to a + # code-gated done, no human follow-up. Wraps the per-workflow loops with a + # macro plan->execute->evaluate->refine->iterate loop. + "mission": { + "max_phases": 12, # hard cap on phase advances (incl. inserts) + "max_iterations_per_phase": 3, # outer repeat budget when a gate says "repeat" + "max_total_cycles": 60, # global cap across all phases (cost cap) + "phase_gate": True, # False -> advance on micro-loop completion + "refine_required": True, # a cycle must run >=1 refine before done-gate + "auto_consensus_on_stall": True, # fan-out consensus only after a stall + "charter_first": True, # author a charter before the plan + "commit_trace": True, # structured trace commits at boundaries + "commit_granularity": "batch", # boundary | batch | coarse + "min_cycles_before_done": 1, + "telemetry_stdout": True, # print the per-cycle summary line + "on_budget_exhausted": "advance", # advance (lenient) | fail (strict) + "default_phases": ["discovery", "product", "engineering", "validation", "deployment"], + }, + # Intra- and inter-agent deliberation (the team "thinks" before acting). + "deliberation": { + "enabled": True, + "encourage_questions": True, + "require_open_questions_resolved": True, + "self_reflect_preamble": True, + "min_debate_rounds": 1, + }, + # FM2 — machine-checkable completion gate. ACTION:done becomes a *request*; + # the loop only stops (writes done_granted.md) when every enabled check + # passes. Adaptive: tests-green is relaxed when no real test command exists. + "done_gate": { + "enabled": True, + "require_tests_green": True, + "require_deliverables": True, + "min_deliverable_files": 1, + "require_qa_pass": True, + "require_ralph_pass": True, + "forbid_todo_markers": False, + "max_finish_attempts": 5, + }, + # FM1 — no-op guard. A producing stage that changed no files is + # re-dispatched with an escalating repair preamble. + "noop_guard": { + "enabled": True, + "max_redispatch": 2, + "producing_agents": ["engineer", "ralph"], + "treat_outputs_stage_as_producing": True, + }, } DEFAULT_PROVIDERS: Dict[str, Any] = { diff --git a/clk_harness/git_ops.py b/clk_harness/git_ops.py index e9197bc..fba1c47 100644 --- a/clk_harness/git_ops.py +++ b/clk_harness/git_ops.py @@ -178,6 +178,49 @@ def commit( return False +def commit_trace( + root: Path, + *, + kind: str, + summary: str, + meta: Optional[dict] = None, + stage_all: bool = True, + allow_empty: bool = True, +) -> bool: + """Write a structured *execution-trace* commit. + + Used at mission boundaries (charter / plan / phase-start / + phase-gate: / cycle:) and per ACTION batch so the git log + itself becomes the audit trail of an autonomous run. The title is + ``[clk:] `` — a machine-greppable marker — and the body + carries the ``meta`` key/values (agent, phase, files, verdicts, tokens). + + ``allow_empty`` defaults True so a boundary marker lands even when the + cycle produced no file change, keeping the trace gap-free. Returns True + on a successful commit (or when there was genuinely nothing to do and + ``allow_empty`` is False). + """ + if not is_repo(root): + return False + if stage_all: + add_all(root) + body_lines: List[str] = [] + for key, value in (meta or {}).items(): + if value is None: + continue + if isinstance(value, (list, tuple)): + value = ", ".join(str(v) for v in value) + body_lines.append(f"{key}: {value}") + body_extra = "\n".join(body_lines) if body_lines else None + return commit( + root, + agent=f"clk:{kind}", + objective=summary, + body_extra=body_extra, + allow_empty=allow_empty, + ) + + def head_sha(root: Path) -> Optional[str]: try: r = subprocess.run( diff --git a/clk_harness/orchestration/__init__.py b/clk_harness/orchestration/__init__.py index a680877..de6bc02 100644 --- a/clk_harness/orchestration/__init__.py +++ b/clk_harness/orchestration/__init__.py @@ -4,18 +4,26 @@ from .workflow import Workflow, WorkflowRunner, is_provider_failure, load_workflow from .ralph_loop import RalphLoop from .autoresearch_loop import AutoresearchLoop -from .evaluator import Evaluator, EvalResult +from .evaluator import Evaluator, EvalResult, derive_validation from .scheduler import Scheduler +from .telemetry import CycleTelemetry +from .done_gate import DoneGateVerdict, evaluate_done_gate +from .charter import Charter, bootstrap_charter, derive_done_criteria, load_charter +from .mission import MissionPlan, MissionRunner, PhaseSpec, load_plan from .casting import ( BASELINE_AGENTS, + CharterProposal, ConsensusProposal, CastingResult, + PlanProposal, RoleProposal, WorkflowProposal, apply_response_proposals, casting_objective, is_baseline, list_roles, + parse_charter_proposal, + parse_plan_proposal, parse_role_proposals, parse_consensus_proposals, parse_workflow_proposals, @@ -36,16 +44,32 @@ "AutoresearchLoop", "Evaluator", "EvalResult", + "derive_validation", "Scheduler", + "CycleTelemetry", + "DoneGateVerdict", + "evaluate_done_gate", + "Charter", + "bootstrap_charter", + "derive_done_criteria", + "load_charter", + "MissionPlan", + "MissionRunner", + "PhaseSpec", + "load_plan", "BASELINE_AGENTS", + "CharterProposal", "ConsensusProposal", "CastingResult", + "PlanProposal", "RoleProposal", "WorkflowProposal", "apply_response_proposals", "casting_objective", "is_baseline", "list_roles", + "parse_charter_proposal", + "parse_plan_proposal", "parse_role_proposals", "parse_consensus_proposals", "parse_workflow_proposals", diff --git a/clk_harness/orchestration/agent.py b/clk_harness/orchestration/agent.py index 9dcd5b0..de8c31c 100644 --- a/clk_harness/orchestration/agent.py +++ b/clk_harness/orchestration/agent.py @@ -28,6 +28,7 @@ from . import actions as _actions from . import blackboard as _blackboard from . import response_quality as _response_quality +from . import noop_guard as _noop_guard def _read_recent_casting_rejections(paths: Paths, *, limit: int = 8) -> str: @@ -95,6 +96,11 @@ class AgentRun: started_at: str finished_at: str files_written: List[str] = field(default_factory=list) + # Number of file-mutating ACTIONs (write/edit/append/delete) the harness + # actually applied for this run. Used by the no-op guard: a producing + # stage that applied zero mutations is re-dispatched with an escalating + # repair preamble (descriptions alone do nothing). + file_mutations_applied: int = 0 # True when the runner already created a git commit for this run's # actions; downstream consumers (WorkflowRunner._commit) skip in # that case to avoid double-committing the same diff. @@ -228,6 +234,11 @@ def render_prompt(self, agent: AgentSpec, objective: str, extra: Optional[Dict[s "qa_answer", "refine_critic", "refine_worker", + # Mission-level chief dispatches: single-shot planning / gating that + # must not recurse into consensus or the quality-retry loop. + "charter", + "mission_plan", + "phase_gate", }) def run( @@ -298,6 +309,19 @@ def _dispatch_with_quality_loop( min_chars = int(cfg.get("min_response_chars") or 40) auto_consensus_mode = str(cfg.get("auto_consensus") or "off").lower() expected_outputs = list(extra.get("stage_outputs") or []) + tel = extra.get("telemetry") + + # FM1 no-op guard: a producing stage that applies zero file mutations + # is re-dispatched with an escalating repair preamble (descriptions + # alone do nothing). Independent budget from the quality retries. + expect_mutation = _noop_guard.is_mutation_expected( + agent_name, + outputs=expected_outputs, + commit=bool(extra.get("commit")), + cfg=self.clk_cfg, + ) + max_noop = _noop_guard.max_redispatch(self.clk_cfg) + noop_redispatches = 0 attempt = 0 current_objective = objective @@ -312,6 +336,40 @@ def _dispatch_with_quality_loop( last_run = run if not run.response.ok: return run + # No-op check: a producing stage that returned substantive prose + # but changed no files. (An empty/near-empty response is left to + # the quality "empty" flag below — that is a different failure.) + substantive = len((run.response.text or "").strip()) >= min_chars + if ( + expect_mutation + and substantive + and run.file_mutations_applied == 0 + and noop_redispatches < max_noop + ): + noop_redispatches += 1 + if tel is not None: + try: + tel.add_noop_redispatch() + except Exception: + pass + log_event( + self.paths, + "agent_noop_redispatch", + agent=agent_name, + attempt=noop_redispatches, + max_attempts=max_noop, + stage_id=extra.get("stage_id"), + workflow=extra.get("workflow"), + ) + self._observer_log( + f"noop :: {agent_name} :: changed no files; " + f"re-dispatch {noop_redispatches}/{max_noop}" + ) + preamble = _noop_guard.repair_preamble( + noop_redispatches, target=str(extra.get("expected_path") or "") + ) + current_objective = preamble + "\n\nOriginal objective:\n" + objective + continue try: q = _response_quality.score( run.response.text, @@ -337,6 +395,11 @@ def _dispatch_with_quality_loop( needs_review=q.needs_review, ) return run + if tel is not None: + try: + tel.add_quality_retry() + except Exception: + pass log_event( self.paths, "agent_quality_retry", @@ -403,6 +466,12 @@ def _dispatch_auto_consensus( cfg = self.clk_cfg.get("consensus") or {} sample_count = max(1, min(int(cfg.get("max_samples") or 3), 6)) max_parallel = max(1, int(cfg.get("max_parallel") or 4)) + tel = extra.get("telemetry") + if tel is not None: + try: + tel.add_consensus_run() + except Exception: + pass name = f"auto_{agent_name}_{datetime.now().strftime('%H%M%S%f')}" log_event( self.paths, @@ -704,7 +773,7 @@ def _on_progress(kind: str, message: str) -> None: # changes. We merge the harness-applied files into the run's # files_written list so the TUI / commit logic see them. if not is_dry: - self._apply_actions(run) + self._apply_actions(run, extra or {}) if self.observer is not None: try: self.observer.end(agent.name, run) @@ -1211,6 +1280,7 @@ def _route_blocking_questions( return if str(extra.get("phase") or "") in self._META_PHASES: return + tel = extra.get("telemetry") cfg = self.clk_cfg.get("robustness") or {} max_depth = int(cfg.get("max_qa_depth") or 3) chain: List[str] = list(extra.get("qa_chain") or []) @@ -1271,6 +1341,11 @@ def _route_blocking_questions( self._observer_log( f"qa :: {run.agent} → {target} :: {q.id[:32]}" ) + if tel is not None: + try: + tel.add_qa_exchange() + except Exception: + pass self._dispatch_once( target, answer_objective, @@ -1285,8 +1360,10 @@ def _route_blocking_questions( dry_run=self.clk_cfg.get("dry_run", False), ) - def _apply_actions(self, run: AgentRun) -> None: + def _apply_actions(self, run: AgentRun, extra: Optional[Dict[str, Any]] = None) -> None: """Execute ACTION blocks; merge harness-written files back into the run.""" + extra = extra or {} + tel = extra.get("telemetry") text = (run.response.text or "") if not text or "ACTION:" not in text and "ACTION :" not in text: return @@ -1296,6 +1373,16 @@ def _apply_actions(self, run: AgentRun) -> None: agent_name=run.agent, clk_cfg=self.clk_cfg, ) + # Record how many file mutations actually landed (drives the no-op + # guard even when ACTION blocks were present but all skipped). + run.file_mutations_applied = len(result.files_written) + len(result.files_deleted) + if tel is not None: + try: + tel.add_actions(len(result.files_written) + len(result.files_deleted) + + len(result.commands_run)) + tel.add_files(len(result.files_written)) + except Exception: + pass if result.is_empty(): return # Merge into run.files_written so downstream consumers (TUI, @@ -1323,9 +1410,14 @@ def _apply_actions(self, run: AgentRun) -> None: # has a per-agent-run granularity. Only fires when this run # actually wrote (or deleted) files. if (result.files_written or result.files_deleted) and self.clk_cfg.get("auto_commit", True): - self._commit_action_batch(run, result) + self._commit_action_batch(run, result, extra) - def _commit_action_batch(self, run: AgentRun, result: "_actions.ActionResult") -> None: + def _commit_action_batch( + self, + run: AgentRun, + result: "_actions.ActionResult", + extra: Optional[Dict[str, Any]] = None, + ) -> None: try: if not is_repo(self.paths.root): return @@ -1363,6 +1455,12 @@ def _commit_action_batch(self, run: AgentRun, result: "_actions.ActionResult") - ) if committed: run.committed = True + tel = (extra or {}).get("telemetry") + if tel is not None: + try: + tel.add_commit() + except Exception: + pass log( f"commit: [{run.agent}] {len(result.files_written)} files, " f"{len(result.files_deleted)} deletes" @@ -1564,7 +1662,13 @@ def _collect_context(self, objective: str, extra: Dict[str, Any]) -> Dict[str, A "notes": notes, "outputs_contract": outputs_contract, } - ctx.update({k: v for k, v in extra.items() if k not in ctx}) + # ``telemetry`` is a live counter object threaded through ``extra`` for + # the dispatch hooks — it is not a template variable, so keep it out of + # the prompt context (otherwise it would be str()'d into the prompt). + ctx.update({ + k: v for k, v in extra.items() + if k not in ctx and k != "telemetry" + }) return ctx def _safe_substitute(self, template_text: str, ctx: Dict[str, Any]) -> str: diff --git a/clk_harness/orchestration/casting.py b/clk_harness/orchestration/casting.py index f911b8c..9d67f8f 100644 --- a/clk_harness/orchestration/casting.py +++ b/clk_harness/orchestration/casting.py @@ -117,6 +117,27 @@ class ConsensusProposal: objective: str = "" +@dataclass +class CharterProposal: + """A chief-authored mission charter (the up-front commitment). + + The plan and the done-gate are derived from this, so "done" is judged + against what the chief committed to rather than drifting. + """ + mission: str = "" + scope: List[str] = field(default_factory=list) + non_goals: List[str] = field(default_factory=list) + success: List[str] = field(default_factory=list) + constraints: List[str] = field(default_factory=list) + assumptions: List[str] = field(default_factory=list) + + +@dataclass +class PlanProposal: + """An ordered list of lifecycle phases for a mission.""" + phases: List[Dict[str, Any]] = field(default_factory=list) + + _ROLE_HEAD_RE = re.compile(r"^\s*PROPOSE_ROLE\s*:\s*([A-Za-z][A-Za-z0-9_\-]*)\s*$", re.MULTILINE) _ROLE_FIELD_RE = re.compile(r"^(ROLE|PROVIDER|CAPABILITIES)\s*:\s*(.*)$", re.IGNORECASE) _ROLE_PROMPT_RE = re.compile(r"^\s*PROMPT\s*:\s*$", re.IGNORECASE) @@ -132,6 +153,153 @@ class ConsensusProposal: _CONS_OBJECTIVE_RE = re.compile(r"^\s*OBJECTIVE\s*:\s*$", re.IGNORECASE) _CONS_END_RE = re.compile(r"^\s*END_CONSENSUS\s*$", re.IGNORECASE) +_CHARTER_HEAD_RE = re.compile(r"^\s*PROPOSE_CHARTER\s*:?\s*$", re.IGNORECASE) +_CHARTER_END_RE = re.compile(r"^\s*END_CHARTER\s*$", re.IGNORECASE) +_CHARTER_FIELD_RE = re.compile( + r"^(MISSION|SCOPE|NON_GOALS|NONGOALS|SUCCESS|SUCCESS_CRITERIA|CONSTRAINTS|ASSUMPTIONS)\s*:\s*(.*)$", + re.IGNORECASE, +) + +_PLAN_HEAD_RE = re.compile(r"^\s*PROPOSE_PLAN\s*:?\s*$", re.IGNORECASE) +_PLAN_PHASES_RE = re.compile(r"^\s*PHASES\s*:\s*$", re.IGNORECASE) +_PLAN_END_RE = re.compile(r"^\s*END_PLAN\s*$", re.IGNORECASE) + + +def _split_items(value: str) -> List[str]: + """Split a charter field value into items. + + Items may be separated by ``;`` (preferred, since success criteria can + contain commas) or, when no semicolon is present, by commas. A leading + ``[`` / trailing ``]`` is tolerated. + """ + v = (value or "").strip() + if v.startswith("[") and v.endswith("]"): + v = v[1:-1] + if not v: + return [] + sep = ";" if ";" in v else "," + return [item.strip().strip("-").strip() for item in v.split(sep) if item.strip().strip("-").strip()] + + +def parse_charter_proposal(text: str) -> Optional[CharterProposal]: + """Extract a single ``PROPOSE_CHARTER`` block from ``text`` (or None).""" + if not text or "PROPOSE_CHARTER" not in text.upper(): + return None + lines = text.splitlines() + i = 0 + while i < len(lines): + if not _CHARTER_HEAD_RE.match(lines[i]): + i += 1 + continue + prop = CharterProposal() + i += 1 + while i < len(lines): + line = lines[i] + if _CHARTER_END_RE.match(line): + i += 1 + break + fm = _CHARTER_FIELD_RE.match(line) + if fm: + key = fm.group(1).upper() + val = fm.group(2).strip() + if key == "MISSION": + prop.mission = val + elif key == "SCOPE": + prop.scope = _split_items(val) + elif key in ("NON_GOALS", "NONGOALS"): + prop.non_goals = _split_items(val) + elif key in ("SUCCESS", "SUCCESS_CRITERIA"): + prop.success = _split_items(val) + elif key == "CONSTRAINTS": + prop.constraints = _split_items(val) + elif key == "ASSUMPTIONS": + prop.assumptions = _split_items(val) + i += 1 + if prop.mission or prop.success or prop.scope: + return prop + return None + + +def _parse_simple_phase_list(body: str) -> List[Dict[str, Any]]: + """Dependency-free parser for the PHASES YAML list. + + Handles the documented shape: a list of ``- key: value`` blocks where + values are scalars or ``[a, b]`` inline lists. Used as a fallback when + PyYAML is unavailable. + """ + phases: List[Dict[str, Any]] = [] + current: Optional[Dict[str, Any]] = None + for raw in body.splitlines(): + line = raw.rstrip() + if not line.strip(): + continue + stripped = line.lstrip() + is_item = stripped.startswith("- ") + if is_item: + stripped = stripped[2:].lstrip() + current = {} + phases.append(current) + if current is None: + current = {} + phases.append(current) + if ":" not in stripped: + continue + key, _, value = stripped.partition(":") + key = key.strip() + value = value.strip() + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1] + current[key] = [x.strip().strip('"\'') for x in inner.split(",") if x.strip()] + elif value: + current[key] = value.strip('"\'') + else: + current[key] = "" + return [p for p in phases if p.get("id")] + + +def parse_plan_proposal(text: str) -> Optional[PlanProposal]: + """Extract a single ``PROPOSE_PLAN`` block (phases list) from ``text``.""" + if not text or "PROPOSE_PLAN" not in text.upper(): + return None + lines = text.splitlines() + i = 0 + while i < len(lines): + if not _PLAN_HEAD_RE.match(lines[i]): + i += 1 + continue + i += 1 + # skip to PHASES: + body_lines: List[str] = [] + seen_phases = False + while i < len(lines): + line = lines[i] + if _PLAN_END_RE.match(line): + i += 1 + break + if not seen_phases: + if _PLAN_PHASES_RE.match(line): + seen_phases = True + i += 1 + continue + body_lines.append(line) + i += 1 + body = "\n".join(body_lines).strip("\n") + if not body: + continue + phases: List[Dict[str, Any]] = [] + try: + import yaml as _yaml # type: ignore + parsed = _yaml.safe_load(body) + if isinstance(parsed, list): + phases = [p for p in parsed if isinstance(p, dict) and p.get("id")] + except Exception: + phases = [] + if not phases: + phases = _parse_simple_phase_list(body) + if phases: + return PlanProposal(phases=phases) + return None + def parse_role_proposals(text: str) -> List[RoleProposal]: """Extract every PROPOSE_ROLE block from ``text``. diff --git a/clk_harness/orchestration/charter.py b/clk_harness/orchestration/charter.py new file mode 100644 index 0000000..c54b7f5 --- /dev/null +++ b/clk_harness/orchestration/charter.py @@ -0,0 +1,232 @@ +"""Mission charter — the chief's up-front commitment, authored before the plan. + +A mission starts by asking the chief to write a *charter*: the mission +statement, scope, explicit non-goals, success criteria, and constraints. The +living plan and the done-gate are derived from it, so "done" is measured against +something the chief committed to at the outset rather than drifting. + +Persisted as ``.clk/state/charter.json`` (machine) + ``.clk/state/CHARTER.md`` +(human mirror), and trace-committed so the git log records the commitment. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Dict, List, Optional + +from ..config import Paths, save_json, load_json +from ..git_ops import commit_trace +from ..utils.activity_log import log_event +from ..utils.logging_utils import log, log_exception +from . import blackboard as _blackboard +from . import casting as _casting + + +# Filename tokens that make a success criterion machine-checkable as "exists". +_FILE_EXT_RE = re.compile( + r"\b([\w./-]+\.(?:py|md|json|txt|ya?ml|js|ts|tsx|html|css|sh|toml|cfg|ini|rs|go|java|rb))\b" +) + + +@dataclass +class Charter: + objective: str = "" + mission_statement: str = "" + scope: List[str] = field(default_factory=list) + non_goals: List[str] = field(default_factory=list) + success_criteria: List[str] = field(default_factory=list) + constraints: List[str] = field(default_factory=list) + assumptions: List[str] = field(default_factory=list) + created_at: str = "" + + def to_dict(self) -> Dict[str, Any]: + return { + "version": 1, + "objective": self.objective, + "mission_statement": self.mission_statement, + "scope": list(self.scope), + "non_goals": list(self.non_goals), + "success_criteria": list(self.success_criteria), + "constraints": list(self.constraints), + "assumptions": list(self.assumptions), + "created_at": self.created_at, + } + + @classmethod + def from_dict(cls, raw: Dict[str, Any]) -> "Charter": + raw = raw or {} + return cls( + objective=str(raw.get("objective") or ""), + mission_statement=str(raw.get("mission_statement") or ""), + scope=[str(x) for x in (raw.get("scope") or [])], + non_goals=[str(x) for x in (raw.get("non_goals") or [])], + success_criteria=[str(x) for x in (raw.get("success_criteria") or [])], + constraints=[str(x) for x in (raw.get("constraints") or [])], + assumptions=[str(x) for x in (raw.get("assumptions") or [])], + created_at=str(raw.get("created_at") or ""), + ) + + def render_md(self) -> str: + def _bullets(items: List[str]) -> str: + return "\n".join(f"- {i}" for i in items) or "- (none)" + return ( + "# Mission Charter\n\n" + f"**Objective:** {self.objective or '(none)'}\n\n" + f"**Mission:** {self.mission_statement or '(none)'}\n\n" + f"## Scope\n{_bullets(self.scope)}\n\n" + f"## Non-goals\n{_bullets(self.non_goals)}\n\n" + f"## Success criteria\n{_bullets(self.success_criteria)}\n\n" + f"## Constraints\n{_bullets(self.constraints)}\n\n" + f"## Assumptions\n{_bullets(self.assumptions)}\n\n" + f"_Authored {self.created_at}_\n" + ) + + +def charter_json_path(paths: Paths): + return paths.state / "charter.json" + + +def charter_md_path(paths: Paths): + return paths.state / "CHARTER.md" + + +def load_charter(paths: Paths) -> Optional[Charter]: + p = charter_json_path(paths) + if not p.exists(): + return None + raw = load_json(p, {}) + if not raw: + return None + return Charter.from_dict(raw) + + +def save_charter(paths: Paths, charter: Charter) -> None: + paths.state.mkdir(parents=True, exist_ok=True) + save_json(charter_json_path(paths), charter.to_dict()) + try: + charter_md_path(paths).write_text(charter.render_md(), encoding="utf-8") + except Exception as exc: + log_exception("orchestration.charter.save_charter.md", exc) + + +def derive_done_criteria(charter: Optional[Charter]) -> List[Dict[str, Any]]: + """Turn the charter's success criteria into machine-checkable checks. + + Conservative: only file-exists criteria are auto-extracted (running + arbitrary commands parsed from prose would be unsafe). A criterion that + names a file (``report.md``, ``src/app.py``) becomes a ``file`` check; + everything else is left for the chief's phase-gate judgment. + """ + out: List[Dict[str, Any]] = [] + if charter is None: + return out + for crit in charter.success_criteria: + m = _FILE_EXT_RE.search(crit or "") + if m: + out.append({"label": crit.strip()[:80], "type": "file", "value": m.group(1)}) + return out + + +def charter_objective(objective: str) -> str: + return ( + "MISSION CHARTER MODE. You are kicking off an autonomous mission with no " + "further human input. Before any work begins, author the charter that the " + "whole mission will be judged against.\n\n" + f"Mission objective:\n{objective}\n\n" + "Emit exactly one PROPOSE_CHARTER block (machine-parsed):\n\n" + " PROPOSE_CHARTER\n" + " MISSION: \n" + " SCOPE: \n" + " NON_GOALS: \n" + " SUCCESS: \n" + " CONSTRAINTS: \n" + " ASSUMPTIONS: \n" + " END_CHARTER\n\n" + "Make SUCCESS criteria concrete and verifiable. Where a criterion implies a " + "file, name the file (e.g. 'README.md documents setup', 'tests/ pass') — the " + "harness enforces file-named criteria mechanically. Keep items short; " + "separate them with semicolons." + ) + + +def _fallback_charter(objective: str) -> Charter: + return Charter( + objective=objective, + mission_statement=objective, + scope=["Deliver a working solution for the objective"], + non_goals=[], + success_criteria=[ + "A runnable deliverable exists", + "Tests or a smoke check pass", + "README documents setup", + ], + constraints=[], + assumptions=[], + created_at=datetime.now().isoformat(timespec="seconds"), + ) + + +def bootstrap_charter(paths: Paths, runner, objective: str, *, dry_run: bool = False) -> Charter: + """Dispatch the chief in CHARTER mode, persist + commit the result. + + Falls back to a sensible default charter when parsing fails or on a + dry run. + """ + charter: Optional[Charter] = None + if not dry_run: + try: + run = runner.run( + "chief", + charter_objective(objective), + extra={"phase": "charter"}, + dry_run=dry_run, + ) + prop = _casting.parse_charter_proposal(run.response.text or "") + if prop is not None: + charter = Charter( + objective=objective, + mission_statement=prop.mission, + scope=prop.scope, + non_goals=prop.non_goals, + success_criteria=prop.success, + constraints=prop.constraints, + assumptions=prop.assumptions, + created_at=datetime.now().isoformat(timespec="seconds"), + ) + except Exception as exc: + log_exception("orchestration.charter.bootstrap_charter", exc) + if charter is None: + charter = _fallback_charter(objective) + + save_charter(paths, charter) + log_event( + paths, + "mission_charter", + objective=objective, + mission=charter.mission_statement, + success_count=len(charter.success_criteria), + scope_count=len(charter.scope), + ) + try: + _blackboard.post( + paths, + author="mission", + body=charter.render_md(), + post_type="charter", + produces=["mission_charter"], + slug_hint="charter", + ) + except Exception as exc: + log_exception("orchestration.charter.bootstrap_charter.post", exc) + try: + commit_trace( + paths.root, + kind="charter", + summary=(charter.mission_statement or objective)[:80], + meta={"success_criteria": len(charter.success_criteria), "scope": len(charter.scope)}, + ) + except Exception as exc: + log_exception("orchestration.charter.bootstrap_charter.commit", exc) + return charter diff --git a/clk_harness/orchestration/deliberation.py b/clk_harness/orchestration/deliberation.py new file mode 100644 index 0000000..875f4a9 --- /dev/null +++ b/clk_harness/orchestration/deliberation.py @@ -0,0 +1,82 @@ +"""Intra- and inter-agent deliberation — "the team thinks". + +Two cheap mechanisms already exist in the harness: + +* inter-agent: directed blackboard Q&A (``POST: question TO: + URGENCY: blocking``), routed inline by the AgentRunner. +* intra-agent / debate: the critic<->worker refine loop in the WorkflowRunner. + +This module makes them first-class and default-on within missions: + +* a self-reflection preamble injected into producing dispatches ("restate the + goal, list approaches, pick one, then act"); +* an invitation to raise blocking questions when stuck; +* a phase-gate guard: a phase cannot ``pass`` while a blocking question for it + is still unanswered. + +Everything is config-gated under ``deliberation.*`` with a single kill switch. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from ..config import Paths +from . import blackboard as _blackboard + + +def _cfg(clk_cfg: Optional[Dict[str, Any]]) -> Dict[str, Any]: + return (clk_cfg or {}).get("deliberation") or {} + + +def enabled(clk_cfg: Optional[Dict[str, Any]]) -> bool: + return bool(_cfg(clk_cfg).get("enabled", True)) + + +def min_debate_rounds(clk_cfg: Optional[Dict[str, Any]]) -> int: + if not enabled(clk_cfg): + return 0 + return int(_cfg(clk_cfg).get("min_debate_rounds", 1) or 0) + + +def require_open_questions_resolved(clk_cfg: Optional[Dict[str, Any]]) -> bool: + return bool(enabled(clk_cfg) and _cfg(clk_cfg).get("require_open_questions_resolved", True)) + + +_SELF_REFLECT = ( + "Think before you act: (1) restate the objective in one line; (2) list 2-3 " + "viable approaches; (3) pick one and say why in a sentence; (4) THEN emit the " + "ACTION blocks that implement it. Do not skip straight to prose." +) + +_ENCOURAGE_QUESTIONS = ( + "If a decision is genuinely blocked on information another agent owns, ask it " + "directly: emit `POST: question TO: URGENCY: blocking` with a focused " + "question. The harness routes it and feeds you the answer. Do not guess on " + "load-bearing unknowns; do not invent questions you can answer yourself." +) + + +def dispatch_preamble(clk_cfg: Optional[Dict[str, Any]]) -> str: + """Preamble injected ahead of producing dispatches in a mission.""" + if not enabled(clk_cfg): + return "" + parts: List[str] = [] + c = _cfg(clk_cfg) + if c.get("self_reflect_preamble", True): + parts.append(_SELF_REFLECT) + if c.get("encourage_questions", True): + parts.append(_ENCOURAGE_QUESTIONS) + return ("\n\n".join(parts) + "\n\n") if parts else "" + + +def unresolved_blocking_questions(paths: Paths) -> List["_blackboard.Post"]: + """Blocking questions on the board that have no matching answer yet.""" + try: + unanswered = _blackboard.find_unanswered_questions(paths) + except Exception: + return [] + return [ + q for q in unanswered + if (q.urgency or "blocking").lower() == "blocking" + ] diff --git a/clk_harness/orchestration/done_gate.py b/clk_harness/orchestration/done_gate.py new file mode 100644 index 0000000..807fd8a --- /dev/null +++ b/clk_harness/orchestration/done_gate.py @@ -0,0 +1,263 @@ +"""Machine-checkable completion gate (the enforced "high bar to stop"). + +Today CLK's done-checklist lives only in the chief's prose prompt: the harness +writes ``done.md`` the instant any agent emits ``ACTION:done`` and never verifies +anything. That is why autonomous runs stop prematurely. This module turns the +checklist into code: ``ACTION:done`` / ``done.md`` become a *request*, and the +mission/supervise loop only grants it (writing ``done_granted.md``) when +:func:`evaluate_done_gate` reports every enabled criterion satisfied. + +All signals are things the system already produces: + +* tests green -> from the cycle's :class:`~clk_harness.orchestration.evaluator.EvalResult` +* deliverables -> real files exist under the project root +* qa pass -> a ``POST: qa`` blackboard entry whose body says PASS +* ralph pass -> ``.clk/state/experiments.jsonl`` has an entry (a refinement ran) +* no TODOs -> changed files contain no TODO/FIXME/placeholder markers (opt-in) +* charter -> the chief's own machine-checkable success criteria + +The gate is *adaptive*: when the evaluator reports a ``weak`` gate (no real test +command could be derived — docs / research / content projects), the tests-green +requirement is relaxed so the mission can still finish. +""" + +from __future__ import annotations + +import re +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..config import Paths +from ..utils.logging_utils import log_exception +from . import blackboard as _blackboard + + +_EXCLUDE_TOP = {".clk", ".git", "node_modules", "__pycache__", ".pytest_cache", "venv", ".venv"} +_STATEISH_ROOT_FILES = {"PROGRESS.md", "DECISIONS.md", "MISSION.md", "CHARTER.md"} +_TODO_RE = re.compile(r"\b(TODO|FIXME|XXX|HACK|PLACEHOLDER)\b", re.IGNORECASE) +_QA_PASS_RE = re.compile(r"\bPASS(ED)?\b", re.IGNORECASE) +_QA_FAIL_RE = re.compile(r"\bFAIL(ED|URE)?\b", re.IGNORECASE) + + +@dataclass +class DoneGateVerdict: + passed: bool = False + failures: List[str] = field(default_factory=list) + checked: Dict[str, Any] = field(default_factory=dict) + reasons: List[str] = field(default_factory=list) + + def summary(self) -> str: + if self.passed: + return "done-gate PASS — every enabled completion criterion satisfied" + return "done-gate REJECT — unmet: " + (", ".join(self.failures) or "?") + + def repair_hint(self) -> str: + """Preamble quoting unmet criteria back to the chief on re-dispatch.""" + if self.passed or not self.reasons: + return "" + bullets = "\n".join(f"- {r}" for r in self.reasons) + return ( + "The harness REJECTED your request to finish (ACTION:done). The " + "following completion criteria are not yet satisfied:\n" + f"{bullets}\n" + "Do NOT emit ACTION:done again until these pass. Keep working: " + "dispatch the work that closes each gap." + ) + + +# --- individual signal helpers -------------------------------------------- + + +def deliverable_files(root: Path, *, limit: int = 5000) -> List[str]: + """Real product files under ``root`` (excludes harness/state/vcs noise).""" + root = Path(root) + out: List[str] = [] + try: + for p in root.rglob("*"): + if not p.is_file(): + continue + try: + rel = p.relative_to(root) + except ValueError: + continue + parts = rel.parts + if parts and parts[0] in _EXCLUDE_TOP: + continue + if len(parts) == 1 and parts[0] in _STATEISH_ROOT_FILES: + continue + out.append(str(rel)) + if len(out) >= limit: + break + except Exception as exc: + log_exception("orchestration.done_gate.deliverable_files", exc) + return out + + +def has_qa_pass(paths: Paths) -> bool: + """True when the most recent ``POST: qa`` entry reads as a PASS.""" + try: + qa_posts = [p for p in _blackboard.list_posts(paths) if (p.post_type or "").lower() == "qa"] + except Exception as exc: + log_exception("orchestration.done_gate.has_qa_pass", exc) + return False + if not qa_posts: + return False + body = qa_posts[-1].body or "" + return bool(_QA_PASS_RE.search(body)) and not _QA_FAIL_RE.search(body) + + +def has_ralph_pass(paths: Paths) -> bool: + """True when at least one refinement iteration has been recorded.""" + exp = paths.state / "experiments.jsonl" + try: + if exp.exists() and exp.stat().st_size > 0: + return any(line.strip() for line in exp.read_text(encoding="utf-8").splitlines()) + except Exception as exc: + log_exception("orchestration.done_gate.has_ralph_pass.exp", exc) + # Fallback: a ralph dispatch in the activity log. + activity = paths.logs / "activity.jsonl" + try: + if activity.exists(): + for line in activity.read_text(encoding="utf-8").splitlines(): + if '"ralph"' in line and "dispatch" in line: + return True + except Exception as exc: + log_exception("orchestration.done_gate.has_ralph_pass.activity", exc) + return False + + +def todo_markers(root: Path, *, max_files: int = 2000) -> List[str]: + """Deliverable files that still contain TODO/FIXME/placeholder markers.""" + hits: List[str] = [] + for rel in deliverable_files(root, limit=max_files): + fp = Path(root) / rel + try: + if fp.suffix.lower() in {".png", ".jpg", ".jpeg", ".gif", ".pdf", ".zip", ".bin"}: + continue + text = fp.read_text(encoding="utf-8", errors="ignore") + except Exception: + continue + if _TODO_RE.search(text): + hits.append(rel) + return hits + + +def _check_extra_criterion(root: Path, crit: Dict[str, Any]) -> bool: + """Evaluate one charter-derived machine-checkable success criterion.""" + ctype = (crit.get("type") or "").lower() + value = crit.get("value") or "" + try: + if ctype == "file": + return (Path(root) / value).exists() + if ctype == "command" and value: + r = subprocess.run( + value, shell=True, cwd=str(root), + capture_output=True, text=True, timeout=180, + ) + return r.returncode == 0 + except Exception as exc: + log_exception("orchestration.done_gate._check_extra_criterion", exc) + return False + + +# --- top-level gate -------------------------------------------------------- + + +def evaluate_done_gate( + paths: Paths, + clk_cfg: Optional[Dict[str, Any]], + eval_result: Any = None, + *, + extra_criteria: Optional[List[Dict[str, Any]]] = None, +) -> DoneGateVerdict: + """Return a verdict on whether the mission is allowed to stop. + + ``extra_criteria`` are charter-derived machine-checkable checks, each a + dict ``{"label":..., "type":"file"|"command", "value":...}``. + """ + cfg = (clk_cfg or {}).get("done_gate") or {} + verdict = DoneGateVerdict() + + if not cfg.get("enabled", True): + verdict.passed = True + verdict.reasons.append("done-gate disabled (done_gate.enabled=false)") + return verdict + + failures: List[str] = [] + reasons: List[str] = [] + checked: Dict[str, Any] = {} + + # 1. tests green (adaptive: relax on a weak gate) + if cfg.get("require_tests_green", True): + weak = bool(getattr(eval_result, "weak", False)) + ok = bool(getattr(eval_result, "ok", False)) + if eval_result is None: + checked["tests_green"] = "skipped (no eval result)" + elif weak: + checked["tests_green"] = "relaxed (weak gate — no real test command)" + elif not ok: + failures.append("tests_red") + reasons.append("Tests are not green. Make the validation command exit 0.") + checked["tests_green"] = False + else: + checked["tests_green"] = True + + # 2. deliverables exist + if cfg.get("require_deliverables", True): + min_n = int(cfg.get("min_deliverable_files", 1) or 1) + files = deliverable_files(paths.root) + checked["deliverable_files"] = len(files) + if len(files) < min_n: + failures.append("no_deliverables") + reasons.append( + f"Only {len(files)} product file(s) on disk (need >= {min_n}). " + "Emit ACTION:write blocks that create real deliverables." + ) + + # 3. qa pass on the blackboard + if cfg.get("require_qa_pass", True): + ok = has_qa_pass(paths) + checked["qa_pass"] = ok + if not ok: + failures.append("no_qa_pass") + reasons.append( + "No QA PASS on the blackboard. Dispatch qa and have it emit a " + "`POST: qa` block whose body states PASS." + ) + + # 4. a ralph refinement pass occurred + if cfg.get("require_ralph_pass", True): + ok = has_ralph_pass(paths) + checked["ralph_pass"] = ok + if not ok: + failures.append("no_ralph_pass") + reasons.append( + "No refinement pass recorded. Run at least one ralph refine " + "iteration before finishing." + ) + + # 5. no TODO / placeholder markers (opt-in) + if cfg.get("forbid_todo_markers", False): + hits = todo_markers(paths.root) + checked["todo_markers"] = len(hits) + if hits: + failures.append("todo_markers") + sample = ", ".join(hits[:5]) + reasons.append(f"TODO/placeholder markers remain in: {sample}.") + + # 6. charter-derived machine-checkable success criteria + for crit in (extra_criteria or []): + label = crit.get("label") or crit.get("value") or "charter criterion" + ok = _check_extra_criterion(paths.root, crit) + checked[f"charter:{label}"] = ok + if not ok: + failures.append(f"charter:{label}") + reasons.append(f"Charter success criterion not met: {label}.") + + verdict.failures = failures + verdict.reasons = reasons + verdict.checked = checked + verdict.passed = not failures + return verdict diff --git a/clk_harness/orchestration/evaluator.py b/clk_harness/orchestration/evaluator.py index 19ff57b..2a216bf 100644 --- a/clk_harness/orchestration/evaluator.py +++ b/clk_harness/orchestration/evaluator.py @@ -6,12 +6,14 @@ from __future__ import annotations +import glob +import json import subprocess import sys import traceback from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Tuple from ..utils.logging_utils import log, log_exception @@ -28,25 +30,101 @@ class CheckResult: class EvalResult: ok: bool checks: List[CheckResult] = field(default_factory=list) + # True when the only gate available was a weak smoke (e.g. compileall) + # because no real test command could be derived. The done-gate uses this + # to adaptively relax its tests-green requirement for projects that have + # no meaningful test suite (docs / research / content). + weak: bool = False def summary(self) -> str: rows = [] for c in self.checks: mark = "PASS" if c.ok else "FAIL" rows.append(f"{mark} rc={c.rc} :: {c.command}") - return "\n".join(rows) or "(no checks)" + out = "\n".join(rows) or "(no checks)" + if self.weak: + out += "\n(weak gate: no real test command derived)" + return out + + +def derive_validation(root: Path) -> Tuple[List[str], bool]: + """Best-effort default validation command(s) inferred from project shape. + + Returns ``(commands, weak)``. ``weak`` is True when the only thing we + could find is a smoke check (no real test suite), so callers can surface + ``eval=weak`` and the done-gate can adaptively relax tests-green. + + Detection order: pytest (tests/ or test_*.py) -> npm test (package.json + with a real test script) -> pytest fallback (pyproject/setup) -> python + compileall smoke -> always-pass weak sentinel. + """ + root = Path(root) + try: + has_tests_dir = (root / "tests").is_dir() + py_test_files = bool( + glob.glob(str(root / "**" / "test_*.py"), recursive=True) + or glob.glob(str(root / "**" / "*_test.py"), recursive=True) + ) + if has_tests_dir or py_test_files: + return (["python -m pytest -q"], False) + + pkg = root / "package.json" + if pkg.is_file(): + try: + data = json.loads(pkg.read_text(encoding="utf-8")) + test_script = ((data.get("scripts") or {}).get("test") or "").strip() + if test_script and "no test specified" not in test_script: + return (["npm test --silent"], False) + except Exception as exc: + log_exception("orchestration.evaluator.derive_validation.pkg", exc) + + if (root / "pyproject.toml").is_file() or (root / "setup.py").is_file(): + return (["python -m pytest -q"], False) + + # No real test command — fall back to a weak smoke gate. + if glob.glob(str(root / "**" / "*.py"), recursive=True): + return (["python -m compileall -q ."], True) + except Exception as exc: + log_exception("orchestration.evaluator.derive_validation", exc) + + # Nothing better: an always-pass sentinel, explicitly weak so callers and + # the done-gate know this was not a real evaluation. + return (["test -d ."], True) class Evaluator: """Run a list of shell commands as validation gates.""" - def __init__(self, root: Path, default_checks: Optional[List[str]] = None, timeout: int = 180) -> None: + def __init__( + self, + root: Path, + default_checks: Optional[List[str]] = None, + timeout: int = 180, + *, + auto_derive: bool = False, + derived_command: Optional[str] = None, + ) -> None: self.root = root self.default_checks = list(default_checks or []) self.timeout = timeout + self.auto_derive = auto_derive + self.derived_command = derived_command def run(self, checks: Optional[List[str]] = None) -> EvalResult: cmds = list(checks if checks is not None else self.default_checks) + weak = False + # FM4: never vacuously pass. When no explicit checks were configured, + # derive a real validation command from the project shape instead of + # treating "no checks" as success. + if not cmds and self.auto_derive: + if self.derived_command: + cmds = [self.derived_command] + else: + cmds, weak = derive_validation(self.root) + if not cmds: + # No evaluation available and auto-derive disabled: report a + # failed (not silently-passing) gate so the loop keeps working. + return EvalResult(ok=False, checks=[], weak=True) results: List[CheckResult] = [] all_ok = True for cmd in cmds: @@ -78,4 +156,4 @@ def run(self, checks: Optional[List[str]] = None) -> EvalResult: log_exception("orchestration.evaluator.run", exc) results.append(CheckResult(command=cmd, ok=False, rc=-1, output=str(exc))) all_ok = False - return EvalResult(ok=all_ok, checks=results) + return EvalResult(ok=all_ok, checks=results, weak=weak) diff --git a/clk_harness/orchestration/mission.py b/clk_harness/orchestration/mission.py new file mode 100644 index 0000000..5e97f08 --- /dev/null +++ b/clk_harness/orchestration/mission.py @@ -0,0 +1,633 @@ +"""Autonomous mission orchestrator. + +One objective in, a genuinely-complete result out — no human follow-up. The +MissionRunner is the macro plan->execute->evaluate->refine->iterate loop that +wraps the existing per-workflow engines: + +1. CHARTER — the chief writes the up-front commitment (see ``charter.py``). +2. PLAN — the chief authors an ordered phase plan (PROPOSE_PLAN), persisted + as a living artifact (``.clk/state/mission.json``). +3. For each phase: run its engine (WorkflowRunner / RalphLoop / Autoresearch), + evaluate a chief PHASE_GATE, and iterate until the gate passes — re-planning + when the gate says the plan itself is wrong. +4. The mission only stops on a machine-checkable done-gate (``done_gate.py``); + any agent's ACTION:done is a *request*, not a command. + +Every boundary is trace-committed so the git log is the execution trail, and a +per-cycle telemetry line makes the loop observable. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional + +from ..config import Paths, save_json, load_json +from ..git_ops import commit_trace +from ..utils.activity_log import log_event +from ..utils.logging_utils import log, log_exception +from . import blackboard as _blackboard +from . import charter as _charter +from . import casting as _casting +from . import deliberation as _deliberation +from . import done_gate as _done_gate +from .agent import AgentRunner +from .evaluator import Evaluator +from .telemetry import CycleTelemetry +from .workflow import WorkflowRunner, load_workflow +from .ralph_loop import RalphLoop +from .autoresearch_loop import AutoresearchLoop + + +_GATE_RE = re.compile(r"^\s*GATE\s*:\s*(pass|repeat|revise|done)\b", re.IGNORECASE | re.MULTILINE) +_DEFAULT_PHASES = ["discovery", "product", "engineering", "validation", "deployment"] + + +@dataclass +class PhaseSpec: + id: str + title: str = "" + workflow: str = "" + engine: str = "workflow" # workflow | ralph | autoresearch + exit_criteria: List[str] = field(default_factory=list) + produced_keys: List[str] = field(default_factory=list) + status: str = "pending" # pending | running | done | skipped | failed + order: int = 0 + iterations_used: int = 0 + max_iterations: int = 0 + gate_history: List[Dict[str, Any]] = field(default_factory=list) + + def to_dict(self) -> Dict[str, Any]: + return { + "id": self.id, + "title": self.title, + "workflow": self.workflow or self.id, + "engine": self.engine, + "exit_criteria": list(self.exit_criteria), + "produced_keys": list(self.produced_keys), + "status": self.status, + "order": self.order, + "iterations_used": self.iterations_used, + "max_iterations": self.max_iterations, + "gate_history": list(self.gate_history), + } + + @classmethod + def from_dict(cls, raw: Dict[str, Any], order: int = 0) -> "PhaseSpec": + raw = raw or {} + return cls( + id=str(raw.get("id") or f"phase{order}"), + title=str(raw.get("title") or ""), + workflow=str(raw.get("workflow") or raw.get("id") or ""), + engine=str(raw.get("engine") or "workflow").lower(), + exit_criteria=[str(x) for x in (raw.get("exit_criteria") or [])], + produced_keys=[str(x) for x in (raw.get("produced_keys") or [])], + status=str(raw.get("status") or "pending"), + order=int(raw.get("order") or order), + iterations_used=int(raw.get("iterations_used") or 0), + max_iterations=int(raw.get("max_iterations") or 0), + gate_history=list(raw.get("gate_history") or []), + ) + + +@dataclass +class MissionPlan: + objective: str = "" + title: str = "" + created_at: str = "" + updated_at: str = "" + status: str = "planning" # planning | running | done | stalled | aborted + current_phase_id: str = "" + total_cycles_used: int = 0 + phases: List[PhaseSpec] = field(default_factory=list) + cycles: List[Dict[str, Any]] = field(default_factory=list) + done_gate_last: Dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> Dict[str, Any]: + return { + "version": 1, + "objective": self.objective, + "title": self.title, + "created_at": self.created_at, + "updated_at": self.updated_at, + "status": self.status, + "current_phase_id": self.current_phase_id, + "total_cycles_used": self.total_cycles_used, + "phases": [p.to_dict() for p in self.phases], + "cycles": list(self.cycles), + "done_gate_last": dict(self.done_gate_last), + } + + @classmethod + def from_dict(cls, raw: Dict[str, Any]) -> "MissionPlan": + raw = raw or {} + phases = [PhaseSpec.from_dict(p, i) for i, p in enumerate(raw.get("phases") or [])] + return cls( + objective=str(raw.get("objective") or ""), + title=str(raw.get("title") or ""), + created_at=str(raw.get("created_at") or ""), + updated_at=str(raw.get("updated_at") or ""), + status=str(raw.get("status") or "planning"), + current_phase_id=str(raw.get("current_phase_id") or ""), + total_cycles_used=int(raw.get("total_cycles_used") or 0), + phases=phases, + cycles=list(raw.get("cycles") or []), + done_gate_last=dict(raw.get("done_gate_last") or {}), + ) + + def next_pending(self) -> Optional[PhaseSpec]: + for p in sorted(self.phases, key=lambda x: x.order): + if p.status in ("pending", "running"): + return p + return None + + def all_done(self) -> bool: + return bool(self.phases) and all( + p.status in ("done", "skipped") for p in self.phases + ) + + def render_md(self) -> str: + lines = [f"# Mission: {self.title or self.objective}", "", + f"**Status:** {self.status} ", f"**Cycles used:** {self.total_cycles_used}", "", + "## Phases", ""] + for p in sorted(self.phases, key=lambda x: x.order): + mark = {"done": "x", "running": ">", "failed": "!", "skipped": "-"}.get(p.status, " ") + lines.append(f"- [{mark}] **{p.id}** ({p.engine}) — {p.title or ''} " + f"[{p.status}, {p.iterations_used} it]") + for c in p.exit_criteria: + lines.append(f" - exit: {c}") + return "\n".join(lines) + "\n" + + +def plan_path(paths: Paths) -> Path: + return paths.state / "mission.json" + + +def mission_md_path(paths: Paths) -> Path: + return paths.state / "MISSION.md" + + +def load_plan(paths: Paths) -> Optional[MissionPlan]: + p = plan_path(paths) + if not p.exists(): + return None + raw = load_json(p, {}) + if not raw: + return None + return MissionPlan.from_dict(raw) + + +def save_plan(paths: Paths, plan: MissionPlan) -> None: + plan.updated_at = datetime.now().isoformat(timespec="seconds") + paths.state.mkdir(parents=True, exist_ok=True) + save_json(plan_path(paths), plan.to_dict()) + try: + mission_md_path(paths).write_text(plan.render_md(), encoding="utf-8") + except Exception as exc: + log_exception("orchestration.mission.save_plan.md", exc) + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")[:40] or "mission" + + +class MissionRunner: + """Drives one objective to a code-gated done with no human follow-up.""" + + def __init__(self, paths: Paths, runner: AgentRunner, evaluator: Evaluator) -> None: + self.paths = paths + self.runner = runner + self.evaluator = evaluator + + # -- config accessors -------------------------------------------------- + + def _cfg(self) -> Dict[str, Any]: + return (self.runner.clk_cfg.get("mission") or {}) + + @property + def max_phases(self) -> int: + return int(self._cfg().get("max_phases", 12) or 12) + + @property + def max_iterations_per_phase(self) -> int: + return int(self._cfg().get("max_iterations_per_phase", 3) or 3) + + @property + def max_total_cycles(self) -> int: + return int(self._cfg().get("max_total_cycles", 60) or 60) + + @property + def phase_gate_enabled(self) -> bool: + return bool(self._cfg().get("phase_gate", True)) + + @property + def charter_first(self) -> bool: + return bool(self._cfg().get("charter_first", True)) + + @property + def telemetry_stdout(self) -> bool: + return bool(self._cfg().get("telemetry_stdout", True)) + + @property + def on_budget_exhausted(self) -> str: + return str(self._cfg().get("on_budget_exhausted", "advance")).lower() + + # -- public ------------------------------------------------------------ + + def run(self, objective: Optional[str] = None, *, resume: bool = False, + dry_run: bool = False) -> MissionPlan: + objective = objective or self._idea_objective() + if not objective: + log("mission: no objective and no captured idea; nothing to do", level="WARN") + return MissionPlan(status="aborted") + + charter = None + plan: Optional[MissionPlan] = load_plan(self.paths) if resume else None + if plan is None: + if self.charter_first: + charter = _charter.bootstrap_charter(self.paths, self.runner, objective, dry_run=dry_run) + plan = self._bootstrap_plan(objective, charter, dry_run=dry_run) + else: + charter = _charter.load_charter(self.paths) + log(f"mission: resuming plan with {len(plan.phases)} phases " + f"({sum(1 for p in plan.phases if p.status == 'done')} done)") + + plan.status = "running" + save_plan(self.paths, plan) + + total = plan.total_cycles_used + for _outer in range(self.max_phases): + if self._mission_complete(plan): + break + phase = plan.next_pending() + if phase is None: + break + phase.status = "running" + plan.current_phase_id = phase.id + save_plan(self.paths, plan) + commit_trace(self.paths.root, kind="phase-start", summary=f"phase {phase.id}", + meta={"engine": phase.engine}) + + advanced = False + budget = phase.max_iterations or self.max_iterations_per_phase + for _it in range(budget): + if total >= self.max_total_cycles: + log(f"mission: total-cycle budget exhausted ({self.max_total_cycles})", level="WARN") + plan.status = "stalled" + break + total += 1 + plan.total_cycles_used = total + tel = CycleTelemetry(n=total, max_cycles=self.max_total_cycles, + phase=phase.id, workflow=phase.workflow or phase.id) + results = self._run_phase(phase, tel, dry_run) + phase.iterations_used += 1 + + eval_result, weak = self._evaluate(dry_run) + tel.record_eval(eval_result, weak=weak) + + verdict = self._phase_gate(plan, phase, results, eval_result, dry_run) + phase.gate_history.append({ + "at": datetime.now().isoformat(timespec="seconds"), + "verdict": verdict, + }) + self._post_phase_summary(plan, phase, verdict) + tel.progress = bool(results) + tel.emit(self.paths, to_stdout=self.telemetry_stdout) + plan.cycles.append(tel.as_dict()) + save_plan(self.paths, plan) + commit_trace(self.paths.root, kind=f"phase-gate:{verdict}", + summary=f"{phase.id} cycle {total}", meta={"phase": phase.id}) + + if verdict == "pass": + phase.status = "done" + advanced = True + break + if verdict == "done": + gate = self._maybe_finish(plan, charter, eval_result) + tel.record_done_gate(gate) + if gate.passed: + phase.status = "done" + plan.status = "done" + save_plan(self.paths, plan) + return plan + # rejected: keep iterating this phase + continue + if verdict == "revise": + self._refine_plan(plan, phase, results, dry_run) + advanced = True + break + # "repeat": loop again on the same phase + else: + # exhausted the per-phase budget without a pass + if self.on_budget_exhausted == "fail": + phase.status = "failed" + else: + phase.status = "done" + advanced = True + + if not advanced and plan.status == "stalled": + break + save_plan(self.paths, plan) + + # Final attempt: if every phase is done, run the done-gate once. + if plan.status not in ("done", "stalled"): + if plan.all_done(): + gate = self._maybe_finish(plan, charter, self._evaluate(dry_run)[0]) + plan.status = "done" if gate.passed else "stalled" + else: + plan.status = "stalled" + save_plan(self.paths, plan) + log(f"mission: finished with status={plan.status} after {plan.total_cycles_used} cycles") + return plan + + # -- internals --------------------------------------------------------- + + def _idea_objective(self) -> str: + idea_path = self.paths.state / "idea.json" + if not idea_path.exists(): + return "" + try: + raw = json.loads(idea_path.read_text(encoding="utf-8")) + return (raw.get("statement") or raw.get("title") or "").strip() + except Exception: + return "" + + def _mission_complete(self, plan: MissionPlan) -> bool: + return (self.paths.state / "done_granted.md").exists() and plan.all_done() + + def _bootstrap_plan(self, objective: str, charter, *, dry_run: bool) -> MissionPlan: + now = datetime.now().isoformat(timespec="seconds") + plan = MissionPlan( + objective=objective, + title=(charter.mission_statement if charter else objective)[:80], + created_at=now, + updated_at=now, + status="planning", + ) + phases: List[PhaseSpec] = [] + if not dry_run: + try: + run = self.runner.run( + "chief", + self._plan_objective(objective, charter), + extra={"phase": "mission_plan"}, + dry_run=dry_run, + ) + prop = _casting.parse_plan_proposal(run.response.text or "") + if prop and prop.phases: + phases = [PhaseSpec.from_dict(p, i) for i, p in enumerate(prop.phases)] + except Exception as exc: + log_exception("orchestration.mission._bootstrap_plan", exc) + if not phases: + default = self._cfg().get("default_phases") or _DEFAULT_PHASES + phases = [ + PhaseSpec(id=name, workflow=name, engine="workflow", order=i) + for i, name in enumerate(default) + ] + for i, p in enumerate(phases): + p.order = i + if not p.workflow: + p.workflow = p.id + plan.phases = phases + save_plan(self.paths, plan) + log_event(self.paths, "mission_plan", objective=objective, + phases=[p.id for p in phases]) + try: + _blackboard.post(self.paths, author="mission", body=plan.render_md(), + post_type="plan", produces=["mission_plan"], slug_hint="plan") + except Exception as exc: + log_exception("orchestration.mission._bootstrap_plan.post", exc) + commit_trace(self.paths.root, kind="plan", summary=f"{len(phases)} phases", + meta={"phases": [p.id for p in phases]}) + return plan + + def _run_phase(self, phase: PhaseSpec, tel: CycleTelemetry, dry_run: bool) -> List[Any]: + engine = (phase.engine or "workflow").lower() + if engine == "ralph": + loop = RalphLoop(self.paths, self.runner, self.evaluator, + max_iterations=phase.max_iterations or 1) + try: + return list(loop.run(dry_run=dry_run)) + except Exception as exc: + log_exception("orchestration.mission._run_phase.ralph", exc) + return [] + if engine == "autoresearch": + loop = AutoresearchLoop(self.paths, self.runner, self.evaluator, + max_iterations=phase.max_iterations or 1) + try: + return list(loop.run(dry_run=dry_run)) + except Exception as exc: + log_exception("orchestration.mission._run_phase.autoresearch", exc) + return [] + # default: workflow engine + wf_name = phase.workflow or phase.id + wf_path = self.paths.workflows / f"{wf_name}.yaml" + if not wf_path.exists(): + wf_path = self.paths.workflows / "engineering.yaml" + try: + wf = load_workflow(wf_path) + except Exception as exc: + log_exception("orchestration.mission._run_phase.load", exc) + return [] + wf_runner = WorkflowRunner(self.paths, self.runner) + wf_runner.mission_mode = True + wf_runner.supervise_cycles_override = 1 # mission owns the outer loop + wf_runner.telemetry = tel + try: + return list(wf_runner.run(wf, dry_run=dry_run)) + except Exception as exc: + log_exception("orchestration.mission._run_phase.run", exc) + return [] + + def _evaluate(self, dry_run: bool): + if dry_run: + from .evaluator import EvalResult + return EvalResult(ok=True, checks=[]), False + try: + result = self.evaluator.run() + return result, bool(getattr(result, "weak", False)) + except Exception as exc: + log_exception("orchestration.mission._evaluate", exc) + from .evaluator import EvalResult + return EvalResult(ok=False, checks=[], weak=True), True + + def _phase_gate(self, plan: MissionPlan, phase: PhaseSpec, results: List[Any], + eval_result, dry_run: bool) -> str: + if dry_run or not self.phase_gate_enabled: + return "pass" + # Deliberation: a phase cannot pass while a blocking question is open. + if _deliberation.require_open_questions_resolved(self.runner.clk_cfg): + open_q = _deliberation.unresolved_blocking_questions(self.paths) + if open_q: + log(f"mission: phase {phase.id} has {len(open_q)} unresolved blocking " + "question(s); repeating", level="INFO") + return "repeat" + try: + run = self.runner.run( + "chief", + self._gate_objective(plan, phase, eval_result), + extra={"phase": "phase_gate"}, + dry_run=dry_run, + ) + m = _GATE_RE.search(run.response.text or "") + if m: + return m.group(1).lower() + except Exception as exc: + log_exception("orchestration.mission._phase_gate", exc) + # Conservative default: keep working on this phase. + return "repeat" + + def _maybe_finish(self, plan: MissionPlan, charter, eval_result) -> "_done_gate.DoneGateVerdict": + extra_criteria = _charter.derive_done_criteria(charter) if charter else [] + verdict = _done_gate.evaluate_done_gate( + self.paths, self.runner.clk_cfg, eval_result, extra_criteria=extra_criteria, + ) + plan.done_gate_last = {"passed": verdict.passed, "failures": list(verdict.failures)} + if verdict.passed: + try: + (self.paths.state / "done_granted.md").write_text( + "# Mission complete\n\n" + verdict.summary() + "\n", encoding="utf-8") + except Exception as exc: + log_exception("orchestration.mission._maybe_finish.write", exc) + log_event(self.paths, "done_gate_granted", checked=verdict.checked, scope="mission") + commit_trace(self.paths.root, kind="done", summary="mission done-gate granted", + meta={"checked": list(verdict.checked.keys())}) + else: + log(f"mission: done requested but gate REJECTED — unmet: " + f"{', '.join(verdict.failures) or '?'}", level="WARN") + log_event(self.paths, "done_gate_rejected", failures=list(verdict.failures), + checked=verdict.checked, scope="mission") + return verdict + + def _refine_plan(self, plan: MissionPlan, phase: PhaseSpec, results: List[Any], + dry_run: bool) -> None: + """Re-plan the not-yet-done phases when the gate says the plan is wrong.""" + if dry_run: + return + try: + run = self.runner.run( + "chief", + self._replan_objective(plan, phase), + extra={"phase": "mission_plan"}, + dry_run=dry_run, + ) + prop = _casting.parse_plan_proposal(run.response.text or "") + except Exception as exc: + log_exception("orchestration.mission._refine_plan", exc) + prop = None + if not prop or not prop.phases: + return + done = [p for p in plan.phases if p.status in ("done", "skipped")] + done_ids = {p.id for p in done} + new_phases = list(done) + order = len(done) + for raw in prop.phases: + ps = PhaseSpec.from_dict(raw, order) + if ps.id in done_ids: + continue + ps.order = order + if not ps.workflow: + ps.workflow = ps.id + new_phases.append(ps) + order += 1 + plan.phases = new_phases + save_plan(self.paths, plan) + log_event(self.paths, "mission_replan", phases=[p.id for p in new_phases]) + commit_trace(self.paths.root, kind="replan", summary=f"{len(new_phases)} phases", + meta={"phases": [p.id for p in new_phases]}) + + def _post_phase_summary(self, plan: MissionPlan, phase: PhaseSpec, verdict: str) -> None: + try: + _blackboard.post( + self.paths, + author="mission", + body=f"Phase `{phase.id}` gate verdict: {verdict}. " + f"Exit criteria: {'; '.join(phase.exit_criteria) or '(none)'}.", + post_type="phase", + produces=[f"phase:{phase.id}"], + stage_id=phase.id, + slug_hint=f"phase-{phase.id}", + ) + except Exception as exc: + log_exception("orchestration.mission._post_phase_summary", exc) + + # -- prompts ----------------------------------------------------------- + + def _plan_objective(self, objective: str, charter) -> str: + charter_block = "" + if charter is not None: + charter_block = ( + "\nCharter (derive the plan from this):\n" + f"- Mission: {charter.mission_statement}\n" + f"- Scope: {'; '.join(charter.scope) or '(none)'}\n" + f"- Success: {'; '.join(charter.success_criteria) or '(none)'}\n" + ) + return ( + "MISSION PLAN MODE. Author the ordered lifecycle plan for this " + "autonomous mission. There will be no human input — plan the whole " + "path to a complete, production-ready result.\n\n" + f"Mission objective:\n{objective}\n" + f"{charter_block}\n" + "Emit exactly one PROPOSE_PLAN block (machine-parsed):\n\n" + " PROPOSE_PLAN\n" + " PHASES:\n" + " - id: discovery\n" + " title: \n" + " workflow: discovery\n" + " engine: workflow\n" + " exit_criteria: [, ]\n" + " - id: engineering\n" + " title: \n" + " workflow: engineering\n" + " engine: workflow\n" + " exit_criteria: []\n" + " END_PLAN\n\n" + "Default to the lifecycle discovery -> product -> engineering -> " + "validation -> deployment, but PRUNE phases this idea does not need " + "(e.g. skip deployment for a pure-research idea). The engineering " + "phase MUST use a workflow that contains a ralph refine stage and a " + "qa stage. You may also emit PROPOSE_ROLE / PROPOSE_WORKFLOW blocks " + "in this same response to define any roles or custom workflows the " + "phases reference." + ) + + def _replan_objective(self, plan: MissionPlan, phase: PhaseSpec) -> str: + return ( + "MISSION PLAN MODE (revision). The phase gate verdict indicated the " + "plan itself needs to change. Re-author the remaining phases.\n\n" + f"Mission objective:\n{plan.objective}\n\n" + f"Phases already completed: " + f"{', '.join(p.id for p in plan.phases if p.status == 'done') or '(none)'}\n" + f"Current phase that triggered the revision: {phase.id}\n\n" + "Emit a PROPOSE_PLAN block with the NEW set of remaining phases " + "(do not repeat completed phases). Same grammar as before." + ) + + def _gate_objective(self, plan: MissionPlan, phase: PhaseSpec, eval_result) -> str: + try: + eval_summary = eval_result.summary() if eval_result else "(no evaluation)" + except Exception: + eval_summary = "(no evaluation)" + criteria = "\n".join(f" - {c}" for c in phase.exit_criteria) or " (none declared)" + digest = _blackboard.digest(self.paths, selectors=[f"stage:{phase.id}"], max_posts=8) + return ( + "PHASE GATE MODE. Decide whether the current phase is complete enough " + "to advance the mission. Be honest: a low bar to keep working, a high " + "bar to advance.\n\n" + f"Mission objective:\n{plan.objective}\n\n" + f"Phase: {phase.id} ({phase.engine})\n" + f"Exit criteria:\n{criteria}\n\n" + f"Latest evaluation:\n{eval_summary}\n\n" + f"Recent blackboard for this phase:\n{digest}\n\n" + "Respond with exactly one line:\n" + " GATE: pass — exit criteria met; advance to the next phase\n" + " GATE: repeat — close, but this phase needs another iteration\n" + " GATE: revise — the PLAN itself is wrong; re-plan remaining phases\n" + " GATE: done — this is the FINAL phase and the whole mission is " + "complete (the harness still verifies the done-gate mechanically)\n" + "Then a one-line REASON:. You may also emit a POST: phase_gate block." + ) diff --git a/clk_harness/orchestration/noop_guard.py b/clk_harness/orchestration/noop_guard.py new file mode 100644 index 0000000..61703f6 --- /dev/null +++ b/clk_harness/orchestration/noop_guard.py @@ -0,0 +1,95 @@ +"""No-op detection: stop agents from "describing work instead of doing it". + +The most common reliability failure: a producing agent emits three paragraphs of +prose (and maybe ``PROGRESS: yes``) but no ``ACTION:write``/``edit`` — so nothing +changes and the loop advances anyway. ``response_quality.score`` never caught this +because it has no notion of "this stage was *expected* to mutate files". + +This module supplies that notion plus an escalating repair preamble the dispatch +loop prepends when a producing stage applied zero file mutations. +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Sequence + + +# Roles that are legitimately prose / verdict only. +_NON_PRODUCING = {"chief", "qa", "critic"} +_DEFAULT_PRODUCING = ("engineer", "ralph") + + +def is_mutation_expected( + agent: str, + *, + outputs: Optional[Sequence[str]] = None, + commit: bool = False, + cfg: Optional[Dict[str, Any]] = None, +) -> bool: + """Whether a stage run by ``agent`` should have changed files on disk. + + True when: the agent is in the configured producing set, OR the stage + declared a non-empty ``outputs`` contract (and that is treated as + producing), OR the stage is set to ``commit`` and the agent is not a + known prose-only role. + """ + g = (cfg or {}).get("noop_guard") or {} + if not g.get("enabled", True): + return False + producing = {str(a).lower() for a in (g.get("producing_agents") or _DEFAULT_PRODUCING)} + agent_l = (agent or "").lower() + + if agent_l in producing: + return True + if outputs and g.get("treat_outputs_stage_as_producing", True): + return True + if commit and agent_l not in _NON_PRODUCING: + return True + return False + + +def max_redispatch(cfg: Optional[Dict[str, Any]] = None) -> int: + g = (cfg or {}).get("noop_guard") or {} + if not g.get("enabled", True): + return 0 + return int(g.get("max_redispatch", 2) or 0) + + +def repair_preamble(attempt: int, *, target: str = "") -> str: + """Escalating instruction prepended when a producing stage made no change. + + ``attempt`` is 1-based (the n-th re-dispatch). ``target`` optionally names + the file/area the stage implies, used in the worked example. + """ + what = target or "the file(s) this objective requires" + if attempt <= 1: + return ( + "Your previous response changed NO files — it only described work. " + f"Descriptions do nothing here. Emit ACTION blocks NOW that create or " + f"edit {what}. Every ACTION must be a real file mutation:\n" + " ACTION: write\n" + " PATH: \n" + " CONTENT:\n" + " \n" + " END_ACTION" + ) + if attempt == 2: + path_hint = target or "src/.py" + return ( + "Still no files changed. This is serious. Output at least one " + "concrete ACTION:write or ACTION:edit block with real content — not a " + "plan, not a description. Worked example (adapt the path/content to " + "this objective):\n" + " ACTION: write\n" + f" PATH: {path_hint}\n" + " CONTENT:\n" + " # real, working code or content here\n" + " END_ACTION\n" + "Do this for every deliverable the objective implies." + ) + return ( + "FINAL ATTEMPT. Your response MUST consist of ACTION blocks only. Output " + "no prose, no preamble, no explanation — just the ACTION:write/edit blocks " + f"that produce {what}. If you cannot, emit a POST: question TO: chief " + "explaining the exact blocker instead." + ) diff --git a/clk_harness/orchestration/response_quality.py b/clk_harness/orchestration/response_quality.py index aa7102f..68aa5b6 100644 --- a/clk_harness/orchestration/response_quality.py +++ b/clk_harness/orchestration/response_quality.py @@ -108,6 +108,9 @@ def repair_hint(self) -> str: ) ) _HEADERLESS_ACTION_RE = re.compile(r"^\s*ACTION\s*:\s*([A-Za-z]+)", re.IGNORECASE | re.MULTILINE) +_FILE_ACTION_RE = re.compile( + r"^\s*ACTION\s*:\s*(write|edit|append|delete)\b", re.IGNORECASE | re.MULTILINE +) _END_ACTION_RE = re.compile(r"^\s*END_ACTION\s*$", re.IGNORECASE | re.MULTILINE) _POST_HEAD_RE = re.compile(r"^\s*POST\s*:\s*([A-Za-z][A-Za-z0-9_]*)\s*$", re.IGNORECASE | re.MULTILINE) _POST_END_RE = re.compile(r"^\s*END_POST\s*$", re.IGNORECASE | re.MULTILINE) @@ -208,6 +211,7 @@ def score( min_chars: int = 40, expected_outputs: Optional[Sequence[str]] = None, require_confidence: bool = False, + expect_file_action: bool = False, ) -> ResponseQuality: """Score a single response text against the harness's quality rules. @@ -224,6 +228,12 @@ def score( When True, missing the ``CONFIDENCE:`` line itself counts as a flag. Off by default so existing agents that have not been re-prompted yet aren't penalised. + expect_file_action: + When True, a response that contains no file-mutating ACTION block + (write / edit / append / delete) is flagged ``noop`` (recoverable) — + the worker described work instead of doing it. Parse-level only; the + dispatch loop also enforces this at apply time via + ``AgentRun.file_mutations_applied``. """ q = ResponseQuality() @@ -289,6 +299,15 @@ def score( "comma-separated on a single line." ) + # 5b. No-op: a producing stage that emitted no file-mutating ACTION. + if expect_file_action and not _FILE_ACTION_RE.search(text or ""): + q.flags.append("noop") + q.reasons.append( + "Your response changed no files — it contained no ACTION:write/" + "edit/append/delete block. Descriptions do nothing here. Emit at " + "least one real file-mutating ACTION block." + ) + # 6. Self-reported low confidence if q.confidence is not None and q.confidence < 0.5: q.flags.append("low_confidence") @@ -320,6 +339,7 @@ def score( "malformed_action": 0.4, "malformed_post": 0.3, "outputs_missing": 0.4, + "noop": 0.5, "low_confidence": 0.3, "needs_review_self": 0.2, "confidence_missing": 0.1, diff --git a/clk_harness/orchestration/telemetry.py b/clk_harness/orchestration/telemetry.py new file mode 100644 index 0000000..65e5fd1 --- /dev/null +++ b/clk_harness/orchestration/telemetry.py @@ -0,0 +1,207 @@ +"""Per-cycle loop telemetry. + +The plan->execute->evaluate->refine->iterate loop is only trustworthy if the +user can *see* it firing. CLK already logs rich events to +``.clk/logs/activity.jsonl``, but there is no per-cycle rollup. This module +provides a small thread-safe counter object that accumulates what happened in +one mission/supervise cycle and renders a single compact line plus a +``loop_cycle_summary`` activity event. + +A ``CycleTelemetry`` is created once per cycle and passed down the dispatch path +via ``extra["telemetry"]``. Because parallel workflow stages share one cycle's +telemetry object, every mutating method takes a lock. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from ..config import Paths +from ..utils.activity_log import log_event + + +@dataclass +class CycleTelemetry: + """Mutable, thread-safe accumulator for a single loop cycle. + + Counters are incremented from hooks already on the dispatch path + (action application, commits, refine rounds, consensus runs, quality + retries, no-op re-dispatches). ``eval`` / ``done_gate`` verdicts and + ``progress`` are stamped by the loop driver before :meth:`emit`. + """ + + n: int = 0 + max_cycles: int = 0 + phase: str = "" + workflow: str = "" + + stages_run: int = 0 + stages_ok: int = 0 + actions_applied: int = 0 + files_written: int = 0 + commits: int = 0 + refine_rounds: int = 0 + consensus_runs: int = 0 + quality_retries: int = 0 + noop_redispatches: int = 0 + qa_exchanges: int = 0 + + eval_ran: bool = False + eval_ok: Optional[bool] = None + eval_failures: int = 0 + eval_weak: bool = False + + done_gate_requested: bool = False + done_gate_passed: Optional[bool] = None + done_gate_failures: List[str] = field(default_factory=list) + + progress: Optional[bool] = None + notes: str = "" + + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False) + + # -- counter helpers (thread-safe) ------------------------------------- + def add_actions(self, n: int = 1) -> None: + with self._lock: + self.actions_applied += int(n) + + def add_files(self, n: int = 1) -> None: + with self._lock: + self.files_written += int(n) + + def add_commit(self, n: int = 1) -> None: + with self._lock: + self.commits += int(n) + + def add_refine_round(self, n: int = 1) -> None: + with self._lock: + self.refine_rounds += int(n) + + def add_consensus_run(self, n: int = 1) -> None: + with self._lock: + self.consensus_runs += int(n) + + def add_quality_retry(self, n: int = 1) -> None: + with self._lock: + self.quality_retries += int(n) + + def add_noop_redispatch(self, n: int = 1) -> None: + with self._lock: + self.noop_redispatches += int(n) + + def add_qa_exchange(self, n: int = 1) -> None: + with self._lock: + self.qa_exchanges += int(n) + + def record_stage(self, *, ok: bool) -> None: + with self._lock: + self.stages_run += 1 + if ok: + self.stages_ok += 1 + + def record_eval(self, eval_result: Any, *, weak: bool = False) -> None: + """Stamp the cycle's evaluation verdict from an ``EvalResult``.""" + with self._lock: + self.eval_ran = True + self.eval_weak = bool(weak) + try: + self.eval_ok = bool(eval_result.ok) + self.eval_failures = sum(1 for c in eval_result.checks if not c.ok) + except Exception: + self.eval_ok = None + self.eval_failures = 0 + + def record_done_gate(self, verdict: Any) -> None: + with self._lock: + self.done_gate_requested = True + try: + self.done_gate_passed = bool(verdict.passed) + self.done_gate_failures = list(verdict.failures or []) + except Exception: + self.done_gate_passed = None + self.done_gate_failures = [] + + # -- rendering --------------------------------------------------------- + def as_dict(self) -> Dict[str, Any]: + with self._lock: + return { + "n": self.n, + "max_cycles": self.max_cycles, + "phase": self.phase, + "workflow": self.workflow, + "stages_run": self.stages_run, + "stages_ok": self.stages_ok, + "actions_applied": self.actions_applied, + "files_written": self.files_written, + "commits": self.commits, + "refine_rounds": self.refine_rounds, + "consensus_runs": self.consensus_runs, + "quality_retries": self.quality_retries, + "noop_redispatches": self.noop_redispatches, + "qa_exchanges": self.qa_exchanges, + "eval_ran": self.eval_ran, + "eval_ok": self.eval_ok, + "eval_failures": self.eval_failures, + "eval_weak": self.eval_weak, + "done_gate_requested": self.done_gate_requested, + "done_gate_passed": self.done_gate_passed, + "done_gate_failures": list(self.done_gate_failures), + "progress": self.progress, + "notes": self.notes, + } + + def render_line(self) -> str: + """Compact one-line summary for stdout / TUI.""" + d = self.as_dict() + cap = f"/{d['max_cycles']}" if d["max_cycles"] else "" + parts: List[str] = [f"cycle {d['n']}{cap}"] + if d["phase"]: + parts.append(f"phase {d['phase']}") + parts.append(f"stages {d['stages_run']} ({d['stages_ok']} ok)") + parts.append(f"actions {d['actions_applied']}") + if d["files_written"]: + parts.append(f"files {d['files_written']}") + if d["commits"]: + parts.append(f"commits {d['commits']}") + if d["refine_rounds"]: + parts.append(f"refine {d['refine_rounds']}r") + if d["consensus_runs"]: + parts.append(f"consensus {d['consensus_runs']}") + if d["qa_exchanges"]: + parts.append(f"q&a {d['qa_exchanges']}") + if d["eval_ran"]: + if d["eval_ok"]: + parts.append("eval PASS" + ("(weak)" if d["eval_weak"] else "")) + else: + parts.append(f"eval FAIL({d['eval_failures']})") + if d["done_gate_requested"]: + if d["done_gate_passed"]: + parts.append("done-gate PASS") + else: + fails = ",".join(d["done_gate_failures"]) or "?" + parts.append(f"done-gate REJECT({fails})") + else: + parts.append("done-gate -") + if d["quality_retries"]: + parts.append(f"retries {d['quality_retries']}") + if d["noop_redispatches"]: + parts.append(f"noop {d['noop_redispatches']}") + return " | ".join(parts) + + def emit(self, paths: Optional[Paths], *, to_stdout: bool = True) -> str: + """Write a ``loop_cycle_summary`` event and return the rendered line. + + ``activity.jsonl`` always receives the event (cheap); stdout printing + is gated by ``to_stdout`` (config ``mission.telemetry_stdout``). + """ + line = self.render_line() + if paths is not None: + try: + log_event(paths, "loop_cycle_summary", **self.as_dict()) + except Exception: + pass + if to_stdout: + print(line, flush=True) + return line diff --git a/clk_harness/orchestration/workflow.py b/clk_harness/orchestration/workflow.py index 0bbe130..60c0fec 100644 --- a/clk_harness/orchestration/workflow.py +++ b/clk_harness/orchestration/workflow.py @@ -30,12 +30,26 @@ from typing import Any, Dict, List, Optional, Set, Tuple from ..config import Paths -from ..git_ops import add_all, commit as git_commit, has_changes, head_sha, revert_to, snapshot_rollback +from ..git_ops import ( + add_all, + commit as git_commit, + commit_trace, + has_changes, + head_sha, + revert_to, + snapshot_rollback, +) from ..utils.activity_log import log_event from ..utils.logging_utils import log, log_exception from . import blackboard as _blackboard from . import response_quality as _response_quality +from . import noop_guard as _noop_guard +from . import deliberation as _deliberation +from . import done_gate as _done_gate +from . import evaluator as _evaluator +from . import charter as _charter from .agent import AgentRunner, AgentRun +from .telemetry import CycleTelemetry _ROUND_STATUS_RE = re.compile(r"^\s*ROUND_STATUS\s*:\s*(continue|done|finished)\s*$", re.IGNORECASE | re.MULTILINE) @@ -384,6 +398,17 @@ class WorkflowRunner: def __init__(self, paths: Paths, runner: AgentRunner) -> None: self.paths = paths self.runner = runner + # When set by the MissionRunner, the per-cycle telemetry object is + # threaded into each stage's dispatch extra so the dispatch-path hooks + # accumulate into the active cycle. When None, ``run`` creates one per + # supervise cycle so standalone ``clk run`` is observable too. + self.telemetry: Optional[CycleTelemetry] = None + # When True, producing dispatches get the deliberation preamble and + # the done-gate / phase semantics lean toward unattended autonomy. + self.mission_mode: bool = False + # When the MissionRunner drives phases, it owns the outer loop, so it + # sets this to 1 to make each WorkflowRunner.run() a single pass. + self.supervise_cycles_override: Optional[int] = None # Default cap on chief recovery dispatches per stage. A stage that # still has unmet deps after this many recovery passes gets a final @@ -416,6 +441,8 @@ def stage_backoff_s(self) -> float: @property def max_supervise_cycles(self) -> int: + if self.supervise_cycles_override is not None: + return int(self.supervise_cycles_override) cfg = (self.runner.clk_cfg.get("supervise") or {}) return int(cfg.get("max_cycles") or self.DEFAULT_MAX_SUPERVISE_CYCLES) @@ -449,6 +476,106 @@ def _should_rollback(self, stage: WorkflowStage) -> bool: return False return bool(stage.careful) + # -- done gate (FM2) --------------------------------------------------- + + def _done_gate_enabled(self) -> bool: + cfg = (self.runner.clk_cfg.get("done_gate") or {}) + return bool(cfg.get("enabled", True)) + + def _telemetry_stdout(self) -> bool: + cfg = (self.runner.clk_cfg.get("mission") or {}) + return bool(cfg.get("telemetry_stdout", True)) + + def _evaluate_done_gate(self) -> "_done_gate.DoneGateVerdict": + """Build a real eval result + charter criteria and run the done gate.""" + val_cfg = (self.runner.clk_cfg.get("validation") or {}) + evaluator = _evaluator.Evaluator( + root=self.paths.root, + default_checks=list(self.runner.clk_cfg.get("validation_checks") or []), + auto_derive=bool(val_cfg.get("auto_derive", True)), + derived_command=val_cfg.get("derived_command"), + ) + try: + eval_result = evaluator.run() + except Exception as exc: + log_exception("orchestration.workflow._evaluate_done_gate.eval", exc) + eval_result = None + try: + charter = _charter.load_charter(self.paths) + extra_criteria = _charter.derive_done_criteria(charter) + except Exception: + extra_criteria = [] + return _done_gate.evaluate_done_gate( + self.paths, self.runner.clk_cfg, eval_result, extra_criteria=extra_criteria, + ) + + def _stop_requested(self, workflow: Workflow) -> bool: + """Whether the loop may stop now. + + ``done_granted.md`` (written only by the gate) is the authoritative + stop signal. A bare ``done.md`` is an agent *request*: when the gate + is enabled it is honored only if every completion criterion passes, + otherwise it is downgraded so the loop keeps working. When the gate + is disabled, ``done.md`` stops the loop as it always did. + """ + state = self.paths.state + if (state / "done_granted.md").exists(): + return True + done_md = state / "done.md" + if not done_md.exists(): + return False + if not self._done_gate_enabled(): + return True + verdict = self._evaluate_done_gate() + if self.telemetry is not None: + try: + self.telemetry.record_done_gate(verdict) + except Exception: + pass + if verdict.passed: + self._grant_done(verdict) + return True + # Reject: downgrade the request so a later cycle can re-earn it. + try: + done_md.rename(state / "done_requested.md") + except Exception: + try: + done_md.unlink() + except Exception: + pass + log( + f"workflow {workflow.name}: ACTION:done REJECTED by done-gate — " + f"unmet: {', '.join(verdict.failures) or '?'}", + level="WARN", + ) + log_event( + self.paths, + "done_gate_rejected", + workflow=workflow.name, + failures=list(verdict.failures), + checked=verdict.checked, + ) + return False + + def _grant_done(self, verdict: "_done_gate.DoneGateVerdict") -> None: + try: + (self.paths.state / "done_granted.md").write_text( + "# Mission complete\n\n" + verdict.summary() + "\n", + encoding="utf-8", + ) + except Exception as exc: + log_exception("orchestration.workflow._grant_done", exc) + log_event(self.paths, "done_gate_granted", checked=verdict.checked) + try: + commit_trace( + self.paths.root, + kind="done", + summary="done-gate granted", + meta={"checked": list(verdict.checked.keys())}, + ) + except Exception: + pass + def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[StageResult]: """Execute the workflow, looping it under chief supervision. @@ -472,11 +599,13 @@ def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[Sta """ all_results: List[StageResult] = [] stopped_for_provider_failure = False + stopped_done = False no_progress = 0 rescue_attempted = False for cycle in range(1, self.max_supervise_cycles + 1): - if (self.paths.state / "done.md").exists(): - log(f"workflow {workflow.name}: done.md present, stopping supervise loop") + if self._stop_requested(workflow): + log(f"workflow {workflow.name}: stop granted, ending supervise loop") + stopped_done = True break cancel_file = self.paths.state / "cancel_requested.txt" @@ -490,6 +619,14 @@ def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[Sta if cycle > 1: log(f"workflow {workflow.name}: supervise cycle {cycle}/{self.max_supervise_cycles}") + # Per-cycle telemetry: when the MissionRunner owns one it is set on + # self.telemetry already; otherwise create one for this cycle so + # standalone `clk run` is observable too (FM5). + owns_telemetry = self.telemetry is None + if owns_telemetry: + self.telemetry = CycleTelemetry( + n=cycle, max_cycles=self.max_supervise_cycles, workflow=workflow.name, + ) try: refreshed = load_workflow(self.paths.workflows / f"{workflow.name}.yaml") except Exception: @@ -515,6 +652,13 @@ def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[Sta explicit = [s for s in signals if s is not None] self_reported_stall = bool(explicit) and not any(explicit) progress = material and not self_reported_stall + # Emit the per-cycle telemetry line (FM5). When the MissionRunner + # owns the telemetry object it records eval/done-gate and emits + # itself, so only emit here for standalone supervise cycles. + if owns_telemetry and self.telemetry is not None: + self.telemetry.progress = progress + self.telemetry.emit(self.paths, to_stdout=self._telemetry_stdout()) + self.telemetry = None if not progress: no_progress += 1 why = "agents reported PROGRESS: no" if (material and self_reported_stall) else "no commits or file writes" @@ -528,8 +672,9 @@ def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[Sta rescue_attempted = True no_progress = 0 self._dispatch_stall_rescue(workflow, cycle, cycle_results) - if (self.paths.state / "done.md").exists(): - log(f"workflow {workflow.name}: chief declared done during stall rescue") + if self._stop_requested(workflow): + log(f"workflow {workflow.name}: stop granted during stall rescue") + stopped_done = True break continue log( @@ -555,7 +700,8 @@ def run(self, workflow: Workflow, *, dry_run: Optional[bool] = None) -> List[Sta break if ( not stopped_for_provider_failure - and not (self.paths.state / "done.md").exists() + and not stopped_done + and not (self.paths.state / "done_granted.md").exists() and self.max_supervise_cycles > 1 ): log( @@ -761,10 +907,30 @@ def _run_stage( "cycle_context": cycle_context, "blackboard_inputs": bb_inputs, "stage_outputs": list(stage.outputs), + # Carried for the no-op guard (commit=producing) and the telemetry + # hooks on the dispatch path. + "commit": bool(stage.commit), + "telemetry": self.telemetry, } if stage.phase: base_extra["phase"] = stage.phase + # Deliberation: in mission mode, prepend the self-reflect + ask-peers + # preamble to producing dispatches so the team "thinks" before acting. + if ( + not dry_run + and self.mission_mode + and stage.phase != "review" + and _deliberation.enabled(self.runner.clk_cfg) + and _noop_guard.is_mutation_expected( + stage.agent, outputs=stage.outputs, commit=stage.commit, + cfg=self.runner.clk_cfg, + ) + ): + preamble = _deliberation.dispatch_preamble(self.runner.clk_cfg) + if preamble: + objective = preamble + objective + stop_when_file = self.paths.state / "stop_when.txt" stop_when = stop_when_file.read_text(encoding="utf-8").strip() if stop_when_file.exists() else "" if stop_when: @@ -820,7 +986,13 @@ def _run_stage( # auto_refine policy), dispatch a critic agent to score the # response; if the critic says revise, re-dispatch the worker # with the critic's feedback until accept or max_rounds. - if not dry_run and run.response.ok and self._refine_enabled(stage): + if not dry_run and run.response.ok and self._debate_enabled(stage): + # Adversarial debate panel takes precedence over the single critic. + try: + run = self._debate_loop(workflow, stage, run, cycle_context, dry_run) + except Exception as exc: + log_exception("orchestration.workflow._run_stage.debate", exc) + elif not dry_run and run.response.ok and self._refine_enabled(stage): try: run = self._refine_loop(workflow, stage, run, cycle_context, dry_run) except Exception as exc: @@ -938,6 +1110,11 @@ def _run_stage( committed=committed, failure_reason=failure_reason, ) + if self.telemetry is not None: + try: + self.telemetry.record_stage(ok=bool(ok and v_ok)) + except Exception: + pass # Per-stage chief checkpoint for sensitive stages. Cheap, gated, # and never blocks: it just keeps the chief in the loop without @@ -1123,6 +1300,11 @@ def _refine_loop( current_run = first_run for round_idx in range(1, max_rounds + 1): + if self.telemetry is not None: + try: + self.telemetry.add_refine_round() + except Exception: + pass verdict, judge_score, feedback = self._dispatch_critic( workflow, stage, current_run, critic_name, round_idx, max_rounds, dry_run, ) @@ -1173,6 +1355,7 @@ def _refine_loop( "stage_outputs": list(stage.outputs), "refine_round": round_idx + 1, "refine_max_rounds": max_rounds, + "telemetry": self.telemetry, }, dry_run=dry_run, ) @@ -1255,6 +1438,239 @@ def _dispatch_critic( score_val = 1.0 if verdict == "accept" else 0.4 return verdict, score_val, text.strip() + # -- adversarial debate panel (multi-critic refinement) -------------- + + _DEBATE_LENS_GUIDANCE: Dict[str, str] = { + "correctness": "logic errors, wrong outputs, unhandled edge cases, broken contracts or APIs.", + "security": "injection, unsafe input handling, secret/credential leakage, unsafe shell/file operations.", + "simplicity": "needless complexity, duplication, dead code, and simpler equivalent designs.", + "performance": "obvious inefficiency, redundant work, N+1 patterns, unbounded loops or memory.", + "robustness": "failure modes, missing error handling, flaky assumptions, and race conditions.", + "tests": "missing or weak tests, untested branches, and assertions that don't actually verify behavior.", + "ux": "confusing interfaces, poor error messages, and undocumented behavior.", + } + + def _debate_enabled(self, stage: WorkflowStage) -> bool: + """Whether the adversarial debate panel should run for this stage. + + Explicit ``refine: {mode: debate}`` always wins; otherwise the + ``robustness.debate`` policy (off | careful_only | all) decides. + chief / qa / critic stages are skipped. + """ + if stage.agent in ("chief", "qa", "critic"): + return False + if isinstance(stage.refine, dict) and str(stage.refine.get("mode") or "").lower() == "debate": + return True + cfg = (self.runner.clk_cfg.get("robustness") or {}) + mode = str(cfg.get("debate") or "off").lower() + if mode in ("", "off", "false", "0"): + return False + if mode == "all": + return True + return bool(stage.careful) # careful_only + + def _debate_lenses(self, stage: WorkflowStage) -> List[str]: + if isinstance(stage.refine, dict) and stage.refine.get("critics"): + lenses = [str(x).strip().lower() for x in stage.refine["critics"] if str(x).strip()] + else: + cfg = (self.runner.clk_cfg.get("robustness") or {}) + lenses = [str(x).strip().lower() for x in (cfg.get("debate_lenses") or []) if str(x).strip()] + return lenses or ["correctness", "security", "simplicity"] + + def _dispatch_lens_critic( + self, + workflow: "Workflow", + stage: WorkflowStage, + worker_run: AgentRun, + critic_name: str, + lens: str, + round_idx: int, + max_rounds: int, + peer_transcript: str, + dry_run: Optional[bool], + ) -> Tuple[str, str, float, str]: + """One adversarial critic pass for a single lens. + + Returns ``(lens, verdict, score, feedback)``. The critic is told to + try to *break* the work from its lens and, in later rounds, to engage + with peers' critiques (reinforce / refute / concede). + """ + worker_text = (worker_run.response.text or "").strip() + if len(worker_text) > 3500: + worker_text = worker_text[:3500].rstrip() + "\n…(truncated)" + guidance = self._DEBATE_LENS_GUIDANCE.get( + lens, f"weaknesses from the {lens} perspective." + ) + peer_block = "" + if peer_transcript.strip(): + peer_block = ( + "\nYour fellow panelists said (engage with them — reinforce, " + "refute, or concede explicitly):\n" + f"{peer_transcript}\n" + ) + objective = ( + f"ADVERSARIAL DEBATE — you are the **{lens}** critic on a review " + f"panel for stage `{stage.id}` (round {round_idx}/{max_rounds}).\n\n" + f"Your lens: hunt for {guidance}\n" + "Try hard to BREAK this work from your lens. Be specific and " + "concrete; cite the exact place. Default to skepticism — only " + "accept if you genuinely cannot find a real problem.\n\n" + f"Worker `{stage.agent}` objective:\n{stage.objective}\n\n" + f"Worker's response:\n---\n{worker_text}\n---\n" + f"{peer_block}\n" + "Keep it to 2-5 concrete bullets. End with exactly two lines:\n" + "VERDICT: accept # or `revise` if any real issue remains\n" + "SCORE: <0..1>\n" + ) + critic_run = self.runner.run( + critic_name, + objective, + extra={ + "phase": "refine_critic", + "stage_id": stage.id, + "workflow": workflow.name, + "refine_round": round_idx, + "debate_lens": lens, + }, + dry_run=dry_run, + ) + text = critic_run.response.text or "" + verdict_m = self._REFINE_VERDICT_RE.search(text) + verdict = (verdict_m.group(1).lower() if verdict_m else "revise") + if verdict not in ("accept", "revise"): + verdict = "revise" + score_m = self._REFINE_SCORE_RE.search(text) + try: + score_val = float(score_m.group(1)) if score_m else (1.0 if verdict == "accept" else 0.4) + except (TypeError, ValueError): + score_val = 0.4 + score_val = max(0.0, min(1.0, score_val)) + return lens, verdict, score_val, text.strip() + + def _debate_loop( + self, + workflow: "Workflow", + stage: WorkflowStage, + first_run: AgentRun, + cycle_context: str, + dry_run: Optional[bool], + ) -> AgentRun: + """Run an adversarial debate panel: N lens-critics → worker revision. + + Each round fans out one critic per lens in parallel; the worker is + kept only if a majority of lenses accept (or the mean score clears the + threshold). Otherwise the combined critiques drive a revision, and the + next round's critics see the prior panel transcript so they can debate + each other. Bounded by ``debate_max_rounds``. + """ + defaults = (self.runner.clk_cfg.get("robustness") or {}) + cfg = dict(stage.refine or {}) if isinstance(stage.refine, dict) else {} + try: + max_rounds = int(cfg.get("max_rounds") or defaults.get("debate_max_rounds") or 2) + except (TypeError, ValueError): + max_rounds = 2 + try: + threshold = float(cfg.get("accept_threshold") or defaults.get("refine_accept_threshold") or 0.8) + except (TypeError, ValueError): + threshold = 0.8 + + agents_cfg = (self.runner.agents_cfg.get("agents") or {}) + critic_name = "critic" if "critic" in agents_cfg else "" + if not critic_name: + # No critic in the roster — fall back to the single-critic loop + # (which itself no-ops when no critic exists). + return self._refine_loop(workflow, stage, first_run, cycle_context, dry_run) + + lenses = self._debate_lenses(stage) + max_parallel = max(1, int((self.runner.clk_cfg.get("consensus") or {}).get("max_parallel") or 4)) + current_run = first_run + peer_transcript = "" + + for round_idx in range(1, max_rounds + 1): + if self.telemetry is not None: + try: + self.telemetry.add_refine_round() + except Exception: + pass + verdicts: List[Tuple[str, str, float, str]] = [] + with ThreadPoolExecutor(max_workers=min(max_parallel, len(lenses))) as pool: + futs = { + pool.submit( + self._dispatch_lens_critic, workflow, stage, current_run, + critic_name, lens, round_idx, max_rounds, peer_transcript, dry_run, + ): lens + for lens in lenses + } + for fut in as_completed(futs): + try: + verdicts.append(fut.result()) + except Exception as exc: + log_exception("orchestration.workflow._debate_loop.critic", exc) + if not verdicts: + return current_run + revise_votes = sum(1 for (_l, v, _s, _f) in verdicts if v == "revise") + scores = [s for (_l, _v, s, _f) in verdicts] + avg_score = sum(scores) / len(scores) if scores else 0.0 + transcript = "\n".join( + f"[{lens}] verdict={v} score={s:.2f}\n{fb}" for (lens, v, s, fb) in verdicts + ) + peer_transcript = transcript + log_event( + self.paths, "debate_round", + agent=stage.agent, workflow=workflow.name, stage_id=stage.id, + round=round_idx, max_rounds=max_rounds, + lenses=[l for (l, *_r) in verdicts], + revise_votes=revise_votes, avg_score=round(avg_score, 3), + accept_threshold=threshold, + ) + self.runner._observer_log( + f"debate :: {stage.id} :: round {round_idx}/{max_rounds} " + f"{len(lenses)} critics, {revise_votes} revise, avg={avg_score:.2f}" + ) + try: + _blackboard.post( + self.paths, author="critic-panel", body=transcript[:4000], + post_type="debate", stage_id=stage.id, workflow=workflow.name, + slug_hint=f"debate-{stage.id}-r{round_idx}", + ) + except Exception as exc: + log_exception("orchestration.workflow._debate_loop.post", exc) + + # Panel accepts when a majority accept AND the mean clears the bar. + if revise_votes * 2 <= len(verdicts) and avg_score >= threshold: + return current_run + if round_idx == max_rounds: + return current_run + + revise_objective = ( + f"Debate round {round_idx + 1}/{max_rounds} of stage `{stage.id}`. " + f"An adversarial review panel ({', '.join(lenses)}) found issues " + f"(mean score {avg_score:.2f}/1.0). Address every concrete point " + "below; keep what already works. Re-emit POST and ACTION blocks " + "the same way so the harness records the updated work.\n\n" + f"Panel critiques:\n{transcript}\n\n" + f"Original objective:\n{stage.objective}" + ) + current_run = self.runner.run( + stage.agent, + revise_objective, + extra={ + "phase": "refine_worker", + "stage_id": stage.id, + "workflow": workflow.name, + "cycle_context": cycle_context, + "blackboard_inputs": list(stage.inputs), + "stage_outputs": list(stage.outputs), + "refine_round": round_idx + 1, + "refine_max_rounds": max_rounds, + "telemetry": self.telemetry, + }, + dry_run=dry_run, + ) + if not current_run.response.ok: + return current_run + return current_run + def _dispatch_checkpoint( self, workflow: Workflow, @@ -1590,8 +2006,23 @@ def _maybe_refresh_workflow( return merged, new_mtime def _validate(self, stage: WorkflowStage) -> tuple[bool, str]: - if not stage.validation: - return True, "" + cmd = stage.validation + # FM4: a producing stage with no explicit validation no longer + # auto-passes — derive a real command from the project shape. Non- + # producing stages (chief/critic prose) keep the auto-pass. + if not cmd: + val_cfg = (self.runner.clk_cfg.get("validation") or {}) + if val_cfg.get("auto_derive", True) and _noop_guard.is_mutation_expected( + stage.agent, outputs=stage.outputs, commit=stage.commit, + cfg=self.runner.clk_cfg, + ): + if val_cfg.get("derived_command"): + cmd = str(val_cfg.get("derived_command")) + else: + derived, _weak = _evaluator.derive_validation(self.paths.root) + cmd = derived[0] if derived else None + if not cmd: + return True, "" try: log_event( self.paths, @@ -1599,12 +2030,12 @@ def _validate(self, stage: WorkflowStage) -> tuple[bool, str]: agent=stage.agent, action="validation", stage_id=stage.id, - cmd=stage.validation, + cmd=cmd, cwd=str(self.paths.root), timeout_s=120, ) r = subprocess.run( - stage.validation, + cmd, shell=True, cwd=str(self.paths.root), capture_output=True, @@ -1618,7 +2049,7 @@ def _validate(self, stage: WorkflowStage) -> tuple[bool, str]: agent=stage.agent, action="validation", stage_id=stage.id, - cmd=stage.validation, + cmd=cmd, ok=r.returncode == 0, returncode=r.returncode, output=output, @@ -1633,7 +2064,7 @@ def _validate(self, stage: WorkflowStage) -> tuple[bool, str]: agent=stage.agent, action="validation", stage_id=stage.id, - cmd=stage.validation, + cmd=cmd, ok=False, error=str(exc), ) diff --git a/clk_harness/templates/prompts.py b/clk_harness/templates/prompts.py index f9f6c11..065b23d 100644 --- a/clk_harness/templates/prompts.py +++ b/clk_harness/templates/prompts.py @@ -113,7 +113,11 @@ The harness validates these mechanically and re-dispatches you on any miss: 1. Deliverables exist as FILES via ACTION blocks. Prose describing work is not work. If you produced content (posts, docs, code), each piece - is inside an ACTION:write/append with a real PATH. + is inside an ACTION:write/append with a real PATH. ENFORCED: a producing + stage whose response changes NO files is rejected and re-dispatched with + an escalating instruction — never answer such a stage with prose alone. + If you are genuinely blocked on info a peer owns, emit + `POST: question TO: URGENCY: blocking` instead of guessing. 2. Every ACTION block ends with END_ACTION on its own line. 3. Every POST block ends with END_POST on its own line. 4. If your context shows a REQUIRED OUTPUT CONTRACT, one of your POST @@ -175,6 +179,10 @@ If ANY item is false → emit PROPOSE_WORKFLOW for the next iteration instead. When in doubt, keep going. Stopping one cycle too early is the most common mistake — emit PROPOSE_WORKFLOW rather than a premature ACTION:done. + ENFORCED: the harness verifies these mechanically (tests green, a qa PASS + post, a ralph pass, deliverables on disk). A premature ACTION:done is + REJECTED and you are re-dispatched until every enabled check passes — so do + not emit it as a shortcut. Paths must resolve inside $project_root. Originals are backed up. Cap is 25 file actions / response. ``run`` rejects sudo and destructive patterns. @@ -437,6 +445,13 @@ [ ] The original idea is fully addressed — every feature, every edge case the idea implies, not just the fastest path to any output. +These boxes are not honor-system: the harness enforces a machine-checkable +done-gate (tests green, a qa PASS post, a ralph pass, deliverables on disk, +plus any file-named charter success criteria). ACTION:done is treated as a +*request* — if the gate fails it is rejected, the request is discarded, and +you are re-dispatched with the unmet items quoted back. Only the gate writes +the terminal `done_granted.md`. + **If every high-bar box is checked with certainty**: emit exactly one ACTION:done block with REASON: . diff --git a/clk_harness/tui.py b/clk_harness/tui.py index cfe262d..de000f7 100644 --- a/clk_harness/tui.py +++ b/clk_harness/tui.py @@ -70,6 +70,7 @@ AgentRunner, AutoresearchLoop, Evaluator, + MissionRunner, RalphLoop, RoleProposal, WorkflowRunner, @@ -880,6 +881,8 @@ def _dispatch(self, job: Job) -> None: self._do_cast() elif job.kind == "run": self._do_workflow(job.payload or "engineering") + elif job.kind == "mission": + self._do_mission() elif job.kind == "loop": payload = job.payload or {} self._do_loop( @@ -1064,6 +1067,40 @@ def _do_workflow(self, name: str) -> None: "/loop ralph 5 to refine, /undo to revert, or /quit to exit." ) + def _do_mission(self) -> None: + """Drive the autonomous mission (charter -> plan -> phases -> done).""" + self.state.clear_stop() + self.state.set_phase("mission", busy=True) + self.state.add_system_message( + "starting autonomous mission — the chief writes a charter and plan, " + "then drives the lifecycle to a code-gated done. Watch the cards above; " + "type /stop to end after the current cycle." + ) + any_failure = False + try: + mr = MissionRunner(self.paths, self.runner, self.evaluator) + plan = mr.run() + self.state.add_system_message( + f"mission {plan.status}: " + f"{sum(1 for p in plan.phases if p.status == 'done')}/{len(plan.phases)} " + f"phases done, {plan.total_cycles_used} cycles." + ) + if plan.status != "done" and (plan.done_gate_last or {}).get("failures"): + self.state.add_system_message( + "done-gate unmet: " + ", ".join(plan.done_gate_last["failures"]) + ) + except Exception as exc: + any_failure = True + log_exception("tui.Worker._do_mission", exc) + self.state.add_log(f"mission hit an error: {exc}", level="ERROR") + finally: + self.state.set_phase("idle", busy=False) + if not any_failure: + self.state.add_system_message( + "next steps: type a follow-up to extend the mission, " + "/loop ralph 5 to refine, /undo to revert, or /quit." + ) + def _do_loop(self, mode: str, n: int) -> None: self.state.clear_stop() self.state.set_phase(f"loop:{mode}", busy=True) @@ -2483,6 +2520,11 @@ def _dispatch(self, msg: str) -> bool: if cmd == "run": self.worker.submit(Job("run", args[0] if args else "engineering")) return True + if cmd in ("mission", "auto"): + if args and not self.state.idea: + self.worker.submit(Job("idea", " ".join(args))) + self.worker.submit(Job("mission")) + return True if cmd == "loop": mode = args[0] if args else "ralph" n = int(args[1]) if len(args) > 1 and args[1].isdigit() else 5 @@ -2569,12 +2611,15 @@ def _dispatch(self, msg: str) -> bool: # back per user message and was the cause of the "chief stuck # at 90+ seconds" symptom. /cast remains as an explicit manual # trigger when you want a re-cast without running engineering. + # Autonomy by default: a free-text message drives the full autonomous + # mission (charter -> plan -> phases -> code-gated done) rather than a + # single workflow pass. Use /run for a one-shot engineering cycle. if not self.state.idea: self.worker.submit(Job("idea", msg)) - self.worker.submit(Job("run", "engineering")) + self.worker.submit(Job("mission")) else: self._append_conversation(msg) - self.worker.submit(Job("run", "engineering")) + self.worker.submit(Job("mission")) return True def _do_abort(self) -> None: diff --git a/kickoff.sh b/kickoff.sh index 07e53a9..036198c 100644 --- a/kickoff.sh +++ b/kickoff.sh @@ -941,6 +941,17 @@ echo "[kickoff] clk init" # CLK_ROBUSTNESS_PLATEAU_WINDOW int >= 2 # CLK_ROBUSTNESS_PLATEAU_ACTION off | escalate_only | # reframe_only | escalate_then_reframe +# CLK_ROBUSTNESS_DEBATE off | careful_only | all +# CLK_ROBUSTNESS_DEBATE_LENSES comma-separated lens names +# CLK_ROBUSTNESS_DEBATE_MAX_ROUNDS int >= 1 +# +# Autonomous-mission knobs (clk run drives a full mission to done): +# CLK_MISSION_* max_phases / max_total_cycles / +# phase_gate / charter_first / ... +# CLK_DONE_GATE_* enabled + require_* completion checks +# CLK_NOOP_GUARD_* enabled / max_redispatch / ... +# CLK_DELIBERATION_* enabled / require_open_questions_resolved / ... +# CLK_VALIDATION_AUTO_DERIVE true | false # # Prior knobs (already supported, surfaced here for parity): # CLK_PROVIDER_TIMEOUT_S int seconds, 0 = harness default @@ -994,6 +1005,10 @@ def _bool(s): return str(s).strip().lower() in {"1", "true", "yes", "y", "on"} +def _csv(s): + return [item.strip() for item in str(s).split(",") if item.strip()] + + # Robustness block _set("CLK_ROBUSTNESS_AUTO_CONSENSUS", "robustness", "auto_consensus") _set("CLK_ROBUSTNESS_AUTO_REFINE", "robustness", "auto_refine") @@ -1005,6 +1020,50 @@ _set("CLK_ROBUSTNESS_QA_PARALLEL_JUDGES", "robustness", "qa_parallel_judges", ca _set("CLK_ROBUSTNESS_MAX_QA_DEPTH", "robustness", "max_qa_depth", cast=int) _set("CLK_ROBUSTNESS_PLATEAU_WINDOW", "robustness", "plateau_window", cast=int) _set("CLK_ROBUSTNESS_PLATEAU_ACTION", "robustness", "plateau_action") +_set("CLK_ROBUSTNESS_DEBATE", "robustness", "debate") +_set("CLK_ROBUSTNESS_DEBATE_LENSES", "robustness", "debate_lenses", cast=_csv) +_set("CLK_ROBUSTNESS_DEBATE_MAX_ROUNDS", "robustness", "debate_max_rounds", cast=int) + +# Autonomous mission block +_set("CLK_MISSION_MAX_PHASES", "mission", "max_phases", cast=int) +_set("CLK_MISSION_MAX_ITERATIONS_PER_PHASE", "mission", "max_iterations_per_phase", cast=int) +_set("CLK_MISSION_MAX_TOTAL_CYCLES", "mission", "max_total_cycles", cast=int) +_set("CLK_MISSION_PHASE_GATE", "mission", "phase_gate", cast=_bool) +_set("CLK_MISSION_REFINE_REQUIRED", "mission", "refine_required", cast=_bool) +_set("CLK_MISSION_AUTO_CONSENSUS_ON_STALL", "mission", "auto_consensus_on_stall", cast=_bool) +_set("CLK_MISSION_CHARTER_FIRST", "mission", "charter_first", cast=_bool) +_set("CLK_MISSION_COMMIT_TRACE", "mission", "commit_trace", cast=_bool) +_set("CLK_MISSION_COMMIT_GRANULARITY", "mission", "commit_granularity") +_set("CLK_MISSION_MIN_CYCLES_BEFORE_DONE", "mission", "min_cycles_before_done", cast=int) +_set("CLK_MISSION_TELEMETRY_STDOUT", "mission", "telemetry_stdout", cast=_bool) +_set("CLK_MISSION_ON_BUDGET_EXHAUSTED", "mission", "on_budget_exhausted") +_set("CLK_MISSION_DEFAULT_PHASES", "mission", "default_phases", cast=_csv) + +# Done-gate block +_set("CLK_DONE_GATE_ENABLED", "done_gate", "enabled", cast=_bool) +_set("CLK_DONE_GATE_REQUIRE_TESTS_GREEN", "done_gate", "require_tests_green", cast=_bool) +_set("CLK_DONE_GATE_REQUIRE_DELIVERABLES", "done_gate", "require_deliverables", cast=_bool) +_set("CLK_DONE_GATE_MIN_DELIVERABLE_FILES", "done_gate", "min_deliverable_files", cast=int) +_set("CLK_DONE_GATE_REQUIRE_QA_PASS", "done_gate", "require_qa_pass", cast=_bool) +_set("CLK_DONE_GATE_REQUIRE_RALPH_PASS", "done_gate", "require_ralph_pass", cast=_bool) +_set("CLK_DONE_GATE_FORBID_TODO_MARKERS", "done_gate", "forbid_todo_markers", cast=_bool) +_set("CLK_DONE_GATE_MAX_FINISH_ATTEMPTS", "done_gate", "max_finish_attempts", cast=int) + +# No-op guard block +_set("CLK_NOOP_GUARD_ENABLED", "noop_guard", "enabled", cast=_bool) +_set("CLK_NOOP_GUARD_MAX_REDISPATCH", "noop_guard", "max_redispatch", cast=int) +_set("CLK_NOOP_GUARD_PRODUCING_AGENTS", "noop_guard", "producing_agents", cast=_csv) +_set("CLK_NOOP_GUARD_TREAT_OUTPUTS_STAGE_AS_PRODUCING", "noop_guard", "treat_outputs_stage_as_producing", cast=_bool) + +# Deliberation block +_set("CLK_DELIBERATION_ENABLED", "deliberation", "enabled", cast=_bool) +_set("CLK_DELIBERATION_ENCOURAGE_QUESTIONS", "deliberation", "encourage_questions", cast=_bool) +_set("CLK_DELIBERATION_REQUIRE_OPEN_QUESTIONS_RESOLVED", "deliberation", "require_open_questions_resolved", cast=_bool) +_set("CLK_DELIBERATION_SELF_REFLECT_PREAMBLE", "deliberation", "self_reflect_preamble", cast=_bool) +_set("CLK_DELIBERATION_MIN_DEBATE_ROUNDS", "deliberation", "min_debate_rounds", cast=int) + +# Validation auto-derive +_set("CLK_VALIDATION_AUTO_DERIVE", "validation", "auto_derive", cast=_bool) # Prior knobs _set("CLK_PROVIDER_TIMEOUT_S", "provider_timeout_s", cast=int) diff --git a/pi-extension/src/quality.ts b/pi-extension/src/quality.ts index 790ed17..ef12976 100644 --- a/pi-extension/src/quality.ts +++ b/pi-extension/src/quality.ts @@ -43,6 +43,14 @@ export interface ScoreOpts { * Default false so existing prompts aren't penalised retroactively. */ requireConfidence?: boolean; + /** + * When true, a response with no file-mutating ACTION block (write / edit / + * append / delete) is flagged `noop` (recoverable) — the worker described + * work instead of doing it. Mirrors the Python harness's no-op guard (FM1). + * Only applied to substantive (non-empty) responses; an empty response is + * already covered by the `empty` flag. + */ + expectFileAction?: boolean; } const CONFIDENCE_RE = /^\s*CONFIDENCE\s*:\s*([0-9]*\.?[0-9]+)\s*$/im; @@ -55,6 +63,7 @@ const REFUSAL_RES: RegExp[] = [ /\bI\s+do\s+not\s+have\s+the\s+ability\b/i, ]; const HEADER_ACTION_RE = /^\s*ACTION\s*:\s*([A-Za-z]+)/gim; +const FILE_ACTION_RE = /^\s*ACTION\s*:\s*(write|edit|append|delete)\b/im; const END_ACTION_RE = /^\s*END_ACTION\s*$/gim; const POST_HEAD_RE = /^\s*POST\s*:\s*([A-Za-z][A-Za-z0-9_]*)\s*$/gim; const POST_END_RE = /^\s*END_POST\s*$/gim; @@ -135,6 +144,7 @@ export function scoreResponse( const minChars = opts.minChars ?? 40; const expected = opts.expectedOutputs ?? []; const requireConfidence = opts.requireConfidence ?? false; + const expectFileAction = opts.expectFileAction ?? false; const raw = text ?? ""; const body = raw.trim(); @@ -197,6 +207,15 @@ export function scoreResponse( "comma-separated on a single line.", ); } + // No-op guard (FM1): a substantive response that emitted no file-mutating + // ACTION block. An empty body is left to the `empty` flag above. + if (expectFileAction && body.length >= Math.max(1, minChars) && !FILE_ACTION_RE.test(raw)) { + flags.push("noop"); + reasons.push( + "Your response changed no files — it contained no ACTION:write/edit/append/delete " + + "block. Descriptions do nothing here. Emit at least one real file-mutating ACTION block.", + ); + } if (confidence !== undefined && confidence < 0.5) { flags.push("low_confidence"); reasons.push( @@ -222,6 +241,7 @@ export function scoreResponse( malformed_action: 0.4, malformed_post: 0.3, outputs_missing: 0.4, + noop: 0.5, low_confidence: 0.3, needs_review_self: 0.2, confidence_missing: 0.1, diff --git a/pi-extension/tests/quality.test.ts b/pi-extension/tests/quality.test.ts index cf5ab77..860398d 100644 --- a/pi-extension/tests/quality.test.ts +++ b/pi-extension/tests/quality.test.ts @@ -14,6 +14,36 @@ import { progressSignal, } from "../src/quality.ts"; +describe("scoreResponse — no-op guard (FM1)", () => { + const prose = + "I would build the parser module and then add tests for it. This plan is " + + "detailed enough to clear the minimum character threshold comfortably."; + + test("substantive prose with no file action is flagged noop when expected", () => { + const q = scoreResponse(prose, { expectFileAction: true }); + assert.ok(q.flags.includes("noop")); + assert.equal(q.ok, false); + assert.equal(isRecoverable(q), true); + }); + + test("a real file-mutating ACTION clears the noop flag", () => { + const text = "ACTION: write\nPATH: src/app.ts\nCONTENT:\nconsole.log(1);\nEND_ACTION"; + const q = scoreResponse(text, { expectFileAction: true }); + assert.ok(!q.flags.includes("noop")); + }); + + test("noop is not raised when not expected", () => { + const q = scoreResponse(prose); + assert.ok(!q.flags.includes("noop")); + }); + + test("empty response gets `empty`, not `noop`", () => { + const q = scoreResponse("", { expectFileAction: true }); + assert.ok(q.flags.includes("empty")); + assert.ok(!q.flags.includes("noop")); + }); +}); + describe("scoreResponse — happy paths", () => { test("substantive prose passes with score 1.0", () => { const text = "This is a substantive response covering the requested work in detail. " + diff --git a/tests/test_debate.py b/tests/test_debate.py new file mode 100644 index 0000000..c901150 --- /dev/null +++ b/tests/test_debate.py @@ -0,0 +1,77 @@ +"""Unit tests for the adversarial debate-panel refinement (offline).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from clk_harness.config import Paths +from clk_harness.orchestration.workflow import WorkflowRunner, WorkflowStage + + +class _StubRunner: + """Minimal stand-in for AgentRunner — debate gating only reads these.""" + + def __init__(self, clk_cfg, agents_cfg=None): + self.clk_cfg = clk_cfg + self.agents_cfg = agents_cfg or {"agents": {"critic": {}}} + + +def _runner(tmp_path: Path, debate="careful_only", lenses=None) -> WorkflowRunner: + robustness = {"debate": debate} + if lenses is not None: + robustness["debate_lenses"] = lenses + p = Paths(root=tmp_path) + return WorkflowRunner(p, _StubRunner({"robustness": robustness})) + + +def _stage(agent="engineer", careful=False, refine=None) -> WorkflowStage: + return WorkflowStage(id="implement", agent=agent, objective="do it", + careful=careful, refine=refine) + + +def test_debate_careful_only_default(tmp_path): + wr = _runner(tmp_path, debate="careful_only") + assert wr._debate_enabled(_stage(careful=True)) + assert not wr._debate_enabled(_stage(careful=False)) + + +def test_debate_all(tmp_path): + wr = _runner(tmp_path, debate="all") + assert wr._debate_enabled(_stage(agent="engineer")) + # prose/verdict agents never debate + assert not wr._debate_enabled(_stage(agent="chief")) + assert not wr._debate_enabled(_stage(agent="qa")) + assert not wr._debate_enabled(_stage(agent="critic")) + + +def test_debate_off(tmp_path): + wr = _runner(tmp_path, debate="off") + assert not wr._debate_enabled(_stage(careful=True)) + + +def test_debate_explicit_mode_overrides_global_off(tmp_path): + wr = _runner(tmp_path, debate="off") + assert wr._debate_enabled(_stage(refine={"mode": "debate"})) + + +def test_debate_lenses_default(tmp_path): + wr = _runner(tmp_path) + assert wr._debate_lenses(_stage()) == ["correctness", "security", "simplicity"] + + +def test_debate_lenses_config_override(tmp_path): + wr = _runner(tmp_path, lenses=["correctness", "performance"]) + assert wr._debate_lenses(_stage()) == ["correctness", "performance"] + + +def test_debate_lenses_stage_override(tmp_path): + wr = _runner(tmp_path) + stage = _stage(refine={"mode": "debate", "critics": ["Security", "tests"]}) + assert wr._debate_lenses(stage) == ["security", "tests"] + + +def test_lens_guidance_table_has_core_lenses(): + for lens in ("correctness", "security", "simplicity", "performance"): + assert lens in WorkflowRunner._DEBATE_LENS_GUIDANCE diff --git a/tests/test_docs_consistency.py b/tests/test_docs_consistency.py index e02bcac..e9faafa 100644 --- a/tests/test_docs_consistency.py +++ b/tests/test_docs_consistency.py @@ -134,6 +134,45 @@ def test_readme_documents_robustness_keys() -> None: ) +# --------------------------------------------------------------------------- +# New config blocks: env-var + kickoff parity +# --------------------------------------------------------------------------- +# The autonomy blocks must be overridable from the environment the same way +# robustness is: every key gets a CLK__ var in .env.example AND a +# matching mapping in kickoff.sh, so an override actually takes effect. + +_AUTONOMY_BLOCKS = ("mission", "done_gate", "noop_guard", "deliberation") + + +def _block_env_var(block: str, key: str) -> str: + return "CLK_" + block.upper() + "_" + key.upper() + + +def test_env_example_documents_new_blocks() -> None: + for block in _AUTONOMY_BLOCKS: + assert block in DEFAULT_CLK_CONFIG, f"DEFAULT_CLK_CONFIG missing '{block}'" + for key in DEFAULT_CLK_CONFIG[block].keys(): + var = _block_env_var(block, key) + assert var in ENV_EXAMPLE, ( + f"{var} not documented in .env.example (DEFAULT_CLK_CONFIG" + f"['{block}']['{key}'] is a public knob)" + ) + + +def test_kickoff_sh_maps_new_blocks() -> None: + for block in _AUTONOMY_BLOCKS: + for key in DEFAULT_CLK_CONFIG[block].keys(): + var = _block_env_var(block, key) + assert var in KICKOFF, ( + f"{var} not handled in kickoff.sh's env→config mapping" + ) + + +def test_validation_auto_derive_wired() -> None: + assert "CLK_VALIDATION_AUTO_DERIVE" in ENV_EXAMPLE + assert "CLK_VALIDATION_AUTO_DERIVE" in KICKOFF + + # --------------------------------------------------------------------------- # Prior-knob parity (just an inventory check) # --------------------------------------------------------------------------- diff --git a/tests/test_done_gate.py b/tests/test_done_gate.py new file mode 100644 index 0000000..cb40140 --- /dev/null +++ b/tests/test_done_gate.py @@ -0,0 +1,162 @@ +"""Unit tests for the machine-checkable done-gate (offline, no provider).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from clk_harness.config import Paths +from clk_harness.orchestration import blackboard as bb +from clk_harness.orchestration.done_gate import evaluate_done_gate, deliverable_files +from clk_harness.orchestration.evaluator import EvalResult, CheckResult +from clk_harness.orchestration.charter import Charter, derive_done_criteria + + +def _paths(tmp_path: Path) -> Paths: + p = Paths(root=tmp_path) + p.ensure() + return p + + +def _full_cfg(**overrides): + gate = { + "enabled": True, + "require_tests_green": True, + "require_deliverables": True, + "min_deliverable_files": 1, + "require_qa_pass": True, + "require_ralph_pass": True, + "forbid_todo_markers": False, + "max_finish_attempts": 5, + } + gate.update(overrides) + return {"done_gate": gate} + + +def _satisfy_all(paths: Paths) -> EvalResult: + """Make every default criterion pass and return a green eval result.""" + (paths.root / "app.py").write_text("print('hi')\n", encoding="utf-8") + bb.post(paths, author="qa", body="PASS — all checks green", post_type="qa") + (paths.state / "experiments.jsonl").write_text( + json.dumps({"index": 1, "improved": True}) + "\n", encoding="utf-8" + ) + return EvalResult(ok=True, checks=[CheckResult(command="pytest", ok=True, rc=0, output="")]) + + +def test_all_criteria_satisfied_passes(tmp_path): + paths = _paths(tmp_path) + eval_result = _satisfy_all(paths) + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result) + assert verdict.passed, verdict.failures + assert verdict.failures == [] + + +def test_tests_red_fails(tmp_path): + paths = _paths(tmp_path) + _satisfy_all(paths) + red = EvalResult(ok=False, checks=[CheckResult(command="pytest", ok=False, rc=1, output="boom")]) + verdict = evaluate_done_gate(paths, _full_cfg(), red) + assert not verdict.passed + assert "tests_red" in verdict.failures + + +def test_weak_eval_relaxes_tests_green(tmp_path): + paths = _paths(tmp_path) + _satisfy_all(paths) + weak = EvalResult(ok=False, checks=[], weak=True) + verdict = evaluate_done_gate(paths, _full_cfg(), weak) + # Adaptive: a weak gate (no real test command) must not block on tests. + assert "tests_red" not in verdict.failures + assert verdict.passed, verdict.failures + + +def test_missing_qa_pass_fails(tmp_path): + paths = _paths(tmp_path) + (paths.root / "app.py").write_text("x=1\n", encoding="utf-8") + (paths.state / "experiments.jsonl").write_text("{}\n", encoding="utf-8") + eval_result = EvalResult(ok=True, checks=[CheckResult("pytest", True, 0, "")]) + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result) + assert not verdict.passed + assert "no_qa_pass" in verdict.failures + + +def test_qa_fail_body_does_not_count_as_pass(tmp_path): + paths = _paths(tmp_path) + _satisfy_all(paths) + bb.post(paths, author="qa", body="FAIL — 2 tests broken", post_type="qa") + eval_result = EvalResult(ok=True, checks=[CheckResult("pytest", True, 0, "")]) + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result) + assert "no_qa_pass" in verdict.failures + + +def test_missing_ralph_pass_fails(tmp_path): + paths = _paths(tmp_path) + (paths.root / "app.py").write_text("x=1\n", encoding="utf-8") + bb.post(paths, author="qa", body="PASS", post_type="qa") + eval_result = EvalResult(ok=True, checks=[CheckResult("pytest", True, 0, "")]) + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result) + assert "no_ralph_pass" in verdict.failures + + +def test_no_deliverables_fails(tmp_path): + paths = _paths(tmp_path) + bb.post(paths, author="qa", body="PASS", post_type="qa") + (paths.state / "experiments.jsonl").write_text("{}\n", encoding="utf-8") + eval_result = EvalResult(ok=True, checks=[CheckResult("pytest", True, 0, "")]) + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result) + assert "no_deliverables" in verdict.failures + + +def test_each_require_flag_disables_its_check(tmp_path): + paths = _paths(tmp_path) + # Nothing satisfied; turn every requirement off -> should pass. + eval_result = EvalResult(ok=False, checks=[], weak=True) + cfg = _full_cfg( + require_tests_green=False, + require_deliverables=False, + require_qa_pass=False, + require_ralph_pass=False, + ) + verdict = evaluate_done_gate(paths, cfg, eval_result) + assert verdict.passed, verdict.failures + + +def test_gate_disabled_always_passes(tmp_path): + paths = _paths(tmp_path) + verdict = evaluate_done_gate(paths, {"done_gate": {"enabled": False}}, None) + assert verdict.passed + + +def test_todo_markers_when_enabled(tmp_path): + paths = _paths(tmp_path) + eval_result = _satisfy_all(paths) + (paths.root / "app.py").write_text("# TODO: finish this\nx=1\n", encoding="utf-8") + verdict = evaluate_done_gate(paths, _full_cfg(forbid_todo_markers=True), eval_result) + assert "todo_markers" in verdict.failures + + +def test_charter_file_criterion(tmp_path): + paths = _paths(tmp_path) + eval_result = _satisfy_all(paths) + charter = Charter(success_criteria=["report.md documents the findings"]) + criteria = derive_done_criteria(charter) + assert criteria and criteria[0]["type"] == "file" and criteria[0]["value"] == "report.md" + # Without report.md present, the charter criterion fails the gate. + verdict = evaluate_done_gate(paths, _full_cfg(), eval_result, extra_criteria=criteria) + assert any(f.startswith("charter:") for f in verdict.failures) + # Create it -> passes. + (paths.root / "report.md").write_text("findings\n", encoding="utf-8") + verdict2 = evaluate_done_gate(paths, _full_cfg(), eval_result, extra_criteria=criteria) + assert verdict2.passed, verdict2.failures + + +def test_deliverable_files_excludes_state(tmp_path): + paths = _paths(tmp_path) + (paths.root / "real.py").write_text("x=1\n", encoding="utf-8") + (paths.root / "PROGRESS.md").write_text("notes\n", encoding="utf-8") + files = deliverable_files(paths.root) + assert "real.py" in files + assert "PROGRESS.md" not in files + assert not any(f.startswith(".clk") for f in files) diff --git a/tests/test_mission_living_plan.py b/tests/test_mission_living_plan.py new file mode 100644 index 0000000..0d4fad3 --- /dev/null +++ b/tests/test_mission_living_plan.py @@ -0,0 +1,125 @@ +"""Unit tests for the mission living-plan, charter, and derived validation.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from clk_harness.config import Paths +from clk_harness.orchestration.mission import ( + MissionPlan, PhaseSpec, load_plan, save_plan, +) +from clk_harness.orchestration.charter import Charter, load_charter, save_charter +from clk_harness.orchestration.evaluator import Evaluator, EvalResult, derive_validation + + +def _paths(tmp_path: Path) -> Paths: + p = Paths(root=tmp_path) + p.ensure() + return p + + +def test_plan_roundtrip(tmp_path): + paths = _paths(tmp_path) + plan = MissionPlan( + objective="build x", + title="build x", + phases=[ + PhaseSpec(id="discovery", engine="workflow", order=0, + exit_criteria=["brief written"]), + PhaseSpec(id="engineering", engine="workflow", order=1), + ], + ) + save_plan(paths, plan) + assert (paths.state / "mission.json").exists() + assert (paths.state / "MISSION.md").exists() + loaded = load_plan(paths) + assert loaded is not None + assert [p.id for p in loaded.phases] == ["discovery", "engineering"] + assert loaded.phases[0].exit_criteria == ["brief written"] + + +def test_next_pending_and_all_done(): + plan = MissionPlan(phases=[ + PhaseSpec(id="a", order=0, status="done"), + PhaseSpec(id="b", order=1, status="pending"), + PhaseSpec(id="c", order=2, status="pending"), + ]) + assert plan.next_pending().id == "b" + assert not plan.all_done() + for p in plan.phases: + p.status = "done" + assert plan.next_pending() is None + assert plan.all_done() + + +def test_plan_from_dict_defaults(): + raw = {"objective": "o", "phases": [{"id": "x"}]} + plan = MissionPlan.from_dict(raw) + assert plan.phases[0].engine == "workflow" + assert plan.phases[0].workflow == "x" # defaults to id + + +def test_charter_roundtrip(tmp_path): + paths = _paths(tmp_path) + charter = Charter( + objective="o", + mission_statement="ship it", + scope=["a", "b"], + success_criteria=["tests pass", "README.md exists"], + ) + save_charter(paths, charter) + assert (paths.state / "charter.json").exists() + assert (paths.state / "CHARTER.md").exists() + loaded = load_charter(paths) + assert loaded is not None + assert loaded.mission_statement == "ship it" + assert loaded.success_criteria == ["tests pass", "README.md exists"] + + +def test_evaluator_empty_checks_not_vacuous_pass(tmp_path): + # No checks + auto_derive off -> must NOT silently pass. + ev = Evaluator(root=tmp_path, default_checks=[], auto_derive=False) + result = ev.run() + assert result.ok is False + assert result.weak is True + + +def test_evaluator_runs_real_check(tmp_path): + ev = Evaluator(root=tmp_path, default_checks=["true"]) + assert ev.run().ok is True + ev2 = Evaluator(root=tmp_path, default_checks=["false"]) + assert ev2.run().ok is False + + +def test_derive_validation_pytest(tmp_path): + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_smoke.py").write_text("def test_x():\n assert True\n") + cmds, weak = derive_validation(tmp_path) + assert any("pytest" in c for c in cmds) + assert weak is False + + +def test_derive_validation_npm(tmp_path): + (tmp_path / "package.json").write_text( + json.dumps({"scripts": {"test": "jest"}}), encoding="utf-8" + ) + cmds, weak = derive_validation(tmp_path) + assert any("npm test" in c for c in cmds) + assert weak is False + + +def test_derive_validation_weak_fallback(tmp_path): + # Bare dir with no tests / package.json -> weak gate. + cmds, weak = derive_validation(tmp_path) + assert cmds # always returns something runnable + assert weak is True + + +def test_evaluator_auto_derive_weak_when_no_tests(tmp_path): + ev = Evaluator(root=tmp_path, default_checks=[], auto_derive=True) + result = ev.run() + # weak smoke (test -d .) passes but is flagged weak. + assert result.weak is True diff --git a/tests/test_noop_guard.py b/tests/test_noop_guard.py new file mode 100644 index 0000000..9e680b0 --- /dev/null +++ b/tests/test_noop_guard.py @@ -0,0 +1,80 @@ +"""Unit tests for the no-op guard (FM1) and its response-quality flag.""" + +from __future__ import annotations + +import pytest + +from clk_harness.orchestration import noop_guard +from clk_harness.orchestration.response_quality import score + + +_CFG = {"noop_guard": {"enabled": True, "max_redispatch": 2, + "producing_agents": ["engineer", "ralph"], + "treat_outputs_stage_as_producing": True}} + + +def test_producing_agent_expected_to_mutate(): + assert noop_guard.is_mutation_expected("engineer", cfg=_CFG) + assert noop_guard.is_mutation_expected("ralph", cfg=_CFG) + + +def test_prose_only_agents_not_expected(): + assert not noop_guard.is_mutation_expected("chief", cfg=_CFG) + assert not noop_guard.is_mutation_expected("qa", cfg=_CFG) + assert not noop_guard.is_mutation_expected("critic", cfg=_CFG) + + +def test_outputs_contract_makes_stage_producing(): + assert noop_guard.is_mutation_expected("analyst", outputs=["brief"], cfg=_CFG) + assert not noop_guard.is_mutation_expected("analyst", outputs=[], cfg=_CFG) + + +def test_commit_stage_is_producing_for_unknown_role(): + assert noop_guard.is_mutation_expected("designer", commit=True, cfg=_CFG) + # but chief/qa/critic stay prose-only even with commit + assert not noop_guard.is_mutation_expected("qa", commit=True, cfg=_CFG) + + +def test_disabled_guard_never_expects(): + cfg = {"noop_guard": {"enabled": False}} + assert not noop_guard.is_mutation_expected("engineer", cfg=cfg) + assert noop_guard.max_redispatch(cfg) == 0 + + +def test_repair_preamble_escalates(): + a1 = noop_guard.repair_preamble(1) + a2 = noop_guard.repair_preamble(2) + a3 = noop_guard.repair_preamble(3) + assert "changed NO files" in a1 + assert "Worked example" in a2 or "worked example" in a2.lower() + assert "FINAL ATTEMPT" in a3 + # later attempts get progressively more forceful / different text + assert a1 != a2 != a3 + + +def test_max_redispatch_reads_config(): + assert noop_guard.max_redispatch(_CFG) == 2 + + +def test_score_flags_noop_on_prose(): + q = score( + "I would create the parser and then add tests. This is a solid plan " + "with plenty of detail to exceed the minimum character threshold.", + expect_file_action=True, + ) + assert "noop" in q.flags + assert not q.ok + assert q.recoverable # worth re-dispatching + + +def test_score_no_noop_with_real_action(): + q = score( + "ACTION: write\nPATH: src/app.py\nCONTENT:\nprint('hi')\nEND_ACTION", + expect_file_action=True, + ) + assert "noop" not in q.flags + + +def test_score_noop_not_flagged_when_not_expected(): + q = score("Just a prose answer, no actions, but not expected to mutate either.") + assert "noop" not in q.flags diff --git a/tests/test_robustness_integration.py b/tests/test_robustness_integration.py index 2833e9b..cdb8e0d 100644 --- a/tests/test_robustness_integration.py +++ b/tests/test_robustness_integration.py @@ -199,6 +199,40 @@ def _make_runner(paths: Paths, provider: _FakeProvider, clk_cfg_overrides: Dict[ return runner +def test_debate_loop_runs_panel_and_revises(paths: Paths) -> None: + """The debate panel fans out one critic per lens, then revises the worker + when the panel asks for it, and accepts on the next round.""" + from clk_harness.orchestration.agent import AgentRun + + # 3 lenses revise (round 1) -> 1 worker revision -> 3 lenses accept (round 2). + responses = ( + [AgentResponse(ok=True, text="Issue found.\nVERDICT: revise\nSCORE: 0.4")] * 3 + + [AgentResponse(ok=True, text="Revised the work substantively per the panel.")] + + [AgentResponse(ok=True, text="Looks good now.\nVERDICT: accept\nSCORE: 0.9")] * 3 + ) + provider = _FakeProvider(responses) + runner = _make_runner(paths, provider, { + # sequential fan-out so the fake provider's call list isn't raced + "consensus": {**DEFAULT_CLK_CONFIG["consensus"], "max_parallel": 1}, + "robustness": {**DEFAULT_CLK_CONFIG["robustness"], "debate": "careful_only", + "debate_max_rounds": 2, "auto_refine": "off"}, + }) + wr = wf.WorkflowRunner(paths, runner) + flow = wf.Workflow(name="engineering", description="", stages=[]) + stage = wf.WorkflowStage(id="implement", agent="engineer", objective="build it", + careful=True) + assert wr._debate_enabled(stage) + first = AgentRun(agent="engineer", objective="build it", + response=AgentResponse(ok=True, text="initial work"), + started_at="t0", finished_at="t1") + final = wr._debate_loop(flow, stage, first, "cycle 1/1", dry_run=False) + assert final.response.ok + # 3 critics (r1) + 1 worker revision + 3 critics (r2) = 7 dispatches. + assert len(provider.calls) == 7, [c["agent"] for c in provider.calls] + # The worker (engineer) was re-dispatched for the revision. + assert any(c["agent"] == "engineer" for c in provider.calls) + + def test_quality_retry_fires_on_empty_response(paths: Paths) -> None: # First call: empty (should trigger retry). Second call: substantive. good = "Here is a substantive engineering plan with concrete steps." @@ -212,7 +246,10 @@ def test_quality_retry_fires_on_empty_response(paths: Paths) -> None: "auto_consensus": "off", "auto_refine": "off", "max_quality_retries": 1, - } + }, + # Isolate the quality-retry path: the no-op guard would otherwise + # re-dispatch the (action-less) engineer response on its own. + "noop_guard": {"enabled": False}, }) run = runner.run("engineer", "Implement feature X.") assert run.response.text == good diff --git a/user_tests/test_mission_e2e.py b/user_tests/test_mission_e2e.py new file mode 100644 index 0000000..f279818 --- /dev/null +++ b/user_tests/test_mission_e2e.py @@ -0,0 +1,129 @@ +"""End-to-end tests for the autonomous mission (shell provider, offline). + +The shell provider echoes prompts and emits no ACTION blocks, so it is ideal +for verifying the *gates* fire: with no real deliverables / qa pass / ralph +pass, the done-gate must REJECT completion (no ``done_granted.md``), proving +premature done is blocked. We also verify the charter-first ordering, the +living-plan artifacts, the per-cycle telemetry event, and the commit trace. + +Everything is capped hard (single phase, one total cycle, refinement off) so +the suite stays fast without any network access. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Any, Dict, List + +import pytest + +from .conftest import run_clk + + +def _read_jsonl(path: Path) -> List[Dict[str, Any]]: + if not path.exists(): + return [] + out = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line: + try: + out.append(json.loads(line)) + except json.JSONDecodeError: + pass + return out + + +def _patch_config(proj: Path, **overrides: Any) -> None: + cfg_path = proj / ".clk" / "config" / "clk.config.json" + cfg = json.loads(cfg_path.read_text(encoding="utf-8")) + for k, v in overrides.items(): + if isinstance(v, dict) and isinstance(cfg.get(k), dict): + cfg[k].update(v) + else: + cfg[k] = v + cfg_path.write_text(json.dumps(cfg, indent=2), encoding="utf-8") + + +def _fast_mission(proj: Path) -> None: + """Cap the mission to one short phase/cycle with refinement off (speed).""" + _patch_config( + proj, + mission={"default_phases": ["engineering"], "max_total_cycles": 1, + "max_iterations_per_phase": 1, "telemetry_stdout": True}, + supervise={"max_cycles": 1}, + robustness={"auto_refine": "off", "auto_consensus": "off", + "max_quality_retries": 0, "plateau_action": "off"}, + meta_prompt={"dispatch": "off", "role": "off"}, + deliberation={"enabled": False}, + ) + + +def _git_log(proj: Path) -> List[str]: + r = subprocess.run(["git", "log", "--oneline"], cwd=str(proj), + capture_output=True, text=True) + return [l.strip() for l in r.stdout.splitlines() if l.strip()] + + +def test_mission_help_lists_subcommand(initialized_project: Path): + res = run_clk("mission", "--help", cwd=initialized_project) + assert res.returncode == 0 + assert "objective" in res.stdout.lower() + + +def test_mission_dry_run_bootstraps_plan(initialized_project: Path): + proj = initialized_project + _fast_mission(proj) + res = run_clk("mission", "build a tiny tool", "--dry-run", cwd=proj, timeout=180) + assert res.returncode == 0, res.stderr + assert (proj / ".clk" / "state" / "mission.json").exists() + assert (proj / ".clk" / "state" / "CHARTER.md").exists() + assert (proj / ".clk" / "state" / "MISSION.md").exists() + plan = json.loads((proj / ".clk" / "state" / "mission.json").read_text()) + assert [p["id"] for p in plan["phases"]] == ["engineering"] + + +def test_mission_blocks_premature_done(initialized_project: Path): + proj = initialized_project + _fast_mission(proj) + res = run_clk("mission", "build a tiny tool", "--max-cycles", "1", + cwd=proj, timeout=600) + # With the shell provider there is no qa PASS / ralph pass / deliverable, + # so the done-gate must NOT grant completion. + assert not (proj / ".clk" / "state" / "done_granted.md").exists() + plan_path = proj / ".clk" / "state" / "mission.json" + assert plan_path.exists() + plan = json.loads(plan_path.read_text()) + assert plan["status"] in ("stalled", "running") + # Telemetry event was emitted for the cycle. + events = _read_jsonl(proj / ".clk" / "logs" / "activity.jsonl") + assert any(e.get("event") == "loop_cycle_summary" for e in events) + + +def test_mission_charter_before_plan_in_trace(initialized_project: Path): + proj = initialized_project + _fast_mission(proj) + run_clk("mission", "build a tiny tool", "--max-cycles", "1", cwd=proj, timeout=600) + log = _git_log(proj) + charter_idx = next((i for i, l in enumerate(log) if "[clk:charter]" in l), None) + plan_idx = next((i for i, l in enumerate(log) if "[clk:plan]" in l), None) + assert charter_idx is not None, f"no charter commit in: {log}" + assert plan_idx is not None, f"no plan commit in: {log}" + # git log is newest-first, so the charter (older) has a HIGHER index. + assert charter_idx > plan_idx, f"charter should precede plan: {log}" + + +def test_mission_writes_plan_post_to_blackboard(initialized_project: Path): + proj = initialized_project + _fast_mission(proj) + run_clk("mission", "build a tiny tool", "--max-cycles", "1", cwd=proj, timeout=600) + bb = proj / ".clk" / "blackboard" + kinds = [] + for f in bb.glob("*.json"): + try: + kinds.append(json.loads(f.read_text()).get("post_type")) + except Exception: + pass + assert "plan" in kinds or "charter" in kinds diff --git a/user_tests/test_orchestration_e2e.py b/user_tests/test_orchestration_e2e.py index ba4f0ed..29bc2cc 100644 --- a/user_tests/test_orchestration_e2e.py +++ b/user_tests/test_orchestration_e2e.py @@ -257,7 +257,7 @@ def test_engineering_workflow_dispatches_all_four_stages( """ _patch_config(initialized_project, supervise={"max_cycles": 1}) - res = run_clk("run", cwd=initialized_project) + res = run_clk("run", "--once", cwd=initialized_project) assert res.returncode == 0, res.stderr agents = _agent_names_in_memory(initialized_project) @@ -276,7 +276,7 @@ def test_engineering_workflow_creates_run_dirs_per_stage( """ _patch_config(initialized_project, supervise={"max_cycles": 1}) - run_clk("run", cwd=initialized_project) + run_clk("run", "--once", cwd=initialized_project) run_dirs = list((initialized_project / ".clk" / "runs").iterdir()) agent_run_dirs = [d for d in run_dirs if d.is_dir() and d.name != "shell-stubs"] @@ -291,7 +291,7 @@ def test_engineering_workflow_prompt_and_response_files_present( """Each run dir contains prompt.txt, response.txt, and meta.json.""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - run_clk("run", cwd=initialized_project) + run_clk("run", "--once", cwd=initialized_project) run_dirs = [ d for d in (initialized_project / ".clk" / "runs").iterdir() @@ -310,7 +310,7 @@ def test_engineering_workflow_stage_ordering_preserved( """agent_memory.jsonl entries appear in depends_on order: chief before engineer.""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - run_clk("run", cwd=initialized_project) + run_clk("run", "--once", cwd=initialized_project) mem = _read_jsonl(initialized_project / ".clk" / "state" / "agent_memory.jsonl") agents_seq = [r["agent"] for r in mem] @@ -331,7 +331,7 @@ def test_engineering_workflow_stdout_summarises_results( """CLI prints a stage summary line (ok count, fail count, total).""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - res = run_clk("run", cwd=initialized_project) + res = run_clk("run", "--once", cwd=initialized_project) assert res.returncode == 0, res.stderr assert "engineering:" in res.stdout, f"expected workflow summary; got: {res.stdout!r}" @@ -348,7 +348,7 @@ def test_supervise_stage_runs_as_chief_agent(self, initialized_project: Path) -> """The supervise stage is dispatched to the chief (appears in agent_memory ≥2×).""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - run_clk("run", cwd=initialized_project) + run_clk("run", "--once", cwd=initialized_project) # Chief runs for: (a) cast stage, (b) supervise stage. agents = _agent_names_in_memory(initialized_project) @@ -363,7 +363,7 @@ def test_supervise_stage_terminates_after_max_cycles( """With max_cycles=1 the workflow terminates without done.md (shell can't write it).""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - res = run_clk("run", cwd=initialized_project) + res = run_clk("run", "--once", cwd=initialized_project) # The workflow runner emits a WARN about cycle limit but still returns 0 assert res.returncode == 0, res.stderr # done.md should NOT exist because the shell provider cannot write it @@ -410,7 +410,7 @@ def test_recovery_dispatch_fires_chief_when_dep_fails( _patch_config(initialized_project, provider_retry={"max_retries": 0, "stage_max_retries": 0}) self._write_failing_workflow(initialized_project) - res = run_clk("run", cwd=initialized_project) + res = run_clk("run", "--once", cwd=initialized_project) # The workflow may exit non-zero (provider failure) or 0 (skipped dep) # but recovery dispatch means chief appears in agent_memory agents = _agent_names_in_memory(initialized_project) @@ -430,7 +430,7 @@ class TestBlackboardIntegration: def test_blackboard_dir_created_after_run(self, initialized_project: Path) -> None: """The .clk/blackboard/ directory exists after any non-dry workflow run.""" _patch_config(initialized_project, supervise={"max_cycles": 1}) - run_clk("run", cwd=initialized_project) + run_clk("run", "--once", cwd=initialized_project) bb = initialized_project / ".clk" / "blackboard" assert bb.is_dir(), ".clk/blackboard/ should exist after a workflow run"