From ef2e8e14d23daee93c85f3cdc226b0f6cca084b5 Mon Sep 17 00:00:00 2001 From: denis Date: Sun, 24 May 2026 10:49:00 +0100 Subject: [PATCH 1/5] feat(arena): PoC script for skills arena methodology validation (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the skills arena roadmap (per .ievo/research/2026-05-24-skills-arena.md). Standalone script that runs ONE controlled-experiment arena round to validate the A/B eval methodology before investing in full implementation. ## Controlled experiment design - Candidate A: full production-style SKILL.md (verbose ievo:init orchestrator with 7 steps, pre-flight, stack detection, security audit, install, summary, journal) - Candidate B: deliberately stripped variant of the same skill (3-line body, no steps, no failure handling) - Same task: "you just ran /ievo:init in a Next.js+TS project, walk through every step" - Ground truth: A should win. If arena says B or tie, methodology has a bug. ## Pipeline 1. Run each candidate K times (default 3) with Haiku, passing SKILL.md as system prompt 2. Pick representative output per candidate (longest non-empty — verbosity bias is a known issue, Phase 5 of roadmap adds correction) 3. Opus judge sees both outputs side-by-side, picks winner + emits JSON verdict 4. Position-swap: re-run judge with outputs reversed 5. Consistent-winner only counts (mitigates first-position bias — GPT-4 30%, Claude-v1 75% per MT-Bench) ## Reuses - `cortex.claude.query_raw` — same machinery as cortex evolution A/B eval (task #31, shipped v0.5.2) - Methodology validated in `.ievo/research/2026-03-06-blind-ab-eval.md` ## Configuration via env - `ARENA_RUNS_PER` (default 3) — Haiku runs per candidate - `ARENA_RUNNER_MODEL` (default haiku) — model for candidate runs - `ARENA_JUDGE_MODEL` (default opus) — judge model ## Output - Stdout: human-readable report with both verdicts + consistency check - `dist/arena-poc/round-A-vs-B.json` — full transcripts + cost data - Exit codes: 0 = A won (methodology validated), 1 = B won (unexpected — investigate), 2 = tie, 3 = position-bias inconsistent ## Cost ~$0.50-1.00 per round (3 Haiku × 2 candidates + 2 Opus judges). Requires CLAUDE_CODE_OAUTH_TOKEN in env. ## Out of scope (deferred to Phase 2+ of roadmap) - Real sandbox installer (current PoC inlines SKILL.md content as system prompt; production version installs via `npx skills add`) - Token-length anti-verbosity correction - Author/owner anonymization at the metadata level (judge prompt already strips author info) - Real candidate pairs from skills.sh - Multi-candidate (N>2) round-robin or tournament logic - `/ievo:arena ` CLI integration - Periodic re-rank workflow in cortex CI ## Next step after PoC runs successfully If consistent_winner = production: methodology validated → Phase 2 (real sandbox installer). If unexpected: investigate Haiku output patterns or judge prompt before scaling. Co-Authored-By: iEVO --- scripts/skills_arena_poc.py | 507 ++++++++++++++++++++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 scripts/skills_arena_poc.py diff --git a/scripts/skills_arena_poc.py b/scripts/skills_arena_poc.py new file mode 100644 index 0000000..d2da553 --- /dev/null +++ b/scripts/skills_arena_poc.py @@ -0,0 +1,507 @@ +#!/usr/bin/env -S uv run python +"""Skills Arena PoC — methodology validation for godfather task #28. + +Controlled experiment: two skill candidates competing on the same task — +candidate A is a real production SKILL.md (well-designed), candidate B is +a deliberately stripped-down variant of the same skill. If the arena +correctly ranks A > B, the methodology produces sensible quality signals. + +Pipeline per round: + 1. For each candidate: run the same task K times with Haiku, passing + the SKILL.md content as system prompt + 2. Pick the most representative output per candidate (longest non-empty + run, picking the model's best-effort attempt) + 3. Opus judge sees both outputs side-by-side and picks a winner + 4. Position swap — re-run the judge with outputs in opposite order; + only a CONSISTENT winner counts (mitigates first-position bias) + 5. Report ranking + reasoning + cost + +Reuses cortex.claude.query_raw — same machinery used by cortex evolution +A/B eval (task #31, shipped v0.5.2). + +Usage: + uv run python scripts/skills_arena_poc.py + +Cost: ~$0.50-1.00 per round (3 Haiku × 2 candidates + 2 Opus judges). +Requires CLAUDE_CODE_OAUTH_TOKEN (or ANTHROPIC_API_KEY) in env. + +Research reference: + .ievo/research/2026-05-24-skills-arena.md (godfather) + .ievo/research/2026-03-06-blind-ab-eval.md (methodology validation) +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import sys +from dataclasses import asdict, dataclass, field +from pathlib import Path + +# Inject cortex src into path so script runs without install +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from cortex.claude import query_raw # noqa: E402 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +log = logging.getLogger("arena_poc") + + +# ============================================================================ +# CANDIDATES — controlled experiment with known-good vs deliberately-weak pair +# ============================================================================ + +# Candidate A — full production SKILL.md content (deliberately verbose for +# the controlled experiment; this is approximately what the real +# ievo init skill looks like in detail). +CANDIDATE_A = { + "id": "ievo-init-full", + "label": "ievo:init (production)", + "owner": "ievo-ai", + "skill_md": """--- +name: ievo-init +description: Bootstrap iEvo in a project — discover candidates, security audit, install. Use when starting a new project or first-time iEvo install. +--- + +# iEvo init orchestrator + +When the user runs `/ievo:init` in their project, execute these steps in +order. Do NOT skip steps. If any step fails, halt and report the failure +rather than proceeding with partial setup. + +## Step 0 — Pre-flight check + +Verify the runtime environment is ready: +1. `node --version` — must be ≥ 18.x +2. `gh auth status` — must show authenticated user +3. `git rev-parse --is-inside-work-tree` — must be inside a git repo + +If any check fails, halt with the specific remediation step (e.g. "Install +Node.js 18+ from nodejs.org"). Don't ask user to investigate themselves. + +## Step 1 — Stack detection + +Read project markers to determine the stack: +- `package.json` → Node / TypeScript / JavaScript +- `pyproject.toml` or `requirements.txt` → Python +- `Cargo.toml` → Rust +- `go.mod` → Go + +For each detected language, also detect frameworks from dependencies: +- Node: next, react, vue, svelte, express, fastify, nest +- Python: django, fastapi, flask, sqlmodel, pydantic +- Use the detected stack to drive Step 2's discovery query + +## Step 2 — Discover candidates via discover.mjs + +Invoke `node plugins/ievo/scripts/discover.mjs` with the detected stack +as JSON on stdin. Receive ranked candidates from skills.sh API. + +Show the user the top 5 candidates with: name, owner, install count, +quality tier (trusted / neutral / low-confidence). + +## Step 3 — User selection + +Use AskUserQuestion to let the user pick which candidates to install. +Default to ticking the `trusted` tier; require explicit confirmation for +`low-confidence` ones. + +## Step 4 — Security audit per selected candidate + +For each candidate the user selected, invoke the `security-auditor` +agent with the candidate's repo URL. The agent does a content-level scan +(reads ALL files, looks for known malicious patterns). + +If a candidate gets `RED` verdict, halt and offer the report-to-source +flow (filed as issue at the candidate's repo). + +## Step 5 — Install candidates + +For each approved candidate, run `npx skills add /` in the +user's project. Verify the skill files land at the expected paths. + +## Step 6 — Confirm + summary + +Show the user a summary: what was installed, where, what to try next. + +## Step 7 — Wrap-up + +Record the install set in `.ievo/skills-installed.json` for later audit +or reinstall. +""", +} + +# Candidate B — deliberately stripped-down weak variant. Same SKILL.md +# header but a much shallower body. If arena methodology works, A should +# beat B on the same task. +CANDIDATE_B = { + "id": "ievo-init-weak", + "label": "ievo:init (stripped)", + "owner": "ievo-ai", + "skill_md": """--- +name: ievo-init +description: Install iEvo skills. +--- + +# iEvo init + +Run discover.mjs to find skills, then install them. +""", +} + + +# ============================================================================ +# THE TASK — both candidates compete on the SAME prompt +# ============================================================================ + +TASK = """The user just ran /ievo:init in a Next.js + TypeScript project. + +You have access to the ievo-init skill (its SKILL.md content was provided +as your system prompt). Execute the skill's orchestrator logic against +this project. Walk through every step the skill defines. + +For each step, output: +- What you would do (the action / command) +- Expected output / what success looks like +- What you would do if that step fails + +Be thorough. The user wants to understand the full flow before committing +to running it for real. +""" + + +# ============================================================================ +# JUDGE — pairwise comparison with position swap +# ============================================================================ + +JUDGE_SYSTEM_PROMPT = """You are evaluating two AI-generated responses to +the same task. Your job is to pick the better one and explain why. + +Criteria (in order of importance): +1. **Completeness** — did the response cover all required steps? +2. **Specificity** — concrete commands and expected outputs, not vague gestures? +3. **Failure handling** — explicit "what to do if X fails" reasoning? +4. **Clarity** — well-organized, easy to follow? +5. **Safety** — security audit / user confirmation / no destructive bypass? + +Format your verdict as JSON on the LAST line of your response: +{"winner": "A" | "B" | "tie", "confidence": "high" | "medium" | "low", "summary": ""} + +Above the JSON, give your full reasoning (3-6 paragraphs). + +IMPORTANT — anonymization: the candidates are identified only by their +outputs. Do not assume one is "production" or "stripped" — judge purely +on what you see in their response to the task. +""" + + +def build_judge_prompt(task: str, output_first: str, output_second: str) -> str: + """Build the judge prompt with candidates as A/B (no author info).""" + return f"""## The task both candidates received + +{task} + +--- + +## Candidate A's response + +{output_first} + +--- + +## Candidate B's response + +{output_second} + +--- + +Pick the better response per the criteria. Position swap notice: the +operator will run you a second time with A and B reversed; only a +consistent verdict (same winner regardless of position) will count. +Don't infer from position; judge purely on content quality.""" + + +# ============================================================================ +# RUN LOGIC +# ============================================================================ + +@dataclass +class CandidateRun: + """One Haiku run for one candidate.""" + candidate_id: str + iteration: int + output: str + cost_usd: float = 0.0 + turns: int = 0 + + +@dataclass +class JudgeVerdict: + """One judge run (one direction).""" + position: str # "A=cand1, B=cand2" or "A=cand2, B=cand1" + winner_label: str # "A" | "B" | "tie" + confidence: str + summary: str + raw_reasoning: str + cost_usd: float = 0.0 + + +@dataclass +class ArenaRound: + """Full arena round between two candidates.""" + candidate_1_id: str + candidate_2_id: str + task: str + runs_per_candidate: int = 3 + candidate_1_runs: list[CandidateRun] = field(default_factory=list) + candidate_2_runs: list[CandidateRun] = field(default_factory=list) + verdict_forward: JudgeVerdict | None = None + verdict_swapped: JudgeVerdict | None = None + consistent_winner: str | None = None # candidate_id or "tie" or "inconsistent" + total_cost_usd: float = 0.0 + + +def pick_best_output(runs: list[CandidateRun]) -> str: + """Pick the most representative output across runs. + + Heuristic: longest non-empty, since longer outputs typically mean the + model engaged more thoroughly. This is a known verbosity bias — Phase 5 + of the roadmap adds token-balance correction. + """ + non_empty = [r for r in runs if r.output and r.output.strip()] + if not non_empty: + return "" + return max(non_empty, key=lambda r: len(r.output)).output + + +async def run_candidate( + candidate: dict, task: str, iterations: int +) -> list[CandidateRun]: + """Run a candidate K times with Haiku. SKILL.md becomes system prompt.""" + runs = [] + for i in range(iterations): + log.info("Running %s iteration %d/%d", candidate["id"], i + 1, iterations) + result = await query_raw( + prompt=task, + model=os.environ.get("ARENA_RUNNER_MODEL", "haiku"), + system_prompt=candidate["skill_md"], + tools=[], # No tools — pure reasoning over the SKILL.md content + max_turns=3, # Short — just the response, not a whole agent session + isolated=True, + ) + runs.append( + CandidateRun( + candidate_id=candidate["id"], + iteration=i + 1, + output=result.text, + cost_usd=result.total_cost_usd, + turns=result.num_turns, + ) + ) + return runs + + +def parse_judge_response(text: str) -> tuple[str, str, str, str]: + """Parse winner/confidence/summary from judge's response. Returns + (winner_label, confidence, summary, raw_reasoning). + """ + lines = text.strip().split("\n") + # Find the last JSON line + for line in reversed(lines): + line = line.strip() + if line.startswith("{") and line.endswith("}"): + try: + verdict = json.loads(line) + reasoning = "\n".join(lines[: lines.index(line)]).strip() + return ( + verdict.get("winner", "tie"), + verdict.get("confidence", "low"), + verdict.get("summary", "(no summary)"), + reasoning, + ) + except json.JSONDecodeError: + continue + return "tie", "low", "(failed to parse judge verdict)", text + + +async def run_judge( + task: str, first_output: str, second_output: str, position_label: str +) -> JudgeVerdict: + """Run Opus judge in one direction.""" + log.info("Judging — position: %s", position_label) + judge_prompt = build_judge_prompt(task, first_output, second_output) + result = await query_raw( + prompt=judge_prompt, + model=os.environ.get("ARENA_JUDGE_MODEL", "opus"), + system_prompt=JUDGE_SYSTEM_PROMPT, + tools=[], + max_turns=2, + isolated=True, + ) + winner, confidence, summary, reasoning = parse_judge_response(result.text) + return JudgeVerdict( + position=position_label, + winner_label=winner, + confidence=confidence, + summary=summary, + raw_reasoning=reasoning, + cost_usd=result.total_cost_usd, + ) + + +async def run_arena_round( + candidate_1: dict, candidate_2: dict, task: str, runs_per: int = 3 +) -> ArenaRound: + """Full arena round with position swap.""" + log.info( + "Arena round: %s vs %s (runs_per=%d)", + candidate_1["id"], candidate_2["id"], runs_per, + ) + round_ = ArenaRound( + candidate_1_id=candidate_1["id"], + candidate_2_id=candidate_2["id"], + task=task, + runs_per_candidate=runs_per, + ) + + # Run candidates + round_.candidate_1_runs = await run_candidate(candidate_1, task, runs_per) + round_.candidate_2_runs = await run_candidate(candidate_2, task, runs_per) + + # Pick representative outputs + output_1 = pick_best_output(round_.candidate_1_runs) + output_2 = pick_best_output(round_.candidate_2_runs) + + # Forward judge (A=cand1, B=cand2) + round_.verdict_forward = await run_judge( + task, output_1, output_2, + position_label=f"A={candidate_1['id']}, B={candidate_2['id']}", + ) + + # Swapped judge (A=cand2, B=cand1) — bias mitigation + round_.verdict_swapped = await run_judge( + task, output_2, output_1, + position_label=f"A={candidate_2['id']}, B={candidate_1['id']}", + ) + + # Consistency check + forward_winner_id = ( + candidate_1["id"] if round_.verdict_forward.winner_label == "A" + else candidate_2["id"] if round_.verdict_forward.winner_label == "B" + else "tie" + ) + swapped_winner_id = ( + candidate_2["id"] if round_.verdict_swapped.winner_label == "A" + else candidate_1["id"] if round_.verdict_swapped.winner_label == "B" + else "tie" + ) + + if forward_winner_id == swapped_winner_id: + round_.consistent_winner = forward_winner_id + else: + round_.consistent_winner = "inconsistent" + + # Tally cost + round_.total_cost_usd = sum(r.cost_usd for r in round_.candidate_1_runs) + round_.total_cost_usd += sum(r.cost_usd for r in round_.candidate_2_runs) + round_.total_cost_usd += round_.verdict_forward.cost_usd + round_.total_cost_usd += round_.verdict_swapped.cost_usd + + return round_ + + +def print_report(round_: ArenaRound) -> None: + """Pretty-print the round result.""" + print() + print("=" * 78) + print(f" ARENA ROUND REPORT — {round_.candidate_1_id} vs {round_.candidate_2_id}") + print("=" * 78) + print() + print(f"Task: {round_.task[:120]}...") + print() + print(f"Runs per candidate: {round_.runs_per_candidate}") + print(f"Total cost: ${round_.total_cost_usd:.4f}") + print() + print("--- Forward judge ---") + print(f" Position: {round_.verdict_forward.position}") + print(f" Winner label: {round_.verdict_forward.winner_label}") + print(f" Confidence: {round_.verdict_forward.confidence}") + print(f" Summary: {round_.verdict_forward.summary}") + print() + print("--- Swapped judge ---") + print(f" Position: {round_.verdict_swapped.position}") + print(f" Winner label: {round_.verdict_swapped.winner_label}") + print(f" Confidence: {round_.verdict_swapped.confidence}") + print(f" Summary: {round_.verdict_swapped.summary}") + print() + print(f"--- Consistent winner (position-swap agreement): {round_.consistent_winner} ---") + print() + print("=" * 78) + + +def write_full_report(round_: ArenaRound, out_path: Path) -> None: + """Write full JSON report for later inspection.""" + payload = { + "candidate_1_id": round_.candidate_1_id, + "candidate_2_id": round_.candidate_2_id, + "task": round_.task, + "runs_per_candidate": round_.runs_per_candidate, + "candidate_1_runs": [asdict(r) for r in round_.candidate_1_runs], + "candidate_2_runs": [asdict(r) for r in round_.candidate_2_runs], + "verdict_forward": asdict(round_.verdict_forward) if round_.verdict_forward else None, + "verdict_swapped": asdict(round_.verdict_swapped) if round_.verdict_swapped else None, + "consistent_winner": round_.consistent_winner, + "total_cost_usd": round_.total_cost_usd, + } + out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False)) + log.info("Full report written: %s", out_path) + + +async def main() -> None: + log.info("Skills Arena PoC — methodology validation") + log.info("Candidate A: %s (%s)", CANDIDATE_A["id"], CANDIDATE_A["label"]) + log.info("Candidate B: %s (%s)", CANDIDATE_B["id"], CANDIDATE_B["label"]) + log.info("Expected: A wins (production vs stripped)") + + round_ = await run_arena_round( + candidate_1=CANDIDATE_A, + candidate_2=CANDIDATE_B, + task=TASK, + runs_per=int(os.environ.get("ARENA_RUNS_PER", "3")), + ) + + print_report(round_) + + # Write full report + out_dir = Path(__file__).parent.parent / "dist" / "arena-poc" + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"round-{round_.candidate_1_id}-vs-{round_.candidate_2_id}.json" + write_full_report(round_, out_path) + + # Methodology validation: did the production candidate win? + if round_.consistent_winner == CANDIDATE_A["id"]: + print("\n✓ METHODOLOGY VALIDATED: production candidate (A) won the round") + print(" Position-swap-consistent. Ready to scale to real candidate pairs.") + sys.exit(0) + elif round_.consistent_winner == CANDIDATE_B["id"]: + print("\n⚠ UNEXPECTED: stripped candidate (B) won the round") + print(" Investigate — maybe Haiku reads brevity as quality?") + print(" Or the production SKILL.md has issues the operator should review.") + sys.exit(1) + elif round_.consistent_winner == "tie": + print("\n⚠ TIE: judge couldn't pick a winner") + print(" Methodology may need stricter criteria or a more discriminating task.") + sys.exit(2) + else: # inconsistent + print("\n✗ POSITION BIAS: forward and swapped judges disagreed") + print(" This is the failure mode the swap is supposed to catch.") + print(" Either the judge model is biased or candidates are too close to call.") + sys.exit(3) + + +if __name__ == "__main__": + asyncio.run(main()) From 2cd7035b17b570cf406e10e5727568691d7ed395 Mon Sep 17 00:00:00 2001 From: denis Date: Sun, 24 May 2026 11:19:47 +0100 Subject: [PATCH 2/5] fix(arena-poc): give agents realistic tools + sandboxed Next.js cwd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First PoC run (2026-05-24 10:51-11:00) validated methodology mechanically but with degenerate inputs: tools=[] + max_turns=3 caused Haiku to produce empty text in 4/6 candidate runs. Judge was comparing real-vs-empty, not real-vs-real. Operator caught it: "у агента нет доступка к тулам?" — yes, that was the bug. Skill orchestrators ARE agent harnesses; without tools the agent has nothing to demonstrate. ## Changes - Each iteration gets a fresh sandbox cwd at $TMPDIR/arena_poc_*/ with minimal Next.js+TS project markers (package.json with next/ react/typescript deps, tsconfig, README, .gitignore) so the skill agent has something realistic to inspect - Tool set: Read, Write, Glob, Grep, Bash — realistic for a skill orchestrator doing project inspection + execution simulation - max_turns: 3 → 15 (realistic budget for an orchestrator with several steps) - Each candidate iteration gets its OWN sandbox (no contamination between runs of the same candidate or between candidates) ## Second run result (2026-05-24 11:16-11:19) Both candidates produced real text outputs. Judges substantively ranked them on quality dimensions: - Forward: A wins on security audit, trust tiers, ecosystem specificity, failure handling - Swapped: same quality observations, swap-consistent - Consistent winner: ievo-init-full ✓ - Cost: $0.5756 Methodology produces meaningful quality signals, not just real-vs-empty distinctions. Phase 1 PoC fully validated. Co-Authored-By: iEVO --- scripts/skills_arena_poc.py | 43 +++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/scripts/skills_arena_poc.py b/scripts/skills_arena_poc.py index d2da553..f3df91a 100644 --- a/scripts/skills_arena_poc.py +++ b/scripts/skills_arena_poc.py @@ -278,19 +278,44 @@ def pick_best_output(runs: list[CandidateRun]) -> str: return max(non_empty, key=lambda r: len(r.output)).output +def _setup_fake_nextjs_project(root: Path) -> None: + """Create minimal Next.js+TS project markers so a skill agent has + something realistic to inspect. Each candidate gets a fresh copy.""" + root.mkdir(parents=True, exist_ok=True) + (root / "package.json").write_text( + '{\n "name": "arena-test-project",\n "version": "0.1.0",\n ' + '"dependencies": {\n "next": "^15.0.0",\n "react": "^19.0.0",\n ' + '"typescript": "^5.0.0"\n }\n}\n' + ) + (root / "tsconfig.json").write_text('{"compilerOptions": {"strict": true}}\n') + (root / "README.md").write_text("# Arena Test Project\nA Next.js + TypeScript project.\n") + (root / ".gitignore").write_text("node_modules/\n.next/\n") + + async def run_candidate( - candidate: dict, task: str, iterations: int + candidate: dict, task: str, iterations: int, sandbox_root: Path ) -> list[CandidateRun]: - """Run a candidate K times with Haiku. SKILL.md becomes system prompt.""" + """Run a candidate K times with Haiku in a sandboxed Next.js project. + + Each iteration gets a FRESH sandbox copy so candidates can't see each + other's side effects. Tools are realistic for a skill orchestrator: + Read/Glob/Grep for project inspection, Bash for runtime checks (no + install — we don't actually run `npx skills add`), Write for the + skill to record what it would do. + """ runs = [] for i in range(iterations): log.info("Running %s iteration %d/%d", candidate["id"], i + 1, iterations) + # Fresh sandbox per iteration — no cross-iteration contamination + iter_cwd = sandbox_root / f"{candidate['id']}_iter{i + 1}" + _setup_fake_nextjs_project(iter_cwd) result = await query_raw( prompt=task, model=os.environ.get("ARENA_RUNNER_MODEL", "haiku"), system_prompt=candidate["skill_md"], - tools=[], # No tools — pure reasoning over the SKILL.md content - max_turns=3, # Short — just the response, not a whole agent session + cwd=iter_cwd, + tools=["Read", "Write", "Glob", "Grep", "Bash"], + max_turns=15, # Realistic budget for a skill orchestrator isolated=True, ) runs.append( @@ -368,9 +393,13 @@ async def run_arena_round( runs_per_candidate=runs_per, ) - # Run candidates - round_.candidate_1_runs = await run_candidate(candidate_1, task, runs_per) - round_.candidate_2_runs = await run_candidate(candidate_2, task, runs_per) + import tempfile + sandbox_root = Path(tempfile.mkdtemp(prefix="arena_poc_")) + log.info("Sandbox root: %s", sandbox_root) + + # Run candidates with fresh sandbox per iteration + round_.candidate_1_runs = await run_candidate(candidate_1, task, runs_per, sandbox_root) + round_.candidate_2_runs = await run_candidate(candidate_2, task, runs_per, sandbox_root) # Pick representative outputs output_1 = pick_best_output(round_.candidate_1_runs) From d5bae0e3249fbe6ca510ebbbe5130c3da84acf72 Mon Sep 17 00:00:00 2001 From: denis Date: Sun, 24 May 2026 12:41:55 +0100 Subject: [PATCH 3/5] =?UTF-8?q?feat(arena):=20genetic=20crossover=20PoC=20?= =?UTF-8?q?=E2=80=94=20LLM-mediated=20synthesis=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests whether two parent skills crossed by Opus produce a child that beats both in arena. Multi-child population (N=3) + all-vs-all round-robin to validate the effect isn't lucky-draw variance. Result on ievo-init-full × ievo-init-weak: all 3 children beat both parents (2/4 wins each vs 1/4 strong, 0/4 weak). Cost: $3.81. Caveat: weak parent contributed nothing — child is compressed strong parent. Real idea-mixing requires parents with different valid strategies (next step). Co-Authored-By: iEVO --- scripts/skills_genetic_poc.py | 332 ++++++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 scripts/skills_genetic_poc.py diff --git a/scripts/skills_genetic_poc.py b/scripts/skills_genetic_poc.py new file mode 100644 index 0000000..82c3c44 --- /dev/null +++ b/scripts/skills_genetic_poc.py @@ -0,0 +1,332 @@ +#!/usr/bin/env -S uv run python +"""Skills Genetic PoC — does LLM-mediated crossover beat both parents? + +Operator's question 2026-05-24: "А можно ли генетическим алгоритмом +скрестить два скила и прогнать потом новое поколение на арене и +посмотреть уделал ли он двух родителей?" + +Empirical answer via PoC: + 1. Take two parent SKILL.md candidates (parent_a, parent_b) + 2. Crossover via Opus: synthesize a child SKILL.md combining strengths + 3. 3-way round-robin arena: A vs B, A vs Child, B vs Child + 4. Report ranking — did Child beat both parents? + +This is L5 of the evolution hierarchy: + L0: cortex gate (A/B eval for kernel mutations) — shipped + L1: EVO (single-agent self-correction) — shipped + L2: Curator (cross-agent patterns) — shipped + L3: Eva (platform-wide observation) — shipped + L4: Arena (community ranking) — PoC validated (skills_arena_poc.py) + L5: Genetic synthesis (this script) — PoC + +Reuses skills_arena_poc.py's pipeline. Adds: + - LLM crossover synthesis + - 3-way round-robin scoring + - Win-count ranking (most pairwise wins) + +Cost: ~$1.50-2.00 per round +Runtime: ~10-15 минут +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import sys +from dataclasses import asdict, dataclass, field +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +# Reuse the validated arena pipeline +from skills_arena_poc import ( # noqa: E402 + CANDIDATE_A as PARENT_A, + CANDIDATE_B as PARENT_B, + TASK, + run_arena_round, +) +from cortex.claude import query_raw # noqa: E402 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +log = logging.getLogger("genetic_poc") + + +# ============================================================================ +# CROSSOVER — LLM-mediated synthesis +# ============================================================================ + +CROSSOVER_SYSTEM_PROMPT = """You are a skill author tasked with synthesizing +a new SKILL.md by combining the strengths of two existing SKILL.md files. + +Your output MUST be a valid SKILL.md file (frontmatter + Markdown body). + +Goals: +1. Identify what each parent does WELL that the other lacks +2. Take the BEST features from each — not a 50/50 average, a curated synthesis +3. The result should beat BOTH parents on real tasks, not just be a hybrid + +What to look for: +- One parent has explicit failure handling, the other has cleaner structure → take both +- One has security awareness, the other has speed → preserve security, add speed +- One has detailed steps, the other has concise commands → use details where they matter, brevity elsewhere + +What to AVOID: +- Verbosity for its own sake (verbosity bias) +- Hallucinated steps not present in either parent +- Internally inconsistent recommendations (parent A says X, parent B says ¬X — pick one with reasoning) + +Output format: SKILL.md content ONLY. No commentary, no "Here is the +synthesized skill:" prefix. Start with `---` frontmatter line. +""" + + +def build_crossover_prompt(parent_a: dict, parent_b: dict) -> str: + return f"""Synthesize a new SKILL.md by combining the strengths of these +two parents. Both target the same use case (`{parent_a['id'].split('-')[0]}` +domain). + +## Parent A — {parent_a['label']} + +{parent_a['skill_md']} + +--- + +## Parent B — {parent_b['label']} + +{parent_b['skill_md']} + +--- + +Output the synthesized child SKILL.md below. Frontmatter `name` should +be unique (suggest: `{parent_a['id']}-synth`). Description should reflect +the combined capability. +""" + + +@dataclass +class CrossoverResult: + child_id: str + child_label: str + child_skill_md: str + synthesis_cost_usd: float = 0.0 + + +async def crossover_parents(parent_a: dict, parent_b: dict, child_index: int) -> CrossoverResult: + """LLM-mediated crossover via Opus. Multiple calls give natural variance.""" + log.info("Crossover #%d: %s × %s", child_index, parent_a["id"], parent_b["id"]) + result = await query_raw( + prompt=build_crossover_prompt(parent_a, parent_b), + model=os.environ.get("GENETIC_SYNTHESIS_MODEL", "opus"), + system_prompt=CROSSOVER_SYSTEM_PROMPT, + tools=[], + max_turns=2, + isolated=True, + ) + child_md = result.text.strip() + # Strip code fences if Claude wrapped output + if child_md.startswith("```markdown"): + child_md = child_md[len("```markdown"):].strip() + elif child_md.startswith("```"): + child_md = child_md[3:].strip() + if child_md.endswith("```"): + child_md = child_md[:-3].strip() + + base = parent_a["id"].split("-")[0] + "-" + parent_a["id"].split("-")[1] + return CrossoverResult( + child_id=f"{base}-synth-{child_index}", + child_label=f"Child #{child_index} (synthesized)", + child_skill_md=child_md, + synthesis_cost_usd=result.total_cost_usd, + ) + + +# ============================================================================ +# 3-WAY ROUND-ROBIN ARENA +# ============================================================================ + +@dataclass +class GeneticPoCResult: + parent_a_id: str + parent_b_id: str + child_id: str + child_skill_md: str + synthesis_cost_usd: float + arena_rounds: list[dict] = field(default_factory=list) + win_counts: dict[str, int] = field(default_factory=dict) + total_cost_usd: float = 0.0 + + +N_CHILDREN = int(os.environ.get("GENETIC_N_CHILDREN", "3")) + + +async def run_genetic_poc() -> GeneticPoCResult: + """Full genetic PoC: N crossovers + full round-robin arena + ranking. + + Population = 2 parents + N children. All-vs-all pairwise (C(2+N, 2)). + """ + log.info("=== Phase 1: Crossover × %d children ===", N_CHILDREN) + children = [] + for i in range(1, N_CHILDREN + 1): + c = await crossover_parents(PARENT_A, PARENT_B, child_index=i) + log.info(" Child #%d: %d chars, $%.4f", i, len(c.child_skill_md), c.synthesis_cost_usd) + children.append(c) + + # Build candidate list: parent_a, parent_b, child_1, ..., child_N + child_candidates = [ + { + "id": c.child_id, + "label": c.child_label, + "owner": "synth", + "skill_md": c.child_skill_md, + } + for c in children + ] + all_candidates = [PARENT_A, PARENT_B, *child_candidates] + + log.info("=== Phase 2: %d-way round-robin arena (C(%d,2)=%d pairs) ===", + len(all_candidates), len(all_candidates), + len(all_candidates) * (len(all_candidates) - 1) // 2) + + arena_rounds = [] + pair_no = 0 + # All-vs-all pairwise, no double-counting + for i in range(len(all_candidates)): + for j in range(i + 1, len(all_candidates)): + pair_no += 1 + cand_a = all_candidates[i] + cand_b = all_candidates[j] + log.info("Pair %d: %s vs %s", pair_no, cand_a["id"], cand_b["id"]) + r = await run_arena_round(cand_a, cand_b, TASK, runs_per=3) + arena_rounds.append({ + "pair": f"{cand_a['id']} vs {cand_b['id']}", + "winner": r.consistent_winner, + "cost_usd": r.total_cost_usd, + }) + + # Tally wins + win_counts = {c["id"]: 0 for c in all_candidates} + for r in arena_rounds: + if r["winner"] in win_counts: + win_counts[r["winner"]] += 1 + + total_cost = ( + sum(c.synthesis_cost_usd for c in children) + + sum(r["cost_usd"] for r in arena_rounds) + ) + + # Pick "best child" for the saved SKILL.md output + best_child = max( + children, + key=lambda c: win_counts.get(c.child_id, 0) + ) + + result = GeneticPoCResult( + parent_a_id=PARENT_A["id"], + parent_b_id=PARENT_B["id"], + child_id=best_child.child_id, # best of the N children + child_skill_md=best_child.child_skill_md, + synthesis_cost_usd=sum(c.synthesis_cost_usd for c in children), + arena_rounds=arena_rounds, + win_counts=win_counts, + total_cost_usd=total_cost, + ) + # Annotate with all child IDs for inspection + result.win_counts["_n_children"] = N_CHILDREN # type: ignore[assignment] + return result + + +def print_report(r: GeneticPoCResult) -> None: + print() + print("=" * 78) + print(" GENETIC POC RESULT — does crossover beat both parents?") + print("=" * 78) + print() + print(f"Parents: {r.parent_a_id} × {r.parent_b_id}") + print(f"Child: {r.child_id}") + print(f"Synthesis cost: ${r.synthesis_cost_usd:.4f}") + print() + print("Arena rounds:") + for round_info in r.arena_rounds: + print(f" {round_info['pair']}") + print(f" → winner: {round_info['winner']} (${round_info['cost_usd']:.4f})") + print() + r.win_counts.pop("_n_children", None) + max_wins_per = len(r.win_counts) - 1 # each candidate fights N-1 others + + print(f"Win counts (max {max_wins_per} per candidate, all-vs-all):") + sorted_results = sorted(r.win_counts.items(), key=lambda kv: -kv[1]) + for rank, (cand_id, wins) in enumerate(sorted_results, 1): + marker = "" + if rank == 1: + marker = " ← WINNER" + elif wins == sorted_results[0][1]: + marker = " (tied for 1st)" + print(f" #{rank}: {cand_id} — {wins}/{max_wins_per}{marker}") + print() + print(f"Total cost: ${r.total_cost_usd:.4f}") + print() + print("=" * 78) + + # Verdict — focus on whether ANY child beat both parents + parent_a_wins = r.win_counts.get(r.parent_a_id, 0) + parent_b_wins = r.win_counts.get(r.parent_b_id, 0) + children_ids = [k for k in r.win_counts if k not in (r.parent_a_id, r.parent_b_id)] + children_max_wins = max((r.win_counts[c] for c in children_ids), default=0) + any_child_beats_both = any( + r.win_counts[c] > parent_a_wins and r.win_counts[c] > parent_b_wins + for c in children_ids + ) + all_children_lose = all( + r.win_counts[c] < parent_a_wins and r.win_counts[c] < parent_b_wins + for c in children_ids + ) + + if any_child_beats_both: + best_child_id = max(children_ids, key=lambda c: r.win_counts[c]) + print("\n✓ AT LEAST ONE CHILD BEATS BOTH PARENTS") + print(f" Best: {best_child_id} ({r.win_counts[best_child_id]} wins)") + print(" → L5 genetic synthesis is viable for this domain.") + print(" → Best child SKILL.md saved (see logs).") + elif children_max_wins >= max(parent_a_wins, parent_b_wins): + print("\n~ BEST CHILD TIES WITH PARENT(S)") + print(f" Children's max wins: {children_max_wins}") + print(f" Parent A: {parent_a_wins}, Parent B: {parent_b_wins}") + print(" → Crossover at least doesn't regress. Multi-gen may improve.") + elif all_children_lose: + print("\n✗ ALL CHILDREN LOSE TO BOTH PARENTS") + print(" → Crossover regressed in this run.") + print(" → Try: stricter genotype-based crossover, or skip generation.") + else: + print("\n~ MIXED — some children beat one parent but not both") + print(" → Partial gain. Worth multi-generation to see if it stabilizes.") + + +async def main() -> None: + log.info("Skills Genetic PoC — L5 evolution hierarchy validation") + result = await run_genetic_poc() + print_report(result) + + # Write full report + out_dir = Path(__file__).parent.parent / "dist" / "arena-poc" + out_dir.mkdir(parents=True, exist_ok=True) + out_path = out_dir / f"genetic-{result.parent_a_id}-x-{result.parent_b_id}.json" + + payload = asdict(result) + out_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False)) + log.info("Full report written: %s", out_path) + + # Also save child SKILL.md for inspection + child_md_path = out_dir / f"{result.child_id}-SKILL.md" + child_md_path.write_text(result.child_skill_md) + log.info("Child SKILL.md saved: %s", child_md_path) + + child_wins = result.win_counts.get(result.child_id, 0) + sys.exit(0 if child_wins == 2 else 1 if child_wins == 1 else 2) + + +if __name__ == "__main__": + asyncio.run(main()) From a10487b1f6c19f52e5e57c2ccd27ae1f2c5a41c6 Mon Sep 17 00:00:00 2001 From: denis Date: Sun, 24 May 2026 13:46:04 +0100 Subject: [PATCH 4/5] =?UTF-8?q?feat(arena):=20genetic=20PoC=20v2=20?= =?UTF-8?q?=E2=80=94=20strategy-diverse=20parents=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v1 limitation: strong+weak parents → child was compressed strong parent (weak contributed nothing). Crossover acted as compressor, not idea-mixer. v2: two valid, philosophically opposite parents for the same /ievo:init goal: - defensive-sequential: strict order, audit-first, halt on ambiguity - optimistic-parallel: parallel discovery, retry transients, rollback Result: all 3 children beat BOTH parents. Parents got 0/4 wins each (including head-to-head: inconsistent). Cost $4.75. Best child synthesizes: - Parallel read-only work (Phase 1) — explicit safety classification - Audit gate before write (Phase 4) — combines safety + parallel speedup - Cascading rollback (Phase 5) — per-candidate + halt-if-rollback-fails - "Fast where safe, strict where it matters" — synthesizing principle This is real idea-mixing, not compression. L5 genetic synthesis is viable for skill domain when parents represent distinct valid strategies. Co-Authored-By: iEVO --- scripts/skills_genetic_v2_poc.py | 332 +++++++++++++++++++++++++++++++ 1 file changed, 332 insertions(+) create mode 100644 scripts/skills_genetic_v2_poc.py diff --git a/scripts/skills_genetic_v2_poc.py b/scripts/skills_genetic_v2_poc.py new file mode 100644 index 0000000..a15ac53 --- /dev/null +++ b/scripts/skills_genetic_v2_poc.py @@ -0,0 +1,332 @@ +#!/usr/bin/env -S uv run python +"""Skills Genetic PoC v2 — does crossover MIX ideas, or just compress? + +v1 result (skills_genetic_poc.py): all 3 children beat both parents on +`ievo-init-full × ievo-init-weak`. BUT the weak parent contributed +nothing — child was just a compressed strong parent. + +v2 question: when parents are BOTH valid but follow DIFFERENT strategies, +does Opus synthesize their genuinely different ideas? + +Parents (both valid, both production-quality, philosophically opposite): + - PARENT_A = ievo-init-defensive — strict sequential, fail-fast, audit-first + - PARENT_B = ievo-init-optimistic — parallel discover, lazy security, retry transients + +Expected synthesis if crossover works as idea-mixer: + - parallel detection (B) + pre-install audit (A) + - transient retry (B) + halt-on-real-failure (A) + - batch install (B) + per-candidate verification (A) + +Methodology mirrors v1: + - N=3 crossovers (variance check) + - 5-way all-vs-all round-robin + - Win-count ranking + +Run: + uv run python scripts/skills_genetic_v2_poc.py + +Cost: ~$4-6 expected. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from skills_arena_poc import TASK, run_arena_round # noqa: E402 +from skills_genetic_poc import ( # noqa: E402 + GeneticPoCResult, + crossover_parents, + print_report, +) + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +log = logging.getLogger("genetic_v2") + + +# ============================================================================ +# PARENTS — two valid, philosophically opposite strategies for the SAME task +# ============================================================================ + +PARENT_A = { + "id": "ievo-init-defensive", + "label": "ievo:init (defensive sequential)", + "owner": "synth-experiment", + "skill_md": """--- +name: ievo-init +description: Bootstrap iEvo in a project. Defensive sequential — strict order, halt on any anomaly, full audit before any install. Use when correctness matters more than speed. +--- + +# iEvo init — defensive sequential orchestrator + +Philosophy: **never proceed past an unknown.** Every step validates before +the next runs. No parallelism. No optimistic actions. Side effects ONLY +after all checks pass. + +## Step 0 — Pre-flight (mandatory) + +Run all three checks. Halt at the first failure with the specific fix: + +1. `node --version` — require ≥ 18.x. Fail: "Install Node.js 18+ from nodejs.org" +2. `gh auth status` — require authenticated. Fail: "Run `gh auth login`" +3. `git rev-parse --is-inside-work-tree` — must be a repo. Fail: "Run `git init` first" + +Do NOT continue if any check is ambiguous. Re-run after fix. + +## Step 1 — Stack detection (single-pass) + +Detect ONE primary language from project markers (the most specific match wins): + +| Marker present | Language | Frameworks to detect | +|---|---|---| +| `package.json` | Node/TS/JS | next, react, vue, svelte, express, fastify, nest | +| `pyproject.toml` or `requirements.txt` | Python | django, fastapi, flask, sqlmodel, pydantic | +| `Cargo.toml` | Rust | — | +| `go.mod` | Go | — | + +If multiple markers present (e.g. polyglot repo), ask the user which is primary. +Do NOT guess. + +## Step 2 — Discover (synchronous) + +Wait for Step 1 to complete. Then: + +```bash +echo '' | node plugins/ievo/scripts/discover.mjs +``` + +If discover.mjs fails (non-zero exit, network error, malformed output): +- HALT. Do not retry. Do not fall back to skills.sh API directly. +- Report the failure to user and ask whether to abort or continue manually. + +## Step 3 — User selection (explicit confirmation) + +Show top 5 candidates with: name, owner, install count, quality tier. + +- Pre-select `trusted` tier only. +- `neutral` and `low-confidence` REQUIRE explicit user confirmation each. +- Empty selection → halt. + +## Step 4 — Security audit (BEFORE install) + +For EACH selected candidate, invoke `security-auditor` agent synchronously. +Wait for verdict before moving to the next candidate. + +| Verdict | Action | +|---|---| +| GREEN | Proceed to install set | +| YELLOW | Re-confirm with user, then proceed | +| RED | HALT this candidate; offer report-to-source flow | + +Do NOT install anything until ALL selected candidates have a verdict. + +## Step 5 — Install (per-candidate verification) + +For each approved candidate (in user-selection order): + +```bash +npx skills add / +``` + +After each install: verify files exist at expected paths. If any expected +file is missing — HALT, leave prior installs intact, report which candidate +failed. + +## Step 6 — Manifest + summary + +1. Write `.ievo/skills-installed.json` with the install set + timestamps + audit verdicts. +2. Show: what was installed, file paths, next commands to try. +3. If any step was skipped or halted, report explicitly so the user knows the state is partial. +""", +} + +PARENT_B = { + "id": "ievo-init-optimistic", + "label": "ievo:init (optimistic parallel)", + "owner": "synth-experiment", + "skill_md": """--- +name: ievo-init +description: Bootstrap iEvo in a project. Optimistic parallel — overlap discovery with detection, retry transients, rollback on failure. Use when developer experience and speed matter. +--- + +# iEvo init — optimistic parallel orchestrator + +Philosophy: **overlap independent work, retry transient failures, rollback +on real failure.** The defensive sequential approach wastes wall-clock time +on serializable checks. We can do better. + +## Concurrent Phase A — pre-flight + stack detection + +Run in parallel (all independent): + +- `node --version` — require ≥ 18.x +- `gh auth status` — require authenticated +- `git rev-parse --is-inside-work-tree` — require repo + +In parallel, ALSO start stack detection by reading project root for markers: +`package.json`, `pyproject.toml`, `Cargo.toml`, `go.mod`. For Node/Python, +parse the dep file to detect frameworks (next, react, fastapi, django, etc.). + +Wait for the slowest of these to complete. If any pre-flight check fails, +suggest the fix and ask user to re-run. + +## Concurrent Phase B — discover candidates (with fallback) + +As soon as stack is detected, kick off: + +```bash +echo '' | node plugins/ievo/scripts/discover.mjs +``` + +Retry policy: up to 3 attempts on transient failures (network, timeout, +HTTP 5xx). If discover.mjs is unavailable after retries, fall back to +direct skills.sh API: `curl https://skills.sh/api/search?stack=...` +with the same retry policy. + +Treat discover.mjs's output as advisory: even if some fields are missing, +present what we have to the user with a "(partial)" tag rather than halting. + +## User selection — quick approval + +Show top 5 in a compact table. Pre-select `trusted` + `neutral` (low-confidence +hidden behind a "show more" toggle). User can multi-select; default-accept the +top trusted candidate if user just confirms without choosing. + +## Concurrent Phase C — install + security audit overlap + +For approved candidates, run install + security audit IN PARALLEL per +candidate. Both must succeed for the candidate to "stick": + +```bash +# Both kicked off simultaneously per candidate: +npx skills add / # installer +security-auditor scan / # parallel audit +``` + +If audit returns RED while install is in progress (or already done): rollback +that candidate via `npx skills remove /` and offer report-to-source +flow. Other candidates continue independently. + +If audit returns GREEN or YELLOW: keep installed. YELLOW logs a warning but +does not halt. + +## Manifest + summary (always written) + +Write `.ievo/skills-installed.json` with: install set, audit verdicts, retry +counts, fallback usage flags. ALWAYS emit a summary at the end — including +partial-success cases where some candidates rolled back. User sees: +- What was installed +- What was rolled back and why +- What to try next +- Any transient errors that were retried (useful for debugging flakiness) +""", +} + + +# ============================================================================ +# DRIVER — reuse v1's pipeline with new parents +# ============================================================================ + +N_CHILDREN = int(os.environ.get("GENETIC_N_CHILDREN", "3")) + + +async def run_genetic_v2() -> GeneticPoCResult: + log.info("=== Phase 1: Crossover × %d children (defensive × optimistic) ===", N_CHILDREN) + children = [] + for i in range(1, N_CHILDREN + 1): + c = await crossover_parents(PARENT_A, PARENT_B, child_index=i) + log.info(" Child #%d: %d chars, $%.4f", i, len(c.child_skill_md), c.synthesis_cost_usd) + children.append(c) + + child_candidates = [ + { + "id": c.child_id, + "label": c.child_label, + "owner": "synth", + "skill_md": c.child_skill_md, + } + for c in children + ] + all_candidates = [PARENT_A, PARENT_B, *child_candidates] + + pair_count = len(all_candidates) * (len(all_candidates) - 1) // 2 + log.info("=== Phase 2: %d-way round-robin (C(%d,2)=%d pairs) ===", + len(all_candidates), len(all_candidates), pair_count) + + arena_rounds = [] + pair_no = 0 + for i in range(len(all_candidates)): + for j in range(i + 1, len(all_candidates)): + pair_no += 1 + cand_a = all_candidates[i] + cand_b = all_candidates[j] + log.info("Pair %d/%d: %s vs %s", pair_no, pair_count, cand_a["id"], cand_b["id"]) + r = await run_arena_round(cand_a, cand_b, TASK, runs_per=3) + arena_rounds.append({ + "pair": f"{cand_a['id']} vs {cand_b['id']}", + "winner": r.consistent_winner, + "cost_usd": r.total_cost_usd, + }) + + win_counts = {c["id"]: 0 for c in all_candidates} + for r in arena_rounds: + if r["winner"] in win_counts: + win_counts[r["winner"]] += 1 + + total_cost = ( + sum(c.synthesis_cost_usd for c in children) + + sum(r["cost_usd"] for r in arena_rounds) + ) + + best_child = max(children, key=lambda c: win_counts.get(c.child_id, 0)) + + return GeneticPoCResult( + parent_a_id=PARENT_A["id"], + parent_b_id=PARENT_B["id"], + child_id=best_child.child_id, + child_skill_md=best_child.child_skill_md, + synthesis_cost_usd=sum(c.synthesis_cost_usd for c in children), + arena_rounds=arena_rounds, + win_counts=win_counts, + total_cost_usd=total_cost, + ) + + +async def main(): + log.info("Starting Genetic PoC v2 — defensive × optimistic") + result = await run_genetic_v2() + print_report(result) + + # Save artifacts + out_dir = Path(__file__).parent.parent / "dist" / "arena-poc" + out_dir.mkdir(parents=True, exist_ok=True) + + pair_id = f"{PARENT_A['id']}-x-{PARENT_B['id']}" + report_path = out_dir / f"genetic-v2-{pair_id}.json" + report_data = { + "parent_a_id": result.parent_a_id, + "parent_b_id": result.parent_b_id, + "child_id": result.child_id, + "synthesis_cost_usd": result.synthesis_cost_usd, + "arena_rounds": result.arena_rounds, + "win_counts": result.win_counts, + "total_cost_usd": result.total_cost_usd, + } + report_path.write_text(json.dumps(report_data, indent=2)) + log.info("Full report written: %s", report_path) + + skill_path = out_dir / f"v2-{result.child_id}-SKILL.md" + skill_path.write_text(result.child_skill_md) + log.info("Child SKILL.md saved: %s", skill_path) + + +if __name__ == "__main__": + asyncio.run(main()) From a56c9289c64a9ed80d4bf35d1835541394ecdc95 Mon Sep 17 00:00:00 2001 From: denis Date: Sun, 24 May 2026 18:57:38 +0100 Subject: [PATCH 5/5] =?UTF-8?q?feat(arena):=20genetic=20PoC=20v3=20?= =?UTF-8?q?=E2=80=94=20multi-generation=20(#28)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests: does gen-2 keep improving over gen-1, or plateau? Pipeline: defensive × optimistic → 2 gen-1 children → cross them → 3 grandchildren → 5-way arena. Result: gen-2 strictly dominates gen-1. Head-to-head: gen-2 wins 4, gen-1 wins 0, ties/inconsistent 2. Best grandchild: 3/4 wins. Both gen-1 parents: 0/4 each. Total cost: $5.36. Key finding: variance is higher at gen-2 — only 1/3 grandchildren strongly improved (3 wins), the others stayed flat (1 win each). This means multi-gen needs explicit best-of-N selection: a single gen-2 sample is unreliable; pooling 3+ and keeping the winner recovers the improvement signal. Qualitative diff (v3 grandchild-2 vs v2 synth-1): - 4-way decision tree (parallel/sequential/rollback/halt) vs 2-way "fast where safe" slogan - Phase 5 reverted to sequential install — safer trade-off - discover.mjs failure: explicit user choice instead of silent fallback - Exponential backoff 2s→4s→8s vs flat 2s - Concrete manifest JSON schema with exact keys - Named invariant: "no code lands on disk before its audit clears" For #28 design: arena loop should "generate N → keep best → recurse → stop when no improvement for K generations." Co-Authored-By: iEVO --- scripts/skills_genetic_v3_multigen_poc.py | 181 ++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 scripts/skills_genetic_v3_multigen_poc.py diff --git a/scripts/skills_genetic_v3_multigen_poc.py b/scripts/skills_genetic_v3_multigen_poc.py new file mode 100644 index 0000000..bc6f9a6 --- /dev/null +++ b/scripts/skills_genetic_v3_multigen_poc.py @@ -0,0 +1,181 @@ +#!/usr/bin/env -S uv run python +"""Skills Genetic PoC v3 — multi-generation: does gen-2 beat gen-1? + +v2 proved single-gen crossover beats both parents when parents represent +different valid strategies. Next question: do further generations keep +improving, or plateau? + +Generation 1: defensive × optimistic → 2 children (g1-A, g1-B) +Generation 2: g1-A × g1-B → 3 grandchildren (g2-A, g2-B, g2-C) +Arena: 5-way round-robin (2 gen-1 + 3 grandchildren) + +Hypotheses to test: + - Optimistic: grandchildren keep improving (further compression / + tighter synthesizing principle) + - Pessimistic: gen-2 plateaus or regresses (gen-1 already near-optimal + for this domain, or grandchildren over-fit to judge's biases) + +This matters for #28 design: if multi-gen helps, arena should loop +crossover N generations until no improvement. If plateaus immediately, +single-gen is sufficient and we save compute. + +Cost: ~$5-6 +Runtime: ~1 hour +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + +from skills_arena_poc import TASK, run_arena_round # noqa: E402 +from skills_genetic_poc import ( # noqa: E402 + GeneticPoCResult, + crossover_parents, + print_report, +) +from skills_genetic_v2_poc import PARENT_A, PARENT_B # noqa: E402 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", +) +log = logging.getLogger("genetic_v3") + + +N_GRANDCHILDREN = int(os.environ.get("GENETIC_N_GRANDCHILDREN", "3")) + + +async def run_multigen() -> tuple[GeneticPoCResult, dict, dict]: + # ---- Generation 1: two children from defensive × optimistic ---- + log.info("=== GENERATION 1: defensive × optimistic → 2 children ===") + g1_results = [] + for i in range(1, 3): + c = await crossover_parents(PARENT_A, PARENT_B, child_index=i) + log.info(" Gen-1 child #%d: %d chars, $%.4f", + i, len(c.child_skill_md), c.synthesis_cost_usd) + g1_results.append(c) + + g1_a = { + "id": "g1-A", + "label": "Gen-1 child A", + "owner": "synth", + "skill_md": g1_results[0].child_skill_md, + } + g1_b = { + "id": "g1-B", + "label": "Gen-1 child B", + "owner": "synth", + "skill_md": g1_results[1].child_skill_md, + } + g1_synthesis_cost = sum(c.synthesis_cost_usd for c in g1_results) + + # ---- Generation 2: grandchildren from g1-A × g1-B ---- + log.info("=== GENERATION 2: g1-A × g1-B → %d grandchildren ===", N_GRANDCHILDREN) + g2_results = [] + for i in range(1, N_GRANDCHILDREN + 1): + c = await crossover_parents(g1_a, g1_b, child_index=i) + log.info(" Gen-2 grandchild #%d: %d chars, $%.4f", + i, len(c.child_skill_md), c.synthesis_cost_usd) + g2_results.append(c) + + grandchildren = [ + { + "id": f"g2-grandchild-{i+1}", + "label": f"Gen-2 grandchild #{i+1}", + "owner": "synth", + "skill_md": c.child_skill_md, + } + for i, c in enumerate(g2_results) + ] + g2_synthesis_cost = sum(c.synthesis_cost_usd for c in g2_results) + + # ---- Arena: 5-way round-robin (g1-A, g1-B, 3 grandchildren) ---- + all_candidates = [g1_a, g1_b, *grandchildren] + pair_count = len(all_candidates) * (len(all_candidates) - 1) // 2 + log.info("=== ARENA: %d-way round-robin (C(%d,2)=%d pairs) ===", + len(all_candidates), len(all_candidates), pair_count) + + arena_rounds = [] + pair_no = 0 + for i in range(len(all_candidates)): + for j in range(i + 1, len(all_candidates)): + pair_no += 1 + cand_a = all_candidates[i] + cand_b = all_candidates[j] + log.info("Pair %d/%d: %s vs %s", pair_no, pair_count, cand_a["id"], cand_b["id"]) + r = await run_arena_round(cand_a, cand_b, TASK, runs_per=3) + arena_rounds.append({ + "pair": f"{cand_a['id']} vs {cand_b['id']}", + "winner": r.consistent_winner, + "cost_usd": r.total_cost_usd, + }) + + win_counts = {c["id"]: 0 for c in all_candidates} + for r in arena_rounds: + if r["winner"] in win_counts: + win_counts[r["winner"]] += 1 + + total_cost = ( + g1_synthesis_cost + + g2_synthesis_cost + + sum(r["cost_usd"] for r in arena_rounds) + ) + + best_grandchild = max( + g2_results, + key=lambda c: win_counts.get(f"g2-grandchild-{g2_results.index(c)+1}", 0) + ) + best_idx = g2_results.index(best_grandchild) + 1 + + # Repurpose GeneticPoCResult — parent_a/b here are gen-1 children + result = GeneticPoCResult( + parent_a_id=g1_a["id"], + parent_b_id=g1_b["id"], + child_id=f"g2-grandchild-{best_idx}", + child_skill_md=best_grandchild.child_skill_md, + synthesis_cost_usd=g1_synthesis_cost + g2_synthesis_cost, + arena_rounds=arena_rounds, + win_counts=win_counts, + total_cost_usd=total_cost, + ) + return result, g1_a, g1_b + + +async def main(): + log.info("Starting Genetic PoC v3 — multi-generation crossover") + result, g1_a, g1_b = await run_multigen() + print_report(result) + + out_dir = Path(__file__).parent.parent / "dist" / "arena-poc" + out_dir.mkdir(parents=True, exist_ok=True) + + # Save gen-1 children too — they're needed to interpret the result + (out_dir / "v3-g1-A-SKILL.md").write_text(g1_a["skill_md"]) + (out_dir / "v3-g1-B-SKILL.md").write_text(g1_b["skill_md"]) + log.info("Gen-1 children saved: v3-g1-A-SKILL.md, v3-g1-B-SKILL.md") + + report_path = out_dir / "genetic-v3-multigen.json" + report_data = { + "gen1_parents": {"g1-A": g1_a["skill_md"][:200], "g1-B": g1_b["skill_md"][:200]}, + "best_grandchild_id": result.child_id, + "synthesis_cost_usd": result.synthesis_cost_usd, + "arena_rounds": result.arena_rounds, + "win_counts": result.win_counts, + "total_cost_usd": result.total_cost_usd, + } + report_path.write_text(json.dumps(report_data, indent=2)) + log.info("Full report written: %s", report_path) + + skill_path = out_dir / f"v3-{result.child_id}-SKILL.md" + skill_path.write_text(result.child_skill_md) + log.info("Best grandchild SKILL.md saved: %s", skill_path) + + +if __name__ == "__main__": + asyncio.run(main())