From 3a6db6365d8075d504c616c4b3b1dd06921e05a0 Mon Sep 17 00:00:00 2001 From: Yutong Dai Date: Fri, 18 Sep 2026 01:48:01 +0000 Subject: [PATCH 1/3] Sync from beagle main 0712c8f MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See CHANGELOG.md — v0.0.2. --- CHANGELOG.md | 95 +++++++ README.md | 24 +- beagle/agents/core/litellm_gateway.py | 11 +- beagle/benchmarks/harness/_common.py | 32 +++ beagle/cli/_preflight.py | 165 ++++++++++++ beagle/cli/evaluate.py | 24 ++ beagle/tools/onboard.py | 26 +- docs/advanced.md | 5 + docs/onboarding-an-agent.md | 268 +++++++++++++++++++ experiments/scripts/dashboard.py | 50 +++- experiments/scripts/generate_eval_configs.py | 8 +- experiments/scripts/results_data.py | 97 ++++++- pyproject.toml | 2 +- scripts/generate_eval_configs.py | 16 +- scripts/onboard_all_agents.sh | 14 +- tests/unit/test_examples.py | 59 ++-- tests/unit/test_gateway_proxy.py | 255 ------------------ tests/unit/test_generate_eval_configs.py | 33 ++- tests/unit/test_git_bootstrap.py | 91 +++++++ tests/unit/test_onboard.py | 24 +- tests/unit/test_onboard_all_script.py | 14 +- tests/unit/test_preflight.py | 194 ++++++++++++++ tests/unit/test_results_data.py | 200 ++++++++++++++ tests/unit/test_version_consistency.py | 77 ++++++ uv.lock | 2 +- 25 files changed, 1467 insertions(+), 319 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 beagle/cli/_preflight.py create mode 100644 docs/onboarding-an-agent.md delete mode 100644 tests/unit/test_gateway_proxy.py create mode 100644 tests/unit/test_git_bootstrap.py create mode 100644 tests/unit/test_preflight.py create mode 100644 tests/unit/test_results_data.py create mode 100644 tests/unit/test_version_consistency.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..41dfd16 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,95 @@ +# Changelog + +> Notable changes to beagle, newest first. Entries describe **what changed for a user and why**. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). + +## [Unreleased] + +_Nothing yet._ + +## [v0.0.2] — 2026-09-17 + +### Fixed + +- **Rollouts no longer fail on task images whose Linux distribution has reached end of life.** + A retired suite can keep serving a package index after its pool has been emptied, so the + in-container git bootstrap resolved package versions whose files no longer exist and every + affected trial died with `git bootstrap failed after 3 attempts` before the agent ever ran. + Debian 11 entered this state in September 2026 — its security index still advertised + `deb11u5` builds that had been purged, and had not yet been republished to + `archive.debian.org` — which broke two terminal-bench tasks (`qemu-alpine-ssh`, + `qemu-startup`) for *every* agent. The bootstrap now falls back to the distribution's `main` archive, which is still served, + after the normal install has already failed. Three properties are deliberate: + + - it is **inert** unless the normal path fails, so images on a supported release are + unaffected; + - the suite codename and the package version to downgrade to are both **read from the + container** at run time, so nothing is pinned to one release; + - apt is redirected at temporary files, so the container's own `/etc/apt` is **left + untouched** — an agent that shells out to `apt` later still sees its image's real sources. + + This is a workaround for a live archive inconsistency and is marked for removal in + `beagle/benchmarks/harness/_common.py`; drop it once the affected packages are reachable + again. + +- **`beagle.tools.onboard` recovers from an interrupted seed.** Creating the experiment-copy + repo and seeding it are separate network operations. When creation succeeded but the upstream + fetch did not, the copy was left existing-but-empty, and re-running the same command failed + with `ref 'baseline' not found on upstream` — a confusing message, since the missing ref is on + the *copy*, not upstream. Recovery needed the destructive `--reseed`. An empty copy is now + treated as a fresh one, so simply re-running the command repairs it. A non-empty copy still + follows the existing skip/reseed rules, so nothing can be silently overwritten. + +- **`onboard` no longer offers a GitHub token to other hosts.** Token injection was applied to + any `https://` URL, which meant an upstream hosted somewhere other than github.com would be + handed a GitHub PAT that cannot authenticate there. Injection is now limited to `github.com`; + other hosts authenticate by their own means (an SSH remote, for instance). + +- **An unreachable LLM gateway now fails before anything is spent.** Previously a wrong or dead + endpoint produced a *silent* full-sweep failure: every trial installed cleanly, started work, + exhausted its own retries and recorded a clean completion with zero tokens, so the run scored + 0.000 and looked like a capability result rather than broken plumbing. `beagle evaluate` now + TCP-connects to the resolved endpoint before acquiring any container, and refuses to start if + nothing is listening. `--dry-run` reports the same check. + + The check is by **route type, not vendor**: a `gateway` route (any OpenAI-compatible + `api_base`) is probed, as is a deployment-native `internal` route; a `direct` first-party + provider is deliberately not, since it is not an endpoint beagle operates and probing it would + false-fail behind an egress proxy. + + When the failing endpoint came from the environment and `.env` disagrees, the error says so + explicitly. That case is otherwise very hard to diagnose: `.env` reads correct, yet a variable + already exported in the shell takes precedence over it, so re-running whatever writes `.env` + cannot help. + +### Added + +- **Optional per-solved-task latency and cost columns in the results dashboard.** A checkbox — + *"Also show latency & cost per SOLVED task"*, off by default — adds `[solved]Latency/task (s)` + and `[solved]Cost/task ($)` beside the existing all-task columns. + + They are **additional**, never a replacement: the existing columns answer "what does attempting + a task cost", which mixes in failures that can be cheap (died early) or expensive (burned the + whole budget getting nowhere); the new ones answer "what does solving one cost", which is the + figure worth comparing across harnesses. Both populations are visible side by side. Cells are + blank rather than `0` where a benchmark solved nothing, since `0` would read as "free". + +- **A `Version` column in the dashboard, immediately after `Harness`**, labelled + `_` — for example `20260826_f0d15a`. + + Both halves earn their place: the declared version alone stops distinguishing runs once + evolution starts, because every candidate inherits its baseline's version, while the commit ref + alone is unreadable and loses the name the harness was onboarded under. Together they stay + meaningful in both modes — several benchmark rows sharing one `_` are visibly + the same candidate. + + The version is read from the canonical config the run recorded, falling back to the onboarding + manifest and finally to a short ref. It is deliberately *not* parsed out of the run directory + name, which only carries a version for generated configs and would be wrong silently otherwise. + +- **An end-to-end runbook for onboarding a new agent harness**, `docs/onboarding-an-agent.md`: + the up-front questions worth answering before writing code, the `install`/`run_in` split for + network-phased harnesses, the contracts an adapter must honour (timeout resolution, token + normalisation, ATIF trajectories, patch capture, egress declaration, provider routing), how to + wire an agent into config generation, and the traps that have cost real runs. + `docs/advanced.md` keeps its short conceptual introduction and links to it. + diff --git a/README.md b/README.md index 4325f58..45b4f91 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Python License uv - status + status

@@ -352,6 +352,23 @@ Categories split on one signal: did the trial record an error? Flags are **independent** — combine to union. Resolved tasks are never re-run. +Example of re-running a failed task [add `--dry-run` to print the plan without rolling out]: +```bash +beagle evaluate \ + --config \ + --run-dir \ + --retry-errors --dry-run +``` +We strongly recommend to use the `--dry-run` to print the rerun-plan without rolling out first. Since the automatical classification of the tasks may not be aligned with the your actual understanding of the scenario. +If you find any tasks you would want to exclude for rerun, we suggest users to use the `--task-ids` together with `--retry-unresolved` flag to specify the tasks to rerun. + +```bash +beagle evaluate \ + --config \ + --run-dir \ + --retry-unresolved --task-ids +``` +
Discussion on the flags @@ -399,6 +416,9 @@ run: runtime: xrlenv-cluster ``` +## Analyze the results +We offer a dashboard to analyze the results of the evaluation runs. It is a Streamlit app that can be run locally. Please refer to [experiments/scripts/README.md](experiments/scripts/README.md) for more details. + --- @@ -510,8 +530,10 @@ best node when the pipeline finishes. | Doc | What's in it | |---|---| | [examples/evolution/README.md](examples/evolution/README.md) | one-pipeline DarwinX smoke: generate config, dry-run, launch, and inspect outputs | +| [CHANGELOG.md](CHANGELOG.md) | what changed in each release, and why | | [docs/darwinx-configuration.md](docs/darwinx-configuration.md) | DarwinX concepts, typed knobs, measurement guidance, and campaign progression | | [docs/advanced.md](docs/advanced.md) | module map, onboarding your own agent adapter, the Python API | +| [docs/onboarding-an-agent.md](docs/onboarding-an-agent.md) | end-to-end runbook for adding a new agent harness: contracts, wiring, checklist, traps | | [docs/benchmark-remarks.md](docs/benchmark-remarks.md) | per-benchmark tasks we suggest excluding, with the measured evidence | | [docs/opencode-prune.md](docs/opencode-prune.md) | what `--prune opencode` drops from a clone, and why it's patch-safe | diff --git a/beagle/agents/core/litellm_gateway.py b/beagle/agents/core/litellm_gateway.py index a4ecf22..5f62464 100644 --- a/beagle/agents/core/litellm_gateway.py +++ b/beagle/agents/core/litellm_gateway.py @@ -61,6 +61,11 @@ def provider_api_host(model: str) -> str | None: return None +#: Env var the deployment-internal route reads its endpoint from. Defined once here — the +#: module that already owns this deployment's gateway knowledge — so nothing else re-declares it. +LOCAL_PROXY_ENV = "LLM_GATEWAY_EXPRESS_LOCAL_PROXY_URL" + + def gateway_key_pool() -> list[str]: """The ordered, de-duped gateway API-key pool: the singular ``LLM_GATEWAY_EXPRESS_API_KEY`` first (explicit), then the ``…_LIST`` entries; blanks (stray commas) skipped. Empty when no @@ -90,7 +95,7 @@ def gateway_litellm_kwargs() -> dict[str, str] | None: inside the container, and the gateway's 200/401 split can differ per replica / per endpoint (``/v1/chat/completions`` vs ``/v1/responses``) and even host-vs-container. A key that flips to "blocked" mid-rollout is a key-pool concern, not something this selection can recover.""" - url = (os.environ.get("LLM_GATEWAY_EXPRESS_LOCAL_PROXY_URL") or "").strip() + url = (os.environ.get(LOCAL_PROXY_ENV) or "").strip() if not url: return None pool = gateway_key_pool() @@ -149,10 +154,10 @@ def resolve_gateway(cfg: Mapping[str, Any] | None) -> dict[str, Any] | None: if gateway is None: raise ValueError( f"internal provider {route.name!r} requires " - "LLM_GATEWAY_EXPRESS_LOCAL_PROXY_URL to be set") + f"{LOCAL_PROXY_ENV} to be set") return gateway return None -__all__ = ["config_gateway_kwargs", "gateway_block", "gateway_key_pool", +__all__ = ["LOCAL_PROXY_ENV", "config_gateway_kwargs", "gateway_block", "gateway_key_pool", "gateway_litellm_kwargs", "provider_api_host", "resolve_gateway"] diff --git a/beagle/benchmarks/harness/_common.py b/beagle/benchmarks/harness/_common.py index adeb265..ca199cb 100644 --- a/beagle/benchmarks/harness/_common.py +++ b/beagle/benchmarks/harness/_common.py @@ -50,6 +50,38 @@ apk add --no-cache git ca-certificates && break elif command -v apt-get >/dev/null 2>&1; then apt-get update -qq && apt-get install -y --no-install-recommends git ca-certificates && break + # RETIRED-SUITE FALLBACK. A Debian suite past EOL can keep serving an index whose pool has + # already been emptied: bullseye-security was purged on 2026-09-12 yet still advertises + # deb11u5, and has not landed on archive.debian.org — so every .deb 404s and the retry loop + # above just repeats the identical failure. Observed on terminal-bench's two bullseye task + # images (qemu-alpine-ssh, qemu-startup); the same sweep's bookworm and Ubuntu images are + # unaffected, which is why this runs ONLY after the normal path has already failed. + # + # Install from `main` instead, which is still served. The images pre-install perl-base from + # *security* (deb11u4) while main carries deb11u3, so apt refuses the implied downgrade — + # hence --allow-downgrades plus an explicit target. The target is READ FROM THE CONTAINER at + # run time rather than hardcoded: these images ship deb11u4 where a bare bullseye-slim ships + # deb11u5, so a pinned constant would be wrong somewhere. + # + # Everything is written to temp files and apt is pointed at them with -o overrides, so the + # container's own /etc/apt is left untouched — an agent that shells out to apt later still + # sees its image's real sources. + # + # DROP THIS once Debian publishes bullseye-security to archive.debian.org (or the affected + # images are rebuilt): the block is inert whenever the normal apt path succeeds. + codename=$(. /etc/os-release 2>/dev/null; echo "${VERSION_CODENAME:-}") + if [ -n "$codename" ]; then + echo "[git-bootstrap] apt failed; retrying against the ${codename} main archive" >&2 + _list=$(mktemp); _lists=$(mktemp -d) + printf 'deb http://deb.debian.org/debian %s main\n' "$codename" > "$_list" + _apt="-o Dir::Etc::SourceList=$_list -o Dir::Etc::SourceParts=/dev/null -o Dir::State::Lists=$_lists" + if apt-get $_apt update -qq; then + _pin=$(apt-cache $_apt madison perl-base 2>/dev/null \ + | awk -v c="$codename" '$0 ~ ("debian " c "/main") {print $3; exit}') + apt-get $_apt install -y --no-install-recommends --allow-downgrades \ + git ca-certificates ${_pin:+perl-base=$_pin} && break + fi + fi elif command -v microdnf >/dev/null 2>&1; then microdnf install -y git ca-certificates && break elif command -v dnf >/dev/null 2>&1; then diff --git a/beagle/cli/_preflight.py b/beagle/cli/_preflight.py new file mode 100644 index 0000000..83bbb09 --- /dev/null +++ b/beagle/cli/_preflight.py @@ -0,0 +1,165 @@ +"""Pre-flight checks that run BEFORE a rollout acquires anything or spends anything. + +The check here exists because of a specific, expensive failure: an 89-task sweep ran to +completion scoring 0.000 on every task, because the agent in each container dialled a gateway +that wasn't there. Every trial installed cleanly, started work, exhausted its LLM retries, and +recorded a clean "completed" with zero tokens. ~25 minutes of cluster time and 89 containers to +learn one thing a single TCP connect answers in milliseconds. + +The endpoint was wrong for a reason worth naming: ``.env`` held the right URL, but a **stale +export in the launching shell outranked it** — :func:`beagle.dotenv.load_project_dotenv` fills +gaps only, so host env wins. Re-running the script that rewrites ``.env`` didn't help, because it +can't touch an already-exported variable. ``.env`` looked correct every time it was inspected +while every run kept using the stale value. :func:`gateway_mismatch_hint` exists to say that out +loud rather than leave it to be deduced. +""" + +from __future__ import annotations + +import socket +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +from beagle.agents.core.litellm_gateway import LOCAL_PROXY_ENV + +#: Re-exported from the module that owns this deployment's gateway knowledge — the name is not +#: re-declared here. Only the ``internal`` route involves it; see :func:`gateway_target`. +INTERNAL_PROXY_ENV = LOCAL_PROXY_ENV +#: How long to wait for a TCP connect. Generous for a healthy local/VPC endpoint, short enough +#: that an unreachable one fails fast — this runs before every live evaluate. +DEFAULT_TIMEOUT_S = 5.0 + + +def gateway_target(cfg: Any) -> tuple[str, str] | None: + """``(url, source)`` for the run's LLM endpoint, or ``None`` when there is nothing to probe. + + ``source`` is human-readable provenance — the env var or config path the URL came from — so + a failure can name *where to go fix it*, which is the part that took longest to work out + when this actually happened. + + **Coverage is by ROUTE TYPE, not by vendor** — nothing here is specific to one org's + gateway: + + * ``gateway`` — any OpenAI-compatible endpoint declared in config as ``api_base``. Fully + generic, and the route an external user configures, so the check is real for them too. + * ``internal`` — a deployment-native route; the URL comes from whatever + :func:`~beagle.agents.core.litellm_gateway.gateway_litellm_kwargs` resolves. + * ``direct`` — ``None``. A first-party vendor API is not an endpoint we own or operate, a + TCP probe of it proves little, and it would false-fail wherever egress is proxied. This + no-op is deliberate, not an oversight. + """ + from beagle.agents.core.litellm_gateway import gateway_litellm_kwargs + from beagle.agents.core.provider import GatewayProvider, InternalProvider, provider_config + + try: + route = provider_config(dict(cfg.agent.config or {})) + except (AttributeError, ValueError): + return None + if isinstance(route, GatewayProvider): + base = route.extra_args.api_base + return (base, "provider.extra_args.api_base in the config") if base else None + if isinstance(route, InternalProvider): + # Delegate to the shared resolver instead of re-reading an env var, so this tracks + # whatever the deployment route actually resolves to. + gw = gateway_litellm_kwargs() + url = (gw or {}).get("api_base", "") + return (url, f"${INTERNAL_PROXY_ENV} in the host environment") if url else None + return None + + +def probe_tcp(url: str, *, timeout: float = DEFAULT_TIMEOUT_S) -> str | None: + """``None`` when a TCP connection to ``url``'s host:port succeeds; a short reason otherwise. + + Deliberately a bare TCP connect, not an HTTP request: it needs no credentials, spends no + tokens, and cannot be confused by an auth or routing error. It answers exactly one question — + *is anything listening where we are about to send every request?* + + **What it does not prove:** that a rollout CONTAINER can reach the endpoint. This runs on the + launch host, so a container-to-host network partition would still slip through. It catches + the far more common case — a wrong, stale, or dead address — which is what actually bit. + """ + parts = urlsplit(url if "//" in url else f"//{url}") + host, port = parts.hostname, parts.port + if not host: + return f"no host in {url!r}" + if port is None: + port = 443 if parts.scheme == "https" else 80 + try: + with socket.create_connection((host, port), timeout=timeout): + return None + except TimeoutError: + return f"timed out after {timeout:g}s connecting to {host}:{port}" + except OSError as exc: + return f"{exc.strerror or exc} connecting to {host}:{port}" + + +def gateway_mismatch_hint(url: str, source: str) -> str: + """The extra line to print when the unreachable URL came from the host env AND ``.env`` + disagrees — i.e. a stale export is silently beating a correct ``.env``. + + This is the single most useful thing to say in that situation, because every other signal + points the wrong way: ``.env`` reads correct, and the script that maintains it reports + success. ``""`` when there's nothing to add. + """ + if INTERNAL_PROXY_ENV not in source: + return "" + from beagle.dotenv import find_dotenv, parse_dotenv + + path = find_dotenv() + if path is None or not Path(path).is_file(): + return "" + try: + in_file = (parse_dotenv(Path(path).read_text(encoding="utf-8")).get(INTERNAL_PROXY_ENV) + or "").strip() + except OSError: + return "" + if not in_file or in_file.rstrip("/") == (url or "").rstrip("/"): + return "" + return ( + f"\n {path.name} says {in_file} — DIFFERENT, and it was ignored: a variable already " + f"exported in your shell takes precedence over {path.name}, which fills gaps only. " + f"Re-running the script that rewrites {path.name} will NOT fix this.\n" + f" Fix the shell: unset {INTERNAL_PROXY_ENV} (or export the correct value)" + ) + + +def check_gateway(cfg: Any, *, timeout: float = DEFAULT_TIMEOUT_S) -> tuple[str, str, str | None] | None: + """``(url, source, failure)`` for the run's endpoint — ``failure`` is ``None`` when reachable. + + ``None`` when there is no endpoint to probe (a direct route). Callers decide what to do: + the dry run reports, a live run refuses to start. + """ + target = gateway_target(cfg) + if target is None: + return None + url, source = target + return url, source, probe_tcp(url, timeout=timeout) + + +def require_gateway_reachable(cfg: Any, *, timeout: float = DEFAULT_TIMEOUT_S) -> None: + """Raise :class:`SystemExit` if the run's LLM endpoint is unreachable. No-op otherwise. + + Called on the live path before any container is acquired, so an unreachable gateway costs one + failed connect instead of a full sweep of zero-scoring trials. + """ + checked = check_gateway(cfg, timeout=timeout) + if checked is None: + return + url, source, failure = checked + if failure is None: + return + raise SystemExit( + f"[beagle evaluate] PREFLIGHT FAILED: the LLM gateway is unreachable.\n" + f" endpoint: {url}\n" + f" error : {failure}\n" + f" from : {source}" + f"{gateway_mismatch_hint(url, source)}\n" + f" Nothing was acquired and nothing was spent. Every trial would have scored 0 with an " + f"agent that never reached a model.\n" + f" Re-check with: beagle evaluate --config --dry-run" + ) + + +__all__ = ["INTERNAL_PROXY_ENV", "check_gateway", "gateway_mismatch_hint", "gateway_target", + "probe_tcp", "require_gateway_reachable"] diff --git a/beagle/cli/evaluate.py b/beagle/cli/evaluate.py index 9f13c04..479d54f 100644 --- a/beagle/cli/evaluate.py +++ b/beagle/cli/evaluate.py @@ -42,6 +42,14 @@ def _cmd_evaluate(args: argparse.Namespace) -> int: resume=args.resume, retry_errors=args.retry_errors, retry_unresolved=args.retry_unresolved, only_task_ids=only_task_ids) + # Before ANY container is acquired: is the LLM endpoint actually there? An unreachable + # gateway does not fail a run loudly — every trial installs, runs, exhausts its retries and + # records a clean "completed" with zero tokens, so a whole sweep scores 0.000 and looks like + # a capability result. One TCP connect turns that into an immediate, explained error. + from beagle.cli._preflight import require_gateway_reachable + + require_gateway_reachable(run_cfg) + from beagle.rollout.interrupt import stop_run_on_sigint from beagle.rollout.run_id import build_run_id, compute_config_hash from beagle.rollout.runtime import RuntimeConfig as RtCfg @@ -200,6 +208,22 @@ def _dry_run(cfg, spec, items, *, run_dir: Path, resume: bool = False, gw = gateway_litellm_kwargs() endpoint = gw["api_base"] if gw else "⚠ deployment gateway URL is not set" print(f" provider : internal {route.name} ({endpoint})") + # Reachability, not just resolution: a resolved-but-dead endpoint is what turns a sweep into + # all-zeros. Reported here and ENFORCED on the live path (see require_gateway_reachable). + from beagle.cli._preflight import check_gateway, gateway_mismatch_hint + + checked = check_gateway(cfg) + if checked is not None: + url, source, failure = checked + if failure is None: + print(f" gateway : ✓ reachable ({url})") + else: + print(f" gateway : ⚠ UNREACHABLE — {failure}") + print(f" endpoint {url}, from {source}") + hint = gateway_mismatch_hint(url, source) + if hint: + print(" " + hint.strip().replace("\n ", "\n ")) + print(" a live run would refuse to start (every trial would score 0)") print(f" agent : {spec.name} @ {src.repo}@{src.ref}" if src else f" agent : {spec.name} ⚠ NO SOURCE resolved") print(f" benchmark : {cfg.benchmark.name}") diff --git a/beagle/tools/onboard.py b/beagle/tools/onboard.py index 5cf0fce..e18f938 100644 --- a/beagle/tools/onboard.py +++ b/beagle/tools/onboard.py @@ -71,13 +71,14 @@ def is_full_sha(ref: str) -> bool: def authed_url(url: str, token: str) -> str: - """Inject a token into an HTTPS GitHub URL for a single command's auth. + """Inject a token into an HTTPS github.com URL for a single command's auth. - Returns the URL unchanged for non-HTTPS (ssh) URLs or when no token is given — - those authenticate by other means (ssh keys). The result is passed as a command - argument only, never stored in a remote. + Returns the URL unchanged for other hosts, non-HTTPS (ssh) URLs, or when no token + is given — those authenticate by other means. Restricting injection to github.com + also prevents accidentally presenting a GitHub PAT to an HTTPS upstream on another + host. The result is passed as a command argument only, never stored in a remote. """ - if token and url.startswith("https://"): + if token and re.match(r"https://github\.com(?:/|\Z)", url, flags=re.IGNORECASE): return url.replace("https://", f"https://x-access-token:{token}@", 1) return url @@ -220,6 +221,11 @@ def create_repo(repo: str, visibility: str) -> None: _run(["gh", "repo", "create", repo, f"--{visibility}"]) +def remote_is_empty(remote_auth: str, token: str) -> bool: + """Whether an accessible git remote has no refs (for recovery after create-before-seed failure).""" + return not _run(["git", "ls-remote", remote_auth], token=token, capture=True).strip() + + #: Default branch the single baseline commit lands on. Unified across agents — the ``--repo`` name #: already carries the version (e.g. ``opencode_v1.18.16``). Override per-onboard with ``--branch-name``. DEFAULT_SEED_BRANCH = "baseline" @@ -425,7 +431,15 @@ def main(argv: list[str] | None = None) -> int: # The copy's baseline SHA — a fresh orphan of the upstream tree (small seed), so it differs from # ``upstream_sha`` and is what the manifest/runtime pin. Recovered from the copy when we skip seeding. - if created or args.reseed: + # Creation and seeding are separate network operations. If creation succeeded on a prior run but + # the upstream fetch failed, the copy exists but has no refs. Treat that state exactly like a new + # repo so rerunning the same command repairs it; this is safe and does not require destructive + # --reseed. A non-empty copy still follows the normal skip/reseed rules below. + empty_copy = not created and remote_is_empty(github_auth, token) + if empty_copy: + print(f"[onboard] github repo {args.repo} is empty — resuming interrupted seed") + + if created or empty_copy or args.reseed: if args.reseed and not created: print("[onboard] --reseed: re-seeding from upstream (OVERWRITES any candidate branches)") prune = PRUNE_PROFILES[args.prune] if args.prune else None diff --git a/docs/advanced.md b/docs/advanced.md index 3d73fac..5766771 100644 --- a/docs/advanced.md +++ b/docs/advanced.md @@ -78,6 +78,11 @@ write harness-specific code. Closed-source CLI you can't evolve: `class MyAgent(Agent, Editor)`. Start from `beagle/agents/core/_template.py`. Benchmarks and algorithms onboard the same way — one file, `@register`, done. +The sketch above is the shape; a real adapter also has to split `install`/`run_in` +for network-phased harnesses, resolve timeouts from the benchmark, normalize token +usage, and emit ATIF. See **[Onboarding an agent harness](onboarding-an-agent.md)** +for the full runbook, contracts, and checklist. + ## Onboard your own benchmark A benchmark is three pluggables behind the `Benchmark` ABC, each independently diff --git a/docs/onboarding-an-agent.md b/docs/onboarding-an-agent.md new file mode 100644 index 0000000..41c1613 --- /dev/null +++ b/docs/onboarding-an-agent.md @@ -0,0 +1,268 @@ +# Onboarding an agent harness + +End-to-end runbook for adding a new agent to beagle. [docs/advanced.md](advanced.md#onboard-your-own-agent) +covers the *concept* (capabilities, `@register`, auto-discovery) in a page; this is the operational +checklist — the contracts an adapter must honor, and the traps that have actually cost us runs. + +beagle **never vendors an agent's code.** You add a thin adapter; the agent itself lives in a git +repo you own (the *experiment copy*) and is cloned into the task container per trial. The adapter is +typically 200-400 lines. + +## 0. Answer five questions first + +Do this before writing code. If you can't answer #1 or #2, the agent cannot be onboarded as a +`Runnable` yet, and you'll discover that 300 lines in. + +| # | Question | Why it decides the design | +| --- | --- | --- | +| 1 | **How does it run headless?** A single non-interactive command, one task, exits on its own? | No headless mode → no `run_in`. A TUI-only agent needs an upstream flag first. | +| 2 | **How does it reach an LLM?** Env var, config file, CLI flag? Can the base URL be overridden? | Decides whether gateway/proxy routing is expressible. An agent that hardcodes a vendor endpoint can't use `provider: gateway`. | +| 3 | **What does it install from?** pip/npm/bun/cargo, which registries, which language runtime? | Becomes `install()` and `install_hosts()`. A container's system Python is often too old. | +| 4 | **What does it emit?** A trajectory/log file, in what format, with token usage in it? | Becomes the ATIF converter and the `Usage` parse. No usage in the stream → no cost accounting. | +| 5 | **What's the evolvable surface?** Prompt YAML, a source tree, a config file? | Becomes `AgentSource.entrypoint` — what an evolver edits. | + +Two answers are worth writing down verbatim: the exact headless command line, and the exact shape of +the usage numbers (are cached tokens a *subset* of input, or reported *in addition*? — see §4.2). + +## 1. Onboard the source + +Stand up the experiment copy — a repo **you own**, seeded with the upstream tree at a pinned commit. +The evolver pushes candidate branches there; upstream is never written to. + +```bash +python -m beagle.tools.onboard \ + --upstream https://github.com// --ref <40-char-sha> \ + --repo "$YOUR_ORG/_" --private --version --branch-name baseline \ + --dir ../beagle-experiments/_ --profile-name _ +``` + +Pin a **full 40-char SHA**, not a branch — a branch drifts, so a re-onboard silently snapshots a +different baseline. This writes `.beagle/agents/.json`, the pointer everything downstream +reads. Add the invocation to [`scripts/onboard_all_agents.sh`](../scripts/onboard_all_agents.sh). + +If the clone is large, consider a `--prune` profile (see [opencode-prune.md](opencode-prune.md)) — +it only ever removes whole paths, so kept blobs stay byte-identical and evolution diffs still apply +to upstream. + +## 2. Write the adapter + +One package: `beagle/agents//__init__.py`. Start from +[`beagle/agents/core/_template.py`](../beagle/agents/core/_template.py). Auto-discovered on +`import beagle` — no registration list to edit. + +Compose the capabilities the agent actually has: + +| Mixin | Implement | Gives you | +| --- | --- | --- | +| `Runnable` | `install` + `run_in` | can attempt benchmark tasks (evolvee) | +| `Evolvable` | `_default_source` | versioned `repo@ref` source = θ | +| `Editor` | `edit` | can drive a workspace (evolver) | + +Evolvee = `Runnable` + `Evolvable`. A closed-source CLI is `Editor` only, with +`transparency = Transparency.BLACK_BOX`. + +### The two-phase lifecycle — implement `install` + `run_in`, not `run` + +This is the part the conceptual docs gloss. A **network-phased** harness (pier/harbor) opens the +network for a trusted install, then locks it down to the LLM endpoint for the run. So split: + +```python +def install(self, handle, task_ctx, *, runtime: ContainerRuntime) -> None: + """INSTALL phase — network open. Clone repo@ref, build. Raise AgentInstallError to fail loud.""" + +def run_in(self, handle, task, task_ctx, *, runtime) -> TaskResult: + """RUN phase — egress restricted to network_hosts(). Caller owns the container; no acquire/destroy.""" +``` + +Inherit the default `run()` ([base.py:190](../beagle/agents/core/base.py)) — it composes the two for +always-open harnesses (docker drop-in) *and* records the harbor-shaped per-phase timing +(`environment_setup` / `agent_setup` / `agent_execution`). Overriding `run()` yourself throws that +away. Need a tweak to the container acquire (e.g. clearing an image `ENTRYPOINT` so `sleep infinity` +runs)? Override `_acquire_run_args()`, not `run()`. + +**`runtime.exec` is `check=False`.** Every step you don't check is a silent failure that reads +downstream as a benign "scored 0". Check each one and raise `AgentInstallError` with a tail of +stderr. + +### Cloning a private experiment copy + +Use the shared helper — it handles the token URL rewrite and fetch-by-SHA: + +```python +from beagle.rollout.runtime.transport import GitClone, clone_with_retry + +clone = GitClone(repo_url=src.repo, ref=src.ref or "", container_path="/agent", token_env=token_env) +cloned = clone_with_retry(runtime, handle, clone, env=clone_env or None, timeout=600) +``` + +`clone_with_retry` exists because concurrent trials cloning one private repo trip GitHub's auth +throttle as a spurious 401; it backs off with jitter while definitive errors still fail fast. + +### Language runtimes + +Don't assume the container's interpreter. Task images pin their own (SWE-bench images ship Python +3.9). Stand up your own under `/agent` — `mini_swe` uses `uv` to create a managed 3.11 venv so the +*orchestrator* runs modern while the agent's tool-calls still execute in the task's own environment +([mini_swe/\_\_init\_\_.py:64](../beagle/agents/mini_swe/__init__.py)). Keep everything you install +out of the task workspace so it can never land in the agent's diff. + +## 3. Honor the six contracts + +These are not optional, and five of the six exist because of a specific past failure. + +### 3.1 Timeout — never hardcode a wall clock + +```python +from beagle.agents.core.base import resolve_agent_timeout +timeout = resolve_agent_timeout(cfg, task_ctx) +``` + +The benchmark's declared budget wins; `agent.timeout` in the run config applies only when nothing +was declared. There is deliberately **no house default** — an unstated budget raises +`AgentBudgetUndeclared` rather than inventing a number. Four adapters once each hardcoded 1800 s +while SWE-rebench tasks declared 3000 s and 6000 s, silently truncating every long task. + +### 3.2 Token usage — normalize into `Usage`, never fold cache away + +Parse the native stream into [`Usage`](../beagle/agents/core/usage.py) — four **disjoint** buckets — +and set `TaskResult.tokens = usage.to_token_counts()`. + +**Per-provider cache semantics differ, and getting this wrong double-counts:** + +| Native shape | Cached tokens are… | Fresh input = | +| --- | --- | --- | +| OpenAI-shaped (`prompt_tokens`, `prompt_tokens_details.cached_tokens`) | a **subset** of `prompt_tokens` | `prompt_tokens - cached_tokens` | +| Anthropic-shaped (`cache_read_input_tokens`, `cache_creation_input_tokens`) | **in addition to** `input_tokens` | `input_tokens` as-is | + +Invariant: `prompt == input_uncached + cache_read + cache_write`. beagle stays pricing-agnostic — it +keeps the split so a downstream estimate can price each bucket. A parse miss must never fail the +run: return `({}, 0)` and let the patch stand. + +### 3.3 Trajectory — convert to ATIF, post-job + +**ATIF is beagle's canonical trajectory format** (harbor's Agent Trajectory Interchange Format). +Don't invent a house format. Register one converter per *format*, not per agent×harness — that's +what keeps this M+N: + +```python +# beagle/benchmarks/trajectory.py +@register_converter("my-agent-json") +def _my_agent_to_atif(logs_dir: Path, *, instruction: str, agent_name: str, + agent_version: str, model_name: str | None): ... +``` + +Point `TaskResult.trajectory` at it: `TrajectoryRef(path=Path("my.traj.json"), format="my-agent-json")`. + +Have the agent write its native stream into `/logs/agent` — pier/harbor sync that to the trial's +`agent/` directory. **Conversion happens POST-JOB** in `HarborHarness._emit_trajectories`, never in +the in-trial shim: on the cluster the native stream is synced out of the container only *after* the +agent step, so a shim-time converter runs too early and finds nothing. + +### 3.4 Patch capture — record the base commit first + +Capture `git rev-parse HEAD` **before** the agent runs, then commit whatever it left and diff +`base..HEAD`: + +```python +base = runtime.exec(handle, ["bash", "-lc", f"{pre}cd {repo} && git rev-parse HEAD 2>/dev/null || true"]).stdout.strip() +# ... agent runs ... +# git add -A && git commit; then: git diff ..HEAD +``` + +This covers both grader styles: a working-tree grader (swe-bench reads `TaskResult.patch`) and a +`base..HEAD` grader (deep-swe/pier's `verifier.collect`). Agents that commit their own work produce +an *empty* post-run working-tree diff — that's the failure this prevents. + +Prepend `task_ctx.shell_preamble` before every `cd` so the benchmark's own environment setup +(activating the task's Python env, etc.) is in scope. + +### 3.5 Egress — declare both host sets + +```python +def install_hosts(self) -> list[str]: # git host + package registries, for the INSTALL phase +def network_hosts(self) -> list[str]: # the LLM endpoint, for the RUN phase +``` + +A filtered-egress benchmark allowlists exactly these. Scheme-qualify entries +(`https://api.example.com`): pier urlparses each one, and a bare hostname has no `.hostname`, so it +drops out of the allowlist silently. + +### 3.6 Provider routing — use the typed route + +Read the route with `provider_config(cfg)` and declare what your adapter can honor via +`supported_provider_types` (default: `{"direct", "gateway", "internal"}`); preflight and runtime +enforce the same set. See [provider-routing.md](provider-routing.md). + +Credentials must **not** ride the command line — argv is visible to every process in the container +via `ps`, and beagle echoes commands into runtime records, error tails, and test output. Pass secrets +through the exec's environment and, if the agent needs them in a file, materialize it with a +`umask 077` heredoc. + +## 4. Keep the agent benchmark-agnostic + +**No per-benchmark prompt templates.** Do not key a template on `benchmark_name`. That's an N×M trap: +a new benchmark then has to touch every agent, and a new agent has to ship a template per benchmark. +The benchmark supplies the task instruction; the agent runs it. Benchmark-specific framing, if any, +lives on the benchmark side. + +One `run` works on every harness — you never write harness-specific code in an adapter. + +## 5. Wire it up + +1. **Onboard entry** — add the invocation to `scripts/onboard_all_agents.sh` (§1). +2. **Eval configs** — add an entry to `AGENTS` in + [`scripts/generate_eval_configs.py`](../scripts/generate_eval_configs.py), keyed by your + `harness.name`, listing the `versions` to generate for. **The join key is `version`** — it must + match the `--version` you passed to `onboard`, which is how a generated config finds its manifest. + + ```python + "my-agent": dict( + versions=["v1.2.3"], + extra_args={"my_agent_args": ["--headless"]}, + note="what a reader needs to know about these args", + ), + ``` +3. **Run the smoke** for one benchmark before anything wider. + +## 6. Test it + +Add `tests/unit/test_.py`. The existing agent tests +([`test_mini_swe.py`](../tests/unit/test_mini_swe.py), [`test_opencode.py`](../tests/unit/test_opencode.py)) +show the pattern: a fake runtime records `exec` calls, and assertions run against the composed command +string and the parsed `TaskResult` — no container needed. Cover at minimum: + +- the headless command is composed correctly (model, task, config, output path); +- usage parsing, **including the cache split** — assert the invariant, and assert an OpenAI-shaped + fixture isn't double-counted; +- a failed run that produced no patch surfaces as `FAILED` with a diagnostic tail; +- a trajectory-parse miss degrades to `({}, 0)` instead of raising; +- the ATIF converter round-trips a native fixture (see [`test_trajectory.py`](../tests/unit/test_trajectory.py)). + +## Checklist + +- [ ] Experiment copy onboarded at a pinned full SHA; `.beagle/agents/.json` exists +- [ ] `scripts/onboard_all_agents.sh` entry added +- [ ] Adapter registered with `@register`, capabilities composed, `transparency` set +- [ ] `install` + `run_in` split (not a monolithic `run`); every `exec` return checked +- [ ] `resolve_agent_timeout` used — no hardcoded wall clock +- [ ] Usage normalized into `Usage`; cache semantics verified against the native format +- [ ] ATIF converter registered; native stream written to `/logs/agent` +- [ ] Patch captured as `base..HEAD`; `shell_preamble` honored +- [ ] `install_hosts()` + `network_hosts()` declared, scheme-qualified +- [ ] No secrets in argv; no per-benchmark prompt templates +- [ ] `generate_eval_configs.py` `AGENTS` entry, version matching the manifest +- [ ] Unit tests added; `pytest tests/` green +- [ ] Smoke run passes on one benchmark + +## Traps + +| Trap | Symptom | +| --- | --- | +| Hardcoded timeout | Long tasks truncated; looks like the agent gave up | +| Unchecked `runtime.exec` | Silent install failure reads as "scored 0" | +| Post-run working-tree diff | Empty patch for any agent that commits its own work | +| Converting the trajectory in the shim | `trajectory.json` missing on the cluster, present locally | +| `+= cached` on an OpenAI-shaped stream | Input tokens double-counted | +| Bare hostname in an allowlist | Egress silently blocked under filtered egress | +| Template keyed on `benchmark_name` | N×M coupling; every new benchmark touches every agent | +| Secrets in argv | Credential visible via `ps` and echoed into logs/test output | diff --git a/experiments/scripts/dashboard.py b/experiments/scripts/dashboard.py index f7add7c..3d62969 100644 --- a/experiments/scripts/dashboard.py +++ b/experiments/scripts/dashboard.py @@ -38,6 +38,15 @@ def overview_page() -> None: all_runs = sorted(s["runname"] for s in summaries) f_run = fc[3].multiselect("Run", all_runs) # empty = all runs; select to narrow (keeps the UI clean) + # Opt-in: solved-only latency/cost are ADDITIONAL columns, never a replacement for the + # all-task ones — the two answer different questions and the table should be able to show + # both side by side. Off by default so the default view stays narrow. + show_solved = st.checkbox( + "Also show latency & cost per SOLVED task", + value=False, + help="Adds two columns whose medians are taken over resolved trials only. The existing " + "Latency/task and Cost/task stay as they are (median over all attempted tasks).") + body = st.container() # reserve the table's slot HERE (above pricing); filled after prices resolve with st.expander("💲 Pricing · $/1M tokens (edit to reprice)"): # renders below the table @@ -68,13 +77,31 @@ def _keep(s: dict) -> bool: f"({100*df['Cached tokens'].sum()/max(1, df['Total tokens'].sum()):.0f}%)") m[3].metric("Est. cost", f"${df['Cost (est. $)'].sum():,.2f}") - disp = df.drop(columns=["In progress"]).copy() + solved_cols = ["[solved]Latency/task (s)", "[solved]Cost/task ($)"] + drop = ["In progress"] + ([] if show_solved else solved_cols) + # errors="ignore": streamlit hot-reloads THIS file but keeps an already-imported + # results_data, so an editing session can pair a new drop-list with an old row builder. + # A missing column should cost the two columns, not take down the whole page. + disp = df.drop(columns=drop, errors="ignore").copy() + if show_solved and not any(c in df.columns for c in solved_cols): + # errors="ignore" above keeps the page alive, but silence is its own bug: the + # checkbox would simply do nothing. Say WHY. Streamlit re-executes this file on + # change yet keeps already-imported modules, so a session started before these + # columns existed serves rows without them until it is restarted. + st.warning( + "The per-solved-task columns aren't in this session's data. Streamlit reloads " + "this page but not already-imported modules — **restart streamlit** " + "(Ctrl-C and re-run) to pick them up.") disp["Score"] = (disp["Score"] * 100).round(1) # → percent disp["Total tokens"] = (disp["Total tokens"] / 1e6).round(2) # → millions disp["Cached tokens"] = (disp["Cached tokens"] / 1e6).round(2) st.dataframe( disp, hide_index=True, width="stretch", column_config={ + "Version": st.column_config.TextColumn( + "Version", + help="the onboarded harness version; a short commit ref when the run used a " + "ref we never onboarded under a version name (e.g. an evolved candidate)"), "Score": st.column_config.NumberColumn("Score", format="%.1f%%", help="resolved / tasks"), "Total tokens": st.column_config.NumberColumn("Total (M)", format="%.2f"), "Cached tokens": st.column_config.NumberColumn("Cached (M)", format="%.2f"), @@ -85,10 +112,25 @@ def _keep(s: dict) -> bool: help="median agent-execution time per task (excludes setup + verifier)"), "Cost/task ($)": st.column_config.NumberColumn( "Cost/task ($)", format="%.2f", help="median est. cost per task (at the prices below)"), + # Label == key, so what the table shows is what the row actually holds. + "[solved]Latency/task (s)": st.column_config.NumberColumn( + "[solved]Latency/task (s)", format="%.0f", + help="median agent-execution time over RESOLVED trials only"), + "[solved]Cost/task ($)": st.column_config.NumberColumn( + "[solved]Cost/task ($)", format="%.2f", + help="median est. cost over RESOLVED trials only (at the prices below)"), }) - st.caption( - "- **Latency/task** and **Cost/task** are medians across the benchmark's tasks.\n" - "- Cost = (prompt−cached)·input-price + cached·cached-price + completion·output-price, at the prices below.") + caption = [ + "- **Latency/task** and **Cost/task** are medians across the benchmark's tasks.", + ("- Cost = (prompt−cached)·input-price + cached·cached-price + " + "completion·output-price, at the prices below."), + ] + if show_solved: + caption.append( + "- **[solved]** columns are the same medians over RESOLVED " + "trials only — what it costs to actually solve a task, rather than to attempt " + "one. They are blank where a benchmark solved nothing.") + st.caption("\n".join(caption)) if df["In progress"].any(): st.info("▶ " + ", ".join(df[df["In progress"]]["Run"].unique()) + " — in progress (no run.json yet)") diff --git a/experiments/scripts/generate_eval_configs.py b/experiments/scripts/generate_eval_configs.py index 68222e7..553468c 100755 --- a/experiments/scripts/generate_eval_configs.py +++ b/experiments/scripts/generate_eval_configs.py @@ -146,9 +146,6 @@ def build_config(agent: str, bench: str, manifest: dict, args: argparse.Namespac "model": {"name": args.model}, "effort": args.effort, "max_turns": args.max_turns, - "forward_env": list(args.forward_env), - "timeout": args.timeout, - "extra_args": a["extra_args"], } if args.provider: if not isinstance(args.provider, dict): @@ -156,6 +153,11 @@ def build_config(agent: str, bench: str, manifest: dict, args: argparse.Namespac agent_config["provider"] = gen.provider_dict( gen.provider_config({"provider": args.provider}) ) + agent_config.update({ + "forward_env": list(args.forward_env), + "timeout": args.timeout, + "extra_args": a["extra_args"], + }) return { "run": { "dir": str(args.results), diff --git a/experiments/scripts/results_data.py b/experiments/scripts/results_data.py index 83e98da..aff1c50 100644 --- a/experiments/scripts/results_data.py +++ b/experiments/scripts/results_data.py @@ -17,14 +17,17 @@ import json import statistics from datetime import datetime +from functools import lru_cache from pathlib import Path + +import yaml from typing import Any, Callable RESULTS_DIR = Path(__file__).resolve().parents[1] / "results" _SKIP_DIRS = {"archive-to-delete"} _TOKEN_KEYS = ("prompt", "completion", "input_uncached", "cache_read", "cache_write", "total") #: Bump when summary.json's shape changes so stale caches auto-rebuild (see load_summaries). -_SCHEMA = 2 +_SCHEMA = 4 # bumped: `version` is now _ (cached ones rebuild) # $/1M tokens: fresh input / cached-read input / output. ESTIMATES — internal gateway models have no # public price; edit here or override live in the app's sidebar. "*" is the fallback. @@ -132,12 +135,76 @@ def classify_error(err: str | None, resolved: bool) -> str: # ── per-run summary build ──────────────────────────────────────────────────────────────────────── -def _cfg_meta(cfg: dict[str, Any]) -> tuple[str, str, str]: +#: ``.beagle/agents/.json`` — onboard files the version it pinned alongside the ref it +#: seeded. That mapping is the only place a run's *human* version name survives: run.json records +#: ``agent.source.ref`` (a SHA) and no version, because RunConfig flattens the canonical +#: ``harness.version`` away. Built once, tolerant of a missing/partial directory. +_MANIFEST_DIR = Path(__file__).resolve().parents[2] / ".beagle" / "agents" + + +@lru_cache(maxsize=1) +def _versions_by_ref() -> dict[str, str]: + out: dict[str, str] = {} + for f in sorted(_MANIFEST_DIR.glob("*.json")) if _MANIFEST_DIR.is_dir() else []: + try: + m = json.loads(f.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + ref, ver = str(m.get("ref") or ""), str(m.get("version") or "") + if ref and ver: + out[ref] = ver + return out + + +def _version_from_config(config_path: str | None) -> str: + """``agent.harness.version`` from the canonical config the run was launched with. + + The most authoritative source: it is what the run DECLARED, e.g. ``20260826``. ``run.json`` + keeps the path but not the value, because RunConfig flattens ``harness`` away. ``""`` when + the path is missing, moved, or unreadable — configs get reorganised, so this must not raise. + """ + if not config_path: + return "" + try: + doc = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) or {} + except (OSError, ValueError, yaml.YAMLError): + return "" + harness = ((doc.get("agent") or {}).get("harness") or {}) + return str(harness.get("version") or "") + + +def harness_version(agent: dict[str, Any], config_path: str | None = None) -> str: + """``_`` for a run — the declared version AND the exact code behind it. + + Both halves earn their place. The version alone can't tell two runs apart once evolution + starts, since every candidate shares the baseline's version; the ref alone is unreadable and + loses the name the harness was onboarded under. Together they stay meaningful in both modes: + ``20260826_53a76a`` for a baseline, and the same version with a different suffix for each + candidate branch derived from it. + + Version resolution, best first: + + 1. ``agent.harness.version`` in the canonical config the run recorded — what it declared. + 2. The onboarding manifest, keyed by ref, when that config has moved or been deleted. + 3. Neither: fall back to the bare short ref. Deliberately NOT parsed out of the run + directory name, which only carries a version for generator-produced runs and would need + every benchmark tag to split correctly — brittle, and wrong silently. + """ + ref = str(((agent.get("source") or {}).get("ref")) or "") + version = _version_from_config(config_path) or _versions_by_ref().get(ref, "") + if version and ref: + return f"{version}_{ref[:6]}" + if version: + return version + return ref[:12] if ref else "?" + + +def _cfg_meta(cfg: dict[str, Any], config_path: str | None = None) -> tuple[str, str, str, str]: agent = cfg.get("agent") or {} harness = agent.get("name") or "?" model = (cfg.get("model") or {}).get("name") or (agent.get("model") or {}).get("name") or "?" effort = (agent.get("config") or {}).get("effort") or "—" - return harness, model, effort + return harness, harness_version(agent, config_path), model, effort def _config_from_disk(run_dir: Path) -> dict[str, Any]: @@ -196,7 +263,7 @@ def build_run_summary(run_dir: Path) -> dict[str, Any]: rj = run_dir / "run.json" run = json.loads(rj.read_text()) if rj.exists() else {} cfg = run.get("config") or _config_from_disk(run_dir) - harness, model, effort = _cfg_meta(cfg) + harness, version, model, effort = _cfg_meta(cfg, run.get("config_path")) bench_meta = run.get("benchmarks") or {} bench_names = list(bench_meta) or [b.name for b in run_dir.iterdir() @@ -227,7 +294,8 @@ def build_run_summary(run_dir: Path) -> dict[str, Any]: summary = { "_schema": _SCHEMA, - "runname": run_dir.name, "harness": harness, "model": model, "effort": effort, + "runname": run_dir.name, "harness": harness, "version": version, + "model": model, "effort": effort, "benchmarks": benchmarks, "totals": {"tokens": grand, "num_tasks": sum(b["num_tasks"] for b in benchmarks.values()), @@ -290,8 +358,20 @@ def overview_rows(summaries: list[dict[str, Any]], prices: dict[str, dict[str, f trial_costs = [cost_of(t["tokens"], model, prices) for t in b.get("trials", []) if t["tokens"].get("total")] median_cost = statistics.median(trial_costs) if trial_costs else None + # SOLVED-ONLY variants, reported as SEPARATE columns rather than folded into the two + # above. The all-task medians answer "what does attempting a task cost"; these answer + # "what does solving one cost" — a failed trial can be cheap (died early) or expensive + # (burned its whole budget getting nowhere), so mixing the two populations hides + # exactly the comparison worth making between harnesses. + solved = [t for t in b.get("trials", []) if t.get("resolved")] + solved_lats = [t["agent_seconds"] for t in solved if t.get("agent_seconds") is not None] + solved_costs = [cost_of(t["tokens"], model, prices) + for t in solved if t["tokens"].get("total")] rows.append({ - "Benchmark": bname, "Harness": s["harness"], "Model": model, "Effort": s["effort"], + "Benchmark": bname, "Harness": s["harness"], + # Right after Harness: which build of it produced these numbers. + "Version": s.get("version") or "?", + "Model": model, "Effort": s["effort"], "Resolved": b["num_resolved"], "Tasks": b["num_tasks"], "Score": round(b["score"], 4), "Total tokens": tok["total"], "Cached tokens": tok["cache_read"], @@ -301,6 +381,11 @@ def overview_rows(summaries: list[dict[str, Any]], prices: dict[str, dict[str, f "Latency/task (s)": (round(b["median_latency_sec"]) if b.get("median_latency_sec") is not None else None), "Cost/task ($)": (round(median_cost, 2) if median_cost is not None else None), + # Hidden in the UI unless the viewer opts in; always computed (it is cheap). + "[solved]Latency/task (s)": (round(statistics.median(solved_lats)) + if solved_lats else None), + "[solved]Cost/task ($)": (round(statistics.median(solved_costs), 2) + if solved_costs else None), "Run": s["runname"], "In progress": s.get("in_progress", False), }) return rows diff --git a/pyproject.toml b/pyproject.toml index f7d7a78..eaf21d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "beagle" -version = "0.0.1" +version = "0.0.2" description = "A PyTorch-like framework for agent-harness evolution: agent factory + evolve algorithms + unified runner over vendored xrlenv rollout infra." requires-python = ">=3.12" license = { text = "Apache-2.0" } diff --git a/scripts/generate_eval_configs.py b/scripts/generate_eval_configs.py index a3e8f75..397a325 100644 --- a/scripts/generate_eval_configs.py +++ b/scripts/generate_eval_configs.py @@ -80,7 +80,7 @@ "provider": {"type": "internal", "name": "llm-gateway-express-local-proxy"}, "forward_env": _INTERNAL_FORWARD_ENV, "runtime": "xrlenv-cluster", - "parallelism": 32, + "parallelism": 16, } #: Each agent, keyed by its beagle `harness.name` (the ADAPTER). ``versions`` lists the experiment @@ -97,6 +97,20 @@ "--permissive-auto-approve", "--no-monet-md", "--output-format", "stream-json"]}, note="monet_args: last two are REQUIRED by beagle's stream parser.", ), + "afcode": dict( + # Latest only. **2.3.0 is an effective floor**: earlier afcode routes gpt-5.6+ to + # /chat/completions, where the gateway's bedrock replica rejects function tools + # alongside reasoning_effort — measured at ~1 call in 10, which a multi-turn rollout + # hits almost every time. 2.3.0+ routes gpt-5.6+ to /responses instead. Onboarding an + # older copy would silently confine you to gpt-5.5. + versions=["v2.3.0_0904"], + internal=True, + # No adapter-level args: the wheelhouse path defaults to the vendored location inside + # the experiment copy, and afcode exposes no turn cap or output-format flag to pin. + extra_args={}, + note="installs the CLONED ref (theta) against the offline wheelhouse vendored in the " + "experiment copy; needs a gateway/internal provider route.", + ), "mini-swe": dict( versions=["v2.4.6"], extra_args={"mini_swe_args": [{"config_path": "src/minisweagent/config/mini.yaml"}]}, diff --git a/scripts/onboard_all_agents.sh b/scripts/onboard_all_agents.sh index d77a3be..d19eb07 100644 --- a/scripts/onboard_all_agents.sh +++ b/scripts/onboard_all_agents.sh @@ -22,14 +22,14 @@ fi # mini-swe — SWE-agent/mini-swe-agent @ v2.4.6 "$python_bin" -m beagle.tools.onboard \ - --upstream https://github.com/SWE-agent/mini-swe-agent --ref a83fcae82d2a08f0ee0c688f9d137b3566c097f8 \ - --repo "$YOUR_ORG/mini_swe_agent_v2.4.6" --private --version v2.4.6 --branch-name baseline \ - --dir ../beagle-experiments/mini_swe_agent_v2.4.6 --profile-name mini_swe_agent_v2.4.6 + --upstream https://github.com/SWE-agent/mini-swe-agent --ref a83fcae82d2a08f0ee0c688f9d137b3566c097f8 \ + --repo "$YOUR_ORG/mini_swe_agent_v2.4.6" --private --version v2.4.6 --branch-name baseline \ + --dir ../beagle-experiments/mini_swe_agent_v2.4.6 --profile-name mini_swe_agent_v2.4.6 # opencode — anomalyco/opencode @ v1.18.16 (release tag). --prune opencode drops the web/desktop/ # marketing apps + demo assets from the seed so every per-container clone is ~12 MB, not ~79 MB # (patch-safe — see docs/opencode-prune.md). "$python_bin" -m beagle.tools.onboard \ - --upstream https://github.com/anomalyco/opencode --ref a3647eb025c7615159d417dcc49fc39fdaeba65b \ - --repo "$YOUR_ORG/opencode_v1.18.16" --private --version 1.18.16 --branch-name baseline \ - --prune opencode \ - --dir ../beagle-experiments/opencode_v1.18.16 --profile-name opencode_v1.18.16 + --upstream https://github.com/anomalyco/opencode --ref a3647eb025c7615159d417dcc49fc39fdaeba65b \ + --repo "$YOUR_ORG/opencode_v1.18.16" --private --version 1.18.16 --branch-name baseline \ + --prune opencode \ + --dir ../beagle-experiments/opencode_v1.18.16 --profile-name opencode_v1.18.16 diff --git a/tests/unit/test_examples.py b/tests/unit/test_examples.py index 1a29808..8602235 100644 --- a/tests/unit/test_examples.py +++ b/tests/unit/test_examples.py @@ -23,6 +23,33 @@ EVOLUTION_EXAMPLES = sorted((ROOT / "examples" / "evolution").glob("config*.yaml")) +# --- ``.oss.`` counterparts ------------------------------------------------------------------ +# A few examples ship twice: the plain file, and a ``.oss.`` counterpart carrying the variant the +# public mirror publishes. The mirror is built by promoting each counterpart ONTO the plain name +# (``config.oss.yaml`` -> ``config.yaml``), so the same content lives under a different name in each +# tree, and only one of the pair survives there. Tests that name these files by hand therefore pass +# here and fail in the mirror against a file the port renamed on purpose. Resolve through the two +# helpers below instead of hardcoding a name. + + +def _oss_or_promoted(relative: str) -> Path: + """The ``.oss.`` counterpart, under whichever name this tree carries it.""" + path = ROOT / relative + if path.exists(): + return path + promoted = path.with_name(path.name.replace(".oss.", ".", 1)) + assert promoted.exists(), f"neither {relative} nor its promoted name {promoted.name} exists" + return promoted + + +def _present(*relatives: str) -> list[Path]: + """Those of ``relatives`` that exist here — a pair collapses to one file in the mirror.""" + paths = [ROOT / relative for relative in relatives] + found = [path for path in paths if path.exists()] + assert found, f"none of {relatives} exist — did the examples layout change?" + return found + + def test_there_are_examples() -> None: assert EXAMPLES, "examples/evaluation/*.yaml is empty — the use-case examples are tracked" @@ -41,7 +68,7 @@ def test_evolution_examples_load_through_the_evolve_seam() -> None: def test_oss_evolution_example_is_local_and_portable() -> None: - path = ROOT / "examples" / "evolution" / "config.oss.yaml" + path = _oss_or_promoted("examples/evolution/config.oss.yaml") raw = yaml.safe_load(path.read_text(encoding="utf-8")) assert raw["run"]["runtime"] == "local" assert raw["run"]["parallelism"] == 1 @@ -53,14 +80,14 @@ def test_oss_evolution_example_is_local_and_portable() -> None: def test_evolution_docs_do_not_reference_the_removed_quick_start_path() -> None: - paths = [ - ROOT / "README.md", - ROOT / "docs" / "advanced.md", - ROOT / "examples" / "evolution" / "README.md", - ROOT / "examples" / "evolution" / "README.oss.md", - ROOT / "examples" / "evolution" / "quick_start_inline.py", - ROOT / "examples" / "evolution" / "quick_start_inline.oss.py", - ] + paths = _present( + "README.md", + "docs/advanced.md", + "examples/evolution/README.md", + "examples/evolution/README.oss.md", + "examples/evolution/quick_start_inline.py", + "examples/evolution/quick_start_inline.oss.py", + ) offenders = [ str(path.relative_to(ROOT)) for path in paths @@ -113,13 +140,13 @@ def test_darwinx_guidance_relative_links_resolve() -> None: """The quick-start and concept guide should not ship links to renamed/missing files.""" import re - paths = [ - ROOT / "docs" / "darwinx-configuration.md", - ROOT / "examples" / "evolution" / "README.md", - ROOT / "examples" / "evolution" / "README.oss.md", - ROOT / "beagle" / "algorithms" / "darwinx" / "vendor" / "README.md", - ROOT / "scripts" / "README.md", - ] + paths = _present( + "docs/darwinx-configuration.md", + "examples/evolution/README.md", + "examples/evolution/README.oss.md", + "beagle/algorithms/darwinx/vendor/README.md", + "scripts/README.md", + ) broken = [] for path in paths: for target in re.findall(r"\[[^\]]+\]\(([^)]+)\)", path.read_text(encoding="utf-8")): diff --git a/tests/unit/test_gateway_proxy.py b/tests/unit/test_gateway_proxy.py deleted file mode 100644 index 7ac12dd..0000000 --- a/tests/unit/test_gateway_proxy.py +++ /dev/null @@ -1,255 +0,0 @@ -"""Unit tests for the standalone gateway-proxy helpers + a live relay round-trip -against a fake upstream (no SSH, no real gateway). The script is stdlib-only and not -importable as a package, so we load it by path.""" - -from __future__ import annotations - -import importlib.util -import socket -import threading -import urllib.error -import urllib.request -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path - -import pytest - - -def _load(): - path = Path(__file__).resolve().parents[2] / "scripts" / "gateway" / "gateway_proxy.py" - spec = importlib.util.spec_from_file_location("gateway_proxy", path) - mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(mod) - return mod - - -gp = _load() - - -# --- pure helpers ------------------------------------------------------------ - - -def test_parse_keys_splits_and_dedupes() -> None: - assert gp.parse_keys("a, b; c d") == ["a", "b", "c", "d"] - assert gp.parse_keys("k1,k1,k2") == ["k1", "k2"] - assert gp.parse_keys("") == [] and gp.parse_keys(None) == [] - - -def test_key_pool_round_robin() -> None: - pool = gp.KeyPool(["k1", "k2", "k3"]) - assert [pool.next() for _ in range(4)] == ["k1", "k2", "k3", "k1"] - assert gp.KeyPool([]).next() is None - - -def test_split_upstream() -> None: - assert gp.split_upstream("https://host.example/") == ("host.example", 443, "") - assert gp.split_upstream("https://h:8443/base/") == ("h", 8443, "/base") - with pytest.raises(ValueError, match="https"): - gp.split_upstream("http://h/") # must be https - - -def test_forward_headers_replaces_auth_and_host_no_dupes() -> None: - incoming = {"Content-Type": "application/json", "Host": "x", "Content-Length": "5", - "Connection": "keep-alive", "authorization": "old"} - out = gp.forward_headers(incoming, authorization="Bearer k", host="up.host") - assert out == {"Content-Type": "application/json", "Authorization": "Bearer k", "Host": "up.host"} - - -def test_ssh_tunnel_args_matches_reverse_tunnel_contract() -> None: - args = gp.ssh_tunnel_args("ubuntu@192.0.2.10", 18080, 19090, - ssh_options=["-J", "jump", "-i", "key"]) - assert args[0] == "ssh" and args[-1] == "ubuntu@192.0.2.10" and "-N" in args - assert "-R" in args and "127.0.0.1:18080:127.0.0.1:19090" in args - assert "ExitOnForwardFailure=yes" in args and "ServerAliveInterval=30" in args - # extra ssh options land before the -R refspec; user@ip is the target verbatim. - assert args.index("-J") < args.index("-R") and "jump" in args and "key" in args - - -def test_upstream_ssl_context_verify_and_insecure() -> None: - import ssl - - default = gp._upstream_ssl_context() - assert default.verify_mode == ssl.CERT_REQUIRED and default.check_hostname is True - insecure = gp._upstream_ssl_context(insecure=True) - assert insecure.verify_mode == ssl.CERT_NONE and insecure.check_hostname is False - - -def test_hostport() -> None: - assert gp.hostport("1.2.3.4:18088") == ("1.2.3.4", 18088) - assert gp.hostport(":18088") == ("0.0.0.0", 18088) - assert gp.hostport("18080", default_host="127.0.0.1") == ("127.0.0.1", 18080) - - -def _free_port() -> int: - s = socket.socket() - s.bind(("127.0.0.1", 0)) - p = s.getsockname()[1] - s.close() - return p - - -def test_forwarder_pipes_bytes() -> None: - # A routable forwarder (the express_forward.sh equivalent) → an echo target. - import socket as _socket - import time - - target = _socket.socket() - target.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) - target.bind(("127.0.0.1", 0)) - target.listen() - tport = target.getsockname()[1] - - def _echo(): - c, _ = target.accept() - c.sendall(b"echo:" + c.recv(1024)) - c.close() - - threading.Thread(target=_echo, daemon=True).start() - fport = _free_port() - threading.Thread(target=gp.run_forwarder, - kwargs={"listen": f"127.0.0.1:{fport}", "to": f"127.0.0.1:{tport}"}, - daemon=True).start() - - conn = None - for _ in range(100): # wait for the forwarder to bind - try: - conn = _socket.create_connection(("127.0.0.1", fport), timeout=1) - break - except OSError: - time.sleep(0.02) - assert conn is not None - conn.sendall(b"hi") - out = conn.recv(1024) - conn.close() - assert out == b"echo:hi" - - -# --- live relay round-trip (fake upstream via monkeypatched HTTPSConnection) -- - - -class _FakeUpstream(BaseHTTPRequestHandler): - def log_message(self, *a): # noqa: ANN002 - pass - - def do_POST(self): # noqa: N802 — echo the Authorization back, streamed - body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) - auth = self.headers.get("Authorization", "") - payload = f"auth={auth};path={self.path};body={body.decode()}".encode() - self.send_response(200) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - -def test_relay_injects_key_and_forwards() -> None: - # Fake "gateway" upstream on plain HTTP; inject it as the relay's connection - # factory (no global monkeypatching of http.client, which would break urllib). - import http.client - - up = ThreadingHTTPServer(("127.0.0.1", 0), _FakeUpstream) - threading.Thread(target=up.serve_forever, daemon=True).start() - up_host, up_port = up.server_address[0], up.server_address[1] - - server, port = gp.start_relay( - local_port=0, upstream="https://fake-gateway/", pool=gp.KeyPool(["KEY1", "KEY2"]), - max_concurrent=8, connect=lambda: http.client.HTTPConnection(up_host, up_port, timeout=5), - ) - try: - # No Authorization header → relay injects a pooled key (round-robin). - req = urllib.request.Request(f"http://127.0.0.1:{port}/chat/completions", - data=b'{"x":1}', method="POST") - out1 = urllib.request.urlopen(req, timeout=5).read().decode() - out2 = urllib.request.urlopen(urllib.request.Request( - f"http://127.0.0.1:{port}/chat/completions", data=b'{}', method="POST"), timeout=5).read().decode() - assert "auth=Bearer KEY1" in out1 and "path=/chat/completions" in out1 and 'body={"x":1}' in out1 - assert "auth=Bearer KEY2" in out2 # round-robin to the next key - # Unsupported path → 404 from the relay itself. - with pytest.raises(urllib.error.HTTPError) as e: - urllib.request.urlopen(urllib.request.Request( - f"http://127.0.0.1:{port}/v1/chat/completions", data=b"{}", method="POST"), timeout=5) - assert e.value.code == 404 - finally: - server.shutdown() - up.shutdown() - - -# --- regression: _json must produce valid JSON for all string values --------- - - -def test_relay_error_response_is_valid_json() -> None: - """The _json helper must use json.dumps, not str().replace() — regression for - the P0 bug where strings containing apostrophes produced malformed JSON.""" - import http.client - import json as _json - - # Start a relay with no upstream (requests will 404 on unsupported path). - up = ThreadingHTTPServer(("127.0.0.1", 0), _FakeUpstream) - threading.Thread(target=up.serve_forever, daemon=True).start() - up_host, up_port = up.server_address[0], up.server_address[1] - - server, port = gp.start_relay( - local_port=0, upstream="https://fake-gateway/", - pool=gp.KeyPool([]), # empty pool — no keys - max_concurrent=8, connect=lambda: http.client.HTTPConnection(up_host, up_port, timeout=5), - ) - try: - # Empty key pool + no Authorization header → 401 with a plain-text error message. - # The response body must be valid JSON, not str(dict).replace("'", '"'). - req = urllib.request.Request( - f"http://127.0.0.1:{port}/chat/completions", - data=b"{}", method="POST", - ) - # Expect 401 (HTTPError) with a JSON body we can parse. - with pytest.raises(urllib.error.HTTPError) as exc_info: - urllib.request.urlopen(req, timeout=5) - body = exc_info.value.read() - parsed = _json.loads(body) # must not raise — regression guard - assert "error" in parsed - - # Unsupported path 404 must also be valid JSON. - req2 = urllib.request.Request( - f"http://127.0.0.1:{port}/v1/chat/completions", - data=b"{}", method="POST", - ) - with pytest.raises(urllib.error.HTTPError) as exc_info2: - urllib.request.urlopen(req2, timeout=5) - parsed2 = _json.loads(exc_info2.value.read()) - assert "error" in parsed2 - finally: - server.shutdown() - up.shutdown() - - -# --- hostport edge cases ------------------------------------------------------- - - -def test_hostport_port_only_bare_string() -> None: - assert gp.hostport("8080") == ("0.0.0.0", 8080) - assert gp.hostport("8080", default_host="127.0.0.1") == ("127.0.0.1", 8080) - - -def test_hostport_colon_only_port() -> None: - assert gp.hostport(":18080") == ("0.0.0.0", 18080) - - -def test_hostport_port_zero_os_assigned() -> None: - assert gp.hostport("0") == ("0.0.0.0", 0) - assert gp.hostport("127.0.0.1:0") == ("127.0.0.1", 0) - - -# --- ssh_tunnel_args: bind override ------------------------------------------- - - -def test_ssh_tunnel_args_bind_0000_for_container_reach() -> None: - """bind=0.0.0.0 produces 0.0.0.0::127.0.0.1: in the -R refspec.""" - args = gp.ssh_tunnel_args("mynode", 18080, 19090, bind="0.0.0.0") - r_idx = args.index("-R") - assert args[r_idx + 1] == "0.0.0.0:18080:127.0.0.1:19090" - - -def test_ssh_tunnel_args_no_options() -> None: - """ssh_options=None omits extra args (no empty list spliced in).""" - args = gp.ssh_tunnel_args("node", 18080, 19090) - # Only the fixed options should be there (ExitOnForwardFailure, ServerAlive×2, -R) - assert "-J" not in args - assert "-i" not in args diff --git a/tests/unit/test_generate_eval_configs.py b/tests/unit/test_generate_eval_configs.py index b6fc04e..17b8d6b 100644 --- a/tests/unit/test_generate_eval_configs.py +++ b/tests/unit/test_generate_eval_configs.py @@ -11,10 +11,19 @@ import pytest import yaml -_PATH = Path(__file__).resolve().parents[2] / "scripts" / "generate_eval_configs.py" -_spec = importlib.util.spec_from_file_location("generate_eval_configs", _PATH) -gen = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gen) # type: ignore[union-attr] +def _load(stem: str): + path = Path(__file__).resolve().parents[2] / "scripts" / f"{stem}.py" + spec = importlib.util.spec_from_file_location(stem, path) + assert spec and spec.loader, f"cannot load {path}" + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +gen = _load("generate_eval_configs") +#: The *evolution* configs are written by a second generator with its own filename suffix. Docs +#: quote both families, so the path check below has to know both — see that test. +evo = _load("generate_evolution_config") #: The join key is ``version``, so tests take it FROM the matrix rather than restating it — a #: literal here goes stale the moment an agent is re-onboarded at a new version. @@ -77,7 +86,9 @@ def test_internal_profile_uses_gateway_cluster_and_includes_monet() -> None: cfg = gen.build_config("monet", "terminal_bench_2_1", _MANIFEST, internal=True) assert cfg["run"]["runtime"] == "xrlenv-cluster" - assert cfg["run"]["parallelism"] == 32 + # Lowered 32 -> 16: at 32 an afcode sweep saturated the provider's org-wide TPM + # budget, and 77 of 89 trials logged rate limits. + assert cfg["run"]["parallelism"] == 16 assert cfg["agent"]["provider"] == { "type": "internal", "name": "llm-gateway-express-local-proxy"} assert cfg["agent"]["forward_env"] == gen._INTERNAL_FORWARD_ENV @@ -474,12 +485,18 @@ def _tracked_docs() -> list[Path]: def test_documented_config_paths_are_ones_the_generator_writes() -> None: """Docs quote generated config paths, which carry the version — so they go stale exactly when - an agent is re-onboarded. Fail here rather than in a user's terminal.""" + an agent is re-onboarded. Fail here rather than in a user's terminal. + + Two generators write into these trees, and the check must know both suffixes: quoting a valid + ``generate_evolution_config.py`` path used to fail here as "no longer written", because the + valid set was built from the eval generator alone.""" import re labels = {label for _n, _v, label in gen.agent_cells()} - # both trees: examples/evaluation//