From f34204531658dffec7025a12cb3cebdbd59a1d36 Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Mon, 20 Jul 2026 03:32:23 -0700 Subject: [PATCH 1/2] Rebuild as a general coding environment with diff-based grading One env, three task flavors on a shared repo lifecycle (vault the real .git outside the workspace, hand the agent a fresh single-commit repo, grade the captured diff hermetically with hidden tests brought in after): - coding-task: the 3-branch convention as template args, graded by a test command's exit code; runs locally or from a baked image - sdlc-task: adds a bare mock-GitHub remote plus github_* MCP tools; the deliverable is a pushed branch with a pull request, graded on the PR head with tests plus a structural or LLM-rubric PR check - SWE-bench Pro: swe_tasks.py fetches instances and builds their prebuilt images into servable envs; grading replays the official evaluator (fail_to_pass and pass_to_pass must all pass) Dockerfile.hud is the single image definition for generic and instance builds. Validation: hermetic golden/baseline rollouts for both local flavors, gold-patch integration tests for SWE-bench instances, and both flavors verified against the deployed platform env. --- .dockerignore | 66 +---- .gitignore | 49 ++-- Dockerfile.hud | 114 ++++---- README.md | 90 ++++-- coding/__init__.py | 5 + coding/github.py | 158 +++++++++++ coding/repo.py | 167 ++++++++++++ coding/swe_bench_pro.py | 149 ++++++++++ env.py | 379 +++++++++++++++++++------- grading/__init__.py | 10 - grading/graders.py | 87 ------ grading/runner.py | 160 ----------- instances/.none/.gitkeep | 0 pyproject.toml | 10 +- swe_tasks.py | 166 +++++++++++ tasks.py | 132 +++++---- tests/conftest.py | 52 +--- tests/fixtures/instance/instance.json | 18 ++ tests/fixtures/instance/parser.py | 8 + tests/fixtures/instance/run_script.sh | 3 + tests/test_env.py | 35 +-- tests/test_github.py | 55 ++++ tests/test_grading.py | 99 +++---- tests/test_integration.py | 51 ++++ tests/test_local_rollout.py | 116 ++++++++ tests/test_tasks.py | 48 ++++ uv.lock | 20 +- 27 files changed, 1527 insertions(+), 720 deletions(-) create mode 100644 coding/__init__.py create mode 100644 coding/github.py create mode 100644 coding/repo.py create mode 100644 coding/swe_bench_pro.py delete mode 100644 grading/__init__.py delete mode 100644 grading/graders.py delete mode 100644 grading/runner.py create mode 100644 instances/.none/.gitkeep create mode 100644 swe_tasks.py create mode 100644 tests/fixtures/instance/instance.json create mode 100644 tests/fixtures/instance/parser.py create mode 100644 tests/fixtures/instance/run_script.sh create mode 100644 tests/test_github.py create mode 100644 tests/test_integration.py create mode 100644 tests/test_local_rollout.py create mode 100644 tests/test_tasks.py diff --git a/.dockerignore b/.dockerignore index 187196f..ae2bc26 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,58 +1,8 @@ -# Git (keep .git/modules/ for submodule offline builds) -.git/* -!.git/modules/ -.gitignore - -# Local runtime substrate (clones for `--runtime local`), never ship to the image -.hud_local/ - -# Python -__pycache__/ -*.py[cod] -*$py.class -*.so -.Python -.pytest_cache/ -.coverage -*.cover -.hypothesis/ -.env -.venv -env/ -venv/ -ENV/ - -# Logs -*.log -logs/ -build.log -backend.log - -# Test outputs -qa_results/ -test_results/ -coverage/ - -# Temporary files -*.tmp -*.temp -tmp/ -temp/ - -# IDE -.vscode/ -.idea/ -*.swp -*.swo -*~ -.DS_Store - -# Node (if any) -node_modules/ -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Keep the build lean -*.md -!README.md \ No newline at end of file +# Builds run from the repo root; the context is only what the images COPY, +# plus the Dockerfile itself (remote builders read it from the context). +* +!Dockerfile.hud +!env.py +!coding +!instances +**/__pycache__ diff --git a/.gitignore b/.gitignore index 7366d77..2344606 100644 --- a/.gitignore +++ b/.gitignore @@ -1,49 +1,32 @@ # Python __pycache__/ *.py[cod] -*$py.class -*.so -.Python -build/ -dist/ -*.egg-info/ -*.egg .pytest_cache/ +.ruff_cache/ .coverage -.env .venv -venv/ - -# Node -node_modules/ +.env +build/ +dist/ +*.egg-info/ -# IDE +# OS / IDE +.DS_Store .vscode/ .idea/ *.swp -*.swo -*~ -.DS_Store -.opencode/ - -# Logs -*.log -logs/ -tmp/ - -# references -reference/ - -# Docker -docker_build.log - -# Test outputs -test_results/ -coverage/ # HUD hud.lock.yaml .hud .hud_local/ .hud_eval.toml -problem-metadata.json +.hud-images.json + +# SWE-bench Pro instance assets are fetched per-user (swe_tasks.py) and carry +# the golden patches; only the empty .none dir (generic image builds) ships. +instances/* +!instances/.none/ + +# Logs +*.log diff --git a/Dockerfile.hud b/Dockerfile.hud index 87084e5..8f5562c 100644 --- a/Dockerfile.hud +++ b/Dockerfile.hud @@ -1,78 +1,68 @@ # syntax=docker/dockerfile:1 -FROM ubuntu:24.04 AS setup +# One image definition for both flavors. +# +# Generic (default) — the repo-clone stage bakes your target repo into /app: +# docker build -f Dockerfile.hud --build-arg REPO_URL=https://github.com/you/repo . +# (private repos: --secret id=CODING_GITHUB_TOKEN,env=CODING_GITHUB_TOKEN) +# +# SWE-bench Pro (driven by swe_tasks.py) — BASE selects the instance's +# prebuilt image (repo already at /app) and INSTANCE_ID bakes its assets: +# docker build -f Dockerfile.hud --build-arg BASE=jefzda/sweap-images: \ +# --build-arg INSTANCE_ID= . +ARG BASE=repo-clone + +# ─── generic head: toolchain + target repo at /app ─── +FROM ubuntu:24.04 AS repo-clone RUN apt-get update -y \ && apt-get install -y --no-install-recommends \ - bash \ - build-essential \ - ca-certificates \ - curl \ - git \ - gnupg \ - libpq5 \ - openssl \ - python-is-python3 \ - python3 \ - python3-dev \ - python3-pip \ - python3-venv \ - python3-wheel \ - sqlite3 \ - vim \ - wget \ + bash build-essential ca-certificates curl git openssl \ + python-is-python3 python3 python3-dev python3-pip python3-venv util-linux \ && rm -rf /var/lib/apt/lists/* \ && update-ca-certificates -# Target repo — override at build with --build-arg REPO_URL=https://github.com/you/repo -# (private: --secret id=CODING_GITHUB_TOKEN,env=CODING_GITHUB_TOKEN). -ARG REPO_URL="https://github.com/hud-evals/coding-template-sample" -ARG FOLDER_NAME="project" -ENV FOLDER_NAME=${FOLDER_NAME} \ - REPO_URL=${REPO_URL} \ - HOME=/home/ubuntu \ - PROJECT_DIR=/home/ubuntu/${FOLDER_NAME} \ - PATCHES_DIR=/home/root/patches +# The sample tasks' test command uses the image's python; adjust per repo. +RUN pip3 install --break-system-packages pytest -USER ubuntu -RUN --mount=type=secret,id=CODING_GITHUB_TOKEN,uid=1000 \ +ARG REPO_URL="https://github.com/hud-evals/coding-template-sample" +RUN --mount=type=secret,id=CODING_GITHUB_TOKEN \ if [ -f /run/secrets/CODING_GITHUB_TOKEN ]; then \ - CODING_GITHUB_TOKEN=$(cat /run/secrets/CODING_GITHUB_TOKEN) && \ - git clone https://${CODING_GITHUB_TOKEN}@${REPO_URL#https://} /home/ubuntu/${FOLDER_NAME}; \ + git clone "https://$(cat /run/secrets/CODING_GITHUB_TOKEN)@${REPO_URL#https://}" /app; \ else \ - git clone ${REPO_URL} /home/ubuntu/${FOLDER_NAME}; \ + git clone ${REPO_URL} /app; \ fi - -# uid wall: the repo is owned by the agent (uid 1000); the answer key (.git -# branches) and generated patches stay root:700 so only the env/grader (root) -# can read them — the dropped agent shell can't. -USER root -RUN chown -R ubuntu:ubuntu /home/ubuntu/${FOLDER_NAME} \ - && mkdir -p /home/root/patches \ - && chmod 700 /home/root \ - && chown -R root:root /home/ubuntu/${FOLDER_NAME}/.git \ - && chmod -R 700 /home/ubuntu/${FOLDER_NAME}/.git \ - && git config --global --add safe.directory /home/ubuntu/${FOLDER_NAME} - # Add a build step for the target repo here if it needs one (deps, compile). +# ─── the served image: BASE + hud serving layer (+ instance assets) ─── +FROM ${BASE} -FROM setup AS runtime - -RUN pip install uv --break-system-packages - -COPY ./pyproject.toml /mcp_server/pyproject.toml -WORKDIR /mcp_server -ENV RUST_LOG=warn \ - PYTHONPATH=/mcp_server/.venv/lib/python3.12/site-packages:/mcp_server \ - PATH=/mcp_server/.venv/bin:$PATH -RUN uv venv && . .venv/bin/activate && uv sync --no-install-project --extra dev && uv pip install -e . - -COPY ./env.py /mcp_server/env.py -COPY ./grading /mcp_server/grading -COPY ./tasks.py /mcp_server/tasks.py -COPY ./tests /mcp_server/tests -RUN chmod 777 /root +# ".none" is a committed empty dir, so generic builds bake no instance and +# env.py serves only the generic coding-task template. +ARG INSTANCE_ID=".none" +COPY env.py /hud/env.py +COPY coding /hud/coding +COPY instances/${INSTANCE_ID}/ /hud/instance/ -ENV PROBLEM_ID="" +# Self-contained hud venv under /hud: bootstrap uv (with its own managed +# Python), never touching the base image's Python or site-packages. chmod 700 +# keeps the vault and instance assets out of the uid-dropped agent's reach. +RUN <<'INSTALL' +set -eu +export PATH="$HOME/.local/bin:$PATH" UV_INSTALL_DIR="$HOME/.local/bin" +command -v uv >/dev/null 2>&1 || { + { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \ + || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \ + || apk add --no-cache curl ca-certificates + { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \ + || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \ + || pip install -q -U uv +} +uv python install 3.12 +uv venv /hud/venv --python 3.12 +uv pip install --python /hud/venv/bin/python 'hud>=0.6.10' +chmod -R go-rwx /hud +INSTALL +ENV REPO_DIR=/app PYTHONPATH=/hud EXPOSE 8765 -CMD ["hud", "serve", "env:env", "--host", "0.0.0.0", "--port", "8765"] +ENTRYPOINT [] +CMD ["/hud/venv/bin/hud", "serve", "/hud/env.py", "--host", "0.0.0.0", "--port", "8765"] diff --git a/README.md b/README.md index 7de7553..0b5d7a3 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,89 @@ # Coding Environment (HUD v6) -A HUD v6 environment where an agent **fixes a bug in a Python web app**, graded by a hidden pytest -suite. The env publishes a sandboxed **`ssh` workspace** — the agent's harness brings its own bash -and file tools and drives it over SSH; the env ships no agent tools itself. +A coding environment: the agent gets a git repo over a sandboxed **`ssh` workspace** (the +harness brings its own bash/file tools), and grading is diff-based — capture the agent's +changes, reset to the pre-agent snapshot, re-apply the diff, bring in the hidden tests, run +them. Generic tasks, SDLC workflow tasks, and SWE-bench Pro are flavors on this core. -Grading uses a 3-branch design: every task has `{task}_baseline` (the agent's starting state), -`{task}_test` (hidden tests), and `{task}_golden` (the reference fix). `setup_task` checks out the -baseline; the grader applies the hidden test patch and runs `pytest` — all pass → reward 1.0. The -agent never sees the tests or the solution. +The agent never sees the answer key. Task setup moves the repo's real `.git` — which may hold +solution branches or the fix commit — into a vault outside the workspace and leaves a fresh +single-commit repo: git works normally, but there is no history, no refs, and no remotes. +Grading discards the agent's `.git`, restores the vault, and checks hidden tests out *after* +the agent's diff. In images, the vault and instance assets live under `/hud` (root, mode 700) +and agent shells drop to a non-root uid via `setpriv`. -## Tasks +## Layout -Four hand-picked bugs in the -[coding-template-sample](https://github.com/hud-evals/coding-template-sample) repo: +- `env.py` — the environment: workspace wiring plus the three task templates. +- `coding/repo.py` — the shared repo lifecycle: vault, snapshot, diff capture, reset, apply. +- `coding/github.py` — mock GitHub for SDLC tasks: issue/PR store served as `github_*` tools. +- `coding/swe_bench_pro.py` — the SWE-bench Pro grading pipeline. +- `tasks.py` — sample generic and SDLC tasks. +- `swe_tasks.py` — the SWE-bench Pro task source: fetches instances and builds their images + when run; task rows when imported. +- `Dockerfile.hud` — the image definition for both flavors: `BASE` selects the generic + repo-clone head or a prebuilt instance image. -| Task | Difficulty | Bug | -|------|-----------|-----| -| `sentry-fix` | Basic | KeyError on a missing/null user profile | -| `notif-bug` | Medium | Event-type delimiter `_` vs `.` breaks routing | -| `settings-v2` | Hard | CompactDict filters falsy values during iteration | -| `webhook-bug` | Hard | Shared-list mutation corrupts channel resolution | +## Generic tasks (`coding-task`) -## Setup +Point the env at a repo (`REPO_URL`; locally it clones per process, or bake it with +`Dockerfile.hud`) and parameterize the template: a `base_ref` to start from, a `test_ref` whose +`test_files` are the hidden tests (checked out from the vaulted history at grade time), and a +`test_command` scored by exit code. Tasks follow the 3-branch convention — `{task}_baseline` / +`{task}_test` / `{task}_golden`; `tasks.py` ships four sample bugs on +[coding-template-sample](https://github.com/hud-evals/coding-template-sample): ```bash uv sync -hud set HUD_API_KEY=your-key-here # CLI auth, get one at hud.ai/project/api-keys +hud set HUD_API_KEY=your-key-here +hud eval tasks.py claude --task-ids sentry-fix -y --runtime local ``` -## Run +## SDLC tasks (`sdlc-task`) + +The generic flavor plus workflow: the repo gets an `origin` remote (a bare mock-GitHub repo the +agent pushes to) and `github_*` MCP tools seeded with the task's issues. The deliverable is a +pushed branch with a pull request — grading checks the PR head out of the remote, brings in the +hidden tests, runs the test command (weight 0.8), and scores the PR itself (0.2): a structural +title/body check by default, or `pr_rubric` judged by `LLMJudgeGrader` when provided. See the +`sentry-fix-pr` sample in `tasks.py`: ```bash -# local — clones the target repo into a per-process temp dir (macOS + Linux) -hud eval tasks.py claude --task-ids sentry-fix -y --runtime local +hud eval tasks.py claude --task-ids sentry-fix-pr -y --runtime local +``` + +## SWE-bench Pro tasks -# deploy once, then run hosted -hud deploy . -hud eval tasks.py claude --runtime hud --full +Each of the 731 public [SWE-bench Pro](https://github.com/scaleapi/SWE-bench_Pro-os) instances +ships a prebuilt image (`jefzda/sweap-images:`, `linux/amd64`) with the repo and toolchain +baked in. Running `swe_tasks.py` fetches the dataset row plus the official +`run_script.sh`/`parser.py` into `instances//` and builds `Dockerfile.hud` with the +instance's image as `BASE`, so the image serves this env from inside. Grading replays the +official evaluator: resolved iff every `fail_to_pass` **and** `pass_to_pass` test passes. + +```bash +uv run swe_tasks.py instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan +hud eval swe_tasks.py claude --task-ids nodebb-04998908 +uv run swe_tasks.py ... --push registry.io/acme # push for cloud runtimes ``` ## Tests ```bash -uv run pytest tests/ -q +uv run pytest tests/ -q --ignore=tests/test_integration.py # offline + hermetic local e2e +uv run pytest tests/test_integration.py -v # SWE-bench gold-patch check (Docker) ``` -Offline tests drive the 3-branch grading (baseline fails, golden passes) with no Docker. +`test_local_rollout.py` runs the generic and SDLC flavors end to end against a fixture 3-branch +repo (no Docker or network): the golden ref grades 1.0 and the untouched baseline 0.0. The +integration suite is the same check for built SWE-bench Pro instances. + +## Caveats + +- Public benchmarks are public: a networked agent could fetch solutions from GitHub. Disable + network egress at the runtime layer if that matters for your run. +- The uid wall needs `setpriv` (util-linux) in the image; the repo path comes from `REPO_DIR` + (`/app` in instance images). ## Documentation diff --git a/coding/__init__.py b/coding/__init__.py new file mode 100644 index 0000000..88280a2 --- /dev/null +++ b/coding/__init__.py @@ -0,0 +1,5 @@ +"""The coding environment's core: repo-lifecycle primitives and task flavors.""" + +from . import repo, swe_bench_pro + +__all__ = ["repo", "swe_bench_pro"] diff --git a/coding/github.py b/coding/github.py new file mode 100644 index 0000000..d07288e --- /dev/null +++ b/coding/github.py @@ -0,0 +1,158 @@ +"""A minimal mock GitHub for SDLC tasks. + +One in-process store holds issues (seeded per task) and the pull requests and +comments the agent creates; :func:`serve` publishes it as ``github_*`` MCP +tools over an ``mcp`` capability. The remote itself is a bare git repo on +disk (see :func:`coding.repo.create_remote`) — the agent pushes branches with +plain git and references them from pull requests here. +""" + +from __future__ import annotations + +import asyncio +import json +import socket +from dataclasses import asdict, dataclass, field +from typing import Any + +from fastmcp import FastMCP +from hud.capabilities import Capability + + +@dataclass +class Issue: + number: int + title: str + body: str + labels: list[str] = field(default_factory=list) + state: str = "open" + comments: list[str] = field(default_factory=list) + + +@dataclass +class PullRequest: + number: int + title: str + body: str + head: str + base: str + state: str = "open" + comments: list[str] = field(default_factory=list) + + +class MockGitHub: + """Issue/PR store with an action log (for rubric grading).""" + + def __init__(self) -> None: + self.issues: dict[int, Issue] = {} + self.pull_requests: dict[int, PullRequest] = {} + self.actions: list[dict[str, Any]] = [] + + def seed(self, issues: list[dict[str, Any]]) -> None: + """Reset the store to the task's fixtures.""" + self.issues = { + int(entry["number"]): Issue( + number=int(entry["number"]), + title=str(entry["title"]), + body=str(entry.get("body", "")), + labels=[str(label) for label in entry.get("labels", [])], + ) + for entry in issues + } + self.pull_requests = {} + self.actions = [] + + def _record(self, action: str, **details: Any) -> None: + self.actions.append({"action": action, **details}) + + # ─── operations (the tool surface) ──────────────────────────────── + + def list_issues(self) -> list[dict[str, Any]]: + return [asdict(issue) for issue in self.issues.values()] + + def get_issue(self, number: int) -> dict[str, Any]: + return asdict(self._issue(number)) + + def close_issue(self, number: int) -> dict[str, Any]: + issue = self._issue(number) + issue.state = "closed" + self._record("close_issue", number=number) + return asdict(issue) + + def comment_on_issue(self, number: int, body: str) -> dict[str, Any]: + issue = self._issue(number) + issue.comments.append(body) + self._record("comment_on_issue", number=number, body=body) + return asdict(issue) + + def create_pull_request(self, title: str, body: str, head: str, base: str) -> dict[str, Any]: + number = len(self.pull_requests) + 1 + pr = PullRequest(number=number, title=title, body=body, head=head, base=base) + self.pull_requests[number] = pr + self._record("create_pull_request", number=number, title=title, head=head) + return asdict(pr) + + def list_pull_requests(self) -> list[dict[str, Any]]: + return [asdict(pr) for pr in self.pull_requests.values()] + + def _issue(self, number: int) -> Issue: + if number not in self.issues: + raise ValueError(f"no such issue: #{number}") + return self.issues[number] + + # ─── grading views ──────────────────────────────────────────────── + + def latest_pull_request(self) -> PullRequest | None: + return next(reversed(self.pull_requests.values()), None) + + def transcript(self) -> str: + """Everything the agent did here, for rubric judging.""" + return json.dumps( + { + "issues": [asdict(issue) for issue in self.issues.values()], + "pull_requests": [asdict(pr) for pr in self.pull_requests.values()], + "actions": self.actions, + }, + indent=2, + ) + + +def serve(github: MockGitHub) -> tuple[asyncio.Task[None], Capability]: + """Serve *github* as ``github_*`` MCP tools; returns the server task + capability.""" + server: FastMCP = FastMCP(name="github") + + @server.tool + def github_list_issues() -> list[dict[str, Any]]: + """List all issues in the repository.""" + return github.list_issues() + + @server.tool + def github_get_issue(number: int) -> dict[str, Any]: + """Get one issue by number.""" + return github.get_issue(number) + + @server.tool + def github_close_issue(number: int) -> dict[str, Any]: + """Close an issue.""" + return github.close_issue(number) + + @server.tool + def github_comment_on_issue(number: int, body: str) -> dict[str, Any]: + """Add a comment to an issue.""" + return github.comment_on_issue(number, body) + + @server.tool + def github_create_pull_request(title: str, body: str, head: str, base: str = "main") -> dict[str, Any]: + """Open a pull request from branch *head* (already pushed to the remote) into *base*.""" + return github.create_pull_request(title, body, head, base) + + @server.tool + def github_list_pull_requests() -> list[dict[str, Any]]: + """List all pull requests.""" + return github.list_pull_requests() + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + port = int(sock.getsockname()[1]) + task = asyncio.create_task(server.run_async(transport="http", host="127.0.0.1", port=port, show_banner=False)) + return task, Capability.mcp(name="github", url=f"http://127.0.0.1:{port}/mcp") diff --git a/coding/repo.py b/coding/repo.py new file mode 100644 index 0000000..1cfb63f --- /dev/null +++ b/coding/repo.py @@ -0,0 +1,167 @@ +"""Git-history vaulting and diff-based grading primitives. + +The repo's real ``.git`` may contain the answer key — solution branches, the +fix commit, the hidden tests. Every task flavor shares this lifecycle: + +- setup (:func:`vault_history`): snapshot the pre-agent worktree into the + real history, move ``.git`` into a vault outside the workspace, and leave a + fresh single-commit repo — git works normally for the agent, but there is + no history, no refs, and no remotes. +- grading (:func:`restore_history` → :func:`capture_agent_diff` → + :func:`reset_worktree` → :func:`apply_diff`): discard the agent's ``.git`` + (hooks and history included), restore the vaulted history, capture the + agent's changes as a diff against the setup snapshot, reset the worktree to + that snapshot, and re-apply the diff. Hidden tests come from vaulted refs + and land after the agent's diff. + +The snapshot covers the whole worktree, so installed dependencies and build +artifacts survive into grading. +""" + +from __future__ import annotations + +import asyncio +import os +import shutil +from pathlib import Path + +_GIT_ID = ("-c", "user.name=hud-grader", "-c", "user.email=grader@hud.ai") +_SETUP_COMMIT = "setup_commit" + + +async def run( + *argv: str, + cwd: Path, + timeout: float = 600.0, + check: bool = True, +) -> tuple[int, str, str]: + """Run a command, returning ``(returncode, stdout, stderr)``. Kills on timeout.""" + proc = await asyncio.create_subprocess_exec( + *argv, + cwd=str(cwd), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + out_bytes, err_bytes = await asyncio.wait_for(proc.communicate(), timeout=timeout) + except TimeoutError: + proc.kill() + await proc.wait() + raise TimeoutError(f"{' '.join(argv[:4])} timed out after {timeout:.0f}s") from None + out = out_bytes.decode("utf-8", "replace") + err = err_bytes.decode("utf-8", "replace") + if check and proc.returncode != 0: + detail = (err.strip() or out.strip())[:2000] + raise RuntimeError(f"{' '.join(argv)} failed ({proc.returncode}): {detail}") + return proc.returncode if proc.returncode is not None else 1, out, err + + +async def git(repo: Path, *args: str, timeout: float = 600.0, check: bool = True) -> tuple[int, str, str]: + return await run("git", *_GIT_ID, *args, cwd=repo, timeout=timeout, check=check) + + +def _append_git_exclude(git_dir: Path) -> None: + """Keep the workspace's SSH credential dir out of snapshots and diffs.""" + exclude = git_dir / "info" / "exclude" + exclude.parent.mkdir(parents=True, exist_ok=True) + with exclude.open("a", encoding="utf-8") as fh: + fh.write("\n.hud/\n") + + +async def vault_history(repo: Path, vault: Path) -> None: + """Snapshot the pre-agent state, then hide the repo's history in *vault*.""" + if (vault / "git").exists(): + raise RuntimeError(f"vault {vault} already holds a history: one task per substrate") + + if hasattr(os, "geteuid") and os.geteuid() == 0: + # Grade-time git runs as root on the (chowned) agent-owned repo; git + # honors safe.directory only from system/global config, never from -c. + await run("git", "config", "--global", "--add", "safe.directory", str(repo), cwd=Path("/")) + + _append_git_exclude(repo / ".git") + await git(repo, "add", "-A") + await git(repo, "commit", "-q", "--allow-empty", "-m", "hud: pre-agent snapshot") + _, sha, _ = await git(repo, "rev-parse", "HEAD") + + vault.mkdir(parents=True, exist_ok=True) + (vault / _SETUP_COMMIT).write_text(sha.strip(), "utf-8") + shutil.move(str(repo / ".git"), str(vault / "git")) + + await git(repo, "init", "-q", ".") + _append_git_exclude(repo / ".git") + await git(repo, "add", "-A") + await git(repo, "commit", "-q", "--allow-empty", "-m", "baseline") + + +async def create_remote(repo: Path, remote: Path, ref: str, *, branch: str = "main") -> None: + """A bare "GitHub" remote holding *ref*'s history as *branch*. + + Pushing a single ref publishes only its ancestry — sibling answer-key + branches (hidden tests, golden fix) stay unreachable, while the agent + still gets realistic past history to work with. + """ + remote.parent.mkdir(parents=True, exist_ok=True) + await git(remote.parent, "init", "-q", "--bare", "-b", branch, str(remote)) + await git(repo, "push", "-q", str(remote), f"{ref}:refs/heads/{branch}") + + +async def attach_to_remote(repo: Path, remote: Path, *, branch: str = "main") -> None: + """Point the agent's fresh repo at the remote, sharing its (sanitized) history. + + Called after :func:`vault_history`: the throwaway single-commit repo is + replaced by the remote's clone state — same worktree, but with ``origin`` + configured and history the agent can branch from and push back to. + """ + shutil.rmtree(repo / ".git") + await git(repo, "init", "-q", "-b", branch, ".") + _append_git_exclude(repo / ".git") + await git(repo, "remote", "add", "origin", str(remote)) + await git(repo, "fetch", "-q", "origin") + # Adopt the remote's history without a checkout: the worktree already + # matches (plus untracked build artifacts a checkout would refuse to + # cross), so point the branch at origin and sync only the index. + await git(repo, "update-ref", f"refs/heads/{branch}", f"refs/remotes/origin/{branch}") + await git(repo, "reset", "-q") + await git(repo, "branch", "-q", f"--set-upstream-to=origin/{branch}", branch) + + +async def restore_history(repo: Path, vault: Path) -> str: + """Swap the agent's throwaway ``.git`` for the vaulted real history. + + Returns the setup-snapshot commit sha recorded by :func:`vault_history`. + """ + agent_git = repo / ".git" + if agent_git.exists(): + shutil.rmtree(agent_git) + shutil.move(str(vault / "git"), str(agent_git)) + return (vault / _SETUP_COMMIT).read_text("utf-8").strip() + + +async def capture_agent_diff(repo: Path, setup_commit: str) -> str: + """The agent's changes: setup snapshot -> final worktree state. + + Runs on the restored real history: the final worktree (including files the + agent added) is committed and diffed against the snapshot. + """ + await git(repo, "add", "-A") + await git(repo, "commit", "-q", "--allow-empty", "-m", "hud: final snapshot") + _, final, _ = await git(repo, "rev-parse", "HEAD") + _, diff, _ = await git(repo, "diff", setup_commit, final.strip(), timeout=1200.0) + return diff + + +async def reset_worktree(repo: Path, setup_commit: str) -> None: + await git(repo, "reset", "--hard", setup_commit) + await git(repo, "clean", "-fd") + + +async def apply_diff(repo: Path, diff: str, patch_path: Path) -> str | None: + """Apply *diff* to the worktree; returns git's error output on failure.""" + if not diff.strip(): + return None + patch_path.parent.mkdir(parents=True, exist_ok=True) + patch_path.write_text(diff, "utf-8") + code, out, err = await git(repo, "apply", "-v", str(patch_path), check=False) + if code != 0: + return (err.strip() or out.strip())[-4000:] + return None diff --git a/coding/swe_bench_pro.py b/coding/swe_bench_pro.py new file mode 100644 index 0000000..da655cf --- /dev/null +++ b/coding/swe_bench_pro.py @@ -0,0 +1,149 @@ +"""The SWE-bench Pro task flavor: dataset-row parsing and the official grading pipeline. + +An instance directory (baked into the instance image by ``swe_tasks.py``) +holds the dataset row (``instance.json``) plus the official per-instance +``run_script.sh`` and ``parser.py`` from scaleapi/SWE-bench_Pro-os. Grading +replays the official evaluator's entryscript on the repo-lifecycle primitives +in :mod:`coding.repo`: reset to the pre-agent snapshot, apply the (agent or +golden) patch, check out the hidden test files, run the script, parse, and +resolve iff every ``fail_to_pass`` and ``pass_to_pass`` test passes. +""" + +from __future__ import annotations + +import ast +import json +import re +import sys +from pathlib import Path +from typing import Any + +from hud.graders import EvaluationResult, SubScore + +from . import repo as repo_lib + + +def load_instance(instance_dir: Path) -> dict[str, Any]: + return json.loads((instance_dir / "instance.json").read_text("utf-8")) + + +def str_list(field: str) -> list[str]: + """Parse the dataset's list-shaped string fields. + + The rows store lists as Python reprs (single quotes; the official + evaluator uses ``eval``), so ``ast.literal_eval`` is the safe equivalent. + """ + value = ast.literal_eval(field) + if not isinstance(value, list): + raise ValueError(f"expected a list, got {type(value).__name__}") + return [str(item) for item in value] + + +def strip_binary_hunks(patch: str) -> str: + """Drop binary diff sections, exactly like the official evaluator.""" + if not patch: + return patch + sections = re.split(r"(?=^diff --git )", patch, flags=re.MULTILINE) + kept = [ + section + for section in sections + if section.strip() + and not re.search(r"^Binary files .* differ$", section, re.MULTILINE) + and not re.search(r"^GIT binary patch$", section, re.MULTILINE) + ] + return "".join(kept) + + +def score(instance: dict[str, Any], tests: list[dict[str, Any]]) -> EvaluationResult: + """The official resolution criterion: every required test passes.""" + passed = {t["name"] for t in tests if t.get("status") == "PASSED"} + fail_to_pass = set(str_list(instance["fail_to_pass"])) + pass_to_pass = set(str_list(instance["pass_to_pass"])) + required = fail_to_pass | pass_to_pass + resolved = required <= passed + + def fraction(subset: set[str]) -> float: + return len(subset & passed) / len(subset) if subset else 1.0 + + missing = sorted(required - passed) + return EvaluationResult( + reward=1.0 if resolved else 0.0, + content="resolved" if resolved else f"unresolved: {len(missing)} required test(s) not passing", + subscores=[ + SubScore(name="resolved", value=1.0 if resolved else 0.0, weight=1.0), + SubScore(name="fail_to_pass", value=fraction(fail_to_pass), weight=0.0), + SubScore(name="pass_to_pass", value=fraction(pass_to_pass), weight=0.0), + ], + info={"tests_reported": len(tests), "missing": missing[:20]}, + ) + + +def build_prompt(instance: dict[str, Any], repo_dir: Path) -> str: + sections = [ + f"You are working in {repo_dir}, a checkout of the {instance['repo']} repository. " + "Resolve the issue described below by modifying the code. Hidden tests grade your " + "work when you finish; do not modify existing tests.", + instance["problem_statement"], + ] + if instance.get("requirements"): + sections.append("## Requirements\n\n" + instance["requirements"]) + if instance.get("interface"): + sections.append("## Interface\n\n" + instance["interface"]) + return "\n\n".join(section.strip() for section in sections if section and section.strip()) + + +async def grade( + instance: dict[str, Any], + instance_dir: Path, + repo_dir: Path, + vault_dir: Path, + logs_dir: Path, + *, + validate_mode: str | None = None, + tests_timeout: float = 3600.0, +) -> EvaluationResult: + """Replay the official evaluator's entryscript in place. + + ``validate_mode="golden"`` grades the dataset's golden patch instead of + the agent's diff (the official gold-patch sanity check). The hidden test + files (the last line of ``before_repo_set_cmd``) are checked out *after* + the patch, so patch edits to them never survive. + """ + logs_dir.mkdir(parents=True, exist_ok=True) + setup_commit = await repo_lib.restore_history(repo_dir, vault_dir) + + if validate_mode == "golden": + diff = instance["patch"] + else: + diff = await repo_lib.capture_agent_diff(repo_dir, setup_commit) + diff = strip_binary_hunks(diff) + + await repo_lib.reset_worktree(repo_dir, setup_commit) + apply_error = await repo_lib.apply_diff(repo_dir, diff, logs_dir / "patch.diff") + if apply_error is not None: + return EvaluationResult(reward=0.0, content="patch failed to apply", info={"git_apply": apply_error}) + + test_checkout = instance["before_repo_set_cmd"].strip().splitlines()[-1] + await repo_lib.run("bash", "-c", test_checkout, cwd=repo_dir) + + files_csv = ",".join(str_list(instance["selected_test_files_to_run"])) + _, out, err = await repo_lib.run( + "bash", + str(instance_dir / "run_script.sh"), + files_csv, + cwd=repo_dir, + timeout=tests_timeout, + check=False, + ) + (logs_dir / "stdout.log").write_text(out, "utf-8") + (logs_dir / "stderr.log").write_text(err, "utf-8") + await repo_lib.run( + sys.executable, + str(instance_dir / "parser.py"), + "stdout.log", + "stderr.log", + "output.json", + cwd=logs_dir, + ) + tests = json.loads((logs_dir / "output.json").read_text("utf-8"))["tests"] + return score(instance, tests) diff --git a/env.py b/env.py index 03eac6d..295006f 100644 --- a/env.py +++ b/env.py @@ -1,164 +1,353 @@ -"""Coding environment (HUD v6): a sandboxed repo the agent fixes bugs in. - -The env publishes one ``ssh`` capability (a uid-walled Workspace); the agent's -harness brings its own bash/file tools. A task is a ``@env.template`` async -generator — yield a prompt, then yield a reward. Grading is 3-branch -(``{task}_baseline``/``_test``/``_golden``): ``setup_task`` checks out the -baseline and generates the hidden test patch; ``AgentPatchGrader`` applies it to -an isolated copy and runs pytest at grade time. +"""Coding environment (HUD v6): a repo workspace with diff-based grading. + +The env publishes one ``ssh`` workspace over a git repo; the agent's harness +brings its own bash/file tools. Task setup vaults the repo's real history +(:mod:`coding.repo`) so the agent gets a fresh single-commit repo with working +git but no answer-key refs; grading captures the agent's changes as a diff, +resets to the pre-agent snapshot, re-applies, brings in the hidden tests, and +runs them. + +Three task templates: + +- ``coding-task`` — hidden tests live as refs in the repo's vaulted history + (the ``{task}_baseline`` / ``{task}_test`` / ``{task}_golden`` branch + convention) and a shell command scores by exit code. Runs locally (clones + ``REPO_URL`` per process) or from an image that bakes the repo. +- ``sdlc-task`` — adds workflow: a bare mock-GitHub remote the agent pushes + to, ``github_*`` issue/PR tools (:mod:`coding.github`), and grading over + the pushed pull-request branch instead of the worktree. +- one SWE-bench Pro instance template, registered only when an instance dir + is baked in (see ``swe_tasks.py``): grading replays the official evaluator + (:mod:`coding.swe_bench_pro`). + +Images set ``REPO_DIR``; locally everything lives in per-process temp dirs. +The vault and instance assets sit outside the workspace — under ``/hud`` +(root, mode 700) in images — and agent shells drop to a non-root uid, so only +the env process can read them. """ +from __future__ import annotations + import asyncio import logging import os import shutil -import subprocess import sys import tempfile from pathlib import Path +from typing import Any from hud.environment import Environment, Workspace -from hud.graders import combine +from hud.graders import BashGrader, EvaluationResult, LLMJudgeGrader, SubScore, combine +from hud.settings import settings -from grading import AgentPatchGrader, ValidateMode +from coding import repo as repo_lib +from coding import swe_bench_pro as swe +from coding.github import MockGitHub +from coding.github import serve as serve_github logger = logging.getLogger(__name__) -FOLDER_NAME = os.environ.get("FOLDER_NAME", "project") -REPO_URL = os.environ.get("REPO_URL", "https://github.com/hud-evals/coding-template-sample") +# ─── substrate layout ──────────────────────────────────────────────────── +# Images set REPO_DIR (and ship the repo); locally everything lives in +# per-process temp dirs and the repo is cloned from REPO_URL at startup. -# The built image pins PROJECT_DIR/PATCHES_DIR to the build-time clone. Locally -# they're unset, so fall back to a per-process dir (keyed on pid so concurrent -# --runtime local rollouts don't clobber each other) under the system temp dir — -# NOT in-repo, or git apply/pytest climb to this repo's root (porting notes §15.D). -_explicit_dir = os.environ.get("PROJECT_DIR") -if _explicit_dir: - PROJECT_DIR = _explicit_dir - PATCHES_DIR = os.environ.get("PATCHES_DIR", "/home/root/patches") +_explicit_repo = os.environ.get("REPO_DIR") +if _explicit_repo: + REPO_DIR = Path(_explicit_repo) + VAULT_DIR = Path(os.environ.get("VAULT_DIR", "/hud/vault")) + LOGS_DIR = Path(os.environ.get("GRADING_LOGS_DIR", "/hud/logs")) _LOCAL_SUBSTRATE = False else: - _LOCAL_ROOT = Path(tempfile.gettempdir()) / f"hud-{FOLDER_NAME}" - PROJECT_DIR = str(_LOCAL_ROOT / f"{FOLDER_NAME}-{os.getpid()}") - PATCHES_DIR = str(_LOCAL_ROOT / f"patches-{os.getpid()}") + _LOCAL_ROOT = Path(tempfile.gettempdir()) / "hud-coding" + REPO_DIR = _LOCAL_ROOT / f"repo-{os.getpid()}" + VAULT_DIR = _LOCAL_ROOT / f"vault-{os.getpid()}" + LOGS_DIR = _LOCAL_ROOT / f"logs-{os.getpid()}" _LOCAL_SUBSTRATE = True +# The SDLC flavor's mock-GitHub remote: a bare repo the agent pushes to. It +# must be writable by the agent, so it lives outside /hud. +REMOTE_DIR = ( + Path(os.environ.get("REMOTE_DIR", "/srv/git/project.git")) + if _explicit_repo + else _LOCAL_ROOT / f"remote-{os.getpid()}" / "project.git" +) + +REPO_URL = os.environ.get("REPO_URL", "https://github.com/hud-evals/coding-template-sample") +TESTS_TIMEOUT = float(os.environ.get("GRADING_TIMEOUT", "3600")) + +INSTANCE_DIR = Path(os.environ.get("INSTANCE_DIR", "/hud/instance")) + +AGENT_UID = 1000 +AGENT_HOME = Path("/tmp/agent-home") # noqa: S108 - container-local, created at setup env = Environment(name="coding") class UidWallWorkspace(Workspace): - """Workspace that drops the agent shell to uid 1000. + """Drop agent shells to a non-root uid. - bwrap is unusable on the hosted runner (no user namespaces), so we run - without it and protect the answer key with a uid wall: ``setpriv`` drops the - shell to the repo-owner uid, so the agent can edit the repo but can't read - the root:700 ``.git`` / patches. No-op off root (local). See notes §15.H. + The env process (root, in images) keeps the vault and instance assets + under ``/hud`` at mode 700; the uid-dropped agent can edit the repo but + never read the answer key. No-op when the env itself is not root (local). """ def shell_argv(self, command=None, *, cwd=None, env=None): argv = super().shell_argv(command, cwd=cwd, env=env) if sys.platform != "win32" and hasattr(os, "geteuid") and os.geteuid() == 0: - argv = ["setpriv", "--reuid", "1000", "--regid", "1000", "--clear-groups", "--", *argv] + uid = str(AGENT_UID) + argv = ["setpriv", "--reuid", uid, "--regid", uid, "--clear-groups", "--", *argv] return argv _ws = UidWallWorkspace( - PROJECT_DIR, - guest_path=PROJECT_DIR, + REPO_DIR, + guest_path=str(REPO_DIR), network=True, - env={"HOME": "/home/ubuntu"}, + env={"HOME": str(AGENT_HOME)}, + track_files=settings.file_tracking_enabled, ) - -async def _ensure_repo() -> None: - if (Path(PROJECT_DIR) / ".git").exists(): - return - Path(PROJECT_DIR).parent.mkdir(parents=True, exist_ok=True) - logger.info("Cloning %s into %s", REPO_URL, PROJECT_DIR) - proc = await asyncio.create_subprocess_exec( - "git", "clone", REPO_URL, PROJECT_DIR, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - raise RuntimeError(f"git clone {REPO_URL} failed: {stderr.decode(errors='replace')}") +_github = MockGitHub() +_github_server: asyncio.Task[None] | None = None @env.initialize async def _up() -> None: - # Clone before starting the workspace: clone needs PROJECT_DIR absent; the - # workspace then writes its ssh credentials into PROJECT_DIR/.hud. - await _ensure_repo() + global _github_server + if _LOCAL_SUBSTRATE and not (REPO_DIR / ".git").exists(): + REPO_DIR.parent.mkdir(parents=True, exist_ok=True) + logger.info("cloning %s into %s", REPO_URL, REPO_DIR) + await repo_lib.run("git", "clone", "-q", REPO_URL, str(REPO_DIR), cwd=REPO_DIR.parent) await _ws.start() env.add_capability(_ws.capability("shell")) + if _ws.tracks_files: + env.add_capability(_ws.file_tracking_capability()) + _github_server, github_capability = serve_github(_github) + env.add_capability(github_capability) @env.shutdown async def _down() -> None: + if _github_server is not None: + _github_server.cancel() await _ws.stop() if _LOCAL_SUBSTRATE: - shutil.rmtree(PROJECT_DIR, ignore_errors=True) - shutil.rmtree(PATCHES_DIR, ignore_errors=True) + for path in (REPO_DIR, VAULT_DIR, LOGS_DIR, REMOTE_DIR.parent): + shutil.rmtree(path, ignore_errors=True) -def setup_task(task_id: str, validate_mode: ValidateMode | None = None) -> None: - """Generate the hidden test.patch and check out the baseline (or golden, to validate).""" - base = f"{task_id}_baseline" - test = f"{task_id}_test" - golden = f"{task_id}_golden" +# ─── shared task lifecycle ─────────────────────────────────────────────── - os.environ["PROBLEM_ID"] = task_id - task_patches_dir = os.path.join(PATCHES_DIR, task_id) - os.makedirs(task_patches_dir, exist_ok=True) - diff = subprocess.run( - ["git", "diff", f"origin/{base}", f"origin/{test}"], - cwd=PROJECT_DIR, capture_output=True, text=True, +async def _setup(base_ref: str | None = None, *, remote: bool = False) -> None: + """Check out the task's baseline, vault the history, hand the repo to the agent. + + With ``remote=True`` (the SDLC flavor) a bare "GitHub" remote is created + from the baseline first, and the agent's repo is attached to it as a + normal clone would be — realistic history, pushes, and branches, but no + reachable answer-key refs. + """ + if base_ref: + await repo_lib.git(REPO_DIR, "checkout", "-qf", base_ref) + if remote: + await repo_lib.create_remote(REPO_DIR, REMOTE_DIR, base_ref or "HEAD") + await repo_lib.vault_history(REPO_DIR, VAULT_DIR) + if remote: + await repo_lib.attach_to_remote(REPO_DIR, REMOTE_DIR) + if hasattr(os, "geteuid") and os.geteuid() == 0: + AGENT_HOME.mkdir(parents=True, exist_ok=True) + uid = f"{AGENT_UID}:{AGENT_UID}" + paths = [str(REPO_DIR), str(AGENT_HOME)] + ([str(REMOTE_DIR)] if remote else []) + await repo_lib.run("chown", "-R", uid, *paths, cwd=Path("/")) + if remote: + # The grader (root) pushes to and clones from the agent-owned remote. + await repo_lib.run("git", "config", "--global", "--add", "safe.directory", str(REMOTE_DIR), cwd=Path("/")) + + +# ─── generic flavor ────────────────────────────────────────────────────── + + +async def _grade_generic( + test_command: str, + base_ref: str | None, + test_ref: str | None, + test_files: list[str] | None, + golden_ref: str | None, + validate_mode: str | None, +) -> EvaluationResult: + setup_commit = await repo_lib.restore_history(REPO_DIR, VAULT_DIR) + if validate_mode == "golden": + if not (base_ref and golden_ref): + raise ValueError('validate_mode="golden" needs base_ref and golden_ref') + _, diff, _ = await repo_lib.git(REPO_DIR, "diff", base_ref, golden_ref) + else: + diff = await repo_lib.capture_agent_diff(REPO_DIR, setup_commit) + + await repo_lib.reset_worktree(REPO_DIR, setup_commit) + apply_error = await repo_lib.apply_diff(REPO_DIR, diff, LOGS_DIR / "patch.diff") + if apply_error is not None: + return EvaluationResult(reward=0.0, content="patch failed to apply", info={"git_apply": apply_error}) + if test_ref: + # Hidden tests come from the vaulted history, after the agent's diff — + # so agent edits to them never survive into grading. + await repo_lib.git(REPO_DIR, "checkout", test_ref, "--", *(test_files or ["."])) + + command = test_command.format(test_files=" ".join(test_files or [])) + return await combine( + BashGrader.grade(weight=1.0, command=command, cwd=str(REPO_DIR), timeout_seconds=int(TESTS_TIMEOUT)) ) - with open(os.path.join(task_patches_dir, "test.patch"), "w") as f: - f.write(diff.stdout) - checkout_branch = golden if validate_mode == "golden_pass" else base - result = subprocess.run( - ["git", "checkout", "-f", f"origin/{checkout_branch}"], - cwd=PROJECT_DIR, capture_output=True, text=True, + +@env.template(id="coding-task", description="Fix a bug in the repo; hidden tests grade the diff.") +async def coding_task( + description: str, + test_command: str, + base_ref: str | None = None, + test_ref: str | None = None, + test_files: list[str] | None = None, + golden_ref: str | None = None, + validate_mode: str | None = None, +): + """Generic coding task. + + ``base_ref`` is the agent's starting state; ``test_ref`` holds the hidden + tests (``test_files`` are checked out from it at grade time); + ``test_command`` scores by exit code, with ``{test_files}`` expanded. + ``validate_mode="golden"`` grades ``base_ref..golden_ref`` instead of + agent edits. + """ + if validate_mode not in (None, "golden"): + raise ValueError(f"unknown validate_mode: {validate_mode!r}") + await _setup(base_ref) + _ = yield ( + f"You are working in a coding repository located at {REPO_DIR}.\n\n" + "Use the tools provided to complete the following task. Hidden tests grade your " + f"work when you finish; do not modify existing tests.\n\n{description}" ) - if result.returncode != 0: - logger.error("Failed to checkout %s: %s", checkout_branch, result.stderr) - elif hasattr(os, "geteuid") and os.geteuid() == 0: - # Re-assert the uid wall after checkout (image only): repo -> agent, .git -> root. - subprocess.run(["chown", "-R", "ubuntu:ubuntu", PROJECT_DIR], capture_output=True) - subprocess.run(["chown", "-R", "root:root", os.path.join(PROJECT_DIR, ".git")], capture_output=True) + yield await _grade_generic(test_command, base_ref, test_ref, test_files, golden_ref, validate_mode) - os.chdir(PROJECT_DIR) +# ─── SDLC flavor: issue-driven fix, graded on the pushed PR branch ─────── -def make_prompt(description: str) -> str: - return f"""You are working in a coding repository located at {PROJECT_DIR}. -Use the tools provided to complete the following task: +async def _fetch_hidden_ref(dest: Path, ref: str) -> None: + """Fetch an answer-key ref from the vaulted history into *dest*.""" + full = f"refs/remotes/{ref}" if ref.startswith("origin/") else ref + await repo_lib.git(dest, "fetch", "-q", str(VAULT_DIR / "git"), full) -{description} -""" +async def _grade_sdlc( + test_command: str, + test_ref: str, + test_files: list[str], + golden_ref: str | None, + pr_rubric: str | None, + validate_mode: str | None, +) -> EvaluationResult: + if validate_mode == "golden": + # Simulate the workflow the agent is graded on: push the reference + # fix as a branch and open a pull request for it. + if not golden_ref: + raise ValueError('validate_mode="golden" needs golden_ref') + await repo_lib.git(VAULT_DIR / "git", "push", "-q", str(REMOTE_DIR), f"{golden_ref}:refs/heads/golden-fix") + _github.create_pull_request( + "Fix the reported bug", "Applies the reference fix.", head="golden-fix", base="main" + ) + + pr = _github.latest_pull_request() + if pr is None: + return EvaluationResult(reward=0.0, content="no pull request was opened") + + # Hermetic checkout of the PR head from the remote (never the worktree). + grading_dir = LOGS_DIR / "pr-checkout" + shutil.rmtree(grading_dir, ignore_errors=True) + LOGS_DIR.mkdir(parents=True, exist_ok=True) + await repo_lib.run("git", "clone", "-q", str(REMOTE_DIR), str(grading_dir), cwd=LOGS_DIR) + code, _, err = await repo_lib.git(grading_dir, "checkout", "-q", pr.head, check=False) + if code != 0: + return EvaluationResult( + reward=0.0, + content=f"pull request head branch {pr.head!r} was never pushed", + info={"git_checkout": err.strip()[-2000:]}, + ) -@env.template(id="coding-bug", description="Fix a bug in the target repository.") -async def coding_bug( - task_id: str, + await _fetch_hidden_ref(grading_dir, test_ref) + await repo_lib.git(grading_dir, "checkout", "FETCH_HEAD", "--", *test_files) + + command = test_command.format(test_files=" ".join(test_files)) + quality: Any + if pr_rubric: + quality = LLMJudgeGrader.grade(weight=0.2, answer=_github.transcript(), criteria=[pr_rubric]) + else: + opened_well = bool(pr.title.strip() and pr.body.strip()) + quality = SubScore(name="pull_request", value=1.0 if opened_well else 0.0, weight=0.2) + return await combine( + BashGrader.grade(weight=0.8, command=command, cwd=str(grading_dir), timeout_seconds=int(TESTS_TIMEOUT)), + quality, + ) + + +@env.template( + id="sdlc-task", + description="Issue-driven bug fix: fix the code, push a branch, open a pull request.", +) +async def sdlc_task( description: str, + test_command: str, + base_ref: str, + test_ref: str, test_files: list[str], - validate_mode: ValidateMode | None = None, + issues: list[dict[str, Any]] | None = None, + golden_ref: str | None = None, + pr_rubric: str | None = None, + validate_mode: str | None = None, ): - setup_task(task_id=task_id, validate_mode=validate_mode) - _ = yield make_prompt(description) - yield await combine( - AgentPatchGrader.grade( - weight=1.0, - problem_id=task_id, - test_files=test_files, + """The generic flavor plus workflow: the repo has an ``origin`` remote (a + bare mock-GitHub repo) and ``github_*`` tools seeded with *issues*; the + deliverable is a pushed branch with a pull request. Grading checks the PR + head out of the remote, brings in the hidden tests, runs ``test_command`` + (weight 0.8), and scores the PR itself (weight 0.2) — structurally, or + against ``pr_rubric`` with an LLM judge when provided. + """ + if validate_mode not in (None, "golden"): + raise ValueError(f"unknown validate_mode: {validate_mode!r}") + _github.seed(issues or []) + await _setup(base_ref, remote=True) + _ = yield ( + f"You are working in a coding repository located at {REPO_DIR}; its `origin` remote " + "is the team's shared repository, and the `github_*` tools give you the team's " + "issue tracker.\n\n" + "Complete the following task. When you are done, push your work to `origin` as a " + "new branch and open a pull request for it with `github_create_pull_request`. " + "Hidden tests grade your work; do not modify existing tests.\n\n" + f"{description}" + ) + yield await _grade_sdlc(test_command, test_ref, test_files, golden_ref, pr_rubric, validate_mode) + + +# ─── SWE-bench Pro flavor (present only in baked instance images) ──────── + +if (INSTANCE_DIR / "instance.json").is_file(): + INSTANCE = swe.load_instance(INSTANCE_DIR) + + @env.template( + id=INSTANCE["instance_id"], + description=f"SWE-bench Pro instance on {INSTANCE['repo']}", + ) + async def swe_bench_pro_task(validate_mode: str | None = None): + """One SWE-bench Pro rollout. ``validate_mode="golden"`` grades the + dataset's golden patch instead of agent edits (gold-patch validation).""" + if validate_mode not in (None, "golden"): + raise ValueError(f"unknown validate_mode: {validate_mode!r}") + await _setup() + _ = yield swe.build_prompt(INSTANCE, REPO_DIR) + yield await swe.grade( + INSTANCE, + INSTANCE_DIR, + REPO_DIR, + VAULT_DIR, + LOGS_DIR, validate_mode=validate_mode, - repo_path=PROJECT_DIR, - patches_dir=PATCHES_DIR, + tests_timeout=TESTS_TIMEOUT, ) - ) diff --git a/grading/__init__.py b/grading/__init__.py deleted file mode 100644 index 503a018..0000000 --- a/grading/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -"""Grading system for coding environment tasks.""" - -from .graders import AgentPatchGrader, ValidateMode -from .runner import GradingRunner - -__all__ = [ - "AgentPatchGrader", - "GradingRunner", - "ValidateMode", -] diff --git a/grading/graders.py b/grading/graders.py deleted file mode 100644 index 786f298..0000000 --- a/grading/graders.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Graders for evaluating agent solutions.""" - -import asyncio -import os -from typing import Literal - -from hud.graders import Grader - -from .runner import GradingRunner - -# Validation modes used by tests/test_grading.py (no agent edits): -# "baseline_fail" — invert the score so a correctly-failing baseline reads 1.0. -# "golden_pass" — grade the golden branch as-is (expected 1.0). -ValidateMode = Literal["baseline_fail", "golden_pass"] - - -class AgentPatchGrader(Grader): - """Apply the hidden ``test.patch`` to an isolated copy of the repo, run pytest. - - Usage:: - - await AgentPatchGrader.grade( - weight=1.0, - problem_id="my_task", - test_files=["test_foo.py"], - ) - - Custom test command:: - - await AgentPatchGrader.grade( - weight=1.0, - problem_id="my_task", - test_files=["test_foo.test.ts"], - test_command="yarn test {test_files}", - ) - """ - - name = "AgentPatchGrader" - DEFAULT_TEST_COMMAND = "uv run pytest {test_files}" - - @classmethod - async def compute_score( - cls, - test_files: list[str], - problem_id: str | None = None, - test_command: str | None = None, - validate_mode: ValidateMode | None = None, - repo_path: str | None = None, - patches_dir: str | None = None, - **kwargs, - ) -> float: - """Run the hidden tests against the current repo state. - - Args: - test_files: Test files to run. - problem_id: Problem ID for patches (default: ``PROBLEM_ID`` env). - test_command: Test command with a ``{test_files}`` placeholder. - validate_mode: ``"baseline_fail"`` inverts the score (validation); - ``"golden_pass"`` grades as-is. - repo_path: Repo to grade (default: ``PROJECT_DIR`` env / fallback). - patches_dir: Where patches were generated (default: ``PATCHES_DIR`` env). - - Returns: - ``1.0`` if the tests pass, ``0.0`` otherwise (inverted under - ``"baseline_fail"``). - """ - pid = problem_id or os.environ.get("PROBLEM_ID") - if not pid: - raise ValueError("problem_id required (or set PROBLEM_ID env)") - - runner = GradingRunner( - problem_id=pid, - test_command=test_command or cls.DEFAULT_TEST_COMMAND, - test_files=test_files, - repo_path=repo_path, - patches_dir=patches_dir, - ) - - # runner.grade() blocks (copytree + git apply + pytest); run it off the - # event loop so the served environment stays responsive. - score = await asyncio.to_thread(runner.grade) - - # baseline_fail: a correct baseline *fails* the hidden tests, so invert. - if validate_mode == "baseline_fail": - score = 1.0 if score == 0.0 else 0.0 - - return score diff --git a/grading/runner.py b/grading/runner.py deleted file mode 100644 index a825f8a..0000000 --- a/grading/runner.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Grading runner for agent patch testing. - -Workflow: -1. grade() calls: - - Copy repo, apply test.patch - - run_tests() [customize this] - - Returns score (0.0 or 1.0) -""" - -import logging -import os -import shutil -import subprocess -import uuid - -logger = logging.getLogger(__name__) - - -class GradingRunner: - """ - Grading runner. - - Usage: - runner = GradingRunner( - problem_id="my_task", - test_command="pytest {test_files}", - test_files=["test_foo.py"], - ) - score = runner.grade() - - To customize, override run_tests(): - - class MyRunner(GradingRunner): - def run_tests(self) -> tuple[bool, dict]: - result = subprocess.run(["make", "test"], cwd=self.working_dir) - return result.returncode == 0, {} - """ - - def __init__( - self, - problem_id: str, - test_command: str = "", - test_files: list[str] | None = None, - patches_dir: str | None = None, - repo_path: str | None = None, - ): - self.problem_id = problem_id - self.test_command = test_command - self.test_files = test_files or [] - # Honor PROJECT_DIR / PATCHES_DIR (set by env.py) so grading uses the same - # paths the agent edited — works both in the built image (absolute paths) - # and locally (a writable dir next to env.py). - self.patches_dir = patches_dir or os.environ.get("PATCHES_DIR", "/home/root/patches") - self.repo_path = ( - repo_path - or os.environ.get("PROJECT_DIR") - or f"/home/ubuntu/{os.environ.get('FOLDER_NAME', 'project')}" - ) - self.working_dir = f"/tmp/grading_{uuid.uuid4()}" - - @property - def test_patch(self) -> str: - return os.path.join(self.patches_dir, self.problem_id, "test.patch") - - def grade(self) -> float: - """ - Run grading and return score. - - Returns: - 1.0 if tests pass, 0.0 otherwise - """ - try: - # Copy repo to grading workspace. (shutil.copytree is cross-platform — - # the previous `cp -rT` used a GNU-only flag that BSD/macOS cp rejects.) - logger.info(f"Copying repo to {self.working_dir}") - shutil.copytree(self.repo_path, self.working_dir, symlinks=True) - - # Refresh git index after cp (stat info is stale in the copy) - refresh = subprocess.run( - ["git", "update-index", "--refresh"], - cwd=self.working_dir, - capture_output=True, - text=True, - ) - if refresh.returncode != 0: - logger.warning( - f"git update-index --refresh failed (exit {refresh.returncode}): " - f"{refresh.stderr.strip()}" - ) - else: - logger.info("Git index refreshed") - - # Apply test patch (adds test files) - logger.info(f"Applying test patch: {self.test_patch}") - with open(self.test_patch) as f: - patch_content = f.read() - if not patch_content.strip(): - raise RuntimeError( - f"Test patch is empty: {self.test_patch}. " - "The test branch likely has no diff from the baseline branch." - ) - - patch_lines = patch_content.splitlines() - logger.info( - f"Patch stats: {len(patch_lines)} lines, " - f"files: {[l for l in patch_lines if l.startswith('diff --git')]}" - ) - - result = subprocess.run( - ["git", "apply", "--verbose"], - cwd=self.working_dir, - input=patch_content, - capture_output=True, - text=True, - ) - if result.returncode != 0: - logger.error(f"git apply stdout: {result.stdout.strip()}") - logger.error(f"git apply stderr: {result.stderr.strip()}") - raise RuntimeError( - f"git apply failed (exit {result.returncode}): " - f"{result.stderr.strip()}" - ) - - # Run tests - success, metadata = self.run_tests() - - return 1.0 if success else 0.0 - finally: - # Never leave the patched copy (which contains the hidden tests) on - # disk — on an unisolated host (local macOS) a later rollout's agent - # could read it via `find /`. - shutil.rmtree(self.working_dir, ignore_errors=True) - - # ========================================================================= - # CUSTOMIZE THIS - # ========================================================================= - - def run_tests(self) -> tuple[bool, dict]: - """ - Run tests and return results. Override this for custom logic. - - Returns: - (success, metadata) - success is True if tests pass - """ - cmd = self.test_command.format(test_files=" ".join(self.test_files)) - logger.info(f"Running: {cmd}") - - result = subprocess.run( - ["bash", "-lc", cmd], - cwd=self.working_dir, - capture_output=True, - text=True, - ) - - return result.returncode == 0, { - "exit_code": result.returncode, - "stdout": result.stdout, - "stderr": result.stderr, - } diff --git a/instances/.none/.gitkeep b/instances/.none/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml index 6f54518..af54025 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,9 @@ [project] name = "coding-env" version = "0.1.0" -description = "Coding environment (HUD v6) for solving programming tasks via a sandboxed ssh workspace" +description = "Native coding environment (HUD v6) with hermetic diff grading; generic tasks and SWE-bench Pro as flavors" requires-python = ">=3.11,<3.13" -dependencies = [ "hud-python[agents]>=0.6.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0",] +dependencies = [ "hud>=0.6.0", "httpx>=0.27.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0",] [build-system] requires = [ "hatchling",] @@ -16,9 +16,6 @@ dev = [ "pytest>=8.0.0", "ruff==0.11.2", "pre-commit>=3.5.0",] target-version = "py311" line-length = 120 -[tool.hud] -image = "coding-template:dev" - [tool.ruff.lint] select = [ "E", "F", "B", "I", "C4", "UP",] ignore = [ "E501",] @@ -30,5 +27,8 @@ quote-style = "double" indent-style = "space" line-ending = "auto" +[tool.pytest.ini_options] +markers = [ "integration: requires Docker and built instance images",] + [tool.hatch.build.targets.wheel] packages = [ ".",] diff --git a/swe_tasks.py b/swe_tasks.py new file mode 100644 index 0000000..b5d46bb --- /dev/null +++ b/swe_tasks.py @@ -0,0 +1,166 @@ +"""The SWE-bench Pro task source: fetch instances, build their images, mint rows. + +Run as a script to add instances — it fetches the dataset row (HuggingFace) +and the official ``run_script.sh`` + ``parser.py`` (scaleapi/SWE-bench_Pro-os) +into ``instances//``, then builds ``Dockerfile.hud`` with the instance's +prebuilt image (``jefzda/sweap-images:``) as ``BASE`` and its +assets baked in; image refs land in ``.hud-images.json``:: + + uv run swe_tasks.py instance_NodeBB__NodeBB-04998908...-vnan + uv run swe_tasks.py ... --push registry.io/acme # push for cloud runtimes + uv run swe_tasks.py ... --fetch-only + +Imported, it exposes one ``Task`` row per fetched instance, stamped with its +image so container placements run each instance in its own image:: + + hud eval swe_tasks.py claude + hud eval swe_tasks.py claude --task-ids nodebb-04998908 + +The environment lives inside the image (``env.py`` plus the baked instance +assets). This module must not import it: rows whose module declares no env +are routed to their images instead of a local serve. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import urllib.parse +from pathlib import Path +from typing import Any + +import httpx +from hud.eval import Task +from hud.eval.runtime import RuntimeConfig + +ROOT = Path(__file__).resolve().parent +INSTANCES_DIR = ROOT / "instances" +IMAGES_MANIFEST = ROOT / ".hud-images.json" + +DATASET_API = "https://datasets-server.huggingface.co/filter" +SCRIPTS_RAW = "https://raw.githubusercontent.com/scaleapi/SWE-bench_Pro-os/main/run_scripts" +BASE_IMAGE = "jefzda/sweap-images" + + +# ─── task rows ─────────────────────────────────────────────────────────── + + +def _slug(row: dict[str, Any]) -> str: + """A short, readable slug: repo tail + commit prefix. + + ``instance_NodeBB__NodeBB-04998908...-vnan`` -> ``nodebb-04998908``. + """ + repo_tail = row["repo"].split("/")[-1].lower() + commit = row["instance_id"].removeprefix(f"instance_{row['repo'].replace('/', '__')}-") + return f"{repo_tail}-{commit[:8]}" + + +def _load() -> list[Task]: + images: dict[str, str] = json.loads(IMAGES_MANIFEST.read_text("utf-8")) if IMAGES_MANIFEST.is_file() else {} + rows = [ + json.loads((instance_dir / "instance.json").read_text("utf-8")) + for instance_dir in sorted(INSTANCES_DIR.iterdir() if INSTANCES_DIR.is_dir() else []) + if (instance_dir / "instance.json").is_file() + ] + return [ + Task( + env="coding", + id=row["instance_id"], + slug=_slug(row), + columns={ + "repo": row["repo"], + "language": row["repo_language"], + "categories": row["issue_categories"], + }, + runtime_config=(RuntimeConfig(image=images[row["instance_id"]]) if row["instance_id"] in images else None), + ) + for row in rows + ] + + +tasks = _load() + + +# ─── instance fetching + image builds (script mode) ────────────────────── + + +def fetch_instance(client: httpx.Client, instance_id: str) -> dict[str, Any]: + """One dataset row by instance id, via the datasets-server filter API.""" + where = urllib.parse.quote(f"\"instance_id\"='{instance_id}'") + url = f"{DATASET_API}?dataset=ScaleAI%2FSWE-bench_Pro&config=default&split=test&where={where}&length=1" + payload = client.get(url).raise_for_status().json() + rows = payload.get("rows", []) + if not rows: + raise SystemExit(f"instance not found in ScaleAI/SWE-bench_Pro: {instance_id}") + return rows[0]["row"] + + +def fetch_assets(instance_id: str) -> dict[str, Any]: + """Fetch the row + official scripts into ``instances//``. Idempotent.""" + target = INSTANCES_DIR / instance_id + row_path = target / "instance.json" + if row_path.is_file(): + print(f"[{instance_id}] assets cached") + return json.loads(row_path.read_text("utf-8")) + target.mkdir(parents=True, exist_ok=True) + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + row = fetch_instance(client, instance_id) + for name in ("run_script.sh", "parser.py"): + text = client.get(f"{SCRIPTS_RAW}/{instance_id}/{name}").raise_for_status().text + (target / name).write_text(text, encoding="utf-8", newline="\n") + row_path.write_text(json.dumps(row, indent=2) + "\n", encoding="utf-8", newline="\n") + print(f"[{instance_id}] fetched row + scripts") + return row + + +def build(instance_id: str, row: dict[str, Any], push: str | None) -> str: + """Build (and optionally push) the instance image: Dockerfile.hud with the + instance's prebuilt image as BASE and its assets baked in.""" + ref = f"{push}/swe-bench-pro:{instance_id}" if push else f"hud-swe-bench-pro:{instance_id}" + subprocess.run( + [ + "docker", + "build", + "--platform", + "linux/amd64", + "--build-arg", + f"BASE={BASE_IMAGE}:{row['dockerhub_tag']}", + "--build-arg", + f"INSTANCE_ID={instance_id}", + "--tag", + ref, + "--file", + str(ROOT / "Dockerfile.hud"), + str(ROOT), + ], + check=True, + ) + if push: + subprocess.run(["docker", "push", ref], check=True) + return ref + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fetch SWE-bench Pro instances and build their images.") + parser.add_argument("instance_ids", nargs="+", help="SWE-bench Pro instance ids") + parser.add_argument("--push", help="registry prefix to push built images to") + parser.add_argument("--fetch-only", action="store_true", help="fetch assets, skip builds") + args = parser.parse_args() + + images: dict[str, str] = json.loads(IMAGES_MANIFEST.read_text("utf-8")) if IMAGES_MANIFEST.is_file() else {} + for instance_id in args.instance_ids: + row = fetch_assets(instance_id) + if args.fetch_only: + continue + images[instance_id] = build(instance_id, row, args.push) + print(f"[{instance_id}] image {images[instance_id]}") + + if not args.fetch_only: + IMAGES_MANIFEST.write_text(json.dumps(images, indent=2) + "\n", encoding="utf-8") + print(f"manifest: {IMAGES_MANIFEST}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks.py b/tasks.py index b2adad2..031aa5f 100644 --- a/tasks.py +++ b/tasks.py @@ -1,32 +1,39 @@ -"""Task definitions for the coding environment (HUD v6). +"""Sample tasks: four bugs in the sample repo, plus one SDLC variant. -The generic ``coding_bug`` template (defined in ``env.py``) accepts all task -content as parameters (description, test files). Each call mints a distinct -``Task`` by passing different values — no new template function needed. +Each row parameterizes a template from ``env.py`` with the 3-branch +convention — ``{task}_baseline`` (starting state), ``{task}_test`` (hidden +tests), ``{task}_golden`` (reference fix) — as refs in the repo the env +serves (``REPO_URL``, default +https://github.com/hud-evals/coding-template-sample):: -Branch names are derived from ``task_id`` by convention: -``{task_id}_baseline``, ``{task_id}_test``, ``{task_id}_golden``. - - hud eval tasks.py claude --task-ids sentry-fix + hud eval tasks.py claude --task-ids sentry-fix -y --runtime local hud eval tasks.py claude --full - hud sync tasks coding-tasks -``env`` is re-exported so ``hud eval tasks.py`` (which serves this source on a -local substrate) can find the Environment; the public ``tasks`` list is what -``hud eval`` / ``hud sync`` collect. +SWE-bench Pro rows live in ``swe_tasks.py``. """ -from env import coding_bug, env # noqa: F401 (env re-exported for `hud eval tasks.py`) +from env import coding_task, env, sdlc_task # noqa: F401 (env re-exported for `hud eval tasks.py`) +TEST_COMMAND = "python3 -m pytest -q {test_files}" -# ============================================================================= -# Four hand-picked bugs: a KeyError, broken event routing, and two subtle -# falsy-value / shared-state bugs in multi-file systems. Span Basic -> Hard. -# ============================================================================= -_sentry_fix = coding_bug( - task_id="sentry_fix", - description=( +def _bug(slug: str, task_id: str, description: str, test_files: list[str]): + task = coding_task( + description=description, + test_command=TEST_COMMAND, + base_ref=f"origin/{task_id}_baseline", + test_ref=f"origin/{task_id}_test", + golden_ref=f"origin/{task_id}_golden", + test_files=test_files, + ) + task.slug = slug + return task + + +tasks = [ + _bug( + "sentry-fix", + "sentry_fix", "Fix a crash in the user profile endpoint.\n\n" "The user profile service crashes with a KeyError for certain users. Some users\n" "have incomplete profile data — their `profile` field may be None or missing\n" @@ -35,60 +42,69 @@ "Expected behavior when a user has no profile (the `profile` field is None or\n" "absent): fall back to the user's top-level `name` for the display name and use an\n" "empty string for the bio. Users with a complete profile keep their existing\n" - "`display_name` and `bio`." + "`display_name` and `bio`.", + ["test_user_service.py"], ), - test_files=["test_user_service.py"], -) -_sentry_fix.slug = "sentry-fix" - -_notif_bug = coding_bug( - task_id="notif_bug", - description=( + _bug( + "notif-bug", + "notif_bug", "Fix the broken notification system.\n\n" "The notification system is completely silent — no notifications are generated when\n" "tasks are created, assigned, or completed. The event handlers are registered and\n" "the notification service is initialized, but events never reach their handlers.\n" - "Investigate the event routing pipeline and fix the issue." + "Investigate the event routing pipeline and fix the issue.", + ["test_notifications.py"], ), - test_files=["test_notifications.py"], -) -_notif_bug.slug = "notif-bug" - -_settings_v2 = coding_bug( - task_id="settings_v2", - description=( + _bug( + "settings-v2", + "settings_v2", "Fix disappearing fields in API responses.\n\n" "API responses for the settings and user endpoints are randomly dropping fields\n" "that have values like 0, false, or empty string. Direct key lookups work fine,\n" "but when responses are serialized to JSON, certain valid fields disappear. The\n" "issue affects multiple endpoints and seems related to how data is iterated over\n" - "during serialization." + "during serialization.", + ["test_settings.py"], ), - test_files=["test_settings.py"], -) -_settings_v2.slug = "settings-v2" - -_webhook_bug = coding_bug( - task_id="webhook_bug", - description=( + _bug( + "webhook-bug", + "webhook_bug", "Fix inconsistent webhook notification channels.\n\n" "Webhook notifications work correctly on the first request for a given event type,\n" "but subsequent requests for the same event type produce incorrect or duplicated\n" "notification channels. The issue gets worse with repeated requests — channels\n" - "accumulate and sort order changes unexpectedly." + "accumulate and sort order changes unexpectedly.", + ["test_notifications.py"], ), - test_files=["test_notifications.py"], -) -_webhook_bug.slug = "webhook-bug" - +] -# ============================================================================= -# Task registry — collected by ``hud eval`` / ``hud sync tasks`` -# ============================================================================= -tasks = [ - _sentry_fix, - _notif_bug, - _settings_v2, - _webhook_bug, -] +# The SDLC variant of sentry-fix: same bug, but the task arrives as a GitHub +# issue and the deliverable is a pushed branch with a pull request. +_sentry_fix_pr = sdlc_task( + description=( + "Issue #42 in the tracker reports a crash in the user profile endpoint. " + "Read the issue with the github tools, fix the bug, and ship the fix " + "through the normal review workflow." + ), + test_command=TEST_COMMAND, + base_ref="origin/sentry_fix_baseline", + test_ref="origin/sentry_fix_test", + golden_ref="origin/sentry_fix_golden", + test_files=["test_user_service.py"], + issues=[ + { + "number": 42, + "title": "KeyError crash on user profile endpoint", + "body": ( + "Some users crash the profile endpoint with a KeyError — their `profile` " + "field can be None or missing. Expected: fall back to the top-level `name` " + "for the display name and an empty string for the bio; users with a " + "complete profile keep their existing `display_name` and `bio`." + ), + "labels": ["bug"], + } + ], +) +_sentry_fix_pr.slug = "sentry-fix-pr" +tasks.append(_sentry_fix_pr) diff --git a/tests/conftest.py b/tests/conftest.py index a279810..3e0a8b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,56 +1,16 @@ -"""Shared fixtures for the coding environment test suite (HUD v6). +"""Shared fixtures for the SWE-bench Pro environment tests. -v6 drives a *served* environment through a placement provider (a ``runtime``) -plus the ``connect`` + ``Run`` client lifecycle — there is no in-process -``Environment`` client (``connect_image`` / ``call_tool`` / ``read_resource`` -are gone). The ``runtime`` fixture resolves a provider from: - - --url tcp://host:port -> Runtime(url) (attach to an env served elsewhere) - --image name-or-url -> DockerRuntime(name) (fresh container per rollout) - else -> [tool.hud].image from pyproject.toml +``env.py`` loads its instance from ``INSTANCE_DIR`` at import (in an instance +image that is ``/hud/instance``); the offline tests point it at a small +fixture instance before anything imports it. """ +import os import sys from pathlib import Path -import pytest - -# Ensure the project root is on sys.path so `from env import ...` / `from tasks -# import ...` resolve when running pytest from the repo root. PROJECT_ROOT = Path(__file__).resolve().parent.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) - -def pytest_addoption(parser): - parser.addoption( - "--image", - default=None, - help="Docker image name to serve the env from (e.g. coding-template:dev)", - ) - parser.addoption( - "--url", - default=None, - help="tcp:// url of an already-served env control channel (e.g. tcp://127.0.0.1:8765)", - ) - - -@pytest.fixture(scope="session") -def runtime(request): - """A v6 placement provider for the environment under test. - - Resolution order: --url (attach) > --image (fresh container per rollout) > - default LocalRuntime (serve from source; clones the substrate per rollout — - no Docker, works on macOS/Linux). - """ - from hud import DockerRuntime, LocalRuntime, Runtime - - url = request.config.getoption("--url") - if url: - return Runtime(url) - - image = request.config.getoption("--image") - if image: - return DockerRuntime(image) - - return LocalRuntime(str(PROJECT_ROOT / "tasks.py")) +os.environ.setdefault("INSTANCE_DIR", str(Path(__file__).parent / "fixtures" / "instance")) diff --git a/tests/fixtures/instance/instance.json b/tests/fixtures/instance/instance.json new file mode 100644 index 0000000..fbe3360 --- /dev/null +++ b/tests/fixtures/instance/instance.json @@ -0,0 +1,18 @@ +{ + "repo": "acme/widgets", + "instance_id": "instance_acme__widgets-0000000000000000000000000000000000000000-vnan", + "base_commit": "0000000000000000000000000000000000000000", + "patch": "diff --git a/widgets.py b/widgets.py\n--- a/widgets.py\n+++ b/widgets.py\n@@ -1 +1 @@\n-BROKEN = True\n+BROKEN = False\n", + "test_patch": "", + "problem_statement": "Widgets are broken.", + "requirements": "- Widgets should not be broken.", + "interface": "", + "repo_language": "py", + "fail_to_pass": "['tests/test_widgets.py | test_widgets_work', \"tests/test_widgets.py | test_widgets_don't_break\"]", + "pass_to_pass": "['tests/test_widgets.py | test_existing_behavior']", + "issue_specificity": "[\"minor_bug\"]", + "issue_categories": "[\"back_end_knowledge\"]", + "before_repo_set_cmd": "git reset --hard 0000000000000000000000000000000000000000\ngit clean -fd \ngit checkout 0000000000000000000000000000000000000000 \ngit checkout 1111111111111111111111111111111111111111 -- tests/test_widgets.py", + "selected_test_files_to_run": "[\"tests/test_widgets.py\"]", + "dockerhub_tag": "acme.widgets-acme__widgets-0000000000000000000000000000000000000000" +} diff --git a/tests/fixtures/instance/parser.py b/tests/fixtures/instance/parser.py new file mode 100644 index 0000000..8e38d73 --- /dev/null +++ b/tests/fixtures/instance/parser.py @@ -0,0 +1,8 @@ +"""Fixture stand-in for an official per-instance parser: reports no tests.""" + +import json +import sys + +if __name__ == "__main__": + with open(sys.argv[3], "w") as f: + json.dump({"tests": []}, f) diff --git a/tests/fixtures/instance/run_script.sh b/tests/fixtures/instance/run_script.sh new file mode 100644 index 0000000..4eebf60 --- /dev/null +++ b/tests/fixtures/instance/run_script.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Fixture stand-in for an official per-instance run script. +echo "running: $@" diff --git a/tests/test_env.py b/tests/test_env.py index 6f4f959..67a48be 100644 --- a/tests/test_env.py +++ b/tests/test_env.py @@ -1,35 +1,24 @@ -"""In-process wiring smoke tests for the v6 environment (no Docker required). +"""In-process wiring smoke tests (no Docker): the served surface is well-formed.""" -These don't serve the env or touch the target repo — they just confirm the -template is registered and the task rows are well-formed, so authoring mistakes -surface fast without a build. -""" +import json +from pathlib import Path from env import env -from tasks import tasks - -def test_template_registered(): - """The generic bug-fix template is registered under its public id.""" - assert "coding-bug" in env.tasks - entry = env.tasks["coding-bug"].manifest_entry() - assert entry["id"] == "coding-bug" +FIXTURE_INSTANCE = json.loads((Path(__file__).parent / "fixtures" / "instance" / "instance.json").read_text("utf-8")) def test_env_identity(): assert env.name == "coding" -def test_tasks_collected(): - """`hud eval` / `hud sync` collect the public ``tasks`` list.""" - assert len(tasks) == 4 +def test_generic_template_registered(): + assert "coding-task" in env.tasks + assert env.tasks["coding-task"].manifest_entry()["id"] == "coding-task" - slugs = {t.slug for t in tasks} - assert slugs == {"sentry-fix", "notif-bug", "settings-v2", "webhook-bug"} - assert len(slugs) == len(tasks), "task slugs must be unique" - for task in tasks: - assert task.env == "coding" - assert task.id == "coding-bug" - assert "task_id" in task.args - assert "test_files" in task.args +def test_swe_bench_template_registered_when_instance_baked(): + """conftest points INSTANCE_DIR at the fixture instance, as an instance image would.""" + instance_id = FIXTURE_INSTANCE["instance_id"] + assert instance_id in env.tasks + assert env.tasks[instance_id].manifest_entry()["id"] == instance_id diff --git a/tests/test_github.py b/tests/test_github.py new file mode 100644 index 0000000..33edf41 --- /dev/null +++ b/tests/test_github.py @@ -0,0 +1,55 @@ +"""Offline tests for the mock GitHub store's contract.""" + +import json + +import pytest + +from coding.github import MockGitHub + +ISSUE = {"number": 42, "title": "Crash", "body": "It crashes.", "labels": ["bug"]} + + +def _seeded() -> MockGitHub: + github = MockGitHub() + github.seed([ISSUE]) + return github + + +def test_seeded_issues_are_listable_and_open(): + github = _seeded() + (issue,) = github.list_issues() + assert issue["number"] == 42 + assert issue["state"] == "open" + assert github.get_issue(42)["title"] == "Crash" + with pytest.raises(ValueError): + github.get_issue(7) + + +def test_issue_workflow_updates_state_and_transcript(): + github = _seeded() + github.comment_on_issue(42, "Investigating.") + github.close_issue(42) + issue = github.get_issue(42) + assert issue["state"] == "closed" + assert issue["comments"] == ["Investigating."] + transcript = json.loads(github.transcript()) + assert [a["action"] for a in transcript["actions"]] == ["comment_on_issue", "close_issue"] + + +def test_pull_requests_number_sequentially_and_latest_wins(): + github = _seeded() + assert github.latest_pull_request() is None + github.create_pull_request("First", "body", head="fix-a", base="main") + second = github.create_pull_request("Second", "body", head="fix-b", base="main") + assert second["number"] == 2 + latest = github.latest_pull_request() + assert latest is not None and latest.head == "fix-b" + assert len(github.list_pull_requests()) == 2 + + +def test_seed_resets_previous_task_state(): + github = _seeded() + github.create_pull_request("Old", "body", head="old", base="main") + github.seed([ISSUE]) + assert github.latest_pull_request() is None + assert github.actions == [] diff --git a/tests/test_grading.py b/tests/test_grading.py index 539d097..b44656f 100644 --- a/tests/test_grading.py +++ b/tests/test_grading.py @@ -1,63 +1,66 @@ -"""Grading validation tests for all tasks (HUD v6). +"""Offline tests for the SWE-bench Pro flavor's pure pieces: the resolution +criterion, dataset field parsing, and patch sanitization.""" -Each task's grading pipeline runs inside the served environment (a built Docker -image by default), with **no agent edits**: - - baseline (buggy) code fails the hidden tests -> grader returns 1.0 (inverted) - - golden (fixed) code passes the hidden tests -> grader returns 1.0 +import json +from pathlib import Path -``setup_task`` runs when the task starts (checks out baseline/golden, generates -patches); the grader applies the hidden ``test.patch`` and runs pytest when the -``Run`` context exits. The agent never acts, so the deliverable is purely the -checked-out state. +import pytest -Usage: - uv run pytest tests/test_grading.py -v --image coding-template:dev - uv run pytest tests/test_grading.py -v --url tcp://127.0.0.1:8765 - uv run pytest tests/test_grading.py -v -k sample-json-bug --image ... -""" +from coding.swe_bench_pro import score, str_list, strip_binary_hunks -import pytest -from hud import Run, connect +INSTANCE = json.loads((Path(__file__).parent / "fixtures" / "instance" / "instance.json").read_text("utf-8")) -from tasks import tasks as ALL_TASKS +F2P = [ + "tests/test_widgets.py | test_widgets_work", + "tests/test_widgets.py | test_widgets_don't_break", +] +P2P = ["tests/test_widgets.py | test_existing_behavior"] -pytestmark = pytest.mark.asyncio(loop_scope="session") -BY_SLUG = {t.slug: t for t in ALL_TASKS} +def _reported(names, status="PASSED"): + return [{"name": name, "status": status} for name in names] + + +def test_resolved_when_all_required_tests_pass(): + result = score(INSTANCE, _reported(F2P + P2P + ["extra | irrelevant_test"])) + assert result.reward == 1.0 + assert result.content == "resolved" + + +def test_unresolved_when_a_fail_to_pass_test_fails(): + reported = _reported(F2P[:1] + P2P) + _reported(F2P[1:], status="FAILED") + result = score(INSTANCE, reported) + assert result.reward == 0.0 + by_name = {s.name: s.value for s in result.subscores} + assert by_name["fail_to_pass"] == 0.5 + assert by_name["pass_to_pass"] == 1.0 + assert F2P[1] in result.info["missing"] -# Non-hint tasks to validate (hint variants share the same branches/grading). -TASK_SLUGS = [ - "sentry-fix", - "notif-bug", - "settings-v2", - "webhook-bug", -] +def test_unresolved_on_pass_to_pass_regression(): + """Fixing the bug while breaking existing behavior is not resolved.""" + result = score(INSTANCE, _reported(F2P) + _reported(P2P, status="ERROR")) + assert result.reward == 0.0 + by_name = {s.name: s.value for s in result.subscores} + assert by_name["fail_to_pass"] == 1.0 + assert by_name["pass_to_pass"] == 0.0 -async def _grade(runtime, slug: str, validate_mode: str) -> float: - """Start the task with a validate_mode, run no agent, return the reward.""" - base = BY_SLUG[slug] - task = base.model_copy(update={"args": {**base.args, "validate_mode": validate_mode}}) - async with runtime(task) as addr, connect(addr) as client: - async with Run(client, task.id, task.args) as run: - pass # no agent work: setup runs on start, grading on exit - return run.reward +def test_missing_report_scores_zero(): + result = score(INSTANCE, []) + assert result.reward == 0.0 -@pytest.mark.parametrize("slug", TASK_SLUGS) -async def test_baseline_fails(runtime, slug): - """Baseline (buggy) code should fail the tests -> grader returns 1.0 (inverted).""" - reward = await _grade(runtime, slug, "baseline_fail") - assert reward == 1.0, ( - f"{slug}: baseline should fail tests (inverted score should be 1.0, got {reward})" - ) +def test_str_list_parses_python_repr_fields(): + """Rows store lists as Python reprs (the official evaluator uses eval).""" + assert str_list(INSTANCE["fail_to_pass"]) == F2P + assert str_list(INSTANCE["selected_test_files_to_run"]) == ["tests/test_widgets.py"] + with pytest.raises(ValueError): + str_list("'not a list'") -@pytest.mark.parametrize("slug", TASK_SLUGS) -async def test_golden_passes(runtime, slug): - """Golden (fixed) code should pass the tests -> grader returns 1.0.""" - reward = await _grade(runtime, slug, "golden_pass") - assert reward == 1.0, ( - f"{slug}: golden branch should pass tests (score should be 1.0, got {reward})" - ) +def test_strip_binary_hunks_drops_only_binary_sections(): + text = "diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -1 +1 @@\n-a\n+b\n" + binary = "diff --git a/img b/img\nBinary files a/img and b/img differ\n" + assert strip_binary_hunks(text + binary) == text + assert strip_binary_hunks("") == "" diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..5f90e35 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,51 @@ +"""Gold-patch validation against built instance images (Docker, linux/amd64). + +The SWE-bench Pro analog of the official gold-patch sanity check, driven +through the full served lifecycle with **no agent edits**: + +- ``validate_mode="golden"``: the dataset's golden patch grades to 1.0 +- no validate_mode: an untouched baseline grades to 0.0 + +Requires images built by ``swe_tasks.py`` (skips otherwise):: + + uv run swe_tasks.py + uv run pytest tests/test_integration.py -v +""" + +import shutil + +import pytest +from hud import DockerRuntime, Run, connect + +from swe_tasks import tasks as ALL_TASKS + +pytestmark = [ + pytest.mark.integration, + pytest.mark.asyncio(loop_scope="session"), +] + +ADAPTED = [t for t in ALL_TASKS if t.runtime_config and t.runtime_config.image] + +if not shutil.which("docker"): + pytest.skip("docker not available", allow_module_level=True) +if not ADAPTED: + pytest.skip("no built instances (run swe_tasks.py first)", allow_module_level=True) + + +async def _grade(task, validate_mode: str | None) -> float: + args = {"validate_mode": validate_mode} if validate_mode else {} + runtime = DockerRuntime(task.runtime_config.image) + async with runtime(task) as addr, connect(addr) as client: + async with Run(client, task.id, args) as run: + pass # no agent work: setup on start, grading on exit + return run.reward + + +@pytest.mark.parametrize("task", ADAPTED, ids=lambda t: t.slug) +async def test_golden_patch_resolves(task): + assert await _grade(task, "golden") == 1.0 + + +@pytest.mark.parametrize("task", ADAPTED, ids=lambda t: t.slug) +async def test_untouched_baseline_unresolved(task): + assert await _grade(task, None) == 0.0 diff --git a/tests/test_local_rollout.py b/tests/test_local_rollout.py new file mode 100644 index 0000000..6132f19 --- /dev/null +++ b/tests/test_local_rollout.py @@ -0,0 +1,116 @@ +"""Hermetic end-to-end validation of the generic flavor (no Docker, no network). + +Builds a tiny 3-branch fixture repo (``bug_baseline`` / ``bug_test`` / +``bug_golden``), serves ``env.py`` on a local substrate cloned from it, and +drives the full lifecycle with no agent edits: + +- ``validate_mode="golden"``: the reference fix grades to 1.0 +- no validate_mode: the untouched baseline grades to 0.0 + +This is the 3-branch analog of the SWE-bench gold-patch sanity check, and it +exercises the whole vault/diff/hidden-test pipeline through real git. +""" + +import os +import subprocess +from pathlib import Path + +import pytest +from hud import LocalRuntime, Run, connect + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +pytestmark = pytest.mark.asyncio(loop_scope="session") + + +def _git(cwd: Path, *args: str) -> None: + subprocess.run( + ["git", "-c", "user.name=t", "-c", "user.email=t@t", *args], + cwd=cwd, + check=True, + capture_output=True, + ) + + +@pytest.fixture(scope="session") +def fixture_repo(tmp_path_factory) -> Path: + repo = tmp_path_factory.mktemp("fixture-repo") + (repo / "widget.py").write_text("BROKEN = True\n") + _git(repo, "init", "-q", "-b", "main") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "buggy widget") + _git(repo, "branch", "bug_baseline") + + _git(repo, "checkout", "-qb", "bug_test") + (repo / "test_widget.py").write_text( + "import unittest\n" + "import widget\n\n\n" + "class TestWidget(unittest.TestCase):\n" + " def test_not_broken(self):\n" + " self.assertFalse(widget.BROKEN)\n" + ) + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "hidden tests") + + _git(repo, "checkout", "-q", "main") + _git(repo, "checkout", "-qb", "bug_golden") + (repo / "widget.py").write_text("BROKEN = False\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-qm", "reference fix") + return repo + + +async def _run_task(fixture_repo: Path, task) -> float: + # The served subprocess inherits the environment; point it at the fixture. + os.environ["REPO_URL"] = str(fixture_repo) + runtime = LocalRuntime(str(PROJECT_ROOT / "env.py")) + async with runtime(task) as addr, connect(addr) as client: + async with Run(client, task.id, task.args) as run: + pass # no agent work: setup on start, grading on exit + return run.reward + + +def _coding_task(validate_mode: str | None): + from env import coding_task + + return coding_task( + description="Fix the widget.", + test_command="python3 -m unittest {test_files}", + base_ref="origin/bug_baseline", + test_ref="origin/bug_test", + test_files=["test_widget.py"], + golden_ref="origin/bug_golden", + validate_mode=validate_mode, + ) + + +def _sdlc_task(validate_mode: str | None): + from env import sdlc_task + + return sdlc_task( + description="Issue #1 reports a broken widget. Fix it and open a PR.", + test_command="python3 -m unittest {test_files}", + base_ref="origin/bug_baseline", + test_ref="origin/bug_test", + test_files=["test_widget.py"], + golden_ref="origin/bug_golden", + issues=[{"number": 1, "title": "Widget broken", "body": "BROKEN should be False."}], + validate_mode=validate_mode, + ) + + +async def test_golden_ref_scores_one(fixture_repo): + assert await _run_task(fixture_repo, _coding_task("golden")) == 1.0 + + +async def test_untouched_baseline_scores_zero(fixture_repo): + assert await _run_task(fixture_repo, _coding_task(None)) == 0.0 + + +async def test_sdlc_golden_pr_scores_one(fixture_repo): + """Golden validation of the full PR workflow: pushed branch + opened PR.""" + assert await _run_task(fixture_repo, _sdlc_task("golden")) == 1.0 + + +async def test_sdlc_no_pull_request_scores_zero(fixture_repo): + assert await _run_task(fixture_repo, _sdlc_task(None)) == 0.0 diff --git a/tests/test_tasks.py b/tests/test_tasks.py new file mode 100644 index 0000000..5798cad --- /dev/null +++ b/tests/test_tasks.py @@ -0,0 +1,48 @@ +"""Task-row tests for both flavors: rows are well-formed and slugs readable.""" + +import json +from pathlib import Path + +import swe_tasks +import tasks + +FIXTURE_ROW = json.loads((Path(__file__).parent / "fixtures" / "instance" / "instance.json").read_text("utf-8")) + + +def test_generic_rows_parameterize_the_coding_task_template(): + slugs = {task.slug for task in tasks.tasks} + assert slugs == {"sentry-fix", "notif-bug", "settings-v2", "webhook-bug", "sentry-fix-pr"} + for task in tasks.tasks: + assert task.env == "coding" + assert task.args["base_ref"].endswith("_baseline") + assert task.args["test_ref"].endswith("_test") + assert task.args["test_files"] + + by_slug = {task.slug: task for task in tasks.tasks} + assert by_slug["sentry-fix"].id == "coding-task" + sdlc = by_slug["sentry-fix-pr"] + assert sdlc.id == "sdlc-task" + assert sdlc.args["issues"][0]["number"] == 42 + + +def test_swe_slug_is_repo_tail_plus_commit_prefix(): + assert swe_tasks._slug(FIXTURE_ROW) == "widgets-00000000" + assert ( + swe_tasks._slug( + { + "repo": "NodeBB/NodeBB", + "instance_id": "instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan", + } + ) + == "nodebb-04998908" + ) + + +def test_swe_rows_are_wellformed(): + """Whatever `swe_tasks.py` has fetched loads as valid, uniquely-slugged rows.""" + slugs = [task.slug for task in swe_tasks.tasks] + assert len(set(slugs)) == len(slugs) + for task in swe_tasks.tasks: + assert task.env == "coding" + assert task.id.startswith("instance_") + assert task.columns and task.columns["repo"] diff --git a/uv.lock b/uv.lock index 51c85d5..0d033cf 100644 --- a/uv.lock +++ b/uv.lock @@ -251,7 +251,8 @@ name = "coding-env" version = "0.1.0" source = { editable = "." } dependencies = [ - { name = "hud-python" }, + { name = "httpx" }, + { name = "hud" }, { name = "pytest" }, { name = "pytest-asyncio" }, ] @@ -265,7 +266,8 @@ dev = [ [package.metadata] requires-dist = [ - { name = "hud-python", extras = ["agents"], specifier = ">=0.6.0" }, + { name = "httpx", specifier = ">=0.27.0" }, + { name = "hud", specifier = ">=0.6.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, @@ -535,8 +537,8 @@ wheels = [ ] [[package]] -name = "hud-python" -version = "0.6.2" +name = "hud" +version = "0.6.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, @@ -557,9 +559,9 @@ dependencies = [ { name = "typer" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9e/d3/ad411df704faeaff21aa956d1bc761ad30729586aa2e777891489807ceda/hud_python-0.6.2.tar.gz", hash = "sha256:a58e64c8d5cda924ec7f58a3584450e1afd1667b4edc96d2491461950c18fb05", size = 310247, upload-time = "2026-06-20T02:42:34.995Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/8c/b94051aacfe311e4a531610c32368d21c99c80644aed8b42e4afbd78c1e3/hud-0.6.10.tar.gz", hash = "sha256:65ceec83fedd86f8d4cb411357a49ac9522d085a7dd65111d46ca6fec5695b44", size = 344418, upload-time = "2026-07-15T23:34:34.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/62/98b202f70ebac9f85f62d86f4e0cb5e57d5e3c7e83195b63ee4f53c42666/hud_python-0.6.2-py3-none-any.whl", hash = "sha256:6f8aabd0a24b89c983105ca0e7780817ffe8af90a60ec1a41e7bfaae4ce8cb51", size = 406268, upload-time = "2026-06-20T02:42:33.537Z" }, + { url = "https://files.pythonhosted.org/packages/78/72/c649ad285bf85d0e7b3b936774efc39c28df2c9572cfa24ea559377bd960/hud-0.6.10-py3-none-any.whl", hash = "sha256:b686e08c464a4cc95fca48ffff4108cc8cbc69408f4e4bd57ffed6c76ac076d9", size = 448102, upload-time = "2026-07-15T23:34:33.475Z" }, ] [[package]] @@ -859,7 +861,7 @@ wheels = [ [[package]] name = "openai" -version = "2.32.0" +version = "2.44.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -871,9 +873,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ed/59/bdcc6b759b8c42dd73afaf5bf8f902c04b37987a5514dbc1c64dba390fef/openai-2.32.0.tar.gz", hash = "sha256:c54b27a9e4cb8d51f0dd94972ffd1a04437efeb259a9e60d8922b8bd26fe55e0", size = 693286, upload-time = "2026-04-15T22:28:19.434Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/c1/d6e64ccd0536bf616556f0cad2b6d94a8125f508d25cfd814b1d2db4e2f1/openai-2.32.0-py3-none-any.whl", hash = "sha256:4dcc9badeb4bf54ad0d187453742f290226d30150890b7890711bda4f32f192f", size = 1162570, upload-time = "2026-04-15T22:28:17.714Z" }, + { url = "https://files.pythonhosted.org/packages/ae/f4/561ed79fd94876160018a5e75254cfcb9b0e62d4dded9dcb20072e86d623/openai-2.44.0-py3-none-any.whl", hash = "sha256:0a2a3ab2e29aeda368700f662ff9ba0f9df17ba4c54577a64e08b8115a3cc0ad", size = 1366216, upload-time = "2026-06-24T20:55:58.882Z" }, ] [[package]] From e26e136b7389991a225f6736872f0d50f1b2238f Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:23:25 -0700 Subject: [PATCH 2/2] Adopt hud 0.6.11: Workspace shell_uid replaces the hand-rolled uid wall The SDK workspace now owns the privilege wall (setpriv drop, external credentials, exec-channel file I/O), so the local UidWallWorkspace subclass is gone. Ownership of the baked repo follows "chown what you stage": the generic build chowns /app in the clone step, and task setup keeps its chown -R for files root mutates per task, so workspace boot stays O(1). --- Dockerfile.hud | 8 ++++++-- README.md | 4 ++++ env.py | 24 +++++------------------- pyproject.toml | 2 +- uv.lock | 8 ++++---- 5 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Dockerfile.hud b/Dockerfile.hud index 8f5562c..a93b4d9 100644 --- a/Dockerfile.hud +++ b/Dockerfile.hud @@ -25,12 +25,16 @@ RUN apt-get update -y \ RUN pip3 install --break-system-packages pytest ARG REPO_URL="https://github.com/hud-evals/coding-template-sample" +# chown in the same RUN as the clone (a separate layer would duplicate /app): +# env.py serves agent shells as uid 1000, so the baked tree must be theirs. +# Doing it at build makes it free at runtime, whatever the repo's size. RUN --mount=type=secret,id=CODING_GITHUB_TOKEN \ if [ -f /run/secrets/CODING_GITHUB_TOKEN ]; then \ git clone "https://$(cat /run/secrets/CODING_GITHUB_TOKEN)@${REPO_URL#https://}" /app; \ else \ git clone ${REPO_URL} /app; \ - fi + fi \ + && chown -R 1000:1000 /app # Add a build step for the target repo here if it needs one (deps, compile). # ─── the served image: BASE + hud serving layer (+ instance assets) ─── @@ -59,7 +63,7 @@ command -v uv >/dev/null 2>&1 || { } uv python install 3.12 uv venv /hud/venv --python 3.12 -uv pip install --python /hud/venv/bin/python 'hud>=0.6.10' +uv pip install --python /hud/venv/bin/python 'hud>=0.6.11' chmod -R go-rwx /hud INSTALL ENV REPO_DIR=/app PYTHONPATH=/hud diff --git a/README.md b/README.md index 0b5d7a3..1b59717 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,10 @@ integration suite is the same check for built SWE-bench Pro instances. network egress at the runtime layer if that matters for your run. - The uid wall needs `setpriv` (util-linux) in the image; the repo path comes from `REPO_DIR` (`/app` in instance images). +- The agent runs as uid 1000, so the baked repo must belong to it. The workspace only chowns + its own directory at start (O(1), keeps boot fast); the tree is owned where it's staged — + the generic build chowns `/app` in the clone step, and task setup re-chowns after root + mutates the worktree (checkout, vaulting). ## Documentation diff --git a/env.py b/env.py index 295006f..cf506d2 100644 --- a/env.py +++ b/env.py @@ -32,7 +32,6 @@ import logging import os import shutil -import sys import tempfile from pathlib import Path from typing import Any @@ -83,29 +82,16 @@ env = Environment(name="coding") - -class UidWallWorkspace(Workspace): - """Drop agent shells to a non-root uid. - - The env process (root, in images) keeps the vault and instance assets - under ``/hud`` at mode 700; the uid-dropped agent can edit the repo but - never read the answer key. No-op when the env itself is not root (local). - """ - - def shell_argv(self, command=None, *, cwd=None, env=None): - argv = super().shell_argv(command, cwd=cwd, env=env) - if sys.platform != "win32" and hasattr(os, "geteuid") and os.geteuid() == 0: - uid = str(AGENT_UID) - argv = ["setpriv", "--reuid", uid, "--regid", uid, "--clear-groups", "--", *argv] - return argv - - -_ws = UidWallWorkspace( +# shell_uid is the privilege wall: the env process (root, in images) keeps the +# vault and instance assets under /hud at mode 700; the uid-dropped agent can +# edit the repo but never read the answer key. No-op off root (local). +_ws = Workspace( REPO_DIR, guest_path=str(REPO_DIR), network=True, env={"HOME": str(AGENT_HOME)}, track_files=settings.file_tracking_enabled, + shell_uid=AGENT_UID, ) _github = MockGitHub() diff --git a/pyproject.toml b/pyproject.toml index af54025..b1134d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "coding-env" version = "0.1.0" description = "Native coding environment (HUD v6) with hermetic diff grading; generic tasks and SWE-bench Pro as flavors" requires-python = ">=3.11,<3.13" -dependencies = [ "hud>=0.6.0", "httpx>=0.27.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0",] +dependencies = [ "hud>=0.6.11", "httpx>=0.27.0", "pytest>=8.0.0", "pytest-asyncio>=0.23.0",] [build-system] requires = [ "hatchling",] diff --git a/uv.lock b/uv.lock index 0d033cf..23dd350 100644 --- a/uv.lock +++ b/uv.lock @@ -267,7 +267,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "httpx", specifier = ">=0.27.0" }, - { name = "hud", specifier = ">=0.6.0" }, + { name = "hud", specifier = ">=0.6.11" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.5.0" }, { name = "pytest", specifier = ">=8.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, @@ -538,7 +538,7 @@ wheels = [ [[package]] name = "hud" -version = "0.6.10" +version = "0.6.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anthropic" }, @@ -559,9 +559,9 @@ dependencies = [ { name = "typer" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d2/8c/b94051aacfe311e4a531610c32368d21c99c80644aed8b42e4afbd78c1e3/hud-0.6.10.tar.gz", hash = "sha256:65ceec83fedd86f8d4cb411357a49ac9522d085a7dd65111d46ca6fec5695b44", size = 344418, upload-time = "2026-07-15T23:34:34.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/3e/d292cbf4e22db78eec966044526fdfa97203b91b11831dffd211c1cd1e6c/hud-0.6.11.tar.gz", hash = "sha256:e71615efe5d4798089e907f5bf4cb24071ed63cc9bfa7e2fdb0b7fe1e75ae5aa", size = 351389, upload-time = "2026-07-20T13:17:24.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/72/c649ad285bf85d0e7b3b936774efc39c28df2c9572cfa24ea559377bd960/hud-0.6.10-py3-none-any.whl", hash = "sha256:b686e08c464a4cc95fca48ffff4108cc8cbc69408f4e4bd57ffed6c76ac076d9", size = 448102, upload-time = "2026-07-15T23:34:33.475Z" }, + { url = "https://files.pythonhosted.org/packages/46/bb/7a500ac60ad7eaa8eee42b12c4e5475c4ecb83cefacc67b8e6a26b02a433/hud-0.6.11-py3-none-any.whl", hash = "sha256:ffaaa349bc57ca7b212c6340b8c7b78c4c789172944ae87d158b58f1e5409b43", size = 455854, upload-time = "2026-07-20T13:17:22.985Z" }, ] [[package]]