hud-evals · jdchawla29 · Jul 20, 2026 · Jul 20, 2026
diff --git a/.dockerignore b/.dockerignore
@@ -1,58 +1,8 @@
-# Git (keep .git/modules/ for submodule offline builds)
-.git/*
-!.git/modules/
-.gitignore
-
-# Local runtime substrate (clones for `--runtime local`), never ship to the image
-.hud_local/
-
-# Python
-__pycache__/
-*.py[cod]
-*$py.class
-*.so
-.Python
-.pytest_cache/
-.coverage
-*.cover
-.hypothesis/
-.env
-.venv
-env/
-venv/
-ENV/
-
-# Logs
-*.log
-logs/
-build.log
-backend.log
-
-# Test outputs
-qa_results/
-test_results/
-coverage/
-
-# Temporary files
-*.tmp
-*.temp
-tmp/
-temp/
-
-# IDE
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-.DS_Store
-
-# Node (if any)
-node_modules/
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Keep the build lean
-*.md
-!README.md 
+# Builds run from the repo root; the context is only what the images COPY,
+# plus the Dockerfile itself (remote builders read it from the context).
+*
+!Dockerfile.hud
+!env.py
+!coding
+!instances
+**/__pycache__
diff --git a/.gitignore b/.gitignore
@@ -1,49 +1,32 @@
 # Python
 __pycache__/
 *.py[cod]
-*$py.class
-*.so
-.Python
-build/
-dist/
-*.egg-info/
-*.egg
 .pytest_cache/
+.ruff_cache/
 .coverage
-.env
 .venv
-venv/
-
-# Node
-node_modules/
+.env
+build/
+dist/
+*.egg-info/
 
-# IDE
+# OS / IDE
+.DS_Store
 .vscode/
 .idea/
 *.swp
-*.swo
-*~
-.DS_Store
-.opencode/
-
-# Logs
-*.log
-logs/
-tmp/
-
-# references
-reference/
-
-# Docker
-docker_build.log
-
-# Test outputs
-test_results/
-coverage/
 
 # HUD
 hud.lock.yaml
 .hud
 .hud_local/
 .hud_eval.toml
-problem-metadata.json
+.hud-images.json
+
+# SWE-bench Pro instance assets are fetched per-user (swe_tasks.py) and carry
+# the golden patches; only the empty .none dir (generic image builds) ships.
+instances/*
+!instances/.none/
+
+# Logs
+*.log
diff --git a/Dockerfile.hud b/Dockerfile.hud
@@ -1,78 +1,72 @@
 # syntax=docker/dockerfile:1
-FROM ubuntu:24.04 AS setup
+# One image definition for both flavors.
+#
+# Generic (default) — the repo-clone stage bakes your target repo into /app:
+#   docker build -f Dockerfile.hud --build-arg REPO_URL=https://github.com/you/repo .
+#   (private repos: --secret id=CODING_GITHUB_TOKEN,env=CODING_GITHUB_TOKEN)
+#
+# SWE-bench Pro (driven by swe_tasks.py) — BASE selects the instance's
+# prebuilt image (repo already at /app) and INSTANCE_ID bakes its assets:
+#   docker build -f Dockerfile.hud --build-arg BASE=jefzda/sweap-images:<tag> \
+#     --build-arg INSTANCE_ID=<id> .
+ARG BASE=repo-clone
+
+# ─── generic head: toolchain + target repo at /app ───
+FROM ubuntu:24.04 AS repo-clone
 
 RUN apt-get update -y \
   && apt-get install -y --no-install-recommends \
-  bash \
-  build-essential \
-  ca-certificates \
-  curl \
-  git \
-  gnupg \
-  libpq5 \
-  openssl \
-  python-is-python3 \
-  python3 \
-  python3-dev \
-  python3-pip \
-  python3-venv \
-  python3-wheel \
-  sqlite3 \
-  vim \
-  wget \
+  bash build-essential ca-certificates curl git openssl \
+  python-is-python3 python3 python3-dev python3-pip python3-venv util-linux \
   && rm -rf /var/lib/apt/lists/* \
   && update-ca-certificates
 
-# Target repo — override at build with --build-arg REPO_URL=https://github.com/you/repo
-# (private: --secret id=CODING_GITHUB_TOKEN,env=CODING_GITHUB_TOKEN).
-ARG REPO_URL="https://github.com/hud-evals/coding-template-sample"
-ARG FOLDER_NAME="project"
-ENV FOLDER_NAME=${FOLDER_NAME} \
-    REPO_URL=${REPO_URL} \
-    HOME=/home/ubuntu \
-    PROJECT_DIR=/home/ubuntu/${FOLDER_NAME} \
-    PATCHES_DIR=/home/root/patches
+# The sample tasks' test command uses the image's python; adjust per repo.
+RUN pip3 install --break-system-packages pytest
 
-USER ubuntu
-RUN --mount=type=secret,id=CODING_GITHUB_TOKEN,uid=1000 \
+ARG REPO_URL="https://github.com/hud-evals/coding-template-sample"
+# chown in the same RUN as the clone (a separate layer would duplicate /app):
+# env.py serves agent shells as uid 1000, so the baked tree must be theirs.
+# Doing it at build makes it free at runtime, whatever the repo's size.
+RUN --mount=type=secret,id=CODING_GITHUB_TOKEN \
     if [ -f /run/secrets/CODING_GITHUB_TOKEN ]; then \
-        CODING_GITHUB_TOKEN=$(cat /run/secrets/CODING_GITHUB_TOKEN) && \
-        git clone https://${CODING_GITHUB_TOKEN}@${REPO_URL#https://} /home/ubuntu/${FOLDER_NAME}; \
+        git clone "https://$(cat /run/secrets/CODING_GITHUB_TOKEN)@${REPO_URL#https://}" /app; \
     else \
-        git clone ${REPO_URL} /home/ubuntu/${FOLDER_NAME}; \
-    fi
-
-# uid wall: the repo is owned by the agent (uid 1000); the answer key (.git
-# branches) and generated patches stay root:700 so only the env/grader (root)
-# can read them — the dropped agent shell can't.
-USER root
-RUN chown -R ubuntu:ubuntu /home/ubuntu/${FOLDER_NAME} \
-    && mkdir -p /home/root/patches \
-    && chmod 700 /home/root \
-    && chown -R root:root /home/ubuntu/${FOLDER_NAME}/.git \
-    && chmod -R 700 /home/ubuntu/${FOLDER_NAME}/.git \
-    && git config --global --add safe.directory /home/ubuntu/${FOLDER_NAME}
-
+        git clone ${REPO_URL} /app; \
+    fi \
+    && chown -R 1000:1000 /app
 # Add a build step for the target repo here if it needs one (deps, compile).
 
+# ─── the served image: BASE + hud serving layer (+ instance assets) ───
+FROM ${BASE}
 
-FROM setup AS runtime
-
-RUN pip install uv --break-system-packages
-
-COPY ./pyproject.toml /mcp_server/pyproject.toml
-WORKDIR /mcp_server
-ENV RUST_LOG=warn \
-    PYTHONPATH=/mcp_server/.venv/lib/python3.12/site-packages:/mcp_server \
-    PATH=/mcp_server/.venv/bin:$PATH
-RUN uv venv && . .venv/bin/activate && uv sync --no-install-project --extra dev && uv pip install -e .
-
-COPY ./env.py /mcp_server/env.py
-COPY ./grading /mcp_server/grading
-COPY ./tasks.py /mcp_server/tasks.py
-COPY ./tests /mcp_server/tests
-RUN chmod 777 /root
+# ".none" is a committed empty dir, so generic builds bake no instance and
+# env.py serves only the generic coding-task template.
+ARG INSTANCE_ID=".none"
+COPY env.py /hud/env.py
+COPY coding /hud/coding
+COPY instances/${INSTANCE_ID}/ /hud/instance/
 
-ENV PROBLEM_ID=""
+# Self-contained hud venv under /hud: bootstrap uv (with its own managed
+# Python), never touching the base image's Python or site-packages. chmod 700
+# keeps the vault and instance assets out of the uid-dropped agent's reach.
+RUN <<'INSTALL'
+set -eu
+export PATH="$HOME/.local/bin:$PATH" UV_INSTALL_DIR="$HOME/.local/bin"
+command -v uv >/dev/null 2>&1 || {
+  { command -v curl >/dev/null 2>&1 || command -v wget >/dev/null 2>&1; } \
+    || { apt-get update -qq && apt-get install -y -qq curl ca-certificates; } \
+    || apk add --no-cache curl ca-certificates
+  { command -v curl >/dev/null 2>&1 && curl -LsSf https://astral.sh/uv/install.sh | sh; } \
+    || { command -v wget >/dev/null 2>&1 && wget -qO- https://astral.sh/uv/install.sh | sh; } \
+    || pip install -q -U uv
+}
+uv python install 3.12
+uv venv /hud/venv --python 3.12
+uv pip install --python /hud/venv/bin/python 'hud>=0.6.11'
+chmod -R go-rwx /hud
+INSTALL
+ENV REPO_DIR=/app PYTHONPATH=/hud
 EXPOSE 8765
-CMD ["hud", "serve", "env:env", "--host", "0.0.0.0", "--port", "8765"]
+ENTRYPOINT []
+CMD ["/hud/venv/bin/hud", "serve", "/hud/env.py", "--host", "0.0.0.0", "--port", "8765"]
diff --git a/README.md b/README.md
@@ -1,51 +1,93 @@
 # Coding Environment (HUD v6)
 
-A HUD v6 environment where an agent **fixes a bug in a Python web app**, graded by a hidden pytest
-suite. The env publishes a sandboxed **`ssh` workspace** — the agent's harness brings its own bash
-and file tools and drives it over SSH; the env ships no agent tools itself.
+A coding environment: the agent gets a git repo over a sandboxed **`ssh` workspace** (the
+harness brings its own bash/file tools), and grading is diff-based — capture the agent's
+changes, reset to the pre-agent snapshot, re-apply the diff, bring in the hidden tests, run
+them. Generic tasks, SDLC workflow tasks, and SWE-bench Pro are flavors on this core.
 
-Grading uses a 3-branch design: every task has `{task}_baseline` (the agent's starting state),
-`{task}_test` (hidden tests), and `{task}_golden` (the reference fix). `setup_task` checks out the
-baseline; the grader applies the hidden test patch and runs `pytest` — all pass → reward 1.0. The
-agent never sees the tests or the solution.
+The agent never sees the answer key. Task setup moves the repo's real `.git` — which may hold
+solution branches or the fix commit — into a vault outside the workspace and leaves a fresh
+single-commit repo: git works normally, but there is no history, no refs, and no remotes.
+Grading discards the agent's `.git`, restores the vault, and checks hidden tests out *after*
+the agent's diff. In images, the vault and instance assets live under `/hud` (root, mode 700)
+and agent shells drop to a non-root uid via `setpriv`.
 
-## Tasks
+## Layout
 
-Four hand-picked bugs in the
-[coding-template-sample](https://github.com/hud-evals/coding-template-sample) repo:
+- `env.py` — the environment: workspace wiring plus the three task templates.
+- `coding/repo.py` — the shared repo lifecycle: vault, snapshot, diff capture, reset, apply.
+- `coding/github.py` — mock GitHub for SDLC tasks: issue/PR store served as `github_*` tools.
+- `coding/swe_bench_pro.py` — the SWE-bench Pro grading pipeline.
+- `tasks.py` — sample generic and SDLC tasks.
+- `swe_tasks.py` — the SWE-bench Pro task source: fetches instances and builds their images
+  when run; task rows when imported.
+- `Dockerfile.hud` — the image definition for both flavors: `BASE` selects the generic
+  repo-clone head or a prebuilt instance image.
 
-| Task | Difficulty | Bug |
-|------|-----------|-----|
-| `sentry-fix` | Basic | KeyError on a missing/null user profile |
-| `notif-bug` | Medium | Event-type delimiter `_` vs `.` breaks routing |
-| `settings-v2` | Hard | CompactDict filters falsy values during iteration |
-| `webhook-bug` | Hard | Shared-list mutation corrupts channel resolution |
+## Generic tasks (`coding-task`)
 
-## Setup
+Point the env at a repo (`REPO_URL`; locally it clones per process, or bake it with
+`Dockerfile.hud`) and parameterize the template: a `base_ref` to start from, a `test_ref` whose
+`test_files` are the hidden tests (checked out from the vaulted history at grade time), and a
+`test_command` scored by exit code. Tasks follow the 3-branch convention — `{task}_baseline` /
+`{task}_test` / `{task}_golden`; `tasks.py` ships four sample bugs on
+[coding-template-sample](https://github.com/hud-evals/coding-template-sample):
 
 ```bash
 uv sync
-hud set HUD_API_KEY=your-key-here   # CLI auth, get one at hud.ai/project/api-keys
+hud set HUD_API_KEY=your-key-here
+hud eval tasks.py claude --task-ids sentry-fix -y --runtime local
 ```
 
-## Run
+## SDLC tasks (`sdlc-task`)
+
+The generic flavor plus workflow: the repo gets an `origin` remote (a bare mock-GitHub repo the
+agent pushes to) and `github_*` MCP tools seeded with the task's issues. The deliverable is a
+pushed branch with a pull request — grading checks the PR head out of the remote, brings in the
+hidden tests, runs the test command (weight 0.8), and scores the PR itself (0.2): a structural
+title/body check by default, or `pr_rubric` judged by `LLMJudgeGrader` when provided. See the
+`sentry-fix-pr` sample in `tasks.py`:
 
 ```bash
-# local — clones the target repo into a per-process temp dir (macOS + Linux)
-hud eval tasks.py claude --task-ids sentry-fix -y --runtime local
+hud eval tasks.py claude --task-ids sentry-fix-pr -y --runtime local
+```
+
+## SWE-bench Pro tasks
 
-# deploy once, then run hosted
-hud deploy .
-hud eval tasks.py claude --runtime hud --full
+Each of the 731 public [SWE-bench Pro](https://github.com/scaleapi/SWE-bench_Pro-os) instances
+ships a prebuilt image (`jefzda/sweap-images:<tag>`, `linux/amd64`) with the repo and toolchain
+baked in. Running `swe_tasks.py` fetches the dataset row plus the official
+`run_script.sh`/`parser.py` into `instances/<id>/` and builds `Dockerfile.hud` with the
+instance's image as `BASE`, so the image serves this env from inside. Grading replays the
+official evaluator: resolved iff every `fail_to_pass` **and** `pass_to_pass` test passes.
+
+```bash
+uv run swe_tasks.py instance_NodeBB__NodeBB-04998908ba6721d64eba79ae3b65a351dcfbc5b5-vnan
+hud eval swe_tasks.py claude --task-ids nodebb-04998908
+uv run swe_tasks.py <id>... --push registry.io/acme   # push for cloud runtimes
 ```
 
 ## Tests
 
 ```bash
-uv run pytest tests/ -q
+uv run pytest tests/ -q --ignore=tests/test_integration.py   # offline + hermetic local e2e
+uv run pytest tests/test_integration.py -v                   # SWE-bench gold-patch check (Docker)
 ```
 
-Offline tests drive the 3-branch grading (baseline fails, golden passes) with no Docker.
+`test_local_rollout.py` runs the generic and SDLC flavors end to end against a fixture 3-branch
+repo (no Docker or network): the golden ref grades 1.0 and the untouched baseline 0.0. The
+integration suite is the same check for built SWE-bench Pro instances.
+
+## Caveats
+
+- Public benchmarks are public: a networked agent could fetch solutions from GitHub. Disable
+  network egress at the runtime layer if that matters for your run.
+- The uid wall needs `setpriv` (util-linux) in the image; the repo path comes from `REPO_DIR`
+  (`/app` in instance images).
+- The agent runs as uid 1000, so the baked repo must belong to it. The workspace only chowns
+  its own directory at start (O(1), keeps boot fast); the tree is owned where it's staged —
+  the generic build chowns `/app` in the clone step, and task setup re-chowns after root
+  mutates the worktree (checkout, vaulting).
 
 ## Documentation
 

diff --git a/coding/__init__.py b/coding/__init__.py
@@ -0,0 +1,5 @@
+"""The coding environment's core: repo-lifecycle primitives and task flavors."""
+
+from . import repo, swe_bench_pro
+
+__all__ = ["repo", "swe_bench_pro"]