From c370488246b69426efead361bc75b3031a69d1b1 Mon Sep 17 00:00:00 2001 From: Vincent Ouyang Date: Mon, 3 Aug 2026 19:30:26 +0000 Subject: [PATCH 1/7] feat(agents): add task quality loop auditor --- .gitignore | 4 + Makefile | 8 +- agents/quality_loop/README.md | 97 +++ agents/quality_loop/__init__.py | 6 + agents/quality_loop/__main__.py | 80 ++ agents/quality_loop/agent_config.yaml | 41 ++ agents/quality_loop/backend.py | 107 +++ agents/quality_loop/config.py | 151 ++++ agents/quality_loop/filesystem.py | 144 ++++ agents/quality_loop/github.py | 346 +++++++++ agents/quality_loop/host.py | 212 ++++++ agents/quality_loop/orchestrator.py | 899 +++++++++++++++++++++++ agents/quality_loop/prompts.py | 82 +++ agents/quality_loop/state.py | 134 ++++ docs/README.md | 1 + docs/how-to/agents.md | 4 + docs/how-to/quality-loop.md | 107 +++ docs/sphinx/_toc.yml.in | 2 + example_configs/quality_loop_mi300.yaml | 28 + example_configs/quality_loop_mi355x.yaml | 28 + src/scripts/docker_benchmark.sh | 97 ++- tests/test_docker_benchmark.sh | 39 +- tests/test_quality_loop.py | 630 ++++++++++++++++ 23 files changed, 3244 insertions(+), 3 deletions(-) create mode 100644 agents/quality_loop/README.md create mode 100644 agents/quality_loop/__init__.py create mode 100644 agents/quality_loop/__main__.py create mode 100644 agents/quality_loop/agent_config.yaml create mode 100644 agents/quality_loop/backend.py create mode 100644 agents/quality_loop/config.py create mode 100644 agents/quality_loop/filesystem.py create mode 100644 agents/quality_loop/github.py create mode 100644 agents/quality_loop/host.py create mode 100644 agents/quality_loop/orchestrator.py create mode 100644 agents/quality_loop/prompts.py create mode 100644 agents/quality_loop/state.py create mode 100644 docs/how-to/quality-loop.md create mode 100644 example_configs/quality_loop_mi300.yaml create mode 100644 example_configs/quality_loop_mi355x.yaml create mode 100644 tests/test_quality_loop.py diff --git a/.gitignore b/.gitignore index a743dc66..b46c952f 100755 --- a/.gitignore +++ b/.gitignore @@ -41,6 +41,10 @@ traj.json # Generated held-out test data (methodology is in src/held_out/, data is private) held_out_tests/ +# Repository-level quality_loop audit state and isolated git worktrees +quality_loop_runs/ +.quality_loop_worktrees/ + # Documentation build environment and output .docvenv/ docs/_build/ diff --git a/Makefile b/Makefile index d1b2c9f0..b55db2ea 100755 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ SHELL := /bin/bash -.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-setup-flydsl docker-setup-geak \ +.PHONY: help docker-shell docker-check-agents docker-smoke docker-run docker-parallel-run docker-quality-loop docker-setup-flydsl docker-setup-geak \ check-docker-runner check-evaluator check-held-out check-visualization \ visualization-build visualization-serve visualization-run \ sync-perf-helpers check-perf-helpers materialize-perf-workspace \ @@ -23,6 +23,7 @@ help: @echo "make docker-smoke - Verify Docker Python, ROCm tools, imports, and GPU access" @echo "make docker-run CONFIG=example_configs/quickstart_claude_mi300.yaml RUN_ARGS=\"--run-suffix test\" - Run an experiment in Docker" @echo "make docker-parallel-run CONFIG=example_configs/benchmark_cursor_mi355x.yaml GPU_IDS=0,1 - Run an experiment across one worker container per GPU" + @echo "make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml - Audit and harden tasks with Codex, then open one draft PR" @echo " Default CONFIG is the MI300/MI300X Claude quickstart" @echo " On other GPUs, pass a matching CONFIG explicitly" @echo " Images: gfx942->mi30x, gfx950->mi35x; override with AKA_DOCKER_IMAGE=..." @@ -45,6 +46,8 @@ help: DOCKER_RUNNER := src/scripts/docker_benchmark.sh CONFIG ?= example_configs/quickstart_claude_mi300.yaml RUN_ARGS ?= +QUALITY_LOOP_CONFIG ?= agents/quality_loop/agent_config.yaml +QUALITY_LOOP_ARGS ?= AGENTS ?= WORKSPACES ?= $(WORKSPACE) TASKS ?= $(TASK) @@ -70,6 +73,9 @@ docker-run: docker-parallel-run: @GPU_IDS="$(GPU_IDS)" $(DOCKER_RUNNER) parallel-run --config_name $(CONFIG) $(RUN_ARGS) +docker-quality-loop: + @$(DOCKER_RUNNER) quality-loop --config $(QUALITY_LOOP_CONFIG) $(QUALITY_LOOP_ARGS) + # Install FlyDSL into the container's persistent pip user-base when the selected # image does not ship it. Needed by all three FlyDSL task types. docker-setup-flydsl: diff --git a/agents/quality_loop/README.md b/agents/quality_loop/README.md new file mode 100644 index 00000000..44ffd3ce --- /dev/null +++ b/agents/quality_loop/README.md @@ -0,0 +1,97 @@ +# quality_loop agent + +`quality_loop` is a repository-level task curator. It audits every selected task, +attempts one repair for blocking validator failures, runs exactly one Codex +optimization iteration, sends the result to an independent Codex reviewer, and +optionally hardens an easy baseline or task cases behind fail-closed correctness +gates. It files idempotent GitHub issues for unrepairable task defects and bundles +all accepted task changes into one draft pull request. + +Unlike normal Arena agents, `quality_loop` is not registered in `AgentType`. +Normal launchers operate once inside one copied task workspace; this workflow owns +the full repository campaign, isolated git worktree, issue deduplication, resume +manifest, and final PR. + +## Hard preflight + +A real run stops before creating a branch or modifying a task unless all of these +pass: + +- `gh auth status -h github.com` +- the authenticated account has repository write permission +- GitHub Issues are enabled +- `git`, `gh`, and `codex` are installed +- Git has a usable author identity for task commits +- the source worktree is clean +- the configured GPU/runtime is available through the Docker runner + +Only the host-side deterministic publisher uses `gh`. The Docker runner performs +GitHub preflight and creates the audit worktree on the host, runs Codex/GPU work +without mounting GitHub credentials, then returns to the host to create issues, +commit accepted task changes, push, and open the draft PR. The main checkout is +read-only inside that container; only this run's artifact and isolated worktree +directories are writable. Codex login state is copied into an ephemeral container +home instead of being writable in place. + +## Run + +Inspect task selection without credentials, GPU work, or mutations: + +```bash +python3 -m agents.quality_loop \ + --config example_configs/quality_loop_mi300.yaml \ + --plan +``` + +Run through the supported Docker environment: + +```bash +make docker-quality-loop QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml +``` + +Use `example_configs/quality_loop_mi355x.yaml` on MI355X (`gfx950`). + +Limit a smoke run to several tasks: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--tasks hip2hip/gpumode/GELU triton2triton/vllm/triton_rms_norm" +``` + +Resume after interruption: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--resume 20260803_120000" +``` + +`--no-publish` still requires the GitHub login/write preflight, but suppresses +issues, push, and PR creation. `--plan` is the only intentionally offline mode. + +## Per-task gates + +1. Run the existing 10-check validator in a fresh workspace. +2. Record WARN findings without repairing them. +3. For FAIL, allow one task-local repair and re-run the full validator in another + fresh workspace. File one fingerprinted issue if it still fails. +4. Measure the baseline, run one Codex optimization candidate, protect the + harness, and use the centralized compile/correctness/performance evaluator. +5. Run an independent read-only Codex review. Deterministic evaluator failures + always override an agent acceptance. +6. Treat the task as easy only when three measurements of the single candidate + have median speedup at least 5x, use consistent benchmark methods and case + counts, and the reviewer accepts logic equivalence. +7. Promote only committed standalone optimization kernels. Translation/generation + tasks report 5x hardening as not applicable. +8. Case changes may touch only test/harness paths and are accepted only when both + the pre-audit kernel and candidate pass the updated cases. +9. Before any host commit, the complete worktree diff must exactly match the + accepted per-task paths recorded in `state.yaml`; unexpected edits abort + publication. + +Run artifacts are written under `quality_loop_runs//`; the isolated audit +branch lives under `.quality_loop_worktrees//`. Both are ignored by Git. +Tasks pinned to another GPU architecture are reported as `platform_deferred`; run +the matching MI300/MI355X campaign to audit those tasks. diff --git a/agents/quality_loop/__init__.py b/agents/quality_loop/__init__.py new file mode 100644 index 00000000..128c95f3 --- /dev/null +++ b/agents/quality_loop/__init__.py @@ -0,0 +1,6 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +"""Repository-level task quality audit and hardening workflow.""" + +from .config import QualityLoopConfig, load_config + +__all__ = ["QualityLoopConfig", "load_config"] diff --git a/agents/quality_loop/__main__.py b/agents/quality_loop/__main__.py new file mode 100644 index 00000000..f25f7f47 --- /dev/null +++ b/agents/quality_loop/__main__.py @@ -0,0 +1,80 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import argparse +import dataclasses +import logging +from pathlib import Path + +import yaml + +from .config import load_config +from .orchestrator import QualityLoop + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = Path(__file__).with_name("agent_config.yaml") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Audit, repair, harden, and publish AgentKernelArena tasks" + ) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument( + "--tasks", + nargs="+", + help="Override task selectors from the config (paths relative to tasks/)", + ) + parser.add_argument( + "--plan", + action="store_true", + help="List runnable/deferred tasks without GitHub, GPU, or agent execution", + ) + parser.add_argument("--resume", metavar="RUN_ID", help="Resume a prior run") + parser.add_argument( + "--no-publish", + action="store_true", + help="Run all hard preflights and audits but do not create issues, push, or open a PR", + ) + parser.add_argument("--defer-github", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--skip-preflight", action="store_true", help=argparse.SUPPRESS) + return parser + + +def configure_logging() -> logging.Logger: + logger = logging.getLogger("quality_loop") + logger.setLevel(logging.INFO) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + return logger + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + config_path = args.config if args.config.is_absolute() else REPO_ROOT / args.config + config = load_config(config_path) + if args.tasks: + config = dataclasses.replace(config, tasks=tuple(args.tasks)) + if args.no_publish: + config = dataclasses.replace( + config, + github=dataclasses.replace(config.github, publish=False), + ) + logger = configure_logging() + workflow = QualityLoop(REPO_ROOT, config, logger=logger, defer_github=args.defer_github) + if args.plan: + print(yaml.safe_dump(workflow.plan(), sort_keys=False, allow_unicode=True)) + return 0 + report = workflow.run( + resume_run_id=args.resume, + skip_preflight=args.skip_preflight, + ) + logger.info("quality_loop report: %s", report) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/quality_loop/agent_config.yaml b/agents/quality_loop/agent_config.yaml new file mode 100644 index 00000000..6b586a3c --- /dev/null +++ b/agents/quality_loop/agent_config.yaml @@ -0,0 +1,41 @@ +# quality_loop is a repository-level audit workflow. It defaults to auditing all +# tasks that are runnable on the selected target GPU. +tasks: + - all +target_gpu_model: MI300 + +quality_loop: + backend: + name: codex + model: null # null uses the Codex CLI default/config + effort: xhigh + timeout_seconds: 3600 + reviewer: + name: codex # independent, read-only Codex session + model: null + effort: xhigh + timeout_seconds: 1800 + + max_repair_attempts: 1 + optimization_iterations: 1 # enforced; any other value is rejected + easy_speedup_threshold: 5.0 + easy_confirmation_runs: 3 + case_enhancement: true + + # Only optimization tasks with a committed runnable baseline can promote a + # first-iteration 5x candidate into the new task baseline. Translation and + # authoring tasks still receive validation, optimization, review, and case audit. + promotion_task_types: + - hip2hip + - triton2triton + - flydsl2flydsl + + artifact_root: quality_loop_runs + worktree_root: .quality_loop_worktrees + + github: + publish: true + draft_pr: true + branch_prefix: quality-loop + base_branch: null # null resolves the repository default branch + issue_labels: [] # labels must already exist in the repository diff --git a/agents/quality_loop/backend.py b/agents/quality_loop/backend.py new file mode 100644 index 00000000..76d3aee9 --- /dev/null +++ b/agents/quality_loop/backend.py @@ -0,0 +1,107 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import json +import logging +import os +import shutil +import subprocess +from pathlib import Path +from typing import Protocol + +from .config import BackendConfig + + +class AgentBackend(Protocol): + def run(self, prompt: str, workspace: Path, *, role: str) -> str: ... + + +def _format_event(line: str) -> str: + try: + payload = json.loads(line) + except json.JSONDecodeError: + return line + if not isinstance(payload, dict): + return line + if payload.get("type") in {"item.completed", "item.updated"}: + item = payload.get("item") or {} + if isinstance(item, dict) and item.get("type") == "agent_message": + return str(item.get("text") or "") + if payload.get("type") in {"turn.failed", "error"}: + return str(payload.get("error") or payload.get("message") or line) + return line + + +class CodexBackend: + """Role-scoped Codex runner with GitHub credentials removed from children.""" + + def __init__(self, config: BackendConfig, logger: logging.Logger): + self.config = config + self.logger = logger + + def run(self, prompt: str, workspace: Path, *, role: str) -> str: + if not shutil.which("codex"): + raise RuntimeError("codex CLI is required for quality_loop") + workspace = workspace.resolve() + no_gh_dir = workspace / ".quality_loop_no_gh" + no_gh_dir.mkdir(exist_ok=True) + env = os.environ.copy() + for key in ( + "GH_TOKEN", + "GITHUB_TOKEN", + "SSH_AUTH_SOCK", + "GIT_ASKPASS", + "GIT_SSH_COMMAND", + ): + env.pop(key, None) + env["GH_CONFIG_DIR"] = str(no_gh_dir) + env["GIT_CONFIG_GLOBAL"] = os.devnull + env["GIT_CONFIG_NOSYSTEM"] = "1" + + command = [ + "codex", + "exec", + "--json", + "--dangerously-bypass-approvals-and-sandbox", + "--skip-git-repo-check", + "-c", + "features.memories=false", + "--cd", + str(workspace), + ] + if self.config.model: + command.extend(["--model", self.config.model]) + if self.config.effort: + command.extend(["-c", f'model_reasoning_effort="{self.config.effort}"']) + command.append(prompt) + + self.logger.info( + "Starting Codex role=%s model=%s effort=%s workspace=%s", + role, + self.config.model or "", + self.config.effort, + workspace, + ) + try: + result = subprocess.run( + command, + cwd=workspace, + env=env, + capture_output=True, + text=True, + timeout=self.config.timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError( + f"Codex role {role} timed out after {self.config.timeout_seconds}s" + ) from exc + output = "\n".join( + _format_event(line) for line in result.stdout.splitlines() if line.strip() + ) + if result.returncode != 0: + detail = (result.stderr or output).strip() + raise RuntimeError(f"Codex role {role} failed ({result.returncode}): {detail[-4000:]}") + if result.stderr.strip(): + self.logger.warning("Codex role=%s stderr: %s", role, result.stderr[-1000:]) + return output diff --git a/agents/quality_loop/config.py b/agents/quality_loop/config.py new file mode 100644 index 00000000..ed0f2313 --- /dev/null +++ b/agents/quality_loop/config.py @@ -0,0 +1,151 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import yaml + + +def _runtime_root(value: Any, name: str) -> str: + path = Path(str(value)) + if path.is_absolute() or not path.parts or any(part == ".." for part in path.parts): + raise ValueError(f"{name} must be a repository-relative path without '..'") + return path.as_posix() + + +@dataclass(frozen=True) +class BackendConfig: + name: str = "codex" + model: str | None = None + effort: str = "xhigh" + timeout_seconds: int = 3600 + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> "BackendConfig": + raw = raw or {} + name = str(raw.get("name", "codex")).strip().lower() + if name != "codex": + raise ValueError("quality_loop currently supports only the codex backend") + timeout = int(raw.get("timeout_seconds", 3600)) + if timeout <= 0: + raise ValueError("backend.timeout_seconds must be positive") + model = raw.get("model") + return cls( + name=name, + model=str(model) if model else None, + effort=str(raw.get("effort", "xhigh")), + timeout_seconds=timeout, + ) + + +@dataclass(frozen=True) +class GitHubConfig: + publish: bool = True + draft_pr: bool = True + issue_labels: tuple[str, ...] = () + branch_prefix: str = "quality-loop" + base_branch: str | None = None + + @classmethod + def from_dict(cls, raw: dict[str, Any] | None) -> "GitHubConfig": + raw = raw or {} + labels = raw.get("issue_labels", []) + if isinstance(labels, str): + labels = [labels] + prefix = str(raw.get("branch_prefix", "quality-loop")).strip(" /-") + if not prefix: + raise ValueError("github.branch_prefix must not be empty") + base = raw.get("base_branch") + return cls( + publish=bool(raw.get("publish", True)), + draft_pr=bool(raw.get("draft_pr", True)), + issue_labels=tuple(str(label) for label in labels), + branch_prefix=prefix, + base_branch=str(base) if base else None, + ) + + +@dataclass(frozen=True) +class QualityLoopConfig: + tasks: tuple[str, ...] = ("all",) + target_gpu_model: str = "MI300" + backend: BackendConfig = field(default_factory=BackendConfig) + reviewer: BackendConfig = field(default_factory=BackendConfig) + github: GitHubConfig = field(default_factory=GitHubConfig) + max_repair_attempts: int = 1 + optimization_iterations: int = 1 + easy_speedup_threshold: float = 5.0 + easy_confirmation_runs: int = 3 + case_enhancement: bool = True + artifact_root: str = "quality_loop_runs" + worktree_root: str = ".quality_loop_worktrees" + promotion_task_types: tuple[str, ...] = ( + "hip2hip", + "triton2triton", + "flydsl2flydsl", + ) + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig": + tasks = raw.get("tasks", ["all"]) + if isinstance(tasks, str): + tasks = [tasks] + if not isinstance(tasks, list) or not tasks: + raise ValueError("tasks must be a non-empty string or list") + target = str(raw.get("target_gpu_model", "")).strip() + if not target: + raise ValueError("target_gpu_model is required") + + audit = raw.get("quality_loop", {}) or {} + if not isinstance(audit, dict): + raise ValueError("quality_loop must be a mapping") + iterations = int(audit.get("optimization_iterations", 1)) + if iterations != 1: + raise ValueError( + "quality_loop enforces exactly one optimization iteration; " + "optimization_iterations must be 1" + ) + repair_attempts = int(audit.get("max_repair_attempts", 1)) + if repair_attempts < 0 or repair_attempts > 1: + raise ValueError("max_repair_attempts must be 0 or 1") + confirmations = int(audit.get("easy_confirmation_runs", 3)) + if confirmations < 1: + raise ValueError("easy_confirmation_runs must be at least 1") + threshold = float(audit.get("easy_speedup_threshold", 5.0)) + if threshold <= 1.0: + raise ValueError("easy_speedup_threshold must be greater than 1.0") + + promotion_types = audit.get( + "promotion_task_types", + ["hip2hip", "triton2triton", "flydsl2flydsl"], + ) + return cls( + tasks=tuple(str(task) for task in tasks), + target_gpu_model=target, + backend=BackendConfig.from_dict(audit.get("backend")), + reviewer=BackendConfig.from_dict(audit.get("reviewer", audit.get("backend"))), + github=GitHubConfig.from_dict(audit.get("github")), + max_repair_attempts=repair_attempts, + optimization_iterations=iterations, + easy_speedup_threshold=threshold, + easy_confirmation_runs=confirmations, + case_enhancement=bool(audit.get("case_enhancement", True)), + artifact_root=_runtime_root( + audit.get("artifact_root", "quality_loop_runs"), + "quality_loop.artifact_root", + ), + worktree_root=_runtime_root( + audit.get("worktree_root", ".quality_loop_worktrees"), + "quality_loop.worktree_root", + ), + promotion_task_types=tuple(str(value) for value in promotion_types), + ) + + +def load_config(path: Path) -> QualityLoopConfig: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict): + raise ValueError(f"quality_loop config must be a mapping: {path}") + return QualityLoopConfig.from_dict(raw) diff --git a/agents/quality_loop/filesystem.py b/agents/quality_loop/filesystem.py new file mode 100644 index 00000000..1bdcff0f --- /dev/null +++ b/agents/quality_loop/filesystem.py @@ -0,0 +1,144 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import hashlib +import shutil +from dataclasses import dataclass +from pathlib import Path + +from src.perf_helper_materialization import ( + MARK_END, + MARK_STARTS, + ROCMBENCH_HELPER_STUB, + VLLM_HELPER_STUB_BLOCK, + replace_marked_region, +) + + +GENERATED_NAMES = { + "validation_report.yaml", + "task_result.yaml", + "baseline_perf.yaml", + "optimized_perf.yaml", + "quality_loop_review.yaml", + "performance_report.json", + "compile_report.json", +} +GENERATED_DIRS = { + ".git", + ".pytest_cache", + ".quality_loop_no_gh", + ".quality_loop_original_sources", + ".rocprofv3", + "__pycache__", + "build", +} + + +def _digest(path: Path) -> str: + if path.is_symlink(): + return "link:" + str(path.readlink()) + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def snapshot_tree(root: Path) -> dict[str, str]: + result: dict[str, str] = {} + for path in sorted(root.rglob("*")): + rel = path.relative_to(root) + if any(part in GENERATED_DIRS for part in rel.parts): + continue + if path.is_file() or path.is_symlink(): + result[rel.as_posix()] = _digest(path) + return result + + +@dataclass(frozen=True) +class TreeChanges: + added: tuple[str, ...] + modified: tuple[str, ...] + deleted: tuple[str, ...] + + @property + def paths(self) -> tuple[str, ...]: + return self.added + self.modified + self.deleted + + @property + def empty(self) -> bool: + return not self.paths + + +def diff_trees(before: dict[str, str], after: dict[str, str]) -> TreeChanges: + return TreeChanges( + added=tuple(sorted(set(after) - set(before))), + modified=tuple(sorted(k for k in set(before) & set(after) if before[k] != after[k])), + deleted=tuple(sorted(set(before) - set(after))), + ) + + +def is_generated_path(relative: str, *, repo_subdir: str | None = None) -> bool: + path = Path(relative) + if path.name in GENERATED_NAMES: + return True + if any(part in GENERATED_DIRS for part in path.parts): + return True + if repo_subdir and path.parts and path.parts[0] == repo_subdir: + return True + return False + + +def is_case_path(relative: str) -> bool: + path = Path(relative) + parts = set(path.parts[:-1]) + name = path.name + return bool( + parts & {"script", "scripts", "test", "tests"} + or name.startswith("test_") + or name.endswith(("_test.py", "_harness.py")) + ) + + +def apply_changes(source: Path, destination: Path, changes: TreeChanges) -> None: + """Apply an already-validated, root-relative change set.""" + source = source.resolve() + destination = destination.resolve() + for relative in changes.added + changes.modified: + src = (source / relative).resolve() + dst = (destination / relative).resolve() + if not src.is_relative_to(source) or not dst.is_relative_to(destination): + raise ValueError(f"unsafe quality_loop path: {relative}") + if not src.is_file() and not src.is_symlink(): + raise ValueError(f"changed path is not a file: {relative}") + dst.parent.mkdir(parents=True, exist_ok=True) + if src.is_symlink(): + if dst.exists() or dst.is_symlink(): + dst.unlink() + dst.symlink_to(src.readlink()) + else: + shutil.copy2(src, dst) + for relative in changes.deleted: + dst = (destination / relative).resolve() + if not dst.is_relative_to(destination): + raise ValueError(f"unsafe quality_loop deletion: {relative}") + if dst.is_file() or dst.is_symlink(): + dst.unlink() + + +def restore_committed_perf_stubs(task_root: Path) -> None: + """Undo runtime helper materialization before a task diff is committed.""" + for helper in task_root.rglob("performance_utils_pytest.py"): + if helper.is_file(): + helper.write_text(ROCMBENCH_HELPER_STUB, encoding="utf-8") + for runner in task_root.rglob("task_runner.py"): + if not runner.is_file(): + continue + current = runner.read_text(encoding="utf-8") + if not (any(marker in current for marker in MARK_STARTS) or MARK_END in current): + continue + replaced = replace_marked_region(current, VLLM_HELPER_STUB_BLOCK) + if replaced is None: + raise RuntimeError(f"invalid AKA-GENERATED markers after audit: {runner}") + runner.write_text(replaced, encoding="utf-8") diff --git a/agents/quality_loop/github.py b/agents/quality_loop/github.py new file mode 100644 index 00000000..74f17526 --- /dev/null +++ b/agents/quality_loop/github.py @@ -0,0 +1,346 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import json +import logging +import re +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from .config import GitHubConfig + + +class CommandError(RuntimeError): + pass + + +def run_command( + args: Sequence[str], + *, + cwd: Path, + check: bool = True, + timeout: int = 120, +) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + list(args), + cwd=cwd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + if check and result.returncode != 0: + rendered = " ".join(args) + detail = (result.stderr or result.stdout).strip() + raise CommandError(f"command failed ({result.returncode}): {rendered}\n{detail}") + return result + + +def parse_github_slug(remote: str) -> str: + remote = remote.strip() + patterns = ( + r"^git@github\.com:([^/]+/[^/]+?)(?:\.git)?$", + r"^ssh://git@github\.com/([^/]+/[^/]+?)(?:\.git)?$", + r"^https?://github\.com/([^/]+/[^/]+?)(?:\.git)?/?$", + ) + for pattern in patterns: + match = re.match(pattern, remote) + if match: + return match.group(1) + raise ValueError(f"origin is not a supported GitHub remote: {remote!r}") + + +@dataclass(frozen=True) +class PreflightResult: + repo_slug: str + default_branch: str + base_sha: str + viewer_permission: str + + +class GitHubPublisher: + """The only quality_loop component allowed to use GitHub credentials.""" + + def __init__(self, repo_root: Path, config: GitHubConfig, logger: logging.Logger): + self.repo_root = repo_root.resolve() + self.config = config + self.logger = logger + + def preflight(self) -> PreflightResult: + for command in ("git", "gh", "codex"): + if not shutil.which(command): + raise RuntimeError(f"required command not found: {command}") + + status = run_command(["git", "status", "--porcelain"], cwd=self.repo_root) + if status.stdout.strip(): + raise RuntimeError( + "quality_loop requires a clean source worktree before creating " + "its isolated audit worktree" + ) + run_command(["git", "var", "GIT_AUTHOR_IDENT"], cwd=self.repo_root) + + run_command(["gh", "auth", "status", "-h", "github.com"], cwd=self.repo_root) + remote = run_command( + ["git", "remote", "get-url", "origin"], cwd=self.repo_root + ).stdout.strip() + slug = parse_github_slug(remote) + repo_data = json.loads( + run_command(["gh", "api", f"repos/{slug}"], cwd=self.repo_root).stdout + ) + permissions = repo_data.get("permissions") or {} + permission = str(repo_data.get("viewer_permission") or "").upper() + push_allowed = bool(permissions.get("push")) or permission in { + "WRITE", + "MAINTAIN", + "ADMIN", + } + if not push_allowed: + raise RuntimeError( + f"authenticated GitHub user lacks write permission for {slug} " + f"(viewer_permission={permission or 'unknown'})" + ) + if repo_data.get("has_issues") is False: + raise RuntimeError(f"GitHub issues are disabled for {slug}") + + default_branch = self.config.base_branch or repo_data.get("default_branch") + if not default_branch: + raise RuntimeError(f"could not determine the default branch for {slug}") + run_command(["git", "fetch", "origin", default_branch], cwd=self.repo_root, timeout=600) + base_sha = run_command( + ["git", "rev-parse", f"origin/{default_branch}"], cwd=self.repo_root + ).stdout.strip() + return PreflightResult(slug, str(default_branch), base_sha, permission or "WRITE") + + def create_worktree( + self, + *, + path: Path, + branch: str, + base_branch: str, + ) -> None: + if path.exists(): + raise RuntimeError(f"quality_loop worktree already exists: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + run_command( + [ + "git", + "worktree", + "add", + "-b", + branch, + str(path), + f"origin/{base_branch}", + ], + cwd=self.repo_root, + timeout=600, + ) + + def commit_task(self, worktree: Path, task_id: str) -> str | None: + relative = f"tasks/{task_id}" + run_command(["git", "add", "--", relative], cwd=worktree) + staged = run_command( + ["git", "diff", "--cached", "--quiet", "--", relative], + cwd=worktree, + check=False, + ) + if staged.returncode == 0: + return None + if staged.returncode != 1: + raise CommandError(f"could not inspect staged changes for {task_id}") + run_command( + ["git", "commit", "-m", f"fix(tasks): quality audit {task_id}"], + cwd=worktree, + timeout=600, + ) + return run_command(["git", "rev-parse", "HEAD"], cwd=worktree).stdout.strip() + + def verify_pending_changes( + self, + *, + worktree: Path, + branch: str, + base_sha: str, + expected_paths: set[str], + ) -> None: + top = Path( + run_command(["git", "rev-parse", "--show-toplevel"], cwd=worktree) + .stdout.strip() + ).resolve() + if top != worktree.resolve(): + raise RuntimeError(f"quality_loop worktree identity changed: {top}") + current_branch = run_command( + ["git", "branch", "--show-current"], cwd=worktree + ).stdout.strip() + if current_branch != branch: + raise RuntimeError( + f"quality_loop worktree branch changed: expected {branch}, got {current_branch}" + ) + ancestor = run_command( + ["git", "merge-base", "--is-ancestor", base_sha, "HEAD"], + cwd=worktree, + check=False, + ) + if ancestor.returncode != 0: + raise RuntimeError("quality_loop worktree no longer descends from its recorded base") + + changed: set[str] = set() + for args in ( + ["git", "diff", "--name-only", "HEAD"], + ["git", "diff", "--cached", "--name-only"], + ["git", "ls-files", "--others", "--exclude-standard"], + ): + output = run_command(args, cwd=worktree).stdout + changed.update(line for line in output.splitlines() if line) + if changed != expected_paths: + unexpected = sorted(changed - expected_paths) + missing = sorted(expected_paths - changed) + raise RuntimeError( + "quality_loop worktree diff does not match accepted task changes; " + f"unexpected={unexpected}, missing={missing}" + ) + + @staticmethod + def issue_marker(task_id: str, fingerprint: str) -> str: + return f"" + + def ensure_issue( + self, + *, + repo_slug: str, + task_id: str, + fingerprint: str, + title: str, + body: str, + artifact_dir: Path, + ) -> str: + marker = self.issue_marker(task_id, fingerprint) + existing_raw = run_command( + [ + "gh", + "issue", + "list", + "--repo", + repo_slug, + "--state", + "all", + "--limit", + "1000", + "--json", + "number,body,state,url", + ], + cwd=self.repo_root, + ).stdout + for issue in json.loads(existing_raw or "[]"): + if marker not in str(issue.get("body") or ""): + continue + if str(issue.get("state", "")).upper() == "CLOSED": + run_command( + ["gh", "issue", "reopen", str(issue["number"]), "--repo", repo_slug], + cwd=self.repo_root, + ) + return str(issue.get("url") or "") + + artifact_dir.mkdir(parents=True, exist_ok=True) + body_path = artifact_dir / "issue_body.md" + body_path.write_text(f"{marker}\n\n{body.rstrip()}\n", encoding="utf-8") + args = [ + "gh", + "issue", + "create", + "--repo", + repo_slug, + "--title", + title, + "--body-file", + str(body_path), + ] + for label in self.config.issue_labels: + args.extend(["--label", label]) + return run_command(args, cwd=self.repo_root).stdout.strip() + + def publish_draft_pr( + self, + *, + worktree: Path, + repo_slug: str, + branch: str, + base_branch: str, + title: str, + body: str, + artifact_dir: Path, + ) -> str | None: + ahead = int( + run_command( + ["git", "rev-list", "--count", f"origin/{base_branch}..HEAD"], + cwd=worktree, + ).stdout.strip() + or "0" + ) + if ahead == 0: + self.logger.info("No accepted task changes; skipping empty pull request") + return None + + # Use gh's credential helper explicitly so an SSH origin does not require + # forwarding private SSH keys into the GPU container. + https_remote = f"https://github.com/{repo_slug}.git" + run_command( + [ + "git", + "-c", + "credential.helper=!gh auth git-credential", + "push", + "--set-upstream", + https_remote, + branch, + ], + cwd=worktree, + timeout=1200, + ) + + artifact_dir.mkdir(parents=True, exist_ok=True) + body_path = artifact_dir / "pull_request_body.md" + body_path.write_text(body.rstrip() + "\n", encoding="utf-8") + args = [ + "gh", + "pr", + "create", + "--repo", + repo_slug, + "--head", + branch, + "--base", + base_branch, + "--title", + title, + "--body-file", + str(body_path), + ] + if self.config.draft_pr: + args.append("--draft") + existing = json.loads( + run_command( + [ + "gh", + "pr", + "list", + "--repo", + repo_slug, + "--head", + branch, + "--state", + "all", + "--limit", + "1", + "--json", + "url", + ], + cwd=worktree, + ).stdout + or "[]" + ) + if existing: + return str(existing[0].get("url") or "") + return run_command(args, cwd=worktree, timeout=300).stdout.strip() diff --git a/agents/quality_loop/host.py b/agents/quality_loop/host.py new file mode 100644 index 00000000..c647a977 --- /dev/null +++ b/agents/quality_loop/host.py @@ -0,0 +1,212 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +"""Host-side GitHub preflight and publication for credential isolation.""" +from __future__ import annotations + +import argparse +import dataclasses +import logging +from datetime import datetime, timezone +from pathlib import Path + +from .config import QualityLoopConfig, load_config +from .github import GitHubPublisher +from .orchestrator import QualityLoop, _task_slug +from .state import AuditState, resolve_worktree, stable_fingerprint, validate_run_id + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = Path(__file__).with_name("agent_config.yaml") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="quality_loop host publication boundary") + parser.add_argument("action", choices=("start", "check", "paths", "finalize")) + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--tasks", nargs="+") + parser.add_argument("--no-publish", action="store_true") + parser.add_argument("--resume") + parser.add_argument("--run-id") + return parser + + +def _effective_config(args: argparse.Namespace) -> QualityLoopConfig: + path = args.config if args.config.is_absolute() else REPO_ROOT / args.config + config = load_config(path) + if args.tasks: + config = dataclasses.replace(config, tasks=tuple(args.tasks)) + if args.no_publish: + config = dataclasses.replace( + config, + github=dataclasses.replace(config.github, publish=False), + ) + return config + + +def _logger() -> logging.Logger: + logger = logging.getLogger("quality_loop.host") + logger.setLevel(logging.INFO) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + return logger + + +def _paths(config: QualityLoopConfig, run_id: str) -> tuple[Path, Path, Path]: + artifact = (REPO_ROOT / config.artifact_root / run_id).resolve() + worktree = (REPO_ROOT / config.worktree_root / run_id).resolve() + return artifact, worktree, artifact / "state.yaml" + + +def start(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + artifact, worktree, state_path = _paths(config, run_id) + if artifact.exists() or worktree.exists(): + raise RuntimeError(f"quality_loop run already exists: {run_id}") + branch = f"{config.github.branch_prefix}/{run_id}" + publisher.create_worktree( + path=worktree, + branch=branch, + base_branch=preflight.default_branch, + ) + AuditState.create( + state_path, + run_id=run_id, + config_fingerprint=stable_fingerprint(config), + repo_slug=preflight.repo_slug, + base_sha=preflight.base_sha, + base_branch=preflight.default_branch, + branch=branch, + worktree=worktree.relative_to(REPO_ROOT), + ) + return run_id + + +def check(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + _, worktree, state_path = _paths(config, run_id) + state = AuditState.load(state_path) + if state.data.get("config_fingerprint") != stable_fingerprint(config): + raise RuntimeError("resume config does not match the original quality_loop run") + if state.data.get("repo_slug") != preflight.repo_slug: + raise RuntimeError("resume repository does not match the original quality_loop run") + if ( + resolve_worktree(REPO_ROOT, str(state.data.get("worktree"))) != worktree + or not worktree.is_dir() + ): + raise RuntimeError(f"resume worktree is missing or changed: {worktree}") + return run_id + + +def finalize(config: QualityLoopConfig, run_id: str, logger: logging.Logger) -> str: + validate_run_id(run_id) + publisher = GitHubPublisher(REPO_ROOT, config.github, logger) + preflight = publisher.preflight() + artifact, worktree, state_path = _paths(config, run_id) + state = AuditState.load(state_path) + if state.data.get("status") != "awaiting_publication": + raise RuntimeError( + f"quality_loop run is not ready for publication: {state.data.get('status')}" + ) + if state.data.get("config_fingerprint") != stable_fingerprint(config): + raise RuntimeError("publication config does not match the original quality_loop run") + if state.data.get("repo_slug") != preflight.repo_slug: + raise RuntimeError("publication repository does not match the original quality_loop run") + if ( + resolve_worktree(REPO_ROOT, str(state.data.get("worktree"))) != worktree + or not worktree.is_dir() + ): + raise RuntimeError(f"publication worktree is missing or changed: {worktree}") + + expected_paths = { + f"tasks/{task_id}/{relative}" + for task_id, record in state.data.get("tasks", {}).items() + if record.get("state") == "completed" and record.get("commit_pending") + for relative in record.get("changes", []) + } + publisher.verify_pending_changes( + worktree=worktree, + branch=str(state.data["branch"]), + base_sha=str(state.data["base_sha"]), + expected_paths=expected_paths, + ) + + for task_id, record in state.data.get("tasks", {}).items(): + if record.get("state") != "completed" or not record.get("commit_pending"): + continue + commit = publisher.commit_task(worktree, task_id) + if not commit: + raise RuntimeError(f"accepted changes disappeared before commit: {task_id}") + record["commit"] = commit + record["commit_pending"] = False + state.save() + + if config.github.publish: + for task_id, record in state.data.get("tasks", {}).items(): + if record.get("state") != "issue_pending": + continue + request = record.get("issue_request") or {} + issue_url = publisher.ensure_issue( + repo_slug=str(state.data["repo_slug"]), + task_id=task_id, + fingerprint=str(request["fingerprint"]), + title=str(request["title"]), + body=str(request["body"]), + artifact_dir=artifact / "tasks" / _task_slug(task_id), + ) + state.transition(task_id, "issue_filed", issue_url=issue_url) + + workflow = QualityLoop(REPO_ROOT, config, logger=logger, publisher=publisher) + workflow.state = state + workflow.artifact_dir = artifact + workflow.worktree = worktree + workflow.preflight = preflight + report_path = workflow._write_report() + pr_url = None + if config.github.publish: + pr_url = publisher.publish_draft_pr( + worktree=worktree, + repo_slug=str(state.data["repo_slug"]), + branch=str(state.data["branch"]), + base_branch=str(state.data["base_branch"]), + title="audit(tasks): quality_loop task quality pass", + body=workflow._pull_request_body(report_path), + artifact_dir=artifact, + ) + state.finish("completed", pull_request_url=pr_url) + return pr_url or "no pull request (no accepted changes or publication disabled)" + + +def main(argv: list[str] | None = None) -> int: + args, unknown = _parser().parse_known_args(argv) + if unknown: + raise ValueError(f"unknown quality_loop host arguments: {unknown}") + config = _effective_config(args) + run_id = args.run_id or args.resume + if args.action == "start": + run_id = run_id or datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + print(start(config, run_id, _logger())) + elif args.action == "check": + if not run_id: + raise ValueError("check requires --resume or --run-id") + print(check(config, run_id, _logger())) + elif args.action == "paths": + if not run_id: + raise ValueError("paths requires --resume or --run-id") + validate_run_id(run_id) + artifact, worktree, _ = _paths(config, run_id) + print(artifact.relative_to(REPO_ROOT)) + print(worktree.relative_to(REPO_ROOT)) + else: + if not run_id: + raise ValueError("finalize requires --run-id") + print(finalize(config, run_id, _logger())) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/agents/quality_loop/orchestrator.py b/agents/quality_loop/orchestrator.py new file mode 100644 index 00000000..239a6a11 --- /dev/null +++ b/agents/quality_loop/orchestrator.py @@ -0,0 +1,899 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import logging +import os +import shutil +import statistics +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + +from agents.quality_loop.backend import AgentBackend, CodexBackend +from agents.quality_loop.config import QualityLoopConfig +from agents.quality_loop.filesystem import ( + TreeChanges, + apply_changes, + diff_trees, + is_case_path, + is_generated_path, + restore_committed_perf_stubs, + snapshot_tree, +) +from agents.quality_loop.github import GitHubPublisher, PreflightResult +from agents.quality_loop.prompts import ( + case_enhancement_prompt, + optimizer_prompt, + repair_prompt, + reviewer_prompt, +) +from agents.quality_loop.state import ( + AuditState, + resolve_worktree, + stable_fingerprint, + validate_run_id, +) +from agents.task_validator.validation_prompt import build_validation_prompt +from src.evaluator import ( + evaluate_compilation, + evaluate_correctness, + evaluate_kernel, + measure_baseline, + write_task_result, +) +from src.harness_guard import snapshot_workspace_harness, verify_workspace_harness +from src.perf_helper_materialization import materialize_perf_helpers_in_workspace +from src.preprocessing import _resolve_gfx_arch, setup_workspace +from src.prompt_builder import prompt_builder +from src.testcases import collect_benchmark_methods + + +def _task_slug(task_id: str) -> str: + return task_id.replace("/", "__") + + +def _source_paths(config: dict[str, Any]) -> tuple[str, ...]: + raw = config.get("source_file_path", []) + if isinstance(raw, str): + raw = [raw] + return tuple(str(path) for path in raw if str(path).strip()) + + +def _repo_subdir(config: dict[str, Any]) -> str | None: + if config.get("repo_subdir"): + return str(config["repo_subdir"]) + if config.get("image_repo_path"): + return Path(str(config["image_repo_path"])).name + if config.get("repo_url"): + name = str(config["repo_url"]).rstrip("/").rsplit("/", 1)[-1] + return name[:-4] if name.endswith(".git") else name + return None + + +def _filtered_changes( + before: dict[str, str], + after: dict[str, str], + *, + repo_subdir: str | None, +) -> TreeChanges: + changes = diff_trees(before, after) + return TreeChanges( + added=tuple(p for p in changes.added if not is_generated_path(p, repo_subdir=repo_subdir)), + modified=tuple( + p for p in changes.modified if not is_generated_path(p, repo_subdir=repo_subdir) + ), + deleted=tuple( + p for p in changes.deleted if not is_generated_path(p, repo_subdir=repo_subdir) + ), + ) + + +def _validation_warnings(report: dict[str, Any]) -> list[str]: + warnings: list[str] = [] + for name, check in (report.get("checks") or {}).items(): + if isinstance(check, dict) and str(check.get("status", "")).upper() == "WARN": + warnings.append(f"{name}: {check.get('details') or check.get('analysis') or 'warning'}") + return warnings + + +def _review_is_valid(review: Any) -> bool: + if not isinstance(review, dict): + return False + for key in ( + "accepted", + "logic_equivalent", + "evidence_sufficient", + "case_enhancement_needed", + ): + if not isinstance(review.get(key), bool): + return False + return isinstance(review.get("summary"), str) and isinstance( + review.get("case_rationale"), str + ) + + +def difficulty_is_easy( + *, + task_type: str, + speedups: list[float], + result: dict[str, Any], + review: dict[str, Any], + config: QualityLoopConfig, +) -> bool: + """Return true only for a reproducible, review-approved first-iteration 5x gain.""" + return bool( + task_type in config.promotion_task_types + and len(speedups) == config.easy_confirmation_runs + and all(value > 0 for value in speedups) + and statistics.median(speedups) >= config.easy_speedup_threshold + and result.get("pass_compilation") is True + and result.get("pass_correctness") is True + and result.get("benchmark_method_consistent") is True + and int(result.get("valid_baseline_cases", 0)) > 0 + and result.get("valid_baseline_cases") == result.get("valid_optimized_cases") + and review.get("accepted") is True + and review.get("logic_equivalent") is True + and review.get("evidence_sufficient") is True + ) + + +class QualityLoop: + def __init__( + self, + repo_root: Path, + config: QualityLoopConfig, + *, + logger: logging.Logger, + backend: AgentBackend | None = None, + reviewer_backend: AgentBackend | None = None, + publisher: GitHubPublisher | None = None, + defer_github: bool = False, + ): + self.repo_root = repo_root.resolve() + self.config = config + self.logger = logger + self.backend = backend or CodexBackend(config.backend, logger) + self.reviewer_backend = reviewer_backend or CodexBackend(config.reviewer, logger) + self.publisher = publisher or GitHubPublisher(self.repo_root, config.github, logger) + self.defer_github = defer_github + self.state: AuditState | None = None + self.artifact_dir: Path | None = None + self.worktree: Path | None = None + self.preflight: PreflightResult | None = None + + def discover_tasks(self, root: Path | None = None) -> dict[str, Path]: + tasks_root = (root or self.repo_root) / "tasks" + discovered = { + str(path.parent.relative_to(tasks_root)): path + for path in tasks_root.rglob("config.yaml") + } + if "all" in self.config.tasks: + return dict(sorted(discovered.items())) + selected: dict[str, Path] = {} + missing: list[str] = [] + for selector in self.config.tasks: + matches = { + task_id: path + for task_id, path in discovered.items() + if task_id == selector or task_id.startswith(selector.rstrip("/") + "/") + } + if not matches: + missing.append(selector) + selected.update(matches) + if missing: + raise ValueError(f"task selector(s) matched nothing: {missing}") + return dict(sorted(selected.items())) + + def plan(self) -> dict[str, Any]: + tasks = self.discover_tasks() + gfx_arch = _resolve_gfx_arch(self.config.target_gpu_model) + runnable: list[str] = [] + deferred: list[str] = [] + for task_id, config_path in tasks.items(): + task_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + if self._platform_matches(task_config, gfx_arch): + runnable.append(task_id) + else: + deferred.append(task_id) + return { + "total": len(tasks), + "runnable": runnable, + "platform_deferred": deferred, + "target_gpu_model": self.config.target_gpu_model, + "gfx_arch": gfx_arch, + "backend": self.config.backend.name, + "optimization_iterations": self.config.optimization_iterations, + } + + @staticmethod + def _platform_matches(task_config: dict[str, Any], gfx_arch: str | None) -> bool: + platform = task_config.get("platform_support") + if not isinstance(platform, dict): + return True + if str(platform.get("status", "active")).strip().lower() == "skip": + return False + required = platform.get("required_arch") + return not required or (gfx_arch is not None and str(required).strip() == gfx_arch) + + def run( + self, + *, + resume_run_id: str | None = None, + skip_preflight: bool = False, + ) -> Path: + run_id = resume_run_id or datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + validate_run_id(run_id) + self.artifact_dir = (self.repo_root / self.config.artifact_root / run_id).resolve() + state_path = self.artifact_dir / "state.yaml" + fingerprint = stable_fingerprint(self.config) + + if resume_run_id: + self.state = AuditState.load(state_path) + if self.state.data.get("config_fingerprint") != fingerprint: + raise RuntimeError("resume config does not match the original quality_loop run") + self.worktree = resolve_worktree( + self.repo_root, str(self.state.data["worktree"]) + ) + if not self.worktree.is_dir(): + raise RuntimeError(f"resume worktree is missing: {self.worktree}") + if skip_preflight: + self.preflight = PreflightResult( + repo_slug=str(self.state.data["repo_slug"]), + default_branch=str(self.state.data["base_branch"]), + base_sha=str(self.state.data["base_sha"]), + viewer_permission="WRITE", + ) + else: + self.preflight = self.publisher.preflight() + else: + if skip_preflight: + raise ValueError("skip_preflight is only valid for a host-initialized resume run") + self.preflight = self.publisher.preflight() + branch = f"{self.config.github.branch_prefix}/{run_id}" + self.worktree = (self.repo_root / self.config.worktree_root / run_id).resolve() + self.publisher.create_worktree( + path=self.worktree, + branch=branch, + base_branch=self.preflight.default_branch, + ) + self.state = AuditState.create( + state_path, + run_id=run_id, + config_fingerprint=fingerprint, + repo_slug=self.preflight.repo_slug, + base_sha=self.preflight.base_sha, + base_branch=self.preflight.default_branch, + branch=branch, + worktree=self.worktree.relative_to(self.repo_root), + ) + + assert self.state is not None and self.worktree is not None + tasks = self.discover_tasks(self.worktree) + gfx_arch = _resolve_gfx_arch(self.config.target_gpu_model) + for index, (task_id, config_path) in enumerate(tasks.items(), 1): + if self.state.is_terminal(task_id): + self.logger.info("Resume: skipping terminal task %s", task_id) + continue + task_config = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {} + if not self._platform_matches(task_config, gfx_arch): + self.state.transition( + task_id, + "platform_deferred", + reason=f"requires a different platform than {gfx_arch or 'unknown'}", + ) + continue + self.logger.info("quality_loop task %d/%d: %s", index, len(tasks), task_id) + try: + self._process_task(task_id, config_path.parent) + except Exception as exc: + self.logger.exception("quality_loop task failed: %s", task_id) + # Tooling/agent/runtime failures are not evidence that the task is + # defective. Keep them resumable and never file a task issue. + self.state.transition(task_id, "infrastructure_failed", error=str(exc)) + + report_path = self._write_report() + infrastructure_failures = [ + task_id + for task_id, record in self.state.data.get("tasks", {}).items() + if record.get("state") == "infrastructure_failed" + ] + if infrastructure_failures: + self.state.finish("incomplete") + raise RuntimeError( + "quality_loop cannot publish until infrastructure failures are resumed: " + + ", ".join(infrastructure_failures) + ) + if self.defer_github: + self.state.finish("awaiting_publication") + return report_path + pr_url = None + if self.config.github.publish and not self.defer_github: + pr_url = self.publisher.publish_draft_pr( + worktree=self.worktree, + repo_slug=str(self.state.data["repo_slug"]), + branch=str(self.state.data["branch"]), + base_branch=str(self.state.data["base_branch"]), + title="audit(tasks): quality_loop task quality pass", + body=self._pull_request_body(report_path), + artifact_dir=self.artifact_dir, + ) + self.state.finish("completed", pull_request_url=pr_url) + return report_path + + def _process_task(self, task_id: str, canonical_task: Path) -> None: + assert self.state is not None + assert self.artifact_dir is not None + assert self.worktree is not None + task_artifacts = self.artifact_dir / "tasks" / _task_slug(task_id) + original_task = task_artifacts / "original_task" + candidate_task = task_artifacts / "candidate_task" + self._reset_path(task_artifacts) + task_artifacts.mkdir(parents=True) + shutil.copytree(canonical_task, original_task) + shutil.copytree(canonical_task, candidate_task) + original_tree = snapshot_tree(original_task) + original_validation_status = "FAIL" + + self.state.transition(task_id, "validating") + validation_workspace, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_initial" + ) + original_validation_status = str(validation.get("overall_status", "FAIL")).upper() + warnings = _validation_warnings(validation) + + if original_validation_status == "FAIL": + if self.config.max_repair_attempts == 0: + self._handle_unrepairable(task_id, validation, task_artifacts) + return + self.state.transition(task_id, "repairing", warnings=warnings) + task_config = self._load_task_config(candidate_task) + before = snapshot_tree(validation_workspace) + self.backend.run( + repair_prompt(validation, task_id), validation_workspace, role="repair" + ) + after = snapshot_tree(validation_workspace) + changes = _filtered_changes( + before, after, repo_subdir=_repo_subdir(task_config) + ) + if changes.empty: + self._handle_unrepairable(task_id, validation, task_artifacts) + return + apply_changes(validation_workspace, candidate_task, changes) + restore_committed_perf_stubs(candidate_task) + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_repaired" + ) + warnings = _validation_warnings(validation) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + self._handle_unrepairable(task_id, validation, task_artifacts) + return + + self.state.transition(task_id, "optimizing", warnings=warnings) + optimization_workspace, baseline_cases, result = self._optimize_once( + task_id, candidate_task, task_artifacts / "optimization" + ) + review = self._review(task_id, optimization_workspace, result) + speedups = [float(result.get("speedup_ratio") or 0.0)] + if speedups[0] >= self.config.easy_speedup_threshold and review.get("accepted"): + for _ in range(1, self.config.easy_confirmation_runs): + eval_result = evaluate_kernel( + optimization_workspace, + self._load_task_config(candidate_task), + baseline_cases, + self.logger, + ) + baseline_methods = set(collect_benchmark_methods(baseline_cases)) + optimized_methods = set(eval_result.get("optimized_benchmark_methods") or []) + repeated_valid = bool( + eval_result.get("pass_compilation") + and eval_result.get("pass_correctness") + and baseline_methods + and baseline_methods == optimized_methods + and int(eval_result.get("valid_baseline_cases", 0)) > 0 + and eval_result.get("valid_baseline_cases") + == eval_result.get("valid_optimized_cases") + ) + speedups.append( + float(eval_result.get("average_speedup") or 0.0) + if repeated_valid + else 0.0 + ) + + hardened = False + task_config = self._load_task_config(candidate_task) + if difficulty_is_easy( + task_type=str(task_config.get("task_type", "")), + speedups=speedups, + result=result, + review=review, + config=self.config, + ): + hardened = self._promote_baseline( + task_id, + original_task, + candidate_task, + optimization_workspace, + task_artifacts, + ) + + cases_enhanced = False + if ( + self.config.case_enhancement + and review.get("accepted") is True + and review.get("case_enhancement_needed") is True + and original_validation_status in {"PASS", "WARN"} + ): + cases_enhanced = self._enhance_cases( + task_id, + original_task, + candidate_task, + str(review.get("case_rationale", "")), + task_artifacts, + ) + + restore_committed_perf_stubs(candidate_task) + candidate_tree = snapshot_tree(candidate_task) + final_changes = _filtered_changes( + original_tree, + candidate_tree, + repo_subdir=_repo_subdir(task_config), + ) + commit = None + commit_pending = False + if not final_changes.empty: + apply_changes(candidate_task, canonical_task, final_changes) + restore_committed_perf_stubs(canonical_task) + if self.defer_github: + # A linked worktree's .git file contains a host-absolute path, + # which is intentionally unavailable inside the GPU container. + # The credential-bearing host finalizer verifies and commits it. + commit_pending = True + else: + commit = self.publisher.commit_task(self.worktree, task_id) + self.state.transition( + task_id, + "completed", + warnings=warnings, + changes=list(final_changes.paths), + commit=commit, + commit_pending=commit_pending, + speedups=speedups, + reviewer=review, + baseline_hardened=hardened, + cases_enhanced=cases_enhanced, + ) + + def _validate( + self, task_id: str, task_dir: Path, stage_dir: Path + ) -> tuple[Path, dict[str, Any]]: + workspace = self._make_workspace(task_id, task_dir, stage_dir) + prompt = build_validation_prompt( + str(task_dir / "config.yaml"), + str(workspace), + self._eval_config(), + ) + self.backend.run(prompt, workspace, role="validator") + report_path = workspace / "validation_report.yaml" + if not report_path.exists(): + report = { + "task_name": task_id, + "overall_status": "FAIL", + "checks": {}, + "summary": "validator backend did not produce validation_report.yaml", + } + report_path.write_text(yaml.safe_dump(report), encoding="utf-8") + return workspace, report + report = yaml.safe_load(report_path.read_text(encoding="utf-8")) or {} + if not isinstance(report, dict) or str(report.get("overall_status", "")).upper() not in { + "PASS", + "WARN", + "FAIL", + }: + raise RuntimeError(f"invalid validator report for {task_id}: {report_path}") + return workspace, report + + def _optimize_once( + self, task_id: str, task_dir: Path, stage_dir: Path + ) -> tuple[Path, list[Any], dict[str, Any]]: + workspace = self._make_workspace(task_id, task_dir, stage_dir) + task_config = self._load_task_config(task_dir) + original_sources = stage_dir / "original_sources" + original_sources.mkdir() + source_manifest: dict[str, str] = {} + for relative in _source_paths(task_config): + source = (workspace / relative).resolve() + if not source.is_relative_to(workspace.resolve()) or not source.is_file(): + source_manifest[relative] = "missing" + continue + destination = original_sources / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + source_manifest[relative] = "copied" + (original_sources / "manifest.yaml").write_text( + yaml.safe_dump(source_manifest, sort_keys=True), encoding="utf-8" + ) + original_source_tree = snapshot_tree(original_sources) + task_type = str(task_config.get("task_type", "")) + if task_type == "torch2hip": + baseline_cases = measure_baseline(workspace, task_config, self.logger) + else: + compiled, error = evaluate_compilation(workspace, task_config, self.logger) + if not compiled: + raise RuntimeError(f"baseline compilation failed after validation: {error}") + baseline_cases = measure_baseline(workspace, task_config, self.logger) + + harness = snapshot_workspace_harness(workspace) + base_prompt = prompt_builder( + str(task_dir / "config.yaml"), + str(workspace), + self._eval_config(), + self.logger, + ) + self.backend.run( + optimizer_prompt(base_prompt, task_id), workspace, role="optimizer" + ) + verify_workspace_harness(harness) + if snapshot_tree(original_sources) != original_source_tree: + raise RuntimeError("optimizer modified the protected original-source snapshot") + materialize_perf_helpers_in_workspace(workspace, logger=self.logger) + evaluation = evaluate_kernel(workspace, task_config, baseline_cases, self.logger) + write_task_result( + workspace, + evaluation, + baseline_cases, + task_id, + "quality_loop/codex", + self.logger, + create_plots=False, + ) + shutil.copytree( + original_sources, + workspace / ".quality_loop_original_sources", + ) + return workspace, baseline_cases, yaml.safe_load( + (workspace / "task_result.yaml").read_text(encoding="utf-8") + ) + + def _review( + self, task_id: str, workspace: Path, result: dict[str, Any] + ) -> dict[str, Any]: + output_name = "quality_loop_review.yaml" + before = snapshot_tree(workspace) + evidence_names = ( + "task_result.yaml", + "baseline_perf.yaml", + "optimized_perf.yaml", + ) + evidence_before = { + name: (workspace / name).read_bytes() + for name in evidence_names + if (workspace / name).is_file() + } + original_sources = workspace / ".quality_loop_original_sources" + original_before = snapshot_tree(original_sources) + self.reviewer_backend.run( + reviewer_prompt(task_id, workspace / "task_result.yaml", output_name), + workspace, + role="reviewer", + ) + after = snapshot_tree(workspace) + evidence_after = { + name: (workspace / name).read_bytes() + for name in evidence_names + if (workspace / name).is_file() + } + if ( + evidence_after != evidence_before + or snapshot_tree(original_sources) != original_before + ): + raise RuntimeError("reviewer modified protected evaluation evidence") + changes = diff_trees(before, after) + unexpected = [path for path in changes.paths if path != output_name] + if unexpected: + raise RuntimeError(f"reviewer modified non-review files: {unexpected}") + review_path = workspace / output_name + review = ( + yaml.safe_load(review_path.read_text(encoding="utf-8")) + if review_path.exists() + else None + ) + if not _review_is_valid(review): + raise RuntimeError(f"reviewer returned an invalid decision for {task_id}") + if not ( + result.get("pass_compilation") + and result.get("pass_correctness") + and result.get("benchmark_method_consistent") + ): + review["accepted"] = False + review["evidence_sufficient"] = False + review["summary"] = ( + "Deterministic evaluator gate rejected the candidate. " + + review["summary"] + ) + return review + + def _promote_baseline( + self, + task_id: str, + original_task: Path, + candidate_task: Path, + optimized_workspace: Path, + task_artifacts: Path, + ) -> bool: + task_config = self._load_task_config(candidate_task) + sources = _source_paths(task_config) + if not sources or any(not (candidate_task / path).is_file() for path in sources): + self.logger.warning("Task %s has no promotable committed source baseline", task_id) + return False + if any(not (optimized_workspace / path).is_file() for path in sources): + self.logger.warning("Task %s optimizer omitted a declared source file", task_id) + return False + source_backup = task_artifacts / "baseline_before_promotion" + self._reset_path(source_backup) + source_backup.mkdir(parents=True) + for relative in sources: + backup_file = source_backup / relative + backup_file.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(candidate_task / relative, backup_file) + for relative in sources: + source = optimized_workspace / relative + if not source.is_file(): + return False + destination = candidate_task / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + restore_committed_perf_stubs(candidate_task) + + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_hardened" + ) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + # Restore the previously accepted source files; an unverified faster + # candidate must never become the task baseline. + for relative in sources: + shutil.copy2(source_backup / relative, candidate_task / relative) + return False + accepted = self._dual_correctness_gate( + task_id, original_task, candidate_task, task_artifacts / "hardening_gate" + ) + if not accepted: + for relative in sources: + shutil.copy2(source_backup / relative, candidate_task / relative) + return accepted + + def _enhance_cases( + self, + task_id: str, + original_task: Path, + candidate_task: Path, + rationale: str, + task_artifacts: Path, + ) -> bool: + case_workspace = self._make_workspace( + task_id, candidate_task, task_artifacts / "case_candidate" + ) + task_config = self._load_task_config(candidate_task) + before = snapshot_tree(case_workspace) + self.backend.run( + case_enhancement_prompt(task_id, rationale), + case_workspace, + role="case_enhancer", + ) + after = snapshot_tree(case_workspace) + changes = _filtered_changes( + before, after, repo_subdir=_repo_subdir(task_config) + ) + if changes.empty: + return False + if any( + not is_case_path(path) or Path(path).name == "performance_utils_pytest.py" + for path in changes.paths + ): + self.logger.warning("Rejecting non-case changes from case enhancer: %s", changes.paths) + return False + + backup = task_artifacts / "candidate_before_cases" + shutil.copytree(candidate_task, backup) + apply_changes(case_workspace, candidate_task, changes) + restore_committed_perf_stubs(candidate_task) + if not self._dual_correctness_gate( + task_id, original_task, candidate_task, task_artifacts / "case_gate" + ): + self._replace_directory(candidate_task, backup) + return False + _, validation = self._validate( + task_id, candidate_task, task_artifacts / "validation_cases" + ) + if str(validation.get("overall_status", "FAIL")).upper() == "FAIL": + self._replace_directory(candidate_task, backup) + return False + return True + + def _dual_correctness_gate( + self, task_id: str, original_task: Path, candidate_task: Path, stage_dir: Path + ) -> bool: + task_config = self._load_task_config(candidate_task) + sources = _source_paths(task_config) + if not sources or any(not (original_task / path).is_file() for path in sources): + self.logger.warning( + "Cannot prove new cases/baseline against a committed original kernel for %s", + task_id, + ) + return False + original_with_cases = stage_dir / "original_with_cases" + self._reset_path(stage_dir) + shutil.copytree(candidate_task, original_with_cases) + for relative in sources: + destination = original_with_cases / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(original_task / relative, destination) + + for label, task_dir in ( + ("original", original_with_cases), + ("candidate", candidate_task), + ): + workspace = self._make_workspace(task_id, task_dir, stage_dir / label) + config = self._load_task_config(task_dir) + compiled, _ = evaluate_compilation(workspace, config, self.logger) + correct, _ = evaluate_correctness(workspace, config, self.logger) + if not compiled or not correct: + self.logger.warning( + "Dual correctness gate rejected %s (%s): compile=%s correctness=%s", + task_id, + label, + compiled, + correct, + ) + return False + return True + + def _handle_unrepairable( + self, task_id: str, report: dict[str, Any], task_artifacts: Path + ) -> None: + assert self.state is not None + fingerprint = stable_fingerprint( + { + "task": task_id, + "base_sha": self.state.data["base_sha"], + "checks": report.get("checks"), + } + ) + body = ( + f"`quality_loop` could not repair task `{task_id}`.\n\n" + f"Base commit: `{self.state.data['base_sha']}`\n\n" + "Validator report:\n\n```yaml\n" + + yaml.safe_dump(report, sort_keys=False) + + "```" + ) + issue_url = None + if self.defer_github and self.config.github.publish: + state = "issue_pending" + elif self.config.github.publish: + issue_url = self.publisher.ensure_issue( + repo_slug=str(self.state.data["repo_slug"]), + task_id=task_id, + fingerprint=fingerprint, + title=f"[quality_loop] Invalid task: {task_id}", + body=body, + artifact_dir=task_artifacts, + ) + state = "issue_filed" + else: + state = "reported_failure" + self.state.transition( + task_id, + state, + issue_url=issue_url, + validation_report=report, + issue_request={ + "task_id": task_id, + "fingerprint": fingerprint, + "title": f"[quality_loop] Invalid task: {task_id}", + "body": body, + } + if state == "issue_pending" + else None, + ) + + def _make_workspace(self, task_id: str, task_dir: Path, stage_dir: Path) -> Path: + self._reset_path(stage_dir) + stage_dir.mkdir(parents=True, exist_ok=True) + return setup_workspace( + str(task_dir / "config.yaml"), + stage_dir, + "qualityloop", + self.logger, + task_name=task_id, + ) + + def _eval_config(self) -> dict[str, Any]: + return { + "target_gpu_model": self.config.target_gpu_model, + "agent": { + "template": "codex", + "python_path": os.environ.get("AGENT_KERNEL_ARENA_PYTHON"), + "compile_timeout": 600, + "correctness_timeout": 600, + "performance_timeout": 600, + "max_iterations": 1, + }, + } + + @staticmethod + def _load_task_config(task_dir: Path) -> dict[str, Any]: + value = yaml.safe_load((task_dir / "config.yaml").read_text(encoding="utf-8")) or {} + if not isinstance(value, dict): + raise ValueError(f"invalid task config: {task_dir / 'config.yaml'}") + return value + + @staticmethod + def _reset_path(path: Path) -> None: + if not path.exists() and not path.is_symlink(): + return + if path.is_symlink() or path.is_file(): + path.unlink() + else: + shutil.rmtree(path) + + @classmethod + def _replace_directory(cls, destination: Path, source: Path) -> None: + cls._reset_path(destination) + shutil.copytree(source, destination) + + def _write_report(self) -> Path: + assert self.state is not None and self.artifact_dir is not None + records = self.state.data.get("tasks", {}) + counts: dict[str, int] = {} + warning_count = 0 + for record in records.values(): + status = str(record.get("state", "unknown")) + counts[status] = counts.get(status, 0) + 1 + warning_count += len(record.get("warnings") or []) + report = { + "run_id": self.state.data["run_id"], + "repo": self.state.data["repo_slug"], + "base_sha": self.state.data["base_sha"], + "target_gpu_model": self.config.target_gpu_model, + "backend": self.config.backend.name, + "optimization_iterations": 1, + "counts": counts, + "warning_count": warning_count, + "tasks": records, + } + path = self.artifact_dir / "audit_report.yaml" + path.write_text( + yaml.safe_dump(report, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + return path + + def _pull_request_body(self, report_path: Path) -> str: + assert self.state is not None + records = self.state.data.get("tasks", {}) + completed = [task for task, value in records.items() if value.get("state") == "completed"] + changed = [task for task in completed if records[task].get("changes")] + issues = [value.get("issue_url") for value in records.values() if value.get("issue_url")] + warnings = sum(len(value.get("warnings") or []) for value in records.values()) + return f"""## Summary + +- Audited tasks: {len(records)} +- Accepted task changes: {len(changed)} +- Validator warnings recorded: {warnings} +- Unrepairable task issues: {len(issues)} +- Optimizer: Codex, exactly one iteration per task +- Easy-task threshold: reproducible {self.config.easy_speedup_threshold:.1f}x + +## Changed tasks + +{chr(10).join(f'- `{task}`' for task in changed) or '- None'} + +## Issues + +{chr(10).join(f'- {url}' for url in issues) or '- None'} + +The full machine-readable report is stored in the local run artifact +`{report_path}`. Every promoted baseline and case change passed the fail-closed +dual correctness gate against the pre-audit kernel. +""" diff --git a/agents/quality_loop/prompts.py b/agents/quality_loop/prompts.py new file mode 100644 index 00000000..2145d2f2 --- /dev/null +++ b/agents/quality_loop/prompts.py @@ -0,0 +1,82 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +from pathlib import Path + +import yaml + + +def repair_prompt(report: dict, task_id: str) -> str: + return f"""# quality_loop validation repair + +Task: `{task_id}` + +The task validator found blocking failures. Fix only the FAIL/TIMEOUT findings in +this task workspace. Do not spend time fixing WARN-only findings. Preserve the +task's intended computation and public contract. Run the relevant compile and +correctness commands after editing. Do not use GitHub, git, network services, or +edit anything outside this workspace. + +Validation report: +```yaml +{yaml.safe_dump(report, sort_keys=False)} +``` +""" + + +def optimizer_prompt(base_prompt: str, task_id: str) -> str: + return base_prompt.rstrip() + f""" + +## quality_loop single-iteration boundary + +This is the one and only optimization iteration for `{task_id}`. Produce at most +one candidate implementation. Complete all analysis, implementation, compile, +correctness, and performance checks needed for that candidate in this single +iteration. You may edit only declared kernel/source files. Do not edit config, +tests, scripts, performance helpers, or any other harness file. Do not use git or +GitHub. Preserve the exact computation, outputs, dtypes, shapes, aliasing, and +side effects of the original task. +""" + + +def reviewer_prompt(task_id: str, result_file: Path, output_name: str) -> str: + return f"""# quality_loop independent evaluation review + +Review task `{task_id}` independently. You are a read-only evaluator, not the +optimizer. Compare the candidate's declared source paths with their pre-optimizer +copies under `.quality_loop_original_sources/`. Inspect the config, test harness, +and centralized evaluator evidence in `{result_file.name}`. Decide whether the +candidate preserves the task's computation and whether the evidence is strong +enough to accept it. Also decide whether task cases have material coverage gaps. + +Do not edit any existing file. Write exactly one new YAML file `{output_name}`: + +```yaml +accepted: true # boolean +logic_equivalent: true # boolean +evidence_sufficient: true # boolean +case_enhancement_needed: false # boolean +case_rationale: "..." +summary: "..." +``` + +Fail closed: set accepted false when behavior is ambiguous, evidence is missing, +the harness changed, performance methods differ, valid case counts shrink, or +the candidate depends on untested assumptions. Do not use git, GitHub, or network +services. +""" + + +def case_enhancement_prompt(task_id: str, rationale: str) -> str: + return f"""# quality_loop task-case hardening + +Task: `{task_id}` +Reviewer rationale: {rationale} + +Strengthen correctness coverage only where the rationale identifies a real gap. +Add a small, targeted set of valid boundary/shape/dtype cases. Do not modify the +kernel/source implementation, computation contract, tolerances merely to accept +wrong answers, benchmark timing helpers, or performance methodology. Every new +case must be valid for and pass the original pre-audit kernel. Run the appropriate +correctness command. Do not use git, GitHub, or network services. +""" diff --git a/agents/quality_loop/state.py b/agents/quality_loop/state.py new file mode 100644 index 00000000..371bdb62 --- /dev/null +++ b/agents/quality_loop/state.py @@ -0,0 +1,134 @@ +# Copyright(C) [2026] Advanced Micro Devices, Inc. All rights reserved. +from __future__ import annotations + +import hashlib +import json +import os +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import yaml + + +TERMINAL_TASK_STATES = { + "completed", + "issue_filed", + "issue_pending", + "reported_failure", + "platform_deferred", +} + +RUN_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$") + + +def validate_run_id(run_id: str) -> str: + if not RUN_ID_PATTERN.fullmatch(run_id) or run_id in {".", ".."}: + raise ValueError( + "quality_loop run ID must contain only letters, digits, '.', '_', or '-'" + ) + return run_id + + +def resolve_worktree(repo_root: Path, stored_path: str | Path) -> Path: + path = Path(stored_path) + return path.resolve() if path.is_absolute() else (repo_root / path).resolve() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def stable_fingerprint(value: Any) -> str: + payload = json.dumps(value, sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +class AuditState: + """Crash-safe YAML manifest used for resume and publication summaries.""" + + def __init__(self, path: Path, data: dict[str, Any]): + self.path = path + self.data = data + + @classmethod + def create( + cls, + path: Path, + *, + run_id: str, + config_fingerprint: str, + repo_slug: str, + base_sha: str, + base_branch: str, + branch: str, + worktree: Path, + ) -> "AuditState": + validate_run_id(run_id) + state = cls( + path, + { + "schema_version": 1, + "run_id": run_id, + "created_at": utc_now(), + "updated_at": utc_now(), + "config_fingerprint": config_fingerprint, + "repo_slug": repo_slug, + "base_sha": base_sha, + "base_branch": base_branch, + "branch": branch, + "worktree": str(worktree), + "status": "running", + "tasks": {}, + "pull_request_url": None, + }, + ) + state.save() + return state + + @classmethod + def load(cls, path: Path) -> "AuditState": + raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(raw, dict) or raw.get("schema_version") != 1: + raise ValueError(f"unsupported or corrupt quality_loop state: {path}") + return cls(path, raw) + + def save(self) -> None: + self.path.parent.mkdir(parents=True, exist_ok=True) + self.data["updated_at"] = utc_now() + tmp = self.path.with_name(f".{self.path.name}.{os.getpid()}.tmp") + tmp.write_text( + yaml.safe_dump(self.data, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + os.replace(tmp, self.path) + + def task(self, task_id: str) -> dict[str, Any]: + tasks = self.data.setdefault("tasks", {}) + return tasks.setdefault( + task_id, + { + "state": "pending", + "events": [], + "warnings": [], + "changes": [], + "issue_url": None, + }, + ) + + def transition(self, task_id: str, state: str, **fields: Any) -> None: + record = self.task(task_id) + record["state"] = state + record.update(fields) + record.setdefault("events", []).append({"at": utc_now(), "state": state}) + self.save() + + def is_terminal(self, task_id: str) -> bool: + return self.task(task_id).get("state") in TERMINAL_TASK_STATES + + def finish(self, status: str, *, pull_request_url: str | None = None) -> None: + self.data["status"] = status + if pull_request_url: + self.data["pull_request_url"] = pull_request_url + self.save() diff --git a/docs/README.md b/docs/README.md index 17bd0d0b..4d43ea5c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ python -m sphinx -T -b html docs docs/_build/html | `how-to/agents.md` | How-to | Supported agents, model providers, and A/B testing. | | `how-to/add-task.md` | How-to | Task directory layout, `config.yaml` fields, and task types. | | `how-to/task-validator.md` | How-to | The task_validator agent and its 10 checks. | +| `how-to/quality-loop.md` | How-to | Repository-wide validation, one-pass optimization, task hardening, issue filing, and draft PR publication. | | `how-to/held-out-evaluation.md` | How-to | Generate private shapes and evaluate completed runs for generalization. | | `how-to/visualization.md` | How-to | Build dashboard data and serve the comparison dashboard. | | `examples/examples.md` | Examples | Step-by-step walkthroughs with expected output. | diff --git a/docs/how-to/agents.md b/docs/how-to/agents.md index 26f1d677..039f3384 100644 --- a/docs/how-to/agents.md +++ b/docs/how-to/agents.md @@ -40,6 +40,10 @@ The Cursor, Claude Code, and Codex integrations reuse their host CLI login state. Specialized integrations have additional setup and configuration under their respective `agents//` directories. +`quality_loop` is intentionally not an `agent.template`. It is a repository-level +task maintenance workflow that invokes Codex roles across many tasks, files issues, +and publishes one draft PR. See [Audit and harden tasks](quality-loop.md). + ## Models, providers, and agent settings AgentKernelArena has no shared model/provider field in the run configuration. diff --git a/docs/how-to/quality-loop.md b/docs/how-to/quality-loop.md new file mode 100644 index 00000000..352fe972 --- /dev/null +++ b/docs/how-to/quality-loop.md @@ -0,0 +1,107 @@ +--- +myst: + html_meta: + "description": "Audit, repair, harden, and publish AgentKernelArena tasks with the Codex-based quality_loop workflow." + "keywords": "AgentKernelArena, quality_loop, task audit, Codex, GitHub issues, pull request, GPU kernel" +--- + +# Audit and harden tasks with quality_loop + +`quality_loop` is a repository-level workflow for maintaining the task corpus. +It differs from a normal `agent.template`: normal agents optimize one copied task, +while `quality_loop` owns a complete multi-task campaign and a single Git branch. + +For every selected, platform-compatible task it: + +1. Runs the existing task validator in a fresh workspace. +2. Records WARN results without repairing them. +3. Attempts one repair for FAIL results, revalidates from a fresh copy, and files + a fingerprinted GitHub issue if the task remains invalid. +4. Runs exactly one Codex optimization iteration and the centralized evaluator. +5. Starts a separate, read-only Codex session to review correctness evidence and + case coverage. +6. Promotes a first-iteration candidate only when three measurements have median + speedup at least 5x and all correctness/method/case-count gates pass. +7. Adds targeted cases only when both the original kernel and candidate pass them. +8. Commits accepted task changes to one isolated branch and creates one draft PR. + +## Prerequisites + +Install and authenticate Codex and GitHub CLI on the host. The GitHub identity +must have write permission to this repository: + +```bash +codex --version +gh auth status -h github.com +gh api repos/AMD-AGI/AgentKernelArena --jq '.permissions.push' +``` + +The Docker launcher performs GitHub preflight and creates the audit worktree on +the host. It mounts Codex state, but never mounts GitHub credentials into the GPU +container. The main checkout is mounted read-only, while only the current run's +artifact and isolated worktree directories are writable. After the task campaign +exits, a host-side deterministic publisher verifies the recorded diff, commits +accepted task changes, creates issues, pushes the branch, and opens the draft PR. + +## Inspect a campaign + +Planning is offline and does not create a branch or require GPU access: + +```bash +python3 -m agents.quality_loop \ + --config example_configs/quality_loop_mi300.yaml \ + --plan +``` + +The output lists runnable and platform-deferred tasks. A task with +`platform_support.required_arch` is run only on the matching architecture. + +## Run and resume + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml +``` + +Select `example_configs/quality_loop_mi355x.yaml` on an MI355X host. + +For a bounded smoke campaign: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--tasks hip2hip/gpumode/GELU triton2triton/vllm/triton_rms_norm" +``` + +Resume with the run ID printed in `quality_loop_runs/`: + +```bash +make docker-quality-loop \ + QUALITY_LOOP_CONFIG=example_configs/quality_loop_mi300.yaml \ + QUALITY_LOOP_ARGS="--resume " +``` + +The crash-safe `state.yaml` skips terminal tasks. `audit_report.yaml` records every +warning, failure, issue URL, speedup confirmation, accepted file change, and commit. + +## Safety boundaries + +- GitHub authentication and write permission are hard preflights. Failure happens + before branch creation or task mutation. +- GitHub credentials never enter the agent container, and Codex state is copied + into an ephemeral writable home. +- The host refuses to commit or publish when the worktree contains a path that is + not in the accepted per-task change manifest. +- Optimizers cannot edit task harness files; the existing harness digest guard is + checked before evaluation. +- Reviewer output is schema checked, and modifications beyond its one YAML result + file invalidate the review. +- External repository/image worktrees and generated benchmark helpers are never + copied into a task commit. +- Translation/authoring tasks do not promote a generated 5x solution as their new + baseline because that would change the task category. +- If equivalence cannot be established against a committed original kernel, the + baseline or case change is rejected. + +See `agents/quality_loop/README.md` and +`agents/quality_loop/agent_config.yaml` for the complete configuration contract. diff --git a/docs/sphinx/_toc.yml.in b/docs/sphinx/_toc.yml.in index 1551e328..f70ff031 100644 --- a/docs/sphinx/_toc.yml.in +++ b/docs/sphinx/_toc.yml.in @@ -33,6 +33,8 @@ subtrees: title: Add a task - file: how-to/task-validator title: Validate tasks + - file: how-to/quality-loop + title: Audit and harden tasks - file: how-to/held-out-evaluation title: Evaluate on held-out shapes - file: how-to/visualization diff --git a/example_configs/quality_loop_mi300.yaml b/example_configs/quality_loop_mi300.yaml new file mode 100644 index 00000000..0d8fe7ac --- /dev/null +++ b/example_configs/quality_loop_mi300.yaml @@ -0,0 +1,28 @@ +tasks: + - all +target_gpu_model: MI300 + +quality_loop: + backend: + name: codex + model: null + effort: xhigh + timeout_seconds: 3600 + reviewer: + name: codex + model: null + effort: xhigh + timeout_seconds: 1800 + max_repair_attempts: 1 + optimization_iterations: 1 + easy_speedup_threshold: 5.0 + easy_confirmation_runs: 3 + case_enhancement: true + artifact_root: quality_loop_runs + worktree_root: .quality_loop_worktrees + github: + publish: true + draft_pr: true + branch_prefix: quality-loop + base_branch: null + issue_labels: [] diff --git a/example_configs/quality_loop_mi355x.yaml b/example_configs/quality_loop_mi355x.yaml new file mode 100644 index 00000000..e398aece --- /dev/null +++ b/example_configs/quality_loop_mi355x.yaml @@ -0,0 +1,28 @@ +tasks: + - all +target_gpu_model: MI355X + +quality_loop: + backend: + name: codex + model: null + effort: xhigh + timeout_seconds: 3600 + reviewer: + name: codex + model: null + effort: xhigh + timeout_seconds: 1800 + max_repair_attempts: 1 + optimization_iterations: 1 + easy_speedup_threshold: 5.0 + easy_confirmation_runs: 3 + case_enhancement: true + artifact_root: quality_loop_runs + worktree_root: .quality_loop_worktrees + github: + publish: true + draft_pr: true + branch_prefix: quality-loop + base_branch: null + issue_labels: [] diff --git a/src/scripts/docker_benchmark.sh b/src/scripts/docker_benchmark.sh index c510cc77..7d4b8182 100755 --- a/src/scripts/docker_benchmark.sh +++ b/src/scripts/docker_benchmark.sh @@ -17,6 +17,10 @@ DEFAULT_RUN_CONFIG="example_configs/quickstart_claude_mi300.yaml" # separate from REQUIRED_AGENTS because geak_v4 is normalized to claude_code # before Docker arguments are built. GEAK_V4_RUNTIME=0 +# quality_loop keeps the repository checkout read-only in the agent container. +# Only these host-validated, run-specific subdirectories are over-mounted rw. +QUALITY_LOOP_ARTIFACT_REL="" +QUALITY_LOOP_WORKTREE_REL="" # /opt/venv/bin is placed before /usr/local/bin and /usr/bin so that a bare # `python3` / `pytest` resolves to the torch-enabled venv interpreter rather than @@ -32,6 +36,7 @@ Usage: src/scripts/docker_benchmark.sh preflight [--config_name ] src/scripts/docker_benchmark.sh shell src/scripts/docker_benchmark.sh check-agents [--config_name ] + src/scripts/docker_benchmark.sh quality-loop [--config ] [quality_loop args...] src/scripts/docker_benchmark.sh smoke Default run config: @@ -580,7 +585,21 @@ build_docker_args() { add_device_if_present /dev/dri add_device_if_present /dev/mem - add_mount "$HOST_ROOT" "$CONTAINER_WORKDIR" + if [[ -n "$QUALITY_LOOP_ARTIFACT_REL" || -n "$QUALITY_LOOP_WORKTREE_REL" ]]; then + [[ -n "$QUALITY_LOOP_ARTIFACT_REL" && -n "$QUALITY_LOOP_WORKTREE_REL" ]] \ + || die "quality_loop requires both artifact and worktree mount paths" + require_path "$HOST_ROOT/$QUALITY_LOOP_ARTIFACT_REL" "quality_loop artifact directory" + require_path "$HOST_ROOT/$QUALITY_LOOP_WORKTREE_REL" "quality_loop worktree" + add_mount "$HOST_ROOT" "$CONTAINER_WORKDIR" ro + add_mount \ + "$HOST_ROOT/$QUALITY_LOOP_ARTIFACT_REL" \ + "$CONTAINER_WORKDIR/$QUALITY_LOOP_ARTIFACT_REL" + add_mount \ + "$HOST_ROOT/$QUALITY_LOOP_WORKTREE_REL" \ + "$CONTAINER_WORKDIR/$QUALITY_LOOP_WORKTREE_REL" + else + add_mount "$HOST_ROOT" "$CONTAINER_WORKDIR" + fi # Persistent pip user-base (PYTHONUSERBASE) so `make docker-setup-flydsl` survives # across runs. It lives INSIDE the repo dir, which is already bind-mounted above and # is owned by the host user — this avoids a separate mount whose source the docker @@ -647,6 +666,46 @@ extract_config_name() { printf '%s\n' "$config" } +extract_quality_loop_config() { + local config="agents/quality_loop/agent_config.yaml" + local arg + while [[ $# -gt 0 ]]; do + arg="$1" + case "$arg" in + --config) + shift + [[ $# -gt 0 ]] || die "--config requires a value" + config="$1" + ;; + --config=*) + config="${arg#--config=}" + ;; + esac + shift || true + done + printf '%s\n' "$config" +} + +extract_quality_loop_resume() { + local arg + while [[ $# -gt 0 ]]; do + arg="$1" + case "$arg" in + --resume) + shift + [[ $# -gt 0 ]] || die "--resume requires a run ID" + printf '%s\n' "$1" + return + ;; + --resume=*) + printf '%s\n' "${arg#--resume=}" + return + ;; + esac + shift || true + done +} + container_smoke() { python - <<'PY' import importlib @@ -1079,6 +1138,42 @@ case "${1:-}" in shift run_parallel "$@" ;; + quality-loop) + shift + quality_loop_config="$(extract_quality_loop_config "$@")" + [[ -f "$quality_loop_config" ]] || die "quality_loop config file not found: $quality_loop_config" + if has_arg --plan "$@"; then + python3 -m agents.quality_loop "$@" + exit + fi + quality_loop_resume="$(extract_quality_loop_resume "$@" || true)" + if [[ -n "$quality_loop_resume" ]]; then + quality_loop_run_id="$(python3 -m agents.quality_loop.host check "$@")" + else + quality_loop_run_id="$(python3 -m agents.quality_loop.host start "$@")" + fi + echo "quality_loop run ID: $quality_loop_run_id" >&2 + mapfile -t quality_loop_paths < <( + python3 -m agents.quality_loop.host paths "$@" --run-id "$quality_loop_run_id" + ) + [[ "${#quality_loop_paths[@]}" -eq 2 ]] \ + || die "quality_loop host returned invalid runtime paths" + QUALITY_LOOP_ARTIFACT_REL="${quality_loop_paths[0]}" + QUALITY_LOOP_WORKTREE_REL="${quality_loop_paths[1]}" + select_runtime_for_config "$quality_loop_config" + REQUIRED_AGENTS="codex" + AGENTS_STRICT=1 + AGENT_HOME_ISOLATION=1 + AKA_CONTAINER_HOME="/tmp/aka-quality-loop-${quality_loop_run_id}" + AKA_CACHE_SUFFIX="quality-loop-${quality_loop_run_id}" + quality_loop_container_args=("$@") + if [[ -z "$quality_loop_resume" ]]; then + quality_loop_container_args+=(--resume "$quality_loop_run_id") + fi + quality_loop_container_args+=(--defer-github --skip-preflight) + docker_exec 0 python3 -m agents.quality_loop "${quality_loop_container_args[@]}" + python3 -m agents.quality_loop.host finalize "$@" --run-id "$quality_loop_run_id" + ;; preflight) shift config_name="$(extract_config_name "$@")" diff --git a/tests/test_docker_benchmark.sh b/tests/test_docker_benchmark.sh index 95eeaf1b..97e33f23 100755 --- a/tests/test_docker_benchmark.sh +++ b/tests/test_docker_benchmark.sh @@ -72,7 +72,7 @@ assert_cache_args_absent() { } TEST_HOME="$(mktemp -d)" -trap 'rm -rf "$TEST_HOME"' EXIT +trap 'rm -rf "$TEST_HOME" "$ROOT/quality_loop_runs/test-run" "$ROOT/.quality_loop_worktrees/test-run"' EXIT UNRELATED_GEAK_WORKFLOW_DIR="$TEST_HOME/unrelated-geak-workflow" GEAK_SDK_PYTHONPATH="PYTHONPATH=/workspace/.aka-pyuserbase/geak-sdk" mkdir -p "$UNRELATED_GEAK_WORKFLOW_DIR" @@ -148,6 +148,43 @@ assert_not_has "$GEAK_SDK_PYTHONPATH" "${args[@]}" assert_not_has "$UNRELATED_GEAK_WORKFLOW_DIR:$UNRELATED_GEAK_WORKFLOW_DIR:ro" "${args[@]}" assert_not_has "GEAK_V4_WORKFLOW_DIR=$UNRELATED_GEAK_WORKFLOW_DIR" "${args[@]}" +# quality_loop provisions only isolated Codex state, never GitHub CLI state, and +# mounts the main checkout read-only while over-mounting only this run's state rw. +QUALITY_HOME="$TEST_HOME/quality-home" +QUALITY_PREFIX="$TEST_HOME/quality-node" +QUALITY_BIN="$TEST_HOME/quality-bin" +QUALITY_CONFIG="$TEST_HOME/quality-loop.yaml" +mkdir -p "$QUALITY_HOME/.codex" "$QUALITY_HOME/.config/gh" "$QUALITY_PREFIX/bin" "$QUALITY_BIN" +touch "$QUALITY_PREFIX/bin/node" "$QUALITY_PREFIX/bin/codex" +printf '#!/usr/bin/env bash\nexit 0\n' > "$QUALITY_BIN/gh" +chmod +x "$QUALITY_BIN/gh" +printf '#!/usr/bin/env bash\ncase "$*" in *"agents.quality_loop.host start"*) echo test-run;; *"agents.quality_loop.host paths"*) printf "quality_loop_runs/test-run\\n.quality_loop_worktrees/test-run\\n";; *"agents.quality_loop.host finalize"*) echo test-pr;; esac\n' > "$QUALITY_BIN/python3" +chmod +x "$QUALITY_BIN/python3" +printf 'tasks:\n - hip2hip/gpumode/GELU\ntarget_gpu_model: MI300\nquality_loop: {}\n' > "$QUALITY_CONFIG" +mkdir -p "$ROOT/quality_loop_runs/test-run" "$ROOT/.quality_loop_worktrees/test-run" + +mapfile -t args < <( + env \ + HOME="$QUALITY_HOME" \ + PATH="$QUALITY_BIN:$PATH" \ + AKA_NODE_PREFIX="$QUALITY_PREFIX" \ + bash "$RUNNER" quality-loop --config "$QUALITY_CONFIG" 2>/dev/null +) +assert_has "$QUALITY_PREFIX:/opt/node:ro" "${args[@]}" +assert_has "$QUALITY_HOME/.codex:/opt/aka-agent-state/.codex:ro" "${args[@]}" +assert_has "$ROOT:/workspace:ro" "${args[@]}" +assert_has "$ROOT/quality_loop_runs/test-run:/workspace/quality_loop_runs/test-run" "${args[@]}" +assert_has "$ROOT/.quality_loop_worktrees/test-run:/workspace/.quality_loop_worktrees/test-run" "${args[@]}" +assert_not_has "$QUALITY_BIN/gh:$QUALITY_BIN/gh:ro" "${args[@]}" +assert_not_has "$QUALITY_HOME/.config/gh:$QUALITY_HOME/.config/gh:ro" "${args[@]}" +assert_has "python3" "${args[@]}" +assert_has "agents.quality_loop" "${args[@]}" +assert_has "--resume" "${args[@]}" +assert_has "test-run" "${args[@]}" +assert_has "--defer-github" "${args[@]}" +assert_has "--skip-preflight" "${args[@]}" +rm -rf "$ROOT/quality_loop_runs/test-run" "$ROOT/.quality_loop_worktrees/test-run" + # A Codex-only config likewise receives neither Claude credentials nor GEAK's # dependency path/mount, even when both are configured on the host. CODEX_HOME="$TEST_HOME/codex-home" diff --git a/tests/test_quality_loop.py b/tests/test_quality_loop.py new file mode 100644 index 00000000..c1cbeff8 --- /dev/null +++ b/tests/test_quality_loop.py @@ -0,0 +1,630 @@ +import json +import logging +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +from agents.quality_loop.config import QualityLoopConfig +from agents.quality_loop.filesystem import ( + apply_changes, + diff_trees, + is_case_path, + snapshot_tree, +) +from agents.quality_loop.github import CommandError, GitHubPublisher, parse_github_slug +from agents.quality_loop.orchestrator import QualityLoop, difficulty_is_easy +from agents.quality_loop.state import AuditState, resolve_worktree, validate_run_id + + +LOGGER = logging.getLogger("quality-loop-test") + + +class FakePublisher: + def __init__(self): + self.issues = [] + self.commits = [] + + def commit_task(self, worktree, task_id): + self.commits.append(task_id) + return "abc123" + + def ensure_issue(self, **kwargs): + self.issues.append(kwargs) + return "https://github.com/AMD-AGI/AgentKernelArena/issues/999" + + +class RepairBackend: + def __init__(self, change_on_repair=True): + self.roles = [] + self.change_on_repair = change_on_repair + + def run(self, prompt, workspace, *, role): + self.roles.append(role) + if role == "repair" and self.change_on_repair: + (workspace / "kernel.py").write_text("def kernel():\n return 1\n") + return "ok" + + +class TamperingReviewerBackend: + def run(self, prompt, workspace, *, role): + (workspace / "task_result.yaml").write_text("pass_correctness: false\n") + (workspace / "quality_loop_review.yaml").write_text( + yaml.safe_dump( + { + "accepted": True, + "logic_equivalent": True, + "evidence_sufficient": True, + "case_enhancement_needed": False, + "case_rationale": "none", + "summary": "accepted", + } + ) + ) + return "tampered" + + +class StubQualityLoop(QualityLoop): + def __init__(self, *args, reports, **kwargs): + super().__init__(*args, **kwargs) + self.reports = list(reports) + + def _validate(self, task_id, task_dir, stage_dir): + stage_dir.mkdir(parents=True, exist_ok=True) + workspace = stage_dir / "workspace" + if workspace.exists(): + self._reset_path(workspace) + workspace.mkdir() + for path in task_dir.iterdir(): + if path.is_file(): + (workspace / path.name).write_bytes(path.read_bytes()) + report = self.reports.pop(0) + (workspace / "validation_report.yaml").write_text(yaml.safe_dump(report)) + return workspace, report + + def _optimize_once(self, task_id, task_dir, stage_dir): + stage_dir.mkdir(parents=True, exist_ok=True) + workspace = stage_dir / "workspace" + workspace.mkdir() + for path in task_dir.iterdir(): + if path.is_file(): + (workspace / path.name).write_bytes(path.read_bytes()) + result = { + "task_name": task_id, + "pass_compilation": True, + "pass_correctness": True, + "speedup_ratio": 2.0, + "benchmark_method_consistent": True, + "valid_baseline_cases": 1, + "valid_optimized_cases": 1, + } + (workspace / "task_result.yaml").write_text(yaml.safe_dump(result)) + return workspace, [], result + + def _review(self, task_id, workspace, result): + return { + "accepted": True, + "logic_equivalent": True, + "evidence_sufficient": True, + "case_enhancement_needed": False, + "case_rationale": "coverage is sufficient", + "summary": "accepted", + } + + +def make_task(root: Path) -> Path: + task = root / "tasks" / "hip2hip" / "sample" + task.mkdir(parents=True) + (task / "config.yaml").write_text( + yaml.safe_dump( + { + "task_type": "hip2hip", + "source_file_path": ["kernel.py"], + "target_kernel_functions": ["kernel"], + "compile_command": ["true"], + "correctness_command": ["true"], + "performance_command": ["true"], + } + ) + ) + (task / "kernel.py").write_text("def kernel():\n raise RuntimeError('broken')\n") + return task + + +def attach_state(workflow: QualityLoop, root: Path, worktree: Path) -> None: + workflow.artifact_dir = root / "artifacts" / "run" + workflow.worktree = worktree + workflow.state = AuditState.create( + workflow.artifact_dir / "state.yaml", + run_id="run", + config_fingerprint="fingerprint", + repo_slug="AMD-AGI/AgentKernelArena", + base_sha="base", + base_branch="main", + branch="quality-loop/run", + worktree=worktree, + ) + + +class QualityLoopTests(unittest.TestCase): + def test_config_defaults_to_codex_and_exactly_one_iteration(self): + config = QualityLoopConfig.from_dict( + {"tasks": ["all"], "target_gpu_model": "MI300", "quality_loop": {}} + ) + self.assertEqual(config.backend.name, "codex") + self.assertEqual(config.reviewer.name, "codex") + self.assertEqual(config.optimization_iterations, 1) + self.assertTrue(config.github.draft_pr) + + with self.assertRaisesRegex(ValueError, "exactly one"): + QualityLoopConfig.from_dict( + { + "tasks": ["all"], + "target_gpu_model": "MI300", + "quality_loop": {"optimization_iterations": 2}, + } + ) + + with self.assertRaisesRegex(ValueError, "repository-relative"): + QualityLoopConfig.from_dict( + { + "target_gpu_model": "MI300", + "quality_loop": {"artifact_root": "../outside"}, + } + ) + + def test_run_ids_and_relative_worktree_paths_are_container_portable(self): + with self.assertRaisesRegex(ValueError, "run ID"): + validate_run_id("../../escape") + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + expected = root / ".quality_loop_worktrees" / "run" + self.assertEqual( + resolve_worktree(root, ".quality_loop_worktrees/run"), + expected.resolve(), + ) + + def test_parse_github_slug(self): + for remote in ( + "git@github.com:AMD-AGI/AgentKernelArena.git", + "https://github.com/AMD-AGI/AgentKernelArena.git", + "ssh://git@github.com/AMD-AGI/AgentKernelArena.git", + ): + with self.subTest(remote=remote): + self.assertEqual(parse_github_slug(remote), "AMD-AGI/AgentKernelArena") + + def test_preflight_rejects_missing_write_permission(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + publisher = GitHubPublisher(root, config.github, LOGGER) + + def fake_run(args, **kwargs): + if args[:3] == ["git", "status", "--porcelain"]: + stdout = "" + elif args[:3] == ["git", "remote", "get-url"]: + stdout = "git@github.com:AMD-AGI/AgentKernelArena.git\n" + elif args[:2] == ["gh", "api"]: + stdout = json.dumps( + { + "permissions": {"push": False}, + "viewer_permission": "READ", + "has_issues": True, + "default_branch": "main", + } + ) + else: + stdout = "" + return subprocess.CompletedProcess(args, 0, stdout=stdout, stderr="") + + with mock.patch( + "agents.quality_loop.github.shutil.which", return_value="/bin/tool" + ), mock.patch( + "agents.quality_loop.github.run_command", side_effect=fake_run + ): + with self.assertRaisesRegex(RuntimeError, "lacks write permission"): + publisher.preflight() + + def test_preflight_stops_on_gh_auth_failure(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + publisher = GitHubPublisher(root, config.github, LOGGER) + + def fake_run(args, **kwargs): + if args[:3] == ["git", "status", "--porcelain"]: + return subprocess.CompletedProcess(args, 0, stdout="", stderr="") + if args[:3] == ["git", "var", "GIT_AUTHOR_IDENT"]: + return subprocess.CompletedProcess( + args, 0, stdout="Quality Loop \n", stderr="" + ) + if args[:3] == ["gh", "auth", "status"]: + raise CommandError("not logged in") + raise AssertionError(f"preflight continued after auth failure: {args}") + + with mock.patch( + "agents.quality_loop.github.shutil.which", return_value="/bin/tool" + ), mock.patch( + "agents.quality_loop.github.run_command", side_effect=fake_run + ): + with self.assertRaisesRegex(CommandError, "not logged in"): + publisher.preflight() + + def test_isolated_worktree_can_live_under_ignored_runtime_root(self): + with tempfile.TemporaryDirectory() as tmp: + parent = Path(tmp) + remote = parent / "remote.git" + subprocess.run(["git", "init", "--bare", str(remote)], check=True, capture_output=True) + root = parent / "repo" + subprocess.run(["git", "clone", str(remote), str(root)], check=True, capture_output=True) + subprocess.run(["git", "switch", "-c", "main"], cwd=root, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "quality-loop@example.invalid"], + cwd=root, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "quality_loop test"], cwd=root, check=True + ) + (root / ".gitignore").write_text(".quality_loop_worktrees/\n") + (root / "README.md").write_text("test\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run(["git", "commit", "-m", "base"], cwd=root, check=True, capture_output=True) + subprocess.run( + ["git", "push", "--set-upstream", "origin", "main"], + cwd=root, + check=True, + capture_output=True, + ) + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + publisher = GitHubPublisher(root, config.github, LOGGER) + worktree = root / ".quality_loop_worktrees" / "run" + publisher.create_worktree(path=worktree, branch="quality-loop/run", base_branch="main") + self.assertTrue((worktree / "README.md").is_file()) + self.assertEqual( + subprocess.run( + ["git", "status", "--porcelain"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout, + "", + ) + + def test_tree_diff_and_apply_are_task_local(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + source = root / "source" + destination = root / "destination" + source.mkdir() + destination.mkdir() + (source / "kernel.py").write_text("old\n") + (destination / "kernel.py").write_text("old\n") + before = snapshot_tree(source) + (source / "kernel.py").write_text("new\n") + (source / "test_kernel.py").write_text("case\n") + changes = diff_trees(before, snapshot_tree(source)) + apply_changes(source, destination, changes) + self.assertEqual((destination / "kernel.py").read_text(), "new\n") + self.assertEqual((destination / "test_kernel.py").read_text(), "case\n") + self.assertTrue(is_case_path("test_kernel.py")) + self.assertFalse(is_case_path("kernel.py")) + + def test_pending_diff_verification_rejects_unrecorded_paths(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + subprocess.run(["git", "init"], cwd=root, check=True, capture_output=True) + subprocess.run( + ["git", "config", "user.email", "quality-loop@example.invalid"], + cwd=root, + check=True, + ) + subprocess.run( + ["git", "config", "user.name", "quality_loop test"], + cwd=root, + check=True, + ) + (root / "README.md").write_text("base\n") + subprocess.run(["git", "add", "."], cwd=root, check=True) + subprocess.run( + ["git", "commit", "-m", "base"], cwd=root, check=True, capture_output=True + ) + branch = subprocess.run( + ["git", "branch", "--show-current"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + base_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=root, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + (root / "README.md").write_text("unexpected\n") + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + publisher = GitHubPublisher(root, config.github, LOGGER) + with self.assertRaisesRegex(RuntimeError, "does not match"): + publisher.verify_pending_changes( + worktree=root, + branch=branch, + base_sha=base_sha, + expected_paths=set(), + ) + + def test_reviewer_cannot_modify_evaluation_evidence(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + workspace = root / "workspace" + workspace.mkdir() + (workspace / "task_result.yaml").write_text( + yaml.safe_dump( + { + "pass_compilation": True, + "pass_correctness": True, + "benchmark_method_consistent": True, + } + ) + ) + original = workspace / ".quality_loop_original_sources" + original.mkdir() + (original / "kernel.py").write_text("def kernel():\n return 1\n") + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + workflow = QualityLoop( + root, + config, + logger=LOGGER, + reviewer_backend=TamperingReviewerBackend(), + ) + with self.assertRaisesRegex(RuntimeError, "protected evaluation evidence"): + workflow._review( + "hip2hip/sample", + workspace, + { + "pass_compilation": True, + "pass_correctness": True, + "benchmark_method_consistent": True, + }, + ) + + def test_repair_then_revalidate_commits_only_successful_task_change(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + worktree = root / "worktree" + task = make_task(worktree) + backend = RepairBackend(change_on_repair=True) + publisher = FakePublisher() + config = QualityLoopConfig.from_dict( + { + "tasks": ["hip2hip/sample"], + "target_gpu_model": "MI300", + "quality_loop": {}, + } + ) + workflow = StubQualityLoop( + root, + config, + logger=LOGGER, + backend=backend, + reviewer_backend=backend, + publisher=publisher, + reports=[ + {"overall_status": "FAIL", "checks": {}, "summary": "broken"}, + {"overall_status": "PASS", "checks": {}, "summary": "fixed"}, + ], + ) + attach_state(workflow, root, worktree) + workflow._process_task("hip2hip/sample", task) + + self.assertIn("return 1", (task / "kernel.py").read_text()) + self.assertEqual(workflow.state.task("hip2hip/sample")["state"], "completed") + self.assertEqual(backend.roles, ["repair"]) + self.assertEqual(publisher.commits, ["hip2hip/sample"]) + self.assertEqual(publisher.issues, []) + + def test_unrepairable_failure_files_issue_without_touching_task(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + worktree = root / "worktree" + task = make_task(worktree) + original = (task / "kernel.py").read_text() + backend = RepairBackend(change_on_repair=False) + publisher = FakePublisher() + config = QualityLoopConfig.from_dict( + { + "tasks": ["hip2hip/sample"], + "target_gpu_model": "MI300", + "quality_loop": {}, + } + ) + workflow = StubQualityLoop( + root, + config, + logger=LOGGER, + backend=backend, + reviewer_backend=backend, + publisher=publisher, + reports=[{"overall_status": "FAIL", "checks": {}, "summary": "broken"}], + ) + attach_state(workflow, root, worktree) + workflow._process_task("hip2hip/sample", task) + + record = workflow.state.task("hip2hip/sample") + self.assertEqual(record["state"], "issue_filed") + self.assertTrue(record["issue_url"].endswith("/999")) + self.assertEqual((task / "kernel.py").read_text(), original) + self.assertEqual(len(publisher.issues), 1) + self.assertEqual(publisher.commits, []) + + def test_container_defers_issue_request_without_github_credentials(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + worktree = root / "worktree" + task = make_task(worktree) + backend = RepairBackend(change_on_repair=False) + publisher = FakePublisher() + config = QualityLoopConfig.from_dict( + { + "tasks": ["hip2hip/sample"], + "target_gpu_model": "MI300", + "quality_loop": {}, + } + ) + workflow = StubQualityLoop( + root, + config, + logger=LOGGER, + backend=backend, + reviewer_backend=backend, + publisher=publisher, + defer_github=True, + reports=[{"overall_status": "FAIL", "checks": {}, "summary": "broken"}], + ) + attach_state(workflow, root, worktree) + workflow._process_task("hip2hip/sample", task) + + record = workflow.state.task("hip2hip/sample") + self.assertEqual(record["state"], "issue_pending") + self.assertEqual(record["issue_url"], None) + self.assertEqual(record["issue_request"]["task_id"], "hip2hip/sample") + self.assertEqual(publisher.issues, []) + + def test_container_defers_task_commit_to_host_finalizer(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + worktree = root / "worktree" + task = make_task(worktree) + backend = RepairBackend(change_on_repair=True) + publisher = FakePublisher() + config = QualityLoopConfig.from_dict( + { + "tasks": ["hip2hip/sample"], + "target_gpu_model": "MI300", + "quality_loop": {}, + } + ) + workflow = StubQualityLoop( + root, + config, + logger=LOGGER, + backend=backend, + reviewer_backend=backend, + publisher=publisher, + defer_github=True, + reports=[ + {"overall_status": "FAIL", "checks": {}, "summary": "broken"}, + {"overall_status": "PASS", "checks": {}, "summary": "fixed"}, + ], + ) + attach_state(workflow, root, worktree) + workflow._process_task("hip2hip/sample", task) + + record = workflow.state.task("hip2hip/sample") + self.assertTrue(record["commit_pending"]) + self.assertIsNone(record["commit"]) + self.assertEqual(publisher.commits, []) + + def test_warn_is_reported_without_repair(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + worktree = root / "worktree" + task = make_task(worktree) + backend = RepairBackend() + publisher = FakePublisher() + config = QualityLoopConfig.from_dict( + { + "tasks": ["hip2hip/sample"], + "target_gpu_model": "MI300", + "quality_loop": {}, + } + ) + workflow = StubQualityLoop( + root, + config, + logger=LOGGER, + backend=backend, + reviewer_backend=backend, + publisher=publisher, + reports=[ + { + "overall_status": "WARN", + "checks": { + "performance": {"status": "WARN", "details": "too few repeats"} + }, + "summary": "warning", + } + ], + ) + attach_state(workflow, root, worktree) + workflow._process_task("hip2hip/sample", task) + + record = workflow.state.task("hip2hip/sample") + self.assertEqual(record["state"], "completed") + self.assertEqual(record["warnings"], ["performance: too few repeats"]) + self.assertNotIn("repair", backend.roles) + self.assertEqual(publisher.issues, []) + + def test_easy_gate_is_fail_closed_and_not_used_for_authoring_tasks(self): + config = QualityLoopConfig.from_dict( + {"target_gpu_model": "MI300", "quality_loop": {}} + ) + result = { + "pass_compilation": True, + "pass_correctness": True, + "benchmark_method_consistent": True, + "valid_baseline_cases": 3, + "valid_optimized_cases": 3, + } + review = { + "accepted": True, + "logic_equivalent": True, + "evidence_sufficient": True, + } + self.assertTrue( + difficulty_is_easy( + task_type="hip2hip", + speedups=[5.1, 5.0, 6.0], + result=result, + review=review, + config=config, + ) + ) + self.assertFalse( + difficulty_is_easy( + task_type="torch2hip", + speedups=[10.0, 10.0, 10.0], + result=result, + review=review, + config=config, + ) + ) + result["benchmark_method_consistent"] = False + self.assertFalse( + difficulty_is_easy( + task_type="hip2hip", + speedups=[6.0, 6.0, 6.0], + result=result, + review=review, + config=config, + ) + ) + + +if __name__ == "__main__": + unittest.main() From 844761ccb45b60661a8eb7933dd3ea888f093664 Mon Sep 17 00:00:00 2001 From: Vincent Ouyang Date: Tue, 4 Aug 2026 01:45:02 +0000 Subject: [PATCH 2/7] feat(eval): add isolated ROCm sanitizer plugins --- .dockerignore | 13 + .gitignore | 1 + agents/quality_loop/config.py | 5 + agents/quality_loop/orchestrator.py | 53 +- docker/eval-tools/gpu-asan/Dockerfile | 30 + docker/eval-tools/gpu-asan/packages.sha256 | 4 + docker/eval-tools/hip-fpsan/Dockerfile | 31 + docker/eval-tools/hip-fpsan/sources.lock.yaml | 1 + docker/eval-tools/images.lock.yaml | 53 + docker/eval-tools/rocjitsu/Dockerfile | 70 ++ docker/eval-tools/rocjitsu/rocjitsu-wrapper | 12 + docker/eval-tools/rocjitsu/sources.lock.yaml | 4 + docker/eval-tools/triton-fpsan/Dockerfile | 21 + .../eval-tools/triton-fpsan/requirements.lock | 4 + docs/README.md | 1 + docs/how-to/run-evaluation.md | 8 +- docs/how-to/use-evaluation-tools.md | 899 ++++++++++++++ docs/index.rst | 1 + docs/reference/api-reference.md | 160 +++ docs/reference/compatibility-matrix.md | 23 + docs/reference/release-notes.md | 36 + docs/sphinx/_toc.yml.in | 2 + .../evaluation_tools_advisory_mi355x.yaml | 55 + main.py | 73 ++ src/eval_tools/__init__.py | 96 ++ src/eval_tools/__main__.py | 54 + src/eval_tools/adapters/__init__.py | 57 + src/eval_tools/adapters/flydsl_aot.py | 152 +++ src/eval_tools/adapters/hip_source.py | 61 + src/eval_tools/adapters/native_launcher.py | 241 ++++ src/eval_tools/adapters/replay_capsule.py | 538 +++++++++ src/eval_tools/adapters/rocjitsu_replay.py | 177 +++ .../adapters/rocjitsu_replay_entrypoint.py | 36 + src/eval_tools/adapters/triton_aot.py | 155 +++ src/eval_tools/config.py | 394 ++++++ src/eval_tools/contracts.py | 829 +++++++++++++ src/eval_tools/evidence.py | 251 ++++ src/eval_tools/execution.py | 315 +++++ src/eval_tools/factory.py | 48 + src/eval_tools/manager.py | 446 +++++++ src/eval_tools/plugins/__init__.py | 63 + src/eval_tools/plugins/attestation.py | 194 +++ src/eval_tools/plugins/base.py | 254 ++++ src/eval_tools/plugins/gpu_asan.py | 212 ++++ src/eval_tools/plugins/hip_fpsan.py | 133 ++ src/eval_tools/plugins/parsers.py | 216 ++++ src/eval_tools/plugins/registry.py | 54 + src/eval_tools/plugins/rocjitsu.py | 330 +++++ src/eval_tools/plugins/triton_fpsan.py | 123 ++ src/eval_tools/probes/__init__.py | 7 + src/eval_tools/probes/gpu_asan_probe.hip | 30 + src/eval_tools/probes/hip_fpsan_probe.hip | 44 + src/eval_tools/probes/rocjitsu_race_probe.hip | 35 + src/eval_tools/probes/triton_asan_probe.py | 34 + src/eval_tools/probes/triton_fpsan_probe.py | 69 ++ src/eval_tools/registry.py | 96 ++ src/eval_tools/reporting.py | 142 +++ src/eval_tools/runtime_client.py | 647 ++++++++++ src/eval_tools/task_profile.py | 411 +++++++ src/eval_tools/worker.py | 1066 +++++++++++++++++ src/evaluator.py | 72 +- src/scripts/docker_benchmark.sh | 485 +++++++- tests/eval_tools/test_attestation.py | 94 ++ tests/eval_tools/test_capabilities.py | 119 ++ tests/eval_tools/test_config_merge.py | 156 +++ tests/eval_tools/test_contracts.py | 161 +++ tests/eval_tools/test_evidence.py | 171 +++ tests/eval_tools/test_execution.py | 110 ++ tests/eval_tools/test_factory.py | 27 + tests/eval_tools/test_manager.py | 407 +++++++ tests/eval_tools/test_parsers.py | 61 + tests/eval_tools/test_plugins.py | 548 +++++++++ tests/eval_tools/test_replay_capsule.py | 266 ++++ tests/eval_tools/test_reporting.py | 157 +++ tests/eval_tools/test_runtime_client.py | 314 +++++ tests/eval_tools/test_task_profile.py | 166 +++ tests/eval_tools/test_worker.py | 193 +++ tests/gpu/eval_tools/aot_sidecar_smoke.py | 90 ++ tests/gpu/eval_tools/candidate_harness.py | 148 +++ .../gpu/eval_tools/candidate_sidecar_smoke.py | 95 ++ tests/gpu/eval_tools/test_integration.py | 54 + tests/test_docker_benchmark.sh | 245 +++- 82 files changed, 13698 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 docker/eval-tools/gpu-asan/Dockerfile create mode 100644 docker/eval-tools/gpu-asan/packages.sha256 create mode 100644 docker/eval-tools/hip-fpsan/Dockerfile create mode 100644 docker/eval-tools/hip-fpsan/sources.lock.yaml create mode 100644 docker/eval-tools/images.lock.yaml create mode 100644 docker/eval-tools/rocjitsu/Dockerfile create mode 100755 docker/eval-tools/rocjitsu/rocjitsu-wrapper create mode 100644 docker/eval-tools/rocjitsu/sources.lock.yaml create mode 100644 docker/eval-tools/triton-fpsan/Dockerfile create mode 100644 docker/eval-tools/triton-fpsan/requirements.lock create mode 100644 docs/how-to/use-evaluation-tools.md create mode 100644 example_configs/evaluation_tools_advisory_mi355x.yaml create mode 100644 src/eval_tools/__init__.py create mode 100644 src/eval_tools/__main__.py create mode 100644 src/eval_tools/adapters/__init__.py create mode 100644 src/eval_tools/adapters/flydsl_aot.py create mode 100644 src/eval_tools/adapters/hip_source.py create mode 100644 src/eval_tools/adapters/native_launcher.py create mode 100644 src/eval_tools/adapters/replay_capsule.py create mode 100644 src/eval_tools/adapters/rocjitsu_replay.py create mode 100644 src/eval_tools/adapters/rocjitsu_replay_entrypoint.py create mode 100644 src/eval_tools/adapters/triton_aot.py create mode 100644 src/eval_tools/config.py create mode 100644 src/eval_tools/contracts.py create mode 100644 src/eval_tools/evidence.py create mode 100644 src/eval_tools/execution.py create mode 100644 src/eval_tools/factory.py create mode 100644 src/eval_tools/manager.py create mode 100644 src/eval_tools/plugins/__init__.py create mode 100644 src/eval_tools/plugins/attestation.py create mode 100644 src/eval_tools/plugins/base.py create mode 100644 src/eval_tools/plugins/gpu_asan.py create mode 100644 src/eval_tools/plugins/hip_fpsan.py create mode 100644 src/eval_tools/plugins/parsers.py create mode 100644 src/eval_tools/plugins/registry.py create mode 100644 src/eval_tools/plugins/rocjitsu.py create mode 100644 src/eval_tools/plugins/triton_fpsan.py create mode 100644 src/eval_tools/probes/__init__.py create mode 100644 src/eval_tools/probes/gpu_asan_probe.hip create mode 100644 src/eval_tools/probes/hip_fpsan_probe.hip create mode 100644 src/eval_tools/probes/rocjitsu_race_probe.hip create mode 100644 src/eval_tools/probes/triton_asan_probe.py create mode 100644 src/eval_tools/probes/triton_fpsan_probe.py create mode 100644 src/eval_tools/registry.py create mode 100644 src/eval_tools/reporting.py create mode 100644 src/eval_tools/runtime_client.py create mode 100644 src/eval_tools/task_profile.py create mode 100644 src/eval_tools/worker.py create mode 100644 tests/eval_tools/test_attestation.py create mode 100644 tests/eval_tools/test_capabilities.py create mode 100644 tests/eval_tools/test_config_merge.py create mode 100644 tests/eval_tools/test_contracts.py create mode 100644 tests/eval_tools/test_evidence.py create mode 100644 tests/eval_tools/test_execution.py create mode 100644 tests/eval_tools/test_factory.py create mode 100644 tests/eval_tools/test_manager.py create mode 100644 tests/eval_tools/test_parsers.py create mode 100644 tests/eval_tools/test_plugins.py create mode 100644 tests/eval_tools/test_replay_capsule.py create mode 100644 tests/eval_tools/test_reporting.py create mode 100644 tests/eval_tools/test_runtime_client.py create mode 100644 tests/eval_tools/test_task_profile.py create mode 100644 tests/eval_tools/test_worker.py create mode 100644 tests/gpu/eval_tools/aot_sidecar_smoke.py create mode 100644 tests/gpu/eval_tools/candidate_harness.py create mode 100644 tests/gpu/eval_tools/candidate_sidecar_smoke.py create mode 100644 tests/gpu/eval_tools/test_integration.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b2c01fa9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Evaluation-tool images need only their locked Docker assets and the immutable +# eval-tools framework. The repository commonly contains multi-gigabyte task +# workspaces under experiments/, so an allow-list keeps builds fast and prevents +# candidate/run artifacts from entering the Docker build context. +** +!docker/ +!docker/eval-tools/ +!docker/eval-tools/** +!src/ +!src/eval_tools/ +!src/eval_tools/** +**/__pycache__/ +**/*.py[cod] diff --git a/.gitignore b/.gitignore index b46c952f..0e3b8365 100755 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ build/ .claude workspace_* /experiments/ +/.eval-tool-artifacts/ # Auto-cloned agent dependencies agents/geak_optimagentv2/GEAK-agent/ diff --git a/agents/quality_loop/config.py b/agents/quality_loop/config.py index ed0f2313..ad92e2b3 100644 --- a/agents/quality_loop/config.py +++ b/agents/quality_loop/config.py @@ -81,6 +81,7 @@ class QualityLoopConfig: case_enhancement: bool = True artifact_root: str = "quality_loop_runs" worktree_root: str = ".quality_loop_worktrees" + evaluation_tools: dict[str, Any] = field(default_factory=dict) promotion_task_types: tuple[str, ...] = ( "hip2hip", "triton2triton", @@ -121,6 +122,9 @@ def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig": "promotion_task_types", ["hip2hip", "triton2triton", "flydsl2flydsl"], ) + evaluation_tools = raw.get("evaluation_tools") or {} + if not isinstance(evaluation_tools, dict): + raise ValueError("evaluation_tools must be a mapping") return cls( tasks=tuple(str(task) for task in tasks), target_gpu_model=target, @@ -140,6 +144,7 @@ def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig": audit.get("worktree_root", ".quality_loop_worktrees"), "quality_loop.worktree_root", ), + evaluation_tools=dict(evaluation_tools), promotion_task_types=tuple(str(value) for value in promotion_types), ) diff --git a/agents/quality_loop/orchestrator.py b/agents/quality_loop/orchestrator.py index 239a6a11..74678a61 100644 --- a/agents/quality_loop/orchestrator.py +++ b/agents/quality_loop/orchestrator.py @@ -48,6 +48,9 @@ from src.preprocessing import _resolve_gfx_arch, setup_workspace from src.prompt_builder import prompt_builder from src.testcases import collect_benchmark_methods +from src.eval_tools.config import EvalToolsConfig +from src.eval_tools.contracts import SourceEvidence +from src.eval_tools.evidence import capture_submission_evidence def _task_slug(task_id: str) -> str: @@ -130,6 +133,8 @@ def difficulty_is_easy( and statistics.median(speedups) >= config.easy_speedup_threshold and result.get("pass_compilation") is True and result.get("pass_correctness") is True + and result.get("pass_tool_gate", True) is True + and result.get("tool_policy_satisfied", True) is True and result.get("benchmark_method_consistent") is True and int(result.get("valid_baseline_cases", 0)) > 0 and result.get("valid_baseline_cases") == result.get("valid_optimized_cases") @@ -499,6 +504,14 @@ def _optimize_once( ) -> tuple[Path, list[Any], dict[str, Any]]: workspace = self._make_workspace(task_id, task_dir, stage_dir) task_config = self._load_task_config(task_dir) + eval_tools_config = EvalToolsConfig.from_mapping(self._eval_config()) + submission_evidence = None + if eval_tools_config.enabled: + submission_evidence = capture_submission_evidence( + workspace, + task_config, + stage_dir / "submission_evidence", + ) original_sources = stage_dir / "original_sources" original_sources.mkdir() source_manifest: dict[str, str] = {} @@ -538,7 +551,40 @@ def _optimize_once( if snapshot_tree(original_sources) != original_source_tree: raise RuntimeError("optimizer modified the protected original-source snapshot") materialize_perf_helpers_in_workspace(workspace, logger=self.logger) - evaluation = evaluate_kernel(workspace, task_config, baseline_cases, self.logger) + tool_manager = None + tool_source_evidence = None + if eval_tools_config.enabled: + assert submission_evidence is not None + submission_evidence.verify() + tool_source_evidence = SourceEvidence( + original_root=str(submission_evidence.files_dir), + original_fingerprint=submission_evidence.fingerprint, + candidate_fingerprint=submission_evidence.candidate_fingerprint(), + metadata={ + "manifest": str(submission_evidence.storage_dir / "manifest.json"), + "quality_loop_task": task_id, + }, + ) + from src.eval_tools.factory import ( + create_default_manager, + task_artifact_root, + ) + + tool_manager = create_default_manager() + tool_report_root = task_artifact_root(workspace) + else: + tool_report_root = None + evaluation = evaluate_kernel( + workspace, + task_config, + baseline_cases, + self.logger, + tool_manager=tool_manager, + eval_tools_config=eval_tools_config, + tool_source_evidence=tool_source_evidence, + tool_artifact_root=tool_report_root, + gpu_arch=_resolve_gfx_arch(self.config.target_gpu_model), + ) write_task_result( workspace, evaluation, @@ -809,7 +855,7 @@ def _make_workspace(self, task_id: str, task_dir: Path, stage_dir: Path) -> Path ) def _eval_config(self) -> dict[str, Any]: - return { + result = { "target_gpu_model": self.config.target_gpu_model, "agent": { "template": "codex", @@ -820,6 +866,9 @@ def _eval_config(self) -> dict[str, Any]: "max_iterations": 1, }, } + if self.config.evaluation_tools: + result["evaluation_tools"] = dict(self.config.evaluation_tools) + return result @staticmethod def _load_task_config(task_dir: Path) -> dict[str, Any]: diff --git a/docker/eval-tools/gpu-asan/Dockerfile b/docker/eval-tools/gpu-asan/Dockerfile new file mode 100644 index 00000000..a3c2bf17 --- /dev/null +++ b/docker/eval-tools/gpu-asan/Dockerfile @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena ROCm GPU ASan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="rocm-7.2.0-asan" + +COPY docker/eval-tools/gpu-asan/packages.sha256 /tmp/packages.sha256 + +RUN mkdir -p /tmp/gpu-asan-debs \ + && cd /tmp/gpu-asan-debs \ + && apt-get update \ + && apt-get download \ + rocm-core-asan=7.2.0.70200-43~22.04 \ + comgr-asan=3.0.0.70200-43~22.04 \ + hsa-rocr-asan=1.18.0.70200-43~22.04 \ + hip-runtime-amd-asan=7.2.26015.70200-43~22.04 \ + && sha256sum --check /tmp/packages.sha256 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ./*.deb \ + && rm -rf /tmp/gpu-asan-debs /tmp/packages.sha256 /var/lib/apt/lists/* + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_GPU_ASAN_RUNTIME_DIR=/opt/rocm-7.2.0/lib/asan \ + AKA_GPU_ASAN_HIP_RUNTIME=/opt/rocm-7.2.0/lib/asan/libamdhip64.so \ + AKA_GPU_ASAN_HOST_PRELOAD=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux/libclang_rt.asan-x86_64.so \ + AKA_GPU_ASAN_HOST_LIB_DIR=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux \ + AKA_GPU_ASAN_NORMAL_ROCM_LIB_DIR=/opt/rocm-7.2.0/lib diff --git a/docker/eval-tools/gpu-asan/packages.sha256 b/docker/eval-tools/gpu-asan/packages.sha256 new file mode 100644 index 00000000..6d8078c4 --- /dev/null +++ b/docker/eval-tools/gpu-asan/packages.sha256 @@ -0,0 +1,4 @@ +3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 rocm-core-asan_7.2.0.70200-43~22.04_amd64.deb +31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 comgr-asan_3.0.0.70200-43~22.04_amd64.deb +c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc hsa-rocr-asan_1.18.0.70200-43~22.04_amd64.deb +fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 hip-runtime-amd-asan_7.2.26015.70200-43~22.04_amd64.deb diff --git a/docker/eval-tools/hip-fpsan/Dockerfile b/docker/eval-tools/hip-fpsan/Dockerfile new file mode 100644 index 00000000..d7b4a4e9 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/Dockerfile @@ -0,0 +1,31 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +ARG HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e + +LABEL org.opencontainers.image.title="AgentKernelArena HIP-FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0ac9be8a1539a473ba21dfa686564c3be33c890e" + +COPY src/eval_tools/probes/hip_fpsan_probe.hip /tmp/hip_fpsan_probe.hip + +RUN git init /opt/hip-fpsan \ + && git -C /opt/hip-fpsan remote add origin https://github.com/ROCm/hip-fpsan.git \ + && git -C /opt/hip-fpsan fetch --depth 1 origin ${HIP_FPSAN_COMMIT} \ + && git -C /opt/hip-fpsan checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/hip-fpsan rev-parse HEAD)" = "${HIP_FPSAN_COMMIT}" \ + && mkdir -p /opt/eval-tools/probes \ + && /opt/rocm/bin/hipcc -O2 --offload-arch=gfx950 \ + -I/opt/hip-fpsan/include \ + /tmp/hip_fpsan_probe.hip \ + -o /opt/eval-tools/probes/hip_fpsan_probe \ + && rm -f /tmp/hip_fpsan_probe.hip + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + HIP_FPSAN_ROOT=/opt/hip-fpsan \ + AKA_HIP_FPSAN_INCLUDE_DIR=/opt/hip-fpsan/include \ + AKA_HIP_FPSAN_PROBE=/opt/eval-tools/probes/hip_fpsan_probe \ + AKA_HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/hip-fpsan/sources.lock.yaml b/docker/eval-tools/hip-fpsan/sources.lock.yaml new file mode 100644 index 00000000..87f39fe3 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/sources.lock.yaml @@ -0,0 +1 @@ +hip_fpsan: 0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/images.lock.yaml b/docker/eval-tools/images.lock.yaml new file mode 100644 index 00000000..bf16a8da --- /dev/null +++ b/docker/eval-tools/images.lock.yaml @@ -0,0 +1,53 @@ +schema_version: 1 + +# The scoring runtime remains unchanged. Every tool image is an immutable child +# of this verified MI355X/gfx950 image. +base: + gfx950: + reference: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 + digest: sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 + rocm: 7.2.0 + +tools: + triton_fpsan: + triton: + version: 3.7.0+amd.rocm7.2.0.gitd0d77a509 + sha256: 3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 + triton_kernels: + version: 1.0.0+amd.rocm7.2.0.gitd0d77a509 + sha256: df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a + + gpu_asan: + packages: + rocm-core-asan: + version: 7.2.0.70200-43~22.04 + sha256: 3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 + comgr-asan: + version: 3.0.0.70200-43~22.04 + sha256: 31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 + hsa-rocr-asan: + version: 1.18.0.70200-43~22.04 + sha256: c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc + hip-runtime-amd-asan: + version: 7.2.26015.70200-43~22.04 + sha256: fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 + + rocjitsu: + repository: https://github.com/ROCm/rocm-systems.git + commit: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 + gcc: 13.4.0-6ubuntu1~22~ppa2 + googletest: + repository: https://github.com/google/googletest.git + commit: b514bdc898e2951020cbdca1304b75f5950d1f59 + flatbuffers: + repository: https://github.com/google/flatbuffers.git + commit: 595bf0007ab1929570c7671f091313c8fc20644e + + hip_fpsan: + repository: https://github.com/ROCm/hip-fpsan.git + commit: 0ac9be8a1539a473ba21dfa686564c3be33c890e + image_probe: src/eval_tools/probes/hip_fpsan_probe.hip + +verification: + gfx950: verified + gfx942: unverified diff --git a/docker/eval-tools/rocjitsu/Dockerfile b/docker/eval-tools/rocjitsu/Dockerfile new file mode 100644 index 00000000..7a0c2c0a --- /dev/null +++ b/docker/eval-tools/rocjitsu/Dockerfile @@ -0,0 +1,70 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} AS builder + +ARG ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +ARG GOOGLETEST_COMMIT=b514bdc898e2951020cbdca1304b75f5950d1f59 +ARG FLATBUFFERS_COMMIT=595bf0007ab1929570c7671f091313c8fc20644e + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates git software-properties-common \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc-13=13.4.0-6ubuntu1~22~ppa2 \ + g++-13=13.4.0-6ubuntu1~22~ppa2 \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN git init /src/rocm-systems \ + && git -C /src/rocm-systems remote add origin https://github.com/ROCm/rocm-systems.git \ + && git -C /src/rocm-systems fetch --depth 1 origin ${ROCJITSU_COMMIT} \ + && git -C /src/rocm-systems checkout --detach FETCH_HEAD \ + && test "$(git -C /src/rocm-systems rev-parse HEAD)" = "${ROCJITSU_COMMIT}" \ + && git init /src/googletest \ + && git -C /src/googletest remote add origin https://github.com/google/googletest.git \ + && git -C /src/googletest fetch --depth 1 origin ${GOOGLETEST_COMMIT} \ + && git -C /src/googletest checkout --detach FETCH_HEAD \ + && git init /src/flatbuffers \ + && git -C /src/flatbuffers remote add origin https://github.com/google/flatbuffers.git \ + && git -C /src/flatbuffers fetch --depth 1 origin ${FLATBUFFERS_COMMIT} \ + && git -C /src/flatbuffers checkout --detach FETCH_HEAD + +RUN cmake -S /src/rocm-systems/emulation/rocjitsu -B /build/rocjitsu -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-13 \ + -DCMAKE_CXX_COMPILER=g++-13 \ + -DCMAKE_INSTALL_PREFIX=/opt/rocjitsu \ + -DFETCHCONTENT_SOURCE_DIR_GOOGLETEST=/src/googletest \ + -DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=/src/flatbuffers \ + -DCMAKE_HIP_ARCHITECTURES=gfx950 \ + -DBUILD_TESTING=ON \ + -DRJ_INSTALL_TESTS=OFF \ + && cmake --build /build/rocjitsu --parallel \ + && cmake --install /build/rocjitsu \ + && mkdir -p /opt/rocjitsu/runtime \ + && cp -a /usr/lib/x86_64-linux-gnu/libstdc++.so.6* /opt/rocjitsu/runtime/ \ + && cp -a /lib/x86_64-linux-gnu/libgcc_s.so.1 /opt/rocjitsu/runtime/ + +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0bf561a0d8a4a6b88954f2c46bd3a50871cda140" + +COPY --from=builder /opt/rocjitsu /opt/rocjitsu +COPY docker/eval-tools/rocjitsu/rocjitsu-wrapper /usr/local/bin/rocjitsu +RUN chmod 0755 /usr/local/bin/rocjitsu \ + && /usr/local/bin/rocjitsu --help >/dev/null + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools +RUN /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/rocjitsu_replay_entrypoint.py \ + --help >/dev/null + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + PATH=/opt/rocjitsu/bin:${PATH} \ + AKA_ROCJITSU_BINARY=/usr/local/bin/rocjitsu \ + AKA_ROCJITSU_CONFIG=/opt/rocjitsu/share/rocjitsu/configs/gfx950_cdna4.json \ + AKA_ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 diff --git a/docker/eval-tools/rocjitsu/rocjitsu-wrapper b/docker/eval-tools/rocjitsu/rocjitsu-wrapper new file mode 100755 index 00000000..d094b3a8 --- /dev/null +++ b/docker/eval-tools/rocjitsu/rocjitsu-wrapper @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +export LD_LIBRARY_PATH="/opt/rocjitsu/runtime:/opt/rocjitsu/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# rocJITsu's current gfx950 runtime exposes one simulated agent numbered zero. +# A physical host GPU mask (for example ROCR_VISIBLE_DEVICES=4) would otherwise +# filter that simulated agent out of the launched HIP process. +export ROCR_VISIBLE_DEVICES=0 +export HIP_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=0 +export GPU_DEVICE_ORDINAL=0 +exec /opt/rocjitsu/bin/rocjitsu "$@" diff --git a/docker/eval-tools/rocjitsu/sources.lock.yaml b/docker/eval-tools/rocjitsu/sources.lock.yaml new file mode 100644 index 00000000..2975cc96 --- /dev/null +++ b/docker/eval-tools/rocjitsu/sources.lock.yaml @@ -0,0 +1,4 @@ +rocm_systems: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +googletest: b514bdc898e2951020cbdca1304b75f5950d1f59 +flatbuffers: 595bf0007ab1929570c7671f091313c8fc20644e +gcc_13: 13.4.0-6ubuntu1~22~ppa2 diff --git a/docker/eval-tools/triton-fpsan/Dockerfile b/docker/eval-tools/triton-fpsan/Dockerfile new file mode 100644 index 00000000..fa7819cc --- /dev/null +++ b/docker/eval-tools/triton-fpsan/Dockerfile @@ -0,0 +1,21 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena Triton FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="triton-3.7.0+amd.rocm7.2.0.gitd0d77a509" + +COPY docker/eval-tools/triton-fpsan/requirements.lock /tmp/eval-tool-requirements.lock + +RUN python -m pip uninstall -y \ + triton pytorch-triton pytorch-triton-rocm triton-rocm amd-triton triton-kernels \ + && python -m pip install --no-cache-dir --no-deps --require-hashes \ + -r /tmp/eval-tool-requirements.lock \ + && python -c "import importlib.metadata as m, triton, triton_kernels; assert m.version('triton') == '3.7.0+amd.rocm7.2.0.gitd0d77a509'; assert m.version('triton-kernels') == '1.0.0+amd.rocm7.2.0.gitd0d77a509'" \ + && rm -f /tmp/eval-tool-requirements.lock + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_TRITON_FPSAN_VERSION=3.7.0+amd.rocm7.2.0.gitd0d77a509 diff --git a/docker/eval-tools/triton-fpsan/requirements.lock b/docker/eval-tools/triton-fpsan/requirements.lock new file mode 100644 index 00000000..d8fc5460 --- /dev/null +++ b/docker/eval-tools/triton-fpsan/requirements.lock @@ -0,0 +1,4 @@ +triton @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton/triton-3.7.0+amd.rocm7.2.0.gitd0d77a509-cp310-cp310-linux_x86_64.whl \ + --hash=sha256:3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 +triton-kernels @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton-kernels/triton_kernels-1.0.0+amd.rocm7.2.0.gitd0d77a509-py3-none-any.whl \ + --hash=sha256:df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a diff --git a/docs/README.md b/docs/README.md index 4d43ea5c..d9c7abc5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ python -m sphinx -T -b html docs docs/_build/html | `reference/api-reference.md` | Configuration and API reference | Run configuration schema, task `config.yaml` schema, CLI flags, scoring, and the agent registry. | | `reference/benchmark-methodology.md` | Reference | Timing methodology, performance-helper materialization, and speedup interpretation. | | `how-to/run-evaluation.md` | How-to | Choose or create a run configuration, run an experiment through Docker, resume runs, and read results. | +| `how-to/use-evaluation-tools.md` | How-to | Configure isolated sanitizer/analysis sidecars, interpret capability and findings, and understand the strict language/GPU support matrix. | | `how-to/parallel-run.md` | How-to | Run one isolated Docker worker per GPU, use the shared `.parallel/` task queue, resume parallel runs, and parallelize `task_validator`. | | `how-to/agents.md` | How-to | Supported agents, model providers, and A/B testing. | | `how-to/add-task.md` | How-to | Task directory layout, `config.yaml` fields, and task types. | diff --git a/docs/how-to/run-evaluation.md b/docs/how-to/run-evaluation.md index 4f2b0d6a..9538e4db 100644 --- a/docs/how-to/run-evaluation.md +++ b/docs/how-to/run-evaluation.md @@ -15,13 +15,14 @@ resume, and inspect a run. ## Choose or create a run configuration A run configuration selects the agent, tasks, and target GPU. The repository -ships three examples: +ships quickstarts and specialized examples, including: | Configuration | Purpose | | --- | --- | | `example_configs/quickstart_claude_mi300.yaml` | One Claude Code GELU task on MI300/MI300X (`gfx942`). | | `example_configs/quickstart_claude_mi355x.yaml` | One Claude Code GELU task on MI355X (`gfx950`). | | `example_configs/benchmark_cursor_mi355x.yaml` | Curated 60-task Cursor Agent benchmark on MI355X; use only after installing and authenticating Cursor Agent. | +| `example_configs/evaluation_tools_advisory_mi355x.yaml` | Default-disabled template for experimental `gfx950` evaluation-tool sidecars. It still requires task-specific adapters before opt-in. | For a first run, select the quickstart that matches the physical GPU: @@ -78,6 +79,11 @@ select tasks at any level of granularity. See [Configuration and API reference](../reference/api-reference.md) for the full set of run-configuration fields. +Optional sanitizer and analysis sidecars run after ordinary correctness and +before performance. They are disabled by default, verified only on `gfx950`, +and require language/task-specific adapters. See [Check kernels with evaluation +tools](use-evaluation-tools.md) before enabling them. + ## Start a run ```bash diff --git a/docs/how-to/use-evaluation-tools.md b/docs/how-to/use-evaluation-tools.md new file mode 100644 index 00000000..05928c1b --- /dev/null +++ b/docs/how-to/use-evaluation-tools.md @@ -0,0 +1,899 @@ +--- +myst: + html_meta: + "description": "Run Triton FpSan, ROCm GPU AddressSanitizer, rocJITsu, and HIP-FpSan as isolated AgentKernelArena evaluation tools." + "keywords": "AgentKernelArena, sanitizer, Triton FpSan, GPU ASan, rocJITsu, HIP-FpSan, ROCm, gfx950" +--- + +# Check kernels with evaluation tools + +AgentKernelArena can run optional kernel-analysis tools after ordinary +compilation and correctness checks and before performance measurement. The +initial tool set is: + +- Triton FpSan for reference-versus-candidate floating-point semantic + comparison. +- ROCm GPU AddressSanitizer (GPU ASan) for invalid device-memory accesses. +- rocJITsu for simulated race detection. +- HIP-FpSan for explicitly ported HIP/C++ floating-point comparisons. + +This feature is experimental and opt-in. Capability and evidence checks fail +closed; whether an incomplete result blocks performance is controlled by the +`advisory` or `required` policy. It is not a general three- or four-tool +sanitizer suite: every result is qualified by the kernel language, generated +artifact, adapter, tool image, GPU architecture, and evidence that the intended +kernel was actually instrumented or dispatched. + +> **Current validation boundary:** sidecar build locks, integrated startup +> controls, and end-to-end fixtures exist only for MI355X (`gfx950`). All four +> startup controls passed in the current hardware qualification. Candidate +> readiness still depends on language, artifact, adapter, and attestation. +> `gfx942` is unverified, and the Docker runner currently rejects +> evaluation-tool sidecars on that architecture. Do not interpret normal +> MI300/MI325 task support as sanitizer support. + +## Keep the scoring runtime unchanged + +Evaluation tools do not get installed into the agent or scoring container. +Each enabled tool runs in a separate run- or worker-scoped sidecar, reused +across that worker's tasks, and communicates with the evaluator over a Unix +socket: + +```mermaid +flowchart LR + A["Pinned scoring image
agent + correctness + scoring"] -->|"health and invocation RPC"| B["One tool sidecar"] + B -->|"bounded logs and structured result"| A + C["Repository tree"] -->|"/input read only"| B + D["Per-worker artifact namespace
.eval-tool-artifacts/label"] <-->|"/artifacts read/write"| B + E["Per-tool socket directory"] -->|"one writable UDS path"| B +``` + +The runner creates a narrow repository-root +`.eval-tool-artifacts/` namespace. It mounts only that directory +as writable `/artifacts` in each sidecar and explicitly bind-mounts the same host +directory read/write at `/workspace/.eval-tool-artifacts/` in the +scoring container. The explicit submount keeps reports writable when the broad +repository mount is read-only, as it is in the quality loop, and avoids asking +Docker to create bind sources on a root-squashed NFS home. Task workspaces and +captured source evidence are not reachable through a writable sidecar alias. +In ordinary and parallel runs, the runner also overlays the top-level +`.eval-tool-artifacts` namespace read-only before overlaying only the current +worker child read/write. This prevents the broad writable repository mount from +becoming a second path to sibling workers' reports or bind sources. +All sidecars for that worker still share this artifact namespace. Each sidecar +receives only its own nested writable socket directory, while the scoring +container receives the socket parent read-only. These changes reduce accidental +cross-tool mutation, but do not create an agent/evaluator trust boundary; the +remaining security consequences are described later. + +All four tool images bake the worker, trusted replay helper, and synthetic +probes into the read-only image at `/opt/aka-eval-tools`. Worker startup verifies +that it imported this image-owned tree rather than the repository mounted at +`/input`. This protects the sidecar control-plane code from a task changing the +checkout used by a running worker; candidate-specific commands and inputs remain +separate, explicitly mounted data. + +The verified `gfx950` scoring image remains: + +```text +lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 +lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +``` + +When any evaluation tool is enabled, the runner resolves Docker's immutable +local image ID for both the selected scoring-image reference and the +content-addressed manifest reference above. Startup fails unless the two IDs are +identical, then launches the scoring container by that verified ID rather than +the mutable tag. A different tag that is an exact alias of the same image is +accepted; an upgraded, rebuilt, or retagged scoring image is rejected even if it +looks compatible. The selected reference and verified image ID are recorded under +`plan.source_evidence.metadata.scoring_runtime`, serialized with the report, +and covered by the plan fingerprint. + +The current design deliberately does **not** upgrade that image, FlyDSL 0.2.2, +or AITER `0.1.17.dev110+g9127c94a1`. The Triton FpSan sidecar replaces Triton +only inside its own container; the other tool dependencies are likewise local +to their sidecars. A sidecar is not a replacement scoring image and must not be +used to establish a new performance baseline. + +FlyDSL does not promise that every generated artifact or task remains compatible +across releases. If the scoring image, FlyDSL, AITER, PyTorch, ROCm, or Triton is +upgraded later, treat that as a benchmark-runtime migration: rerun compilation, +correctness, held-out, sanitizer positive/negative fixtures, and performance +baselines rather than assuming backward compatibility. + +The pinned sidecar dependencies are recorded in +`docker/eval-tools/images.lock.yaml`: + +| Sidecar | Isolated dependency change | +| --- | --- | +| `triton_fpsan` | AMD Triton `3.7.0+amd.rocm7.2.0.gitd0d77a509` and matching `triton-kernels` wheels. | +| `gpu_asan` | ROCm 7.2 ASan runtime packages, including `hip-runtime-amd-asan`. | +| `rocjitsu` | rocJITsu from pinned `rocm-systems` commit `0bf561a0...`, built with GCC 13 for `gfx950`. | +| `hip_fpsan` | HIP-FpSan headers/source from pinned commit `0ac9be8a...`. | + +## Understand support levels + +“The engine can execute a code object” and “the evaluator can safely wrap a +Python submission” are different claims. The report therefore records four +capability dimensions: + +| Dimension | Question | +| --- | --- | +| `engine` | Can this analysis engine reason about the language or generated ISA? | +| `adapter` | Is there a task-specific build, comparison, or replay path that identifies the exact candidate? | +| `runtime` | Is the isolated sidecar reachable, architecture-matched, and carrying the required assets? | +| `effective` | Is the tool ready after combining the other three dimensions? | + +Possible states are `ready`, `adapter_required`, `unsupported`, +`not_applicable`, and `unavailable_runtime`. A non-ready tool is not executed. +Unsupported tooling is also not converted into an ordinary kernel correctness +failure. + +### Use evaluator plugins, not agent skills, for enforcement + +A sanitizer belongs in the evaluator control plane. An agent skill, prompt, or +tool-use recipe may help an optimizer author an adapter or interpret a report, +but it runs on the untrusted optimization side and cannot prove that a required +check happened. It must never be the source of a scoring gate. + +The extension boundary deliberately separates four responsibilities: + +| Layer | Responsibility | +| --- | --- | +| Core manager | Resolve task profiles, combine capability states, enforce policy, fingerprint plans, and serialize one stable report schema. It contains no sanitizer-specific parsing. | +| Tool plugin | Implement deterministic `assess`, `build_invocation`, and `parse` behavior for one tool. It converts raw output into independent execution/finding states. | +| Sidecar runtime provider | Own the pinned image, dependencies, health evidence, runtime-internal paths, architecture guard, and synthetic startup control. | +| Task adapter | Compile or replay the exact candidate and emit candidate-specific build/dispatch attestation. It is selected by language/artifact kind, not merely by tool name. | + +This makes a future analyzer additive: add its typed plugin/parser, isolated +image and lock, worker health/startup control, runner allowlist, task adapters, +support-matrix entry, and positive/negative tests without changing ordinary +correctness or performance code. The current registry is an explicit built-in +allowlist; arbitrary third-party plugin discovery is not implemented. That is +intentional while report provenance and the evaluator/agent isolation boundary +remain experimental. + +### Strict support matrix + +The following table describes the current end-to-end evaluator, not just a +successful standalone experiment. “Ready with adapter” means the task must +provide the dedicated argv/harness and required attestation described later; +it does not mean the ordinary correctness command is automatically reused. + +| Kernel path | Triton FpSan | GPU ASan | rocJITsu | HIP-FpSan | +| --- | --- | --- | --- | --- | +| Editable Triton Python/JIT | Ready with comparison adapter and instrumentation attestation | Ready with dedicated command, fresh JIT cache, XNACK, and build attestation | Trusted `triton_aot` capsule replay is implemented on `gfx950`; whole-Python JIT remains unsupported, and capsule capture/binding to the correctness run is not automatic, so use it only as advisory evidence | Not applicable | +| HIP source controlled by the task | Not applicable | Ready only after recompiling the candidate with `-fsanitize=address -shared-libsan --offload-arch=gfx950:xnack+`, then attesting that artifact | Ready with a dedicated native launcher | Source port and comparison adapter required; both reference and candidate paths must explicitly use `fpsan::Value` | +| FlyDSL 0.2.2 Python/JIT | Unsupported; FlyDSL does not use the Triton FpSan pipeline | Unsupported; the current ROCDL pipeline does not insert AMD GPU ASan instrumentation | Trusted `flydsl_aot` capsule replay is implemented on `gfx950` and detects the seeded LDS race; automatic capsule capture/binding to the correctness run is not ready, so use it only as advisory evidence | Not applicable | +| Editable Triton source inside AITER | Engine may be eligible for the explicitly selected source only, with a dedicated comparison adapter; this does not sanitize AITER library kernels | Unsupported by the current default AITER runtime path | Unsupported by the current Python/AITER runtime | Not applicable | +| AITER or another precompiled HSACO/library kernel | Cannot retrofit instrumentation | Unsupported unless the exact kernel source is rebuilt and attested; preloading the runtime is insufficient | Unsupported by the current evaluator runtime | Cannot retrofit value semantics | +| rocBLAS or RCCL internal kernel | Do not enable; library internals are outside the selected submission | The stock library is not instrumented and is not covered | Not a supported general library-runtime path | Do not enable | + +This matrix describes engine and adapter support once the corresponding runtime +is qualified. The current `gfx950` startup qualification is stricter: + +| Tool runtime | Current startup qualification | +| --- | --- | +| Triton FpSan | Passing on hardware; eligible task paths can proceed to candidate attestation. | +| HIP-FpSan | Passing on hardware; explicitly ported task paths can proceed to candidate attestation. | +| GPU ASan | Passing on hardware for both HIP and Triton safe/OOB lanes; an applicable candidate still needs its own instrumentation/build attestation. | +| rocJITsu | Passing on hardware with barrier-safe and deliberately racy LDS fixtures; an applicable candidate still needs a native HIP launcher or validated AOT replay capsule. | + +Additional boundaries: + +- Static Triton HSACO fixtures, including dynamic matmul, buffer-async matmul, + and flash-attention fixtures, have executed under the `gfx950` rocJITsu + engine. That evidence does not make `rocjitsu -- python task.py` supported. +- A FlyDSL HSACO with a deliberately missing LDS barrier produced a rocJITsu + race report. Directly wrapping the FlyDSL Python process failed before a + usable dispatch. The implemented adapter therefore extracts that boundary + into a strict, single-dispatch replay capsule rather than wrapping Python. +- For Triton/FlyDSL, the rocJITsu plugin rejects arbitrary launchers, validates + the language-specific capsule and all manifest hashes, requires `gfx950`, and + invokes an image-owned helper that generates and compiles a native launcher. + The parser requires the expected kernel dispatch, capsule/code-object digest + attestation, and a successful replay/golden-output marker. The missing piece + is automatic evaluator-owned capsule capture and proof that the capsule came + from the exact correctness run; a task-supplied capsule is still weak + candidate provenance. +- A deliberately out-of-bounds, **uninstrumented** HIP HSACO exited normally in + a complete GPU ASan runtime. This is why a runtime preload alone is never + accepted as GPU ASan coverage. +- `gfx942` has not completed the same image, positive-control, adapter, and + end-to-end validation. Its status is unverified, not unsupported by theory. + +## Build and check the sidecars + +Building requires Docker and network access to the pinned package and source +locations. Runtime sidecars themselves start with networking disabled. + +Build all four local `gfx950` images from the repository root: + +```bash +src/scripts/docker_benchmark.sh build-eval-tool-images +``` + +The default local tags are: + +```text +agent-kernel-arena/eval-tool-triton-fpsan:gfx950 +agent-kernel-arena/eval-tool-gpu-asan:gfx950 +agent-kernel-arena/eval-tool-rocjitsu:gfx950 +agent-kernel-arena/eval-tool-hip-fpsan:gfx950 +``` + +Check that the workers start and report their pinned assets: + +```bash +src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +To check a subset: + +```bash +AKA_EVAL_TOOLS=gpu_asan,rocjitsu \ + src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +`AKA_EVAL_TOOLS` is a host-side subset override. When it is set for either a +smoke test or a normal run, the runner starts exactly that normalized set and +publishes it through the internal `AKA_EVAL_TOOLS_SELECTED` variable; the +scoring process then plans the same set instead of the YAML `enabled` value. +This prevents a sidecar/plan mismatch. Leave the override unset when YAML should +remain authoritative, and record any override as part of the run invocation. + +Worker startup runs a tool-specific synthetic positive control before the Unix +socket appears. Health output includes its verdict, commands, durations, +bounded log paths/excerpts, and the immutable Docker image ID reported by the +worker: + +| Tool | Startup positive control | +| --- | --- | +| `triton_fpsan` | Compile instrumented reference/candidate kernels and require a known numerical mismatch to produce different digests plus FpSan compiler metadata. | +| `gpu_asan` | Compile and run safe/OOB HIP fixtures and safe/OOB Triton fixtures; the task profile selects the relevant lane. | +| `rocjitsu` | Require a barrier-protected fixture to remain clean and a deliberately racy LDS fixture to report a race. | +| `hip_fpsan` | Require explicitly ported equivalent expressions to match and a known-wrong expression to produce a different digest. | + +`eval-tools-smoke` prints this evidence, but its CLI exit status currently means +that the health RPC succeeded, not that every nested `positive_control.passed` +value is true. Inspect the JSON summaries before promotion. A normal evaluation +with `positive_control: required` performs the fail-closed check during the +typed runtime probe. + +As of the current `gfx950` qualification run, all four integrated startup +controls pass on hardware. This qualifies the installed tool runtimes only. It +does not promote a candidate path without the language-specific adapter and +attestation in the strict support matrix. + +The same final image set also passed evaluator-manager-to-sidecar candidate +fixtures on the physical MI355X host: + +| Tool and language | Safe fixture | Seeded bug fixture | +| --- | --- | --- | +| Triton FpSan, Triton | `clean` | Numerical mismatch `found` | +| GPU ASan, HIP | `clean` | Out-of-bounds access `found` | +| GPU ASan, Triton | `clean` | Out-of-bounds access `found` | +| HIP-FpSan, explicitly ported HIP | `clean` | Wrong expression `found` | +| rocJITsu, Triton AOT replay | `clean` | Not exercised in this candidate pair | +| rocJITsu, FlyDSL AOT replay | Not exercised in this candidate pair | Missing-barrier LDS race `found` | + +These are controlled synthetic fixtures that validate the current adapters, +transport, parsing, and attestation paths. They do not qualify any bundled +production task, broaden the strict support matrix, or close the AOT +correctness-dispatch provenance gap. + +## Recommended rollout plan + +Promote one language/tool path at a time. Do not make “all sanitizers enabled” a +global milestone. + +| Phase | Work | Exit criterion | +| --- | --- | --- | +| 0. Freeze baselines | Keep the pinned scoring image, FlyDSL 0.2.2, and AITER version unchanged; build each tool from its lock into a sidecar. | Existing compilation, correctness, held-out, and performance baselines remain unchanged with tools disabled. Sidecar image IDs and the verified scoring-image ID/reference are captured in plans. | +| 1. Qualify installations | Run automatic safe/known-bug startup controls on `gfx950`; repeat the now-passing four-tool qualification on clean hosts. | Both positive and negative lanes pass repeatedly. `eval-tools-smoke` evidence is archived and independently reviewed. | +| 2. Build trusted pilot adapters | Start with one editable Triton task for Triton FpSan, one Triton and one HIP task for GPU ASan, one native HIP task for rocJITsu, and one explicitly ported HIP-FpSan task. Put harnesses under protected `scripts/` paths and declare all inputs. | Each pilot distinguishes a safe fixture from a seeded bug, identifies the selected candidate, and produces bounded structured artifacts. No precompiled AITER/library kernel is claimed as covered. | +| 3. Finish AOT capture and binding | The trusted `triton_aot`/`flydsl_aot` replay path now validates one-dispatch capsules and generates the launcher. Add evaluator-owned extraction immediately after correctness and bind the capsule to that exact candidate/case. | Safe and racy fixtures pass end to end, malformed capsules fail closed, and a task cannot substitute a different valid capsule for the correctness dispatch. | +| 4. Harden provenance and phase isolation | The runner now uses per-tool writable socket directories, a read-only socket parent in scoring, a narrow per-worker artifact mount, a complete serialized plan, and capsule digests in the fingerprint. Next run tools only after the agent exits, freeze the candidate, use evaluator-only/authenticated RPC and per-task/tool artifact ownership, strengthen artifact/dispatch binding, and wire resume to plan freshness. | An adversarial task cannot call a worker, overwrite evidence, reach another task's artifacts, spoof a clean result, or reuse a stale report. This phase is required before sanitizer output becomes a reward signal. | +| 5. Advisory campaign | Run qualified paths with `policy: advisory` across representative and private held-out shapes; measure overhead, timeouts, log volume, flakes, false positives, and GPU recovery behavior. | Each task/tool pair has reviewed coverage cases, stable resource limits, and an explicit owner/runbook. Incomplete results remain visible and never score as clean. | +| 6. Narrow required gates | Change only individually qualified task/tool pairs to `required`; leave unsupported and not-yet-qualified paths advisory or disabled. | Required gates block seeded findings and infrastructure failures without changing ordinary correctness semantics or the scoring performance baseline. | +| 7. Add `gfx942` separately | Build architecture-specific images/configs and rerun every startup, adapter, security, and workload fixture on MI300X/MI325X. | Only mark `gfx942` supported after independent qualification; do not infer it from `gfx950`. | + +Phase 0, the four integrated startup controls, complete plan serialization, the +narrower runner mounts, and the trusted AOT replay core exist. No bundled task +has completed production qualification through phases 2–6. Automatic capsule +capture, exact candidate/dispatch provenance, top-level resume freshness, and +the same-phase agent/evaluator boundary remain blockers. + +## Start from the disabled example + +Copy `example_configs/evaluation_tools_advisory_mi355x.yaml`. It is a normal +MI355X run configuration but has `evaluation_tools.enabled: false`, so copying +and running it does not build or start any sidecar when the host +`AKA_EVAL_TOOLS` override is unset: + +```bash +cp example_configs/evaluation_tools_advisory_mi355x.yaml my_sanitized_run.yaml +``` + +After building the images and adding task-specific adapters, opt in to only the +tools that can actually inspect the selected task: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + - rocjitsu + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} + rocjitsu: + options: {} +``` + +Do not enable all four tools merely because all four images exist. On a +heterogeneous task set, irrelevant tools become `not_applicable`, unsupported +paths remain visible as unsupported, and missing adapters remain +`adapter_required`. + +### Select and attest each tool image separately + +The host runner selects the Docker image using the default local tag or an +environment override. For example: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +make docker-run CONFIG=my_sanitized_run.yaml RUN_ARGS='--run-suffix asan_advisory' +``` + +The override names are `AKA_EVAL_TOOL_IMAGE_TRITON_FPSAN`, +`AKA_EVAL_TOOL_IMAGE_GPU_ASAN`, `AKA_EVAL_TOOL_IMAGE_ROCJITSU`, and +`AKA_EVAL_TOOL_IMAGE_HIP_FPSAN`. + +After selecting a tool-image reference, the runner resolves its local immutable +image ID with `docker image inspect`, launches the sidecar by that bare +`sha256:...` ID, and injects the same ID as runtime identity evidence. Runtime +health reports that ID, and the typed probe fails with +`RUNTIME_REF_MISMATCH` if the planned and observed values differ. The resolved +ID belongs in the plan fingerprint. Scoring-image verification likewise launches +the scoring container by its verified local ID. A registry reference such as +`name@sha256:...` is not string-equal to Docker's local image ID and must not be +used as a manual substitute. + +YAML `runtime_ref`/`image_digest` fields are identity assertions, not image +selectors. Omit them when using automatic host injection. If supplied manually, +use the exact bare ID returned by: + +```bash +docker image inspect --format '{{.Id}}' +``` + +## Add a task adapter + +Run-level configuration chooses tools, policy, optional image-identity +assertions, and maximum timeout. A task can only add options for an +already-enabled tool and lower its timeout. It cannot enable a tool, change the +image or top-level policy, or raise the run-level timeout. +Reserved framework options are rejected at both run and task level. They are +`positive_control_required`; GPU ASan's `asan_runtime_dir`, +`hip_asan_runtime`, `host_asan_preload`, `host_asan_lib_dir`, and +`normal_rocm_lib_dir`; rocJITsu's `rocjitsu_binary` and `config_path`; and +HIP-FpSan's `include_dir` and `public_header`. The host/runtime probe is the +only authority for those values. + +Commands must be argv lists, never shell strings. A dedicated tool command is +required because reusing `correctness_command` could instrument the reference, +load a precompiled library kernel, or sanitize the wrong candidate. + +For example, a HIP task can declare the shape of its adapters as follows: + +```yaml +evaluation_profile: + language: hip + artifact_kind: source_aot + framework: standalone + instrumentation_control: recompile + source_available: true + submission_paths: + - optimized_kernel.hip + - scripts/eval_tools/run_gpu_asan.py + - scripts/eval_tools/rocjitsu_launcher + +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] + rocjitsu: + options: + launcher: [scripts/eval_tools/rocjitsu_launcher] + expected_kernel: my_kernel +``` + +These are adapter contracts, not automatically generated files. The task +wrapper must build and launch the optimized candidate, exercise representative +inputs, and emit the required evidence. Sidecar health is the only authority +for container-internal ASan libraries and preload, the rocJITsu binary and +architecture config, and the HIP-FpSan include directory. The runtime probe +attests and injects those values into the plugin context; neither run nor task +configuration may supply or override them. + +Common built-in option keys are: + +| Tool | Required adapter options | Additional evidence/options | +| --- | --- | --- | +| `triton_fpsan` | `comparison_command` or `command` | `attestation_path`; command must emit one `AKA_FPSAN_RESULT` JSON line | +| `gpu_asan` | `command` | Candidate `attestation_path`; a HIP command must use the required compile flags. Runtime/preload/library paths come from health. | +| `rocjitsu` | HIP: `launcher` or `command`. Triton/FlyDSL: `capsule` plus an exact profile adapter of `triton_aot` or `flydsl_aot`; user launchers are forbidden on these AOT paths. | HIP may set `expected_kernel` and `race_report`. AOT capsule path must stay below the task workspace and target `gfx950`; the executable/config and trusted replay helper come from the sidecar image. | +| `hip_fpsan` | `comparison_command` or `command`, plus `evaluation_profile.fpsan_ported: true` | Candidate `attestation_path`; both paths must be instrumented. The include directory comes from health. | + +For repository or image-kernel tasks, declare every candidate file whose change +must invalidate evidence with `evaluation_profile.submission_paths`. Paths must +be workspace-relative and cannot contain `..`. If this field is absent, capture +falls back to `source_file_path` and `target_file_path`; silently hashing an +entire multi-gigabyte repository is intentionally avoided. + +Put evaluator-owned adapter code under a harness-protected path such as +`scripts/`, not an arbitrary agent-editable `eval_tools/` directory. Also list +adapter scripts, HSACO files, and input blobs in `submission_paths` when their +contents must affect the general candidate fingerprint. A configured replay +capsule receives additional handling: immediately before plan construction the +manager records its SHA-256 and size under +`source_evidence.metadata.option_artifacts`, so that digest is covered by the +plan fingerprint. The validated capsule manifest contains and verifies the +HSACO and blob digests. This binds the plan to the supplied capsule bytes; it +does not prove who captured the capsule or that it came from the same dispatch +as ordinary correctness. + +## Require evidence before accepting “clean” + +Process success is not proof that an analysis ran. The built-in parsers require +tool-specific attestation: + +- GPU ASan requires `build_attestation.json` for the declared artifact. HIP + attestation must include all sanitizer/XNACK flags and `HSA_XNACK=1`; Triton + attestation must include `TRITON_ENABLE_ASAN=1` and `HSA_XNACK=1`. +- Triton FpSan and HIP-FpSan currently require one build attestation whose + evidence contains the self-declared `reference_instrumented` and + `candidate_instrumented` booleans, plus an `AKA_FPSAN_RESULT` payload with the + two digests. They do not validate two independently attested artifacts. +- Native HIP rocJITsu requires an observed simulator dispatch, optionally + matched to `expected_kernel`. This path accepts a task launcher, so its output + text remains weak evidence and can be forged by that launcher. +- Triton/FlyDSL rocJITsu uses the image-owned replay helper instead of a task + launcher. It revalidates the capsule and manifest, generates the native + launcher, and requires an exact capsule/code-object attestation, expected + kernel dispatch, and `AKA_REPLAY_RESULT pass`. Missing or changed evidence is + inconclusive. This is stronger replay integrity, but the task-supplied capsule + is not yet automatically tied to the correctness run. +- Build attestation records the compiler, compiler version, and target + architecture. The current validator directly checks tool identity, + `instrumented: true`, required build flags/environment, artifact existence, + and artifact SHA-256. The host/runtime `gfx950` guards provide the current + architecture boundary; stricter compiler/version/target comparisons remain + future attestation hardening. + +These checks establish integrity evidence, not complete candidate provenance. +Build-attestation validation does not compare every compiler/version/target +field with an expected build or cryptographically bind the declared artifact to +the actual dispatch. It does require the artifact path to be relative to, and +contained below, the directory holding the attestation. A user-controlled build +JSON and native HIP wrapper output can therefore still satisfy checks without +proving which candidate ran. AOT replay binds its generated launcher to a +validated capsule and digest, but the capsule can still have been supplied for +a different candidate/case. Current `required` policy is suitable for trusted +integration diagnostics, not an adversarial reward boundary. + +The default build-attestation location is the tool's directory below the +external per-task artifact root described later. A wrapper executing in a +sidecar must write it through the writable `/artifacts` mount; the +repository/workspace input mount is read-only. GPU ASan, Triton FpSan, and +HIP-FpSan invocations inject `AKA_BUILD_ATTESTATION_PATH`, and the runtime client +translates that output path into the sidecar namespace. The adapter must place +the built artifact beside or below that JSON file and write `artifact_path` as a +relative path such as `build/candidate.hsaco`. The scoring-side parser resolves +the same relative path below its corresponding artifact directory, rejects +absolute/escaping paths, and checks the declared SHA-256. Do not embed either +the sidecar `/artifacts/...` prefix or a scoring-container absolute path. + +### Positive controls and candidate attestation are both required + +`positive_control: required` is the default. Each sidecar runs its synthetic +known-bug control once at worker startup and writes an audit summary and bounded +logs. Health returns that evidence. For every applicable task, the typed runtime +probe selects the relevant control (HIP or Triton for GPU ASan) and returns +`unavailable_runtime/POSITIVE_CONTROL_FAILED` unless it passed. The requirement +and host-resolved tool runtime identities are covered by the plan inputs. The +separately verified scoring-image ID/reference are recorded in plan source +evidence as well. +Control evidence is retained in runtime capability/reporting, but its startup +artifact content is not itself part of the plan fingerprint. + +This proves that the isolated installation detected its synthetic bug; it does +not prove that the optimized candidate was instrumented or replayed. Candidate +build/dispatch attestation remains a separate requirement. A minimally +meaningful clean diagnostic result needs both: + +1. a passing startup positive control for the applicable language lane; and +2. a task result with the declared candidate's build or dispatch attestation. + +Setting `positive_control: optional`, `disabled`, or `false` records +`positive_control_required: false`. The startup probe still runs and remains +visible in health evidence, but a failure no longer blocks runtime capability. +Use that only for tool bring-up, not trusted benchmark results. + +## Choose the policy + +The policy governs whether performance measurement may proceed. It does not +rewrite `pass_correctness`. + +| Policy | Finding, tool error, inconclusive result, missing adapter, unsupported applicable path, or missing runtime | Performance | +| --- | --- | --- | +| `advisory` | Recorded; `policy_satisfied: false` | Continues (`allowed: true`) | +| `required` | Recorded with a reason | Skipped (`allowed: false`) | + +An explicitly `not_applicable` tool is ignored by the gate. Under `required`, +every other selected tool must have effective capability `ready`, execution +`completed`, and finding status `clean`. Start new integrations in `advisory` +mode. Move to `required` only after task adapters, positive controls, and +architecture-specific fixtures are independently reviewed. + +## Read execution and findings separately + +`execution` answers whether the tool invocation completed. `finding` answers +what the parser concluded about the selected kernel: + +| `execution` | Meaning | +| --- | --- | +| `not_run` | No invocation was made. | +| `completed` | The parser obtained a complete tool outcome; a detected bug can still have this state. | +| `tool_error` | The runtime, wrapper, or parser failed without a valid finding. | +| `timeout` | The isolated process group exceeded its timeout. | + +| `finding` | Meaning | +| --- | --- | +| `not_evaluated` | No supported evaluation ran. | +| `clean` | The intended instrumented/simulated kernel ran and no finding was observed. | +| `found` | One or more structured issues were reported. | +| `inconclusive` | Absence of a finding cannot be trusted, often because attestation is missing. | + +A sanitizer can deliberately terminate with a nonzero process status while +still producing a valid finding. Conversely, an uninstrumented out-of-bounds +kernel can return zero and must remain inconclusive. Never use return code alone +as the sanitizer result. + +The report keeps ordinary scoring fields and adds: + +```yaml +pass_tool_gate: true +tool_policy_satisfied: false +tool_evaluation: + schema_version: 1 + plan_fingerprint: "..." + plan: + schema_version: 1 + policy: advisory + profile: {} + tools: + - tool: gpu_asan + runtime_ref: "sha256:..." + plugin_version: "1" + timeout_s: 600 + options: {positive_control_required: true} + fingerprint: "..." + source_evidence: + metadata: + scoring_runtime: + image_id: "sha256:..." + reference: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705" + policy: advisory + overall_status: incomplete + resolved_task_profile: {} + source_evidence: {} # legacy mirror of plan.source_evidence (abbreviated) + decision: + allowed: true + policy_satisfied: false + reasons: ["gpu_asan:capability:adapter_required"] + tools: + gpu_asan: + capability: + engine: {state: ready} + adapter: {state: adapter_required} + runtime: {state: ready} + effective: {state: adapter_required} + result: null +``` + +Large stdout/stderr streams are not embedded in `task_result.yaml`; they remain +bounded artifacts. The report serializes the complete immutable `plan`, +including each tool's runtime reference, plugin version, timeout, options, +profile, source evidence, and fingerprint. This makes a result reconstructable +for audit without relying on the digest alone. + +## Artifact and evidence layout + +For a normal run, the relevant files are: + +```text +/ +├── .eval-tool-artifacts/ +│ └── / # dedicated sidecar/scoring RW submount +│ ├── _eval_tool_runtime/ +│ │ └── //positive-control/ +│ │ ├── summary.json # startup verdict and step metadata +│ │ └── .{stdout,stderr}.log +│ └── -/ +│ ├── gpu_asan/ +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json # when the adapter supplies it +│ │ └── triton-gpu-asan-cache/ # Triton only +│ ├── triton_fpsan/ +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json +│ │ └── triton-fpsan-cache/ +│ ├── rocjitsu/ +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── rocjitsu-report/race.log +│ │ └── rocjitsu-replay/ # generated AOT launcher, when used +│ └── hip_fpsan/ +│ ├── stdout.log +│ ├── stderr.log +│ └── build_attestation.json +└── experiments/workspace__/run_/ + ├── .eval-tool-evidence/ + │ └── / + │ ├── manifest.json + │ └── files/ # captured original declared files + └── / + └── task_result.yaml # nested tool_evaluation summary +``` + +Each stdout and stderr file is limited to 64 MiB by default and records +truncation metadata. Tool processes run in a new process group; timeout cleanup +sends termination and then kill signals to descendants. Per-sidecar scratch and +cache directories outside `.eval-tool-artifacts` are deleted when the sidecar +stops. The report records absolute scoring-side paths below +`/workspace/.eval-tool-artifacts/`. The runner explicitly mounts +that path read/write even when `/workspace` itself is read-only. The per-worker +namespace prevents a writable sidecar alias to task workspaces but is not a +secret or an adversarial integrity boundary. +Raw parser excerpts are omitted from normal JSON/YAML summaries and remain in +the bounded log artifacts. If either stream is truncated, a result that would +otherwise be clean is changed to `inconclusive`; an already observed finding is +preserved. + +## Resume and plan fingerprints + +Every tool report has a SHA-256 `plan_fingerprint` covering: + +- normalized tool configuration, policy, timeouts, options, and configured + runtime references; +- the selected scoring-image reference and its immutable local image ID, after + the runner resolves the pinned SGLang manifest reference to its local image ID + and verifies that the two local IDs match; +- resolved task profile and explicit overrides; +- enabled plugin versions; +- captured-original and optimized-candidate fingerprints for declared paths; +- the SHA-256 and size of a configured replay-capsule JSON, whose validated + manifest in turn binds the referenced HSACO and blobs. + +This prevents a report from being considered current after a material input +change **when the caller checks it**. The reporting API exposes an exact +fingerprint check for this purpose. + +Most coverage is declaration-based. A pathname stored in ordinary tool options +contributes only its string; adapter, HSACO, and input contents must be declared +in submission evidence or represented by an explicit digest. Replay `capsule` +is the exception: the manager captures its content digest during plan +construction. The current `has_current_plan` helper checks only the fingerprint +and presence of a tools mapping; it does not prove that every tool completed or +that its result is clean. The serialized `plan` makes the inputs auditable but +does not by itself change resume scheduling. + +The top-level `--resume-run` and `--resume-latest` paths currently skip a task +when its `task_result.yaml` already exists; they do not yet rebuild the tool plan +and call that fingerprint check. Until this is wired into run scheduling: + +- use a new `--run-suffix` after changing tool configuration, plugin code, + sidecar image, adapter, source declaration, or positive-control policy; +- do not change `AKA_EVAL_TOOL_IMAGE_*` while resuming a run; +- do not change the selected scoring-image reference while resuming, even to + another tag for the same image ID, because the reference is plan evidence; +- do not assume a changed `runtime_ref` causes a completed task to rerun; +- if a tool-only rerun is required, archive the old report and start a fresh run + rather than silently combining evidence from two plans. + +## AOT replay capsules for Triton and FlyDSL + +rocJITsu can execute supported generated `gfx950` HSACO, but it cannot safely +wrap the current Triton/FlyDSL Python JIT process. The implemented adapter +boundary is a versioned replay capsule containing the exact HSACO and SHA-256, +kernel symbol, launch geometry, declared lowered ABI, allocation snapshots, +pointer relocations, scratch requirements, target architecture, producer +versions, and case identity. Configure the matching adapter and a +workspace-contained capsule: + +```yaml +evaluation_profile: + language: triton # use flydsl for a FlyDSL artifact + artifact_kind: python_jit + adapter: triton_aot # or flydsl_aot + +evaluation_tools: + tools: + rocjitsu: + options: + capsule: eval_capsule/capsule.json +``` + +The plugin validates the capsule in the scoring process, rejects `launcher` or +`command` on AOT paths, and hashes the capsule JSON into the plan. The +image-owned helper rechecks its digest and manifest inside the sidecar, verifies +the adapter identity and `gfx950` target, generates a native HIP launcher, +compiles it with the sidecar toolchain, and reconstructs exactly one dispatch +inside rocJITsu. Post-execution parsing revalidates the capsule and requires the +expected dispatch, exact capsule/code-object marker, and replay success marker. + +The current validator fails closed for at least: + +- more than one kernel dispatch; +- opaque/tensor descriptors; +- empty, misordered, or unsupported ABI arguments and unknown implicit refs; +- invalid relocations and out-of-bounds allocation views; +- missing input blobs, mismatched hashes, or an architecture mismatch. + +Do not reduce a Python cache entry to just “HSACO + kernel name.” Without ABI, +launch, allocation, and framework-version evidence, a clean rocJITsu result may +belong to a different execution than the scored task. The replay adapter itself +is implemented and its safe/racy Triton and FlyDSL capsules have run end to end. +What is not implemented is automatic, evaluator-owned extraction from the +ordinary correctness dispatch and a trusted binding between that capture and +the scored candidate/case. Until that provenance exists, treat a clean AOT +result as advisory diagnostics rather than a production-qualified reward gate. + +## Resource, security, and held-out risks + +### Resource controls + +- The worker is sequential: one sidecar executes one GPU command at a time. + Parallel runs create a separate sidecar set per worker/GPU. +- Do not schedule another benchmark or sanitizer on the same physical GPU. + GPU ASan changes allocation behavior and rocJITsu can be much slower than + native execution; neither runtime is a performance measurement environment. +- Begin with one representative case and a bounded timeout. Expand coverage + only after measuring simulator time, HBM/host memory, and artifact growth. +- A timeout or truncated log is not clean evidence. + +### Isolation is a boundary, not proof of safe untrusted execution + +Runtime sidecars use no network, a read-only root filesystem, dropped Linux +capabilities, `no-new-privileges`, the host UID/GID, a read-only repository +mount, image-owned framework/probe code, and separate writable scratch/artifact +mounts. The scoring container gets the per-worker socket parent read-only, not +Docker access or tool-image credentials. Each sidecar gets only its own nested +writable socket directory, so it cannot replace or connect through a sibling's +mounted UDS path. RPC paths are relative and containment-checked, and normal +commands are argv lists. + +Artifact isolation is narrower but not yet per-task/tool. Every sidecar for one +worker gets the same writable repository-root +`.eval-tool-artifacts/` namespace. It no longer receives a +writable alias of the whole `experiments` tree, task workspaces, or +`.eval-tool-evidence`, but candidate code in one sidecar can still modify another +task/tool report in that worker's diagnostic namespace. The runner explicitly +mounts this namespace writable in the scoring container even when the broad +repository mount is read-only. + +However, a GPU kernel can still hang or reset the device, exploit a driver bug, +consume host/GPU memory, or affect another process sharing the GPU. Run +untrusted submissions on dedicated workers with external time, memory, process, +and machine-recovery controls. Do not mount secrets into a tool sidecar. An +explicit `bash -lc` command should be allowed only in a reviewed, trusted +adapter. + +There is also no evaluator-versus-agent trust boundary inside the current +scoring container. Sidecars start before the agent, and their per-tool Unix +sockets remain mounted for the whole container run without client +authentication. An agent process can therefore observe or call the worker RPC +even though it cannot replace the socket path. The external tool-artifact path +is also writable during optimization, so an adversarial submission could +pre-create evidence unless the adapter independently verifies its provenance. +Filesystem placement and a SHA-256 inside an agent-writable JSON file are +integrity checks, not signatures. + +The pre-agent `.eval-tool-evidence` copy is checked against an in-memory +manifest during an uninterrupted task, which detects ordinary mutation. It is +still stored on the current writable run mount and is not signed. After a crash, +an interrupted-run resume loads that on-disk manifest as its starting point, so +it is not a durable trust anchor against an adversarial process that can rewrite +both the manifest and copied files. + +For adversarial or reward-bearing evaluation, use a separate evaluation phase: +stop the agent container, snapshot the candidate, start fresh sidecars with +evaluator-only sockets, mount the candidate read-only, give each task/tool only +its own evaluator-owned artifact directory, authenticate the RPC client, and +generate attestations and AOT capsules there. Per-tool socket directories and a +narrow per-worker artifact mount are already implemented, but the current +single-container runner does not automate this phase boundary, so its tool +results should be treated as diagnostic evidence rather than tamper-resistant +rewards. + +### Held-out integrity + +Sanitizers exercise only the cases launched by their adapter. A clean visible +case is not proof of safety for all shapes, strides, dtypes, schedules, or race +interleavings, and sanitizer runs do not automatically invoke the separate +held-out evaluator. + +Task-local adapter code, commands, capsules, seeds, input blobs, expected +outputs, and positive-control structure may be visible to the optimizing agent. +The sidecar input mount currently contains the repository root, and the socket +is present during agent execution; merely moving a hidden file to another path +inside that mount does not make it private. +For a trusted held-out campaign: + +1. Keep private cases outside the agent-editable workspace and configuration. +2. Generate replay capsules only after the candidate is frozen. +3. Mount private inputs read-only only for the evaluation phase. +4. Keep private seeds and golden outputs out of committed task files and public + `.eval-tool-artifacts` reports. +5. Run ordinary held-out correctness and sanitizer coverage as separate signals; + neither substitutes for the other. + +## Known limitations checklist + +Before treating evaluation tools as a required scoring gate, account for all of +the following: + +- Only `gfx950` has pinned sidecar locks, passing integrated startup controls, + and end-to-end fixtures; `gfx942` is unverified and rejected by the host + runner. +- Every useful task still needs a reviewed adapter command. Tool installation + alone usually produces `adapter_required`. +- All four startup positive controls pass on the current `gfx950` host. This + qualifies tool installation, not candidate coverage. +- Runtime-internal asset paths are injected from verified sidecar health and + cannot be supplied by task configuration. +- Build-attestation artifact paths must be relative to the attestation file; + absolute paths and paths escaping that per-tool artifact directory are + rejected. +- Build attestations and native-HIP rocJITsu dispatch text are weak, + self-reported integrity evidence. Triton/FlyDSL replay validates and attests + the capsule more strongly, but automatic trusted capture still does not bind + it to the ordinary correctness dispatch. +- Sidecars have isolated writable socket directories, a read-only top-level + artifact namespace in scoring, and a narrow writable mount for only the + current worker. Sockets and that worker's agent-writable report paths remain + visible in the same optimizing/scoring container phase, and its tool sidecars + still share the worker artifact namespace. +- Tool startup resolves both the selected scoring-image reference and the + pinned SGLang content-addressed manifest reference to local image IDs, then + rejects the selected image unless those IDs match. The verified ID and + selected reference are recorded in plan evidence; using an alias does not + authorize a different image build. +- YAML `runtime_ref` does not select the image; it is an assertion compared with + the host-injected, worker-reported local image ID. +- Top-level resume does not yet enforce `plan_fingerprint` freshness. +- Fingerprints cover declared file content plus a configured capsule digest; + other option-referenced files need submission evidence or explicit digests. + The report serializes the complete tool plan. +- Triton/FlyDSL AOT replay is implemented for validated, single-dispatch + `gfx950` capsules and forbids arbitrary launchers. Automatic evaluator-owned + capsule extraction and correctness-run provenance are not implemented. +- FlyDSL GPU ASan and Triton FpSan instrumentation are unavailable. +- Precompiled AITER, rocBLAS, RCCL, and other library kernels are not covered + unless the exact source is rebuilt through a supported and attested path. +- “Clean” means no finding in the executed, attested cases; it is not a proof of + memory safety, race freedom, numerical equivalence for all inputs, or + generalization. + +Use `advisory` while any applicable item above remains unresolved. See the +[configuration and API reference](../reference/api-reference.md#evaluation-tools) +for the canonical field schema. diff --git a/docs/index.rst b/docs/index.rst index d3086ef2..7731218f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ repository. .. grid-item-card:: How to * :doc:`Run an experiment ` + * :doc:`Check kernels with evaluation tools ` * :doc:`Run tasks in parallel across multiple GPUs ` * :doc:`Configure agents and models ` * :doc:`Add a task ` diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index ccc1e668..9aa3a215 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -50,6 +50,73 @@ log_directory: logs workspace_directory_prefix: workspace ``` +## Evaluation tools + +`evaluation_tools` configures optional, isolated kernel-analysis sidecars. The +section is disabled when it is absent, `null`, or `false`. A configured mapping +with no enabled tools is also disabled unless the host runner supplies +`AKA_EVAL_TOOLS`; for a mapping, that host subset replaces its `enabled` value. +The built-in IDs are `triton_fpsan`, `gpu_asan`, `rocjitsu`, and `hip_fpsan`. + +Sidecar build locks, integrated positive controls, and end-to-end fixtures +currently exist only for `gfx950`; all four startup controls pass in the current +MI355X qualification. Each applicable candidate still needs a task-specific +adapter and attestation, and enabling an image alone does not imply that a +kernel was analyzed. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md) for the support matrix and operational +requirements. + +When tools are enabled, the selected scoring image must resolve to the same +immutable local Docker image ID as the pinned +`lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78` +manifest. The runner rejects a different build, launches by the verified ID, +and records both the selected reference and verified ID in plan source evidence. + +Worker reports live at repository-root +`.eval-tool-artifacts/`. The runner mounts that specific host +directory read/write into both sidecars and the scoring container; the latter +submount remains writable when the quality-loop repository root is read-only. + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `evaluation_tools.enabled` | boolean, string, or list of strings | empty | Tool IDs to plan. `true` expands to all four built-ins; `false` disables the feature. A single string is accepted. Hyphens are normalized to underscores. When the host sets `AKA_EVAL_TOOLS`, its normalized subset is authoritative for both started sidecars and the in-container plan. | +| `evaluation_tools.policy` | `advisory` or `required` | `advisory` | `advisory` always permits performance but records an unsatisfied policy. `required` permits performance only when every applicable selected tool is ready, completes, and reports `clean`. | +| `evaluation_tools.positive_control` | `required`, `optional`, `disabled`, or boolean | `required` | Requires the applicable synthetic known-bug startup control to pass before runtime capability is ready. `optional`, `disabled`, and `false` normalize to not required; the worker still runs and reports its control. | +| `evaluation_tools.timeout_s` | positive integer | `3600` | Default maximum execution time for each selected tool. | +| `evaluation_tools.runtime_profile` | string or `null` | `null` | Fallback runtime-identity assertion for plans whose tool has no `runtime_ref`. It does not select an image and must exactly match worker health when set. | +| `evaluation_tools.tools` | mapping | `{}` | Per-tool configuration keyed by normalized tool ID. Entries do not enable tools. | +| `evaluation_tools.tools..runtime_ref` | string or `null` | automatic host image ID | Exact bare local Docker image ID (`sha256:...`) asserted by the plan and compared with worker health. The `image_digest` key is an alias. This field does not select an image; omit it when using automatic host injection. | +| `evaluation_tools.tools..timeout_s` | positive integer | top-level timeout | Per-tool timeout. | +| `evaluation_tools.tools..options` | mapping | `{}` | Adapter options, including argv lists and candidate-evidence paths. Reserved framework keys are rejected at run and task level: `positive_control_required`; GPU ASan runtime/preload/library keys; rocJITsu binary/config keys; and HIP-FpSan include/header keys. | + +The exact reserved option keys are `positive_control_required` for every tool; +`asan_runtime_dir`, `hip_asan_runtime`, `host_asan_preload`, +`host_asan_lib_dir`, and `normal_rocm_lib_dir` for `gpu_asan`; +`rocjitsu_binary` and `config_path` for `rocjitsu`; and `include_dir` and +`public_header` for `hip_fpsan`. They are selected and attested by worker health, +not YAML. + +Example run-level section: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} +``` + +The image is selected separately with a host override. The runner resolves and +attests its immutable local image ID automatically: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +``` + ## Command-line flags The in-container `main.py` entrypoint accepts these flags: @@ -157,6 +224,65 @@ For repository-level tasks (`task_type: repository`): See [Add a task](../how-to/add-task.md) for layout and authoring rules. +### Evaluation profile + +The evaluator infers a profile from `task_type`, `repository_language`, source +suffixes, and repository paths. Add `evaluation_profile` only when that +inference is insufficient. Recognized profile overrides, including +`submission_paths`, are recorded in +`resolved_task_profile.explicit_overrides`. Unknown profile fields are currently +ignored, so use only the documented keys. + +| Field | Type | Description | +| --- | --- | --- | +| `evaluation_profile.language` | string | Canonical values are `triton`, `hip`, `flydsl`, and `unknown`. | +| `evaluation_profile.artifact_kind` | string | `source_aot`, `python_jit`, `hsaco_precompiled`, or `unknown`. | +| `evaluation_profile.framework` | string | Framework identity such as `standalone`, `aiter`, `rocblas`, or `rccl`. | +| `evaluation_profile.instrumentation_control` | string | `compiler_controlled`, `recompile`, `none`, or `unknown`. This describes whether the selected candidate can be rebuilt/instrumented. | +| `evaluation_profile.adapter` | string or `null` | Explicit adapter identity, for example `triton_aot`, `flydsl_aot`, or `hip_fpsan_manual`. It is a claim that must still be supported by adapter options/evidence. | +| `evaluation_profile.source_available` | boolean | Whether source for the selected candidate is available to the evaluator. | +| `evaluation_profile.submission_paths` | string or list of strings | Workspace-relative candidate files captured before agent edits and fingerprinted after optimization. Required when repository/image tasks change files beyond the normal source fields. Absolute paths and `..` are rejected. | +| `evaluation_profile.fpsan_ported` | boolean | Explicit evidence that the HIP reference and candidate were manually ported to HIP-FpSan value semantics. | +| `evaluation_profile.rebuilt_from_source` | boolean | Explicit evidence used when a framework/library path is rebuilt from controlled source. It does not replace artifact attestation. | + +### Task-level tool adapters + +A task can add adapter options only for tools enabled by the run. The only +allowed task-level structure is: + +```yaml +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] +``` + +`timeout_s` must be between 1 and the run-level value. Task configuration cannot +enable another tool, change the top-level `policy` or `positive_control`, select +another runtime image, increase a timeout, or set any reserved framework option +listed above. Other options are merged over the run-level options. + +Commands are argv lists, not shell strings. The built-in adapter keys are: + +| Tool | Adapter keys | +| --- | --- | +| `triton_fpsan` | `comparison_command` or `command`; optional `attestation_path`. | +| `gpu_asan` | `command`; optional candidate `attestation_path`. ASan runtime/preload/library paths come only from verified sidecar health. | +| `rocjitsu` | HIP uses `launcher` or `command`, with optional `expected_kernel` and candidate `race_report`. Triton/FlyDSL requires `capsule` and the exact `triton_aot`/`flydsl_aot` profile adapter; arbitrary launchers are rejected. The capsule must be workspace-contained, single-dispatch, manifest-valid, and target `gfx950`. Binary/config and the trusted replay helper come only from the image/health. Automatic trusted capsule capture from correctness is not implemented. | +| `hip_fpsan` | `comparison_command` or `command`; optional candidate `attestation_path`; requires `evaluation_profile.fpsan_ported: true`. The include path comes only from verified sidecar health. | + +Sidecar health attests and injects runtime-internal assets. Candidate/task +configuration cannot override or supply ASan preload/library paths, the +rocJITsu binary/config path, or the HIP-FpSan include path. + +Build-attestation JSON must store `artifact_path` relative to the directory +containing that JSON. The artifact must be beside or below the attestation; +absolute paths, `..`, and symlink resolutions that escape the directory are +rejected. The parser resolves the relative path in the scoring namespace and +verifies the declared SHA-256. + ### Platform support `platform_support.status: skip` excludes a task unconditionally. An active task @@ -175,6 +301,9 @@ Each task produces a `task_result.yaml` in its workspace: | `compilation_error_message` | Error text if compilation failed, else `null` | | `pass_correctness` | Whether correctness passed | | `correctness_error_message` | Error text if correctness failed, else `null` | +| `pass_tool_gate` | Whether the selected evaluation-tool policy allows performance to proceed. Defaults to `true` when tools are disabled. This is independent of `pass_correctness`. | +| `tool_policy_satisfied` | Whether every applicable selected tool was ready and completed with a `clean` finding status. Under `advisory`, this can be `false` while `pass_tool_gate` remains `true`. | +| `tool_evaluation` | Versioned complete plan, plan fingerprint, profile, capability, execution, finding, evidence, and decision data. Omitted when no tool is enabled. | | `base_execution_time` | Baseline runtime in ms | | `best_optimized_execution_time` | Best optimized runtime in ms | | `speedup_ratio` | Speedup over baseline | @@ -187,6 +316,37 @@ Each task produces a `task_result.yaml` in its workspace: | `optimization_summary` | Framework-generated note identifying the optimizing agent and centralized evaluator | | `score` | Computed score (see below) | +`tool_evaluation` uses this high-level shape: + +| Field | Description | +| --- | --- | +| `schema_version` | Evaluation-tool result schema version. | +| `plan_fingerprint` | SHA-256 over normalized configuration, resolved task profile, plugin versions, captured original/candidate evidence for declared paths, verified scoring-image reference/ID, and the content digest of a configured replay capsule. | +| `plan` | Complete immutable plan: schema, policy, profile, ordered tool records (runtime reference, plugin version, timeout, and options), fingerprint, and source evidence. | +| `policy` | `advisory` or `required`. | +| `overall_status` | `clean`, `finding`, `incomplete`, or `not_applicable`. | +| `resolved_task_profile` | Inferred profile plus auditable explicit overrides. | +| `source_evidence` | Captured-original and candidate fingerprints plus manifest metadata, including `metadata.scoring_runtime.image_id` and `.reference` when tools run. | +| `decision` | `allowed`, `policy_satisfied`, and machine-readable reason strings. | +| `tools..capability` | Separate `engine`, `adapter`, `runtime`, and resolved `effective` checks. | +| `tools..result.execution` | `not_run`, `completed`, `tool_error`, or `timeout`. | +| `tools..result.finding` | `not_evaluated`, `clean`, `found`, or `inconclusive`. | +| `tools..result.findings` | Structured finding records. | +| `tools..result.artifacts` | Paths to retained reports/attestations. Raw stdout/stderr is omitted from the YAML summary by default. | + +Execution status and finding status are deliberately independent. A sanitizer +can terminate the candidate while producing a valid finding, and a process can +exit zero without proving that an instrumented kernel ran. + +The complete `plan.tools` records make the selected runtime, plugin version, +timeout, and options reconstructable from the report. Ordinary option path +strings still do not hash referenced adapter, HSACO, or input contents; include +those files in `evaluation_profile.submission_paths` or add explicit digests. A +configured rocJITsu `capsule` is handled specially: its JSON SHA-256/size are +added to source evidence and the fingerprint, while capsule validation verifies +the manifest's HSACO and blob hashes. This does not prove the capsule came from +the ordinary correctness run. + ## Scoring The score is the sum of three components: diff --git a/docs/reference/compatibility-matrix.md b/docs/reference/compatibility-matrix.md index c5225bcd..fe659531 100644 --- a/docs/reference/compatibility-matrix.md +++ b/docs/reference/compatibility-matrix.md @@ -37,6 +37,29 @@ The following software versions are required or verified. | AITER | `0.1.17.dev110+g9127c94a1` in the verified `gfx950` image | Required by AITER-backed task oracles and kernels. | | FlyDSL | `0.2.2` in the verified `gfx950` image (or `make docker-setup-flydsl` when absent) | Required for `flydsl2flydsl`, `torch2flydsl`, and `triton2flydsl` tasks. | +## Evaluation-tool sidecars + +Optional Triton FpSan, GPU ASan, rocJITsu, and HIP-FpSan dependencies are kept +out of the scoring image and installed in one isolated sidecar image per tool. +The scoring image, FlyDSL, and AITER versions in the preceding table remain +unchanged. + +| GPU architecture | Sidecar status | Notes | +| --- | --- | --- | +| `gfx950` (MI355X) | Runtime-qualified, candidate-dependent | Pinned image/build locks and all four integrated startup controls pass on the current hardware. End-to-end readiness still depends on language, artifact, adapter, and candidate attestation. Trusted single-dispatch Triton/FlyDSL rocJITsu capsule replay is implemented, but automatic evaluator-owned capsule capture and binding to the correctness run remain advisory-only gaps. | +| `gfx942` (MI300X/MI325X) | Unverified | No equivalent image/adapter/positive-control qualification has completed; the host runner currently rejects evaluation-tool sidecars. | + +The runtime base digest and per-tool package/source locks are recorded in +`docker/eval-tools/images.lock.yaml`. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md#strict-support-matrix) for the strict +Triton, HIP, FlyDSL, AITER, rocBLAS, and RCCL matrix. Normal task compatibility +does not imply sanitizer coverage. Tool startup resolves both the selected +scoring-image reference and the pinned `gfx950` SGLang content-addressed +manifest reference to immutable local image IDs and requires those local IDs to +match. Aliases of that exact image are allowed, but rebuilt, upgraded, or +retagged images are rejected. The scoring container is launched by the verified +image ID. + ## Agents The following templates are selectable in the current `AgentType` registry. See diff --git a/docs/reference/release-notes.md b/docs/reference/release-notes.md index 6d229c38..b3f07ed5 100644 --- a/docs/reference/release-notes.md +++ b/docs/reference/release-notes.md @@ -21,6 +21,11 @@ execution, and RL-ready GPU kernel evaluation. - Added first-class A/B experimentation workflows with labeled baseline and treatment runs. - Exposed compilation, correctness, latency, speedup, and score fields as structured signals for external agent-RL systems. +- Added opt-in, per-tool sidecar plumbing and typed reports for Triton FpSan, + ROCm GPU ASan, rocJITsu, and HIP-FpSan. The initial sidecar locks are verified + only for `gfx950`. Workers automatically execute synthetic startup controls, + while useful candidate runs still require task-specific adapters and + attestations. - Added run comparison through `src/tools/compare_runs.py` and the standalone visualization dashboard. - Added held-out evaluation for testing kernel generalization on unseen shapes. - Centralized compilation, correctness, performance measurement, result generation, and scoring outside agent-editable code. @@ -115,6 +120,37 @@ The task validator now includes Codex backend support, repository-task validatio - `cuda2hip` is recognized by the prompt system, but no bundled cuda2hip task suite is currently included. - Local vLLM provider configuration remains specific to the selected agent integration. - GPU task execution requires compatible physical AMD hardware and ROCm driver access. +- Evaluation-tool sidecars are experimental and `gfx950`-only. Startup controls + prove a tool installation can detect its synthetic bug, not that a candidate + was instrumented. No bundled task currently supplies a production-qualified + adapter/attestation. All four integrated startup controls pass on the current + MI355X qualification host. Synthetic manager-to-sidecar candidate pairs also + distinguished clean from seeded-bug Triton FpSan, HIP/Triton GPU ASan, and + HIP-FpSan runs; trusted AOT replay produced a clean Triton result and found the + seeded FlyDSL LDS race. These fixtures do not qualify a bundled task. Keep the + policy advisory until each selected candidate path is independently qualified. +- The current runner exposes evaluation-tool sockets and agent-writable report + paths during the same container run. Per-tool writable socket directories and + a read-only top-level artifact namespace plus one writable + `.eval-tool-artifacts/