feat(eval): add isolated ROCm sanitizer plugins

.dockerignore

@@ -0,0 +1,13 @@
+# Evaluation-tool images need only their locked Docker assets and the immutable
+# eval-tools framework.  The repository commonly contains multi-gigabyte task
+# workspaces under experiments/, so an allow-list keeps builds fast and prevents
+# candidate/run artifacts from entering the Docker build context.
+**
+!docker/
+!docker/eval-tools/
+!docker/eval-tools/**
+!src/
+!src/eval_tools/
+!src/eval_tools/**
+**/__pycache__/
+**/*.py[cod]

.gitignore

@@ -7,6 +7,7 @@ build/
 .claude
 workspace_*
 /experiments/
+/.eval-tool-artifacts/
 
 # Auto-cloned agent dependencies
 agents/geak_optimagentv2/GEAK-agent/

agents/quality_loop/config.py

@@ -81,6 +81,7 @@ class QualityLoopConfig:
     case_enhancement: bool = True
     artifact_root: str = "quality_loop_runs"
     worktree_root: str = ".quality_loop_worktrees"
+    evaluation_tools: dict[str, Any] = field(default_factory=dict)
     promotion_task_types: tuple[str, ...] = (
         "hip2hip",
         "triton2triton",
@@ -121,6 +122,9 @@ def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig":
             "promotion_task_types",
             ["hip2hip", "triton2triton", "flydsl2flydsl"],
         )
+        evaluation_tools = raw.get("evaluation_tools") or {}
+        if not isinstance(evaluation_tools, dict):
+            raise ValueError("evaluation_tools must be a mapping")
         return cls(
             tasks=tuple(str(task) for task in tasks),
             target_gpu_model=target,
@@ -140,6 +144,7 @@ def from_dict(cls, raw: dict[str, Any]) -> "QualityLoopConfig":
                 audit.get("worktree_root", ".quality_loop_worktrees"),
                 "quality_loop.worktree_root",
             ),
+            evaluation_tools=dict(evaluation_tools),
             promotion_task_types=tuple(str(value) for value in promotion_types),
         )
 

agents/quality_loop/orchestrator.py

@@ -48,6 +48,9 @@
 from src.preprocessing import _resolve_gfx_arch, setup_workspace
 from src.prompt_builder import prompt_builder
 from src.testcases import collect_benchmark_methods
+from src.eval_tools.config import EvalToolsConfig
+from src.eval_tools.contracts import SourceEvidence
+from src.eval_tools.evidence import capture_submission_evidence
 
 
 def _task_slug(task_id: str) -> str:
@@ -130,6 +133,8 @@ def difficulty_is_easy(
         and statistics.median(speedups) >= config.easy_speedup_threshold
         and result.get("pass_compilation") is True
         and result.get("pass_correctness") is True
+        and result.get("pass_tool_gate", True) is True
+        and result.get("tool_policy_satisfied", True) is True
         and result.get("benchmark_method_consistent") is True
         and int(result.get("valid_baseline_cases", 0)) > 0
         and result.get("valid_baseline_cases") == result.get("valid_optimized_cases")
@@ -499,6 +504,14 @@ def _optimize_once(
     ) -> tuple[Path, list[Any], dict[str, Any]]:
         workspace = self._make_workspace(task_id, task_dir, stage_dir)
         task_config = self._load_task_config(task_dir)
+        eval_tools_config = EvalToolsConfig.from_mapping(self._eval_config())
+        submission_evidence = None
+        if eval_tools_config.enabled:
+            submission_evidence = capture_submission_evidence(
+                workspace,
+                task_config,
+                stage_dir / "submission_evidence",
+            )
         original_sources = stage_dir / "original_sources"
         original_sources.mkdir()
         source_manifest: dict[str, str] = {}
@@ -538,7 +551,40 @@ def _optimize_once(
         if snapshot_tree(original_sources) != original_source_tree:
             raise RuntimeError("optimizer modified the protected original-source snapshot")
         materialize_perf_helpers_in_workspace(workspace, logger=self.logger)
-        evaluation = evaluate_kernel(workspace, task_config, baseline_cases, self.logger)
+        tool_manager = None
+        tool_source_evidence = None
+        if eval_tools_config.enabled:
+            assert submission_evidence is not None
+            submission_evidence.verify()
+            tool_source_evidence = SourceEvidence(
+                original_root=str(submission_evidence.files_dir),
+                original_fingerprint=submission_evidence.fingerprint,
+                candidate_fingerprint=submission_evidence.candidate_fingerprint(),
+                metadata={
+                    "manifest": str(submission_evidence.storage_dir / "manifest.json"),
+                    "quality_loop_task": task_id,
+                },
+            )
+            from src.eval_tools.factory import (
+                create_default_manager,
+                task_artifact_root,
+            )
+
+            tool_manager = create_default_manager()
+            tool_report_root = task_artifact_root(workspace)
+        else:
+            tool_report_root = None
+        evaluation = evaluate_kernel(
+            workspace,
+            task_config,
+            baseline_cases,
+            self.logger,
+            tool_manager=tool_manager,
+            eval_tools_config=eval_tools_config,
+            tool_source_evidence=tool_source_evidence,
+            tool_artifact_root=tool_report_root,
+            gpu_arch=_resolve_gfx_arch(self.config.target_gpu_model),
+        )
         write_task_result(
             workspace,
             evaluation,
@@ -809,7 +855,7 @@ def _make_workspace(self, task_id: str, task_dir: Path, stage_dir: Path) -> Path
         )
 
     def _eval_config(self) -> dict[str, Any]:
-        return {
+        result = {
             "target_gpu_model": self.config.target_gpu_model,
             "agent": {
                 "template": "codex",
@@ -820,6 +866,9 @@ def _eval_config(self) -> dict[str, Any]:
                 "max_iterations": 1,
             },
         }
+        if self.config.evaluation_tools:
+            result["evaluation_tools"] = dict(self.config.evaluation_tools)
+        return result
 
     @staticmethod
     def _load_task_config(task_dir: Path) -> dict[str, Any]:

docker/eval-tools/gpu-asan/Dockerfile

@@ -0,0 +1,30 @@
+ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78
+FROM ${BASE_IMAGE}
+
+LABEL org.opencontainers.image.title="AgentKernelArena ROCm GPU ASan tool runtime" \
+      org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \
+      org.opencontainers.image.version="rocm-7.2.0-asan"
+
+COPY docker/eval-tools/gpu-asan/packages.sha256 /tmp/packages.sha256
+
+RUN mkdir -p /tmp/gpu-asan-debs \
+    && cd /tmp/gpu-asan-debs \
+    && apt-get update \
+    && apt-get download \
+        rocm-core-asan=7.2.0.70200-43~22.04 \
+        comgr-asan=3.0.0.70200-43~22.04 \
+        hsa-rocr-asan=1.18.0.70200-43~22.04 \
+        hip-runtime-amd-asan=7.2.26015.70200-43~22.04 \
+    && sha256sum --check /tmp/packages.sha256 \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ./*.deb \
+    && rm -rf /tmp/gpu-asan-debs /tmp/packages.sha256 /var/lib/apt/lists/*
+
+COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools
+
+ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \
+    PYTHONPATH=/opt/aka-eval-tools \
+    AKA_GPU_ASAN_RUNTIME_DIR=/opt/rocm-7.2.0/lib/asan \
+    AKA_GPU_ASAN_HIP_RUNTIME=/opt/rocm-7.2.0/lib/asan/libamdhip64.so \
+    AKA_GPU_ASAN_HOST_PRELOAD=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux/libclang_rt.asan-x86_64.so \
+    AKA_GPU_ASAN_HOST_LIB_DIR=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux \
+    AKA_GPU_ASAN_NORMAL_ROCM_LIB_DIR=/opt/rocm-7.2.0/lib

docker/eval-tools/gpu-asan/packages.sha256

@@ -0,0 +1,4 @@
+3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83  rocm-core-asan_7.2.0.70200-43~22.04_amd64.deb
+31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179  comgr-asan_3.0.0.70200-43~22.04_amd64.deb
+c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc  hsa-rocr-asan_1.18.0.70200-43~22.04_amd64.deb
+fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300  hip-runtime-amd-asan_7.2.26015.70200-43~22.04_amd64.deb

docker/eval-tools/hip-fpsan/Dockerfile

@@ -0,0 +1,31 @@
+ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78
+FROM ${BASE_IMAGE}
+
+ARG HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e
+
+LABEL org.opencontainers.image.title="AgentKernelArena HIP-FpSan tool runtime" \
+      org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \
+      org.opencontainers.image.revision="0ac9be8a1539a473ba21dfa686564c3be33c890e"
+
+COPY src/eval_tools/probes/hip_fpsan_probe.hip /tmp/hip_fpsan_probe.hip
+
+RUN git init /opt/hip-fpsan \
+    && git -C /opt/hip-fpsan remote add origin https://github.com/ROCm/hip-fpsan.git \
+    && git -C /opt/hip-fpsan fetch --depth 1 origin ${HIP_FPSAN_COMMIT} \
+    && git -C /opt/hip-fpsan checkout --detach FETCH_HEAD \
+    && test "$(git -C /opt/hip-fpsan rev-parse HEAD)" = "${HIP_FPSAN_COMMIT}" \
+    && mkdir -p /opt/eval-tools/probes \
+    && /opt/rocm/bin/hipcc -O2 --offload-arch=gfx950 \
+        -I/opt/hip-fpsan/include \
+        /tmp/hip_fpsan_probe.hip \
+        -o /opt/eval-tools/probes/hip_fpsan_probe \
+    && rm -f /tmp/hip_fpsan_probe.hip
+
+COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools
+
+ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \
+    PYTHONPATH=/opt/aka-eval-tools \
+    HIP_FPSAN_ROOT=/opt/hip-fpsan \
+    AKA_HIP_FPSAN_INCLUDE_DIR=/opt/hip-fpsan/include \
+    AKA_HIP_FPSAN_PROBE=/opt/eval-tools/probes/hip_fpsan_probe \
+    AKA_HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e

docker/eval-tools/hip-fpsan/sources.lock.yaml

@@ -0,0 +1 @@
+hip_fpsan: 0ac9be8a1539a473ba21dfa686564c3be33c890e

docker/eval-tools/images.lock.yaml

@@ -0,0 +1,74 @@
+schema_version: 1
+
+# The scoring runtime remains unchanged.  Every tool image is an immutable child
+# of this verified MI355X/gfx950 image.
+base:
+  gfx950:
+    reference: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705
+    digest: sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78
+    rocm: 7.2.0
+
+tools:
+  triton_fpsan:
+    triton:
+      version: 3.7.0+amd.rocm7.2.0.gitd0d77a509
+      sha256: 3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687
+    triton_kernels:
+      version: 1.0.0+amd.rocm7.2.0.gitd0d77a509
+      sha256: df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a
+
+  gpu_asan:
+    packages:
+      rocm-core-asan:
+        version: 7.2.0.70200-43~22.04
+        sha256: 3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83
+      comgr-asan:
+        version: 3.0.0.70200-43~22.04
+        sha256: 31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179
+      hsa-rocr-asan:
+        version: 1.18.0.70200-43~22.04
+        sha256: c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc
+      hip-runtime-amd-asan:
+        version: 7.2.26015.70200-43~22.04
+        sha256: fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300
+
+  rocjitsu:
+    repository: https://github.com/ROCm/rocm-systems.git
+    commit: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140
+    gcc: 13.4.0-6ubuntu1~22~ppa2
+    googletest:
+      repository: https://github.com/google/googletest.git
+      commit: b514bdc898e2951020cbdca1304b75f5950d1f59
+    flatbuffers:
+      repository: https://github.com/google/flatbuffers.git
+      commit: 595bf0007ab1929570c7671f091313c8fc20644e
+
+  rocjitsu_waitcheck:
+    repository: https://github.com/ROCm/rocm-systems.git
+    commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae
+    target: gfx950
+    interface: waitcheck-c-api-v1
+    gcc: 13.4.0-6ubuntu1~22~ppa2
+    zstd:
+      version: 1.5.7
+      sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
+
+  rocjitsu_consan:
+    repository: https://github.com/ROCm/rocm-systems.git
+    commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae
+    target: gfx950
+    mode: record-replay
+    policy: strict
+    gcc: 13.4.0-6ubuntu1~22~ppa2
+    zstd:
+      version: 1.5.7
+      sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3
+
+  hip_fpsan:
+    repository: https://github.com/ROCm/hip-fpsan.git
+    commit: 0ac9be8a1539a473ba21dfa686564c3be33c890e
+    image_probe: src/eval_tools/probes/hip_fpsan_probe.hip
+
+verification:
+  gfx950: verified
+  gfx942: unverified