diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..b2c01fa9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +# Evaluation-tool images need only their locked Docker assets and the immutable +# eval-tools framework. The repository commonly contains multi-gigabyte task +# workspaces under experiments/, so an allow-list keeps builds fast and prevents +# candidate/run artifacts from entering the Docker build context. +** +!docker/ +!docker/eval-tools/ +!docker/eval-tools/** +!src/ +!src/eval_tools/ +!src/eval_tools/** +**/__pycache__/ +**/*.py[cod] diff --git a/.gitignore b/.gitignore index 021f8bc9..90577184 100755 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build/ .idea .claude workspace_* +/.eval-tool-artifacts/ # Auto-cloned agent dependencies agents/geak_optimagentv2/GEAK-agent/ diff --git a/docker/eval-tools/gpu-asan/Dockerfile b/docker/eval-tools/gpu-asan/Dockerfile new file mode 100644 index 00000000..a3c2bf17 --- /dev/null +++ b/docker/eval-tools/gpu-asan/Dockerfile @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena ROCm GPU ASan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="rocm-7.2.0-asan" + +COPY docker/eval-tools/gpu-asan/packages.sha256 /tmp/packages.sha256 + +RUN mkdir -p /tmp/gpu-asan-debs \ + && cd /tmp/gpu-asan-debs \ + && apt-get update \ + && apt-get download \ + rocm-core-asan=7.2.0.70200-43~22.04 \ + comgr-asan=3.0.0.70200-43~22.04 \ + hsa-rocr-asan=1.18.0.70200-43~22.04 \ + hip-runtime-amd-asan=7.2.26015.70200-43~22.04 \ + && sha256sum --check /tmp/packages.sha256 \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ./*.deb \ + && rm -rf /tmp/gpu-asan-debs /tmp/packages.sha256 /var/lib/apt/lists/* + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_GPU_ASAN_RUNTIME_DIR=/opt/rocm-7.2.0/lib/asan \ + AKA_GPU_ASAN_HIP_RUNTIME=/opt/rocm-7.2.0/lib/asan/libamdhip64.so \ + AKA_GPU_ASAN_HOST_PRELOAD=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux/libclang_rt.asan-x86_64.so \ + AKA_GPU_ASAN_HOST_LIB_DIR=/opt/rocm-7.2.0/lib/llvm/lib/clang/22/lib/linux \ + AKA_GPU_ASAN_NORMAL_ROCM_LIB_DIR=/opt/rocm-7.2.0/lib diff --git a/docker/eval-tools/gpu-asan/packages.sha256 b/docker/eval-tools/gpu-asan/packages.sha256 new file mode 100644 index 00000000..6d8078c4 --- /dev/null +++ b/docker/eval-tools/gpu-asan/packages.sha256 @@ -0,0 +1,4 @@ +3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 rocm-core-asan_7.2.0.70200-43~22.04_amd64.deb +31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 comgr-asan_3.0.0.70200-43~22.04_amd64.deb +c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc hsa-rocr-asan_1.18.0.70200-43~22.04_amd64.deb +fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 hip-runtime-amd-asan_7.2.26015.70200-43~22.04_amd64.deb diff --git a/docker/eval-tools/hip-fpsan/Dockerfile b/docker/eval-tools/hip-fpsan/Dockerfile new file mode 100644 index 00000000..d7b4a4e9 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/Dockerfile @@ -0,0 +1,31 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +ARG HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e + +LABEL org.opencontainers.image.title="AgentKernelArena HIP-FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0ac9be8a1539a473ba21dfa686564c3be33c890e" + +COPY src/eval_tools/probes/hip_fpsan_probe.hip /tmp/hip_fpsan_probe.hip + +RUN git init /opt/hip-fpsan \ + && git -C /opt/hip-fpsan remote add origin https://github.com/ROCm/hip-fpsan.git \ + && git -C /opt/hip-fpsan fetch --depth 1 origin ${HIP_FPSAN_COMMIT} \ + && git -C /opt/hip-fpsan checkout --detach FETCH_HEAD \ + && test "$(git -C /opt/hip-fpsan rev-parse HEAD)" = "${HIP_FPSAN_COMMIT}" \ + && mkdir -p /opt/eval-tools/probes \ + && /opt/rocm/bin/hipcc -O2 --offload-arch=gfx950 \ + -I/opt/hip-fpsan/include \ + /tmp/hip_fpsan_probe.hip \ + -o /opt/eval-tools/probes/hip_fpsan_probe \ + && rm -f /tmp/hip_fpsan_probe.hip + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + HIP_FPSAN_ROOT=/opt/hip-fpsan \ + AKA_HIP_FPSAN_INCLUDE_DIR=/opt/hip-fpsan/include \ + AKA_HIP_FPSAN_PROBE=/opt/eval-tools/probes/hip_fpsan_probe \ + AKA_HIP_FPSAN_COMMIT=0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/hip-fpsan/sources.lock.yaml b/docker/eval-tools/hip-fpsan/sources.lock.yaml new file mode 100644 index 00000000..87f39fe3 --- /dev/null +++ b/docker/eval-tools/hip-fpsan/sources.lock.yaml @@ -0,0 +1 @@ +hip_fpsan: 0ac9be8a1539a473ba21dfa686564c3be33c890e diff --git a/docker/eval-tools/images.lock.yaml b/docker/eval-tools/images.lock.yaml new file mode 100644 index 00000000..90a243ab --- /dev/null +++ b/docker/eval-tools/images.lock.yaml @@ -0,0 +1,74 @@ +schema_version: 1 + +# The scoring runtime remains unchanged. Every tool image is an immutable child +# of this verified MI355X/gfx950 image. +base: + gfx950: + reference: lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 + digest: sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 + rocm: 7.2.0 + +tools: + triton_fpsan: + triton: + version: 3.7.0+amd.rocm7.2.0.gitd0d77a509 + sha256: 3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 + triton_kernels: + version: 1.0.0+amd.rocm7.2.0.gitd0d77a509 + sha256: df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a + + gpu_asan: + packages: + rocm-core-asan: + version: 7.2.0.70200-43~22.04 + sha256: 3bd5b98b3ae2cb8fbfd10c248682feb86d0f5136914e7a226b701340f9b90f83 + comgr-asan: + version: 3.0.0.70200-43~22.04 + sha256: 31118ea2dc79fe9d8c69ad7ff2176a2f1c822128ece604d4438b6c13a1aa3179 + hsa-rocr-asan: + version: 1.18.0.70200-43~22.04 + sha256: c5d6e48846f6163b5c8d7168949e1c24dc41f69bd871705621b0972c2f1a03cc + hip-runtime-amd-asan: + version: 7.2.26015.70200-43~22.04 + sha256: fa56c2192f28adb022dc323d2836dfbe4f12211287575ea27978dc96b10a5300 + + rocjitsu: + repository: https://github.com/ROCm/rocm-systems.git + commit: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 + gcc: 13.4.0-6ubuntu1~22~ppa2 + googletest: + repository: https://github.com/google/googletest.git + commit: b514bdc898e2951020cbdca1304b75f5950d1f59 + flatbuffers: + repository: https://github.com/google/flatbuffers.git + commit: 595bf0007ab1929570c7671f091313c8fc20644e + + rocjitsu_waitcheck: + repository: https://github.com/ROCm/rocm-systems.git + commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae + target: gfx950 + interface: waitcheck-c-api-v1 + gcc: 13.4.0-6ubuntu1~22~ppa2 + zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + + rocjitsu_consan: + repository: https://github.com/ROCm/rocm-systems.git + commit: ed35c0b54547c98bab359c8732529d9f5e8fd1ae + target: gfx950 + mode: record-replay + policy: strict + gcc: 13.4.0-6ubuntu1~22~ppa2 + zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + + hip_fpsan: + repository: https://github.com/ROCm/hip-fpsan.git + commit: 0ac9be8a1539a473ba21dfa686564c3be33c890e + image_probe: src/eval_tools/probes/hip_fpsan_probe.hip + +verification: + gfx950: verified + gfx942: unverified diff --git a/docker/eval-tools/rocjitsu-sanitizers/Dockerfile b/docker/eval-tools/rocjitsu-sanitizers/Dockerfile new file mode 100644 index 00000000..9178b765 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/Dockerfile @@ -0,0 +1,117 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} AS builder + +ARG ROCJITSU_SANITIZERS_COMMIT=ed35c0b54547c98bab359c8732529d9f5e8fd1ae +ARG GOOGLETEST_COMMIT=b514bdc898e2951020cbdca1304b75f5950d1f59 +ARG FLATBUFFERS_COMMIT=595bf0007ab1929570c7671f091313c8fc20644e +ARG ZSTD_VERSION=1.5.7 +ARG ZSTD_SHA256=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates curl git software-properties-common zlib1g-dev \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc-13=13.4.0-6ubuntu1~22~ppa2 \ + g++-13=13.4.0-6ubuntu1~22~ppa2 \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN git init /src/rocm-systems \ + && git -C /src/rocm-systems remote add origin https://github.com/ROCm/rocm-systems.git \ + && git -C /src/rocm-systems fetch --depth 1 origin ${ROCJITSU_SANITIZERS_COMMIT} \ + && git -C /src/rocm-systems checkout --detach FETCH_HEAD \ + && test "$(git -C /src/rocm-systems rev-parse HEAD)" = "${ROCJITSU_SANITIZERS_COMMIT}" \ + && git init /src/googletest \ + && git -C /src/googletest remote add origin https://github.com/google/googletest.git \ + && git -C /src/googletest fetch --depth 1 origin ${GOOGLETEST_COMMIT} \ + && git -C /src/googletest checkout --detach FETCH_HEAD \ + && git init /src/flatbuffers \ + && git -C /src/flatbuffers remote add origin https://github.com/google/flatbuffers.git \ + && git -C /src/flatbuffers fetch --depth 1 origin ${FLATBUFFERS_COMMIT} \ + && git -C /src/flatbuffers checkout --detach FETCH_HEAD + +RUN curl -sSfL -o /tmp/zstd.tar.gz \ + "https://github.com/facebook/zstd/releases/download/v${ZSTD_VERSION}/zstd-${ZSTD_VERSION}.tar.gz" \ + && printf '%s %s\n' "${ZSTD_SHA256}" /tmp/zstd.tar.gz | sha256sum --check --quiet \ + && mkdir -p /src/zstd \ + && tar -xzf /tmp/zstd.tar.gz -C /src/zstd --strip-components=1 \ + && cmake -S /src/zstd/build/cmake -B /build/zstd -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/zstd \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DZSTD_BUILD_SHARED=OFF \ + -DZSTD_BUILD_STATIC=ON \ + -DZSTD_BUILD_PROGRAMS=OFF \ + -DZSTD_BUILD_TESTS=OFF \ + && cmake --build /build/zstd --parallel \ + && cmake --install /build/zstd \ + && rm -f /tmp/zstd.tar.gz + +RUN cmake -S /src/rocm-systems/emulation/rocjitsu -B /build/rocjitsu -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-13 \ + -DCMAKE_CXX_COMPILER=g++-13 \ + -DCMAKE_INSTALL_PREFIX=/opt/rocjitsu \ + -Dzstd_DIR=/opt/zstd/lib/cmake/zstd \ + -DFETCHCONTENT_SOURCE_DIR_GOOGLETEST=/src/googletest \ + -DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=/src/flatbuffers \ + -DBUILD_TESTING=OFF \ + && cmake --build /build/rocjitsu --parallel \ + --target rj_waitcheck rocjitsu_waitcheck rocjitsu_dbi_hooks \ + && mkdir -p /opt/rocjitsu/include /opt/rocjitsu/lib \ + && cp -a /src/rocm-systems/emulation/rocjitsu/lib/rocjitsu/include/rocjitsu \ + /opt/rocjitsu/include/ \ + && cp -a /build/rocjitsu/librocjitsu_waitcheck.so* /opt/rocjitsu/lib/ \ + && install -D -m 0755 \ + /build/rocjitsu/lib/rocjitsu/src/rocjitsu/hooks/librocjitsu_dbi_hooks.so \ + /opt/rocjitsu/lib/librocjitsu_dbi_hooks.so \ + && install -D -m 0755 /build/rocjitsu/tools/rj_waitcheck /opt/rocjitsu/bin/rj_waitcheck \ + && mkdir -p /opt/rocjitsu/runtime \ + && cp -a /usr/lib/x86_64-linux-gnu/libstdc++.so.6* /opt/rocjitsu/runtime/ \ + && cp -a /lib/x86_64-linux-gnu/libgcc_s.so.1 /opt/rocjitsu/runtime/ + +COPY docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp /src/waitcheck-capi.cpp +RUN g++-13 -std=c++17 -O2 -Wall -Wextra -Werror \ + -I/opt/rocjitsu/include /src/waitcheck-capi.cpp \ + -L/opt/rocjitsu/lib -Wl,-rpath,/opt/rocjitsu/lib \ + -lrocjitsu_waitcheck -o /opt/rocjitsu/bin/aka-waitcheck-capi \ + && /opt/rocjitsu/bin/aka-waitcheck-capi --help >/dev/null + +FROM ${BASE_IMAGE} AS runtime-common + +LABEL org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="ed35c0b54547c98bab359c8732529d9f5e8fd1ae" + +COPY --from=builder /opt/rocjitsu /opt/rocjitsu +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + LD_LIBRARY_PATH=/opt/rocjitsu/lib:/opt/rocjitsu/runtime \ + PATH=/opt/rocjitsu/bin:${PATH} \ + AKA_ROCJITSU_SANITIZERS_COMMIT=ed35c0b54547c98bab359c8732529d9f5e8fd1ae + +FROM runtime-common AS rocjitsu-waitcheck-runtime + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu Waitcheck runtime" + +ENV AKA_WAITCHECK_BINARY=/opt/rocjitsu/bin/rj_waitcheck \ + AKA_WAITCHECK_CAPI_WRAPPER=/opt/rocjitsu/bin/aka-waitcheck-capi + +RUN /opt/rocjitsu/bin/rj_waitcheck --help >/dev/null \ + && /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/waitcheck_entrypoint.py \ + --help >/dev/null + +FROM runtime-common AS rocjitsu-consan-runtime + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu ConSan runtime" + +ENV AKA_CONSAN_HOOK=/opt/rocjitsu/lib/librocjitsu_dbi_hooks.so + +RUN test -f /opt/rocjitsu/lib/librocjitsu_dbi_hooks.so \ + && /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/consan_entrypoint.py \ + --help >/dev/null diff --git a/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml b/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml new file mode 100644 index 00000000..700509c6 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/sources.lock.yaml @@ -0,0 +1,7 @@ +rocm_systems: ed35c0b54547c98bab359c8732529d9f5e8fd1ae +googletest: b514bdc898e2951020cbdca1304b75f5950d1f59 +flatbuffers: 595bf0007ab1929570c7671f091313c8fc20644e +gcc_13: 13.4.0-6ubuntu1~22~ppa2 +zstd: + version: 1.5.7 + sha256: eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 diff --git a/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp b/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp new file mode 100644 index 00000000..9e86b060 --- /dev/null +++ b/docker/eval-tools/rocjitsu-sanitizers/waitcheck-capi.cpp @@ -0,0 +1,213 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. +// SPDX-License-Identifier: MIT + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Diagnostic { + std::string code; + std::string kernel_name; + uint64_t kernel_entry = 0; + int counter = 0; + int access = 0; + int register_class = 0; + unsigned register_index = 0; + unsigned register_width = 0; + std::string section_name; + uint64_t section_offset = 0; + uint64_t file_offset = 0; + std::string instruction; + uint64_t producer_section_offset = 0; + uint64_t producer_file_offset = 0; + std::string producer_instruction; + uint32_t required_count = 0; + std::string message; +}; + +std::string copy_string(const char *value) { return value == nullptr ? "" : value; } + +void write_json_string(std::ostream &output, std::string_view value) { + static constexpr char hex[] = "0123456789abcdef"; + output << '"'; + for (const unsigned char character : value) { + switch (character) { + case '"': output << "\\\""; break; + case '\\': output << "\\\\"; break; + case '\b': output << "\\b"; break; + case '\f': output << "\\f"; break; + case '\n': output << "\\n"; break; + case '\r': output << "\\r"; break; + case '\t': output << "\\t"; break; + default: + if (character < 0x20) { + output << "\\u00" << hex[character >> 4] << hex[character & 0xf]; + } else { + output << character; + } + } + } + output << '"'; +} + +void diagnostic_callback(const rj_waitcheck_diagnostic_t *input, void *opaque) { + if (input == nullptr || opaque == nullptr) + return; + auto &diagnostics = *static_cast *>(opaque); + Diagnostic value; + value.code = copy_string(rj_waitcheck_diagnostic_code_name(input->code)); + value.kernel_name = copy_string(input->kernel_name); + value.kernel_entry = input->kernel_entry_offset; + value.counter = static_cast(input->counter); + value.access = static_cast(input->access); + value.register_class = static_cast(input->reg.register_class); + value.register_index = input->reg.index; + value.register_width = input->reg.width; + value.section_name = copy_string(input->section_name); + value.section_offset = input->section_offset; + value.file_offset = input->file_offset; + value.instruction = copy_string(input->instruction); + value.producer_section_offset = input->producer_section_offset; + value.producer_file_offset = input->producer_file_offset; + value.producer_instruction = copy_string(input->producer_instruction); + value.required_count = input->required_count; + value.message = copy_string(input->message); + diagnostics.push_back(std::move(value)); +} + +void error_callback(const char *message, void *opaque) { + if (opaque != nullptr) + *static_cast(opaque) = copy_string(message); +} + +bool parse_uint64(std::string_view text, uint64_t &value) { + try { + size_t consumed = 0; + value = std::stoull(std::string(text), &consumed, 0); + return consumed == text.size(); + } catch (...) { + return false; + } +} + +} // namespace + +int main(int argc, char **argv) { + std::string path; + uint64_t kernel_entry = 0; + bool has_entry = false; + for (int index = 1; index < argc; ++index) { + const std::string_view argument(argv[index]); + if (argument == "--code-object" && index + 1 < argc) { + path = argv[++index]; + } else if (argument == "--kernel-entry" && index + 1 < argc) { + has_entry = parse_uint64(argv[++index], kernel_entry); + } else if (argument == "--help") { + std::cout << "usage: aka-waitcheck-capi --code-object PATH --kernel-entry OFFSET\n"; + return 0; + } else { + std::cerr << "invalid argument: " << argument << "\n"; + return 2; + } + } + if (path.empty() || !has_entry) { + std::cerr << "code object and kernel entry are required\n"; + return 2; + } + + std::ifstream input(path, std::ios::binary); + if (!input) { + std::cerr << "could not open code object\n"; + return 2; + } + const std::vector bytes((std::istreambuf_iterator(input)), {}); + if (bytes.empty()) { + std::cerr << "code object is empty\n"; + return 2; + } + + std::vector diagnostics; + std::string analysis_error; + rj_waitcheck_options_t options{}; + rj_waitcheck_result_t result{}; + if (rj_waitcheck_options_init(&options, sizeof(options)) != ROCJITSU_STATUS_SUCCESS || + rj_waitcheck_result_init(&result, sizeof(result)) != ROCJITSU_STATUS_SUCCESS) { + std::cerr << "could not initialize waitcheck ABI structures\n"; + return 2; + } + options.max_diagnostics = 0; + options.stop_after_first_diagnostic = 0; + options.diagnostic_callback = diagnostic_callback; + options.error_callback = error_callback; + // Both callbacks need distinct storage, so switch user_data immediately + // around the API call through a small aggregate. + struct CallbackState { + std::vector *diagnostics; + std::string *error; + } state{&diagnostics, &analysis_error}; + options.user_data = &state; + options.diagnostic_callback = [](const rj_waitcheck_diagnostic_t *value, void *opaque) { + auto *callback_state = static_cast(opaque); + diagnostic_callback(value, callback_state->diagnostics); + }; + options.error_callback = [](const char *value, void *opaque) { + auto *callback_state = static_cast(opaque); + error_callback(value, callback_state->error); + }; + + const rj_status_t status = rj_waitcheck_analyze_kernel( + bytes.data(), bytes.size(), kernel_entry, &options, &result); + std::cout << "AKA_WAITCHECK_CAPI_RESULT {\"schema_version\":1,\"api_status\":" + << static_cast(status) << ",\"analysis_complete\":" + << (status == ROCJITSU_STATUS_SUCCESS ? "true" : "false") << ",\"target\":"; + write_json_string(std::cout, rj_waitcheck_target_name(result.target)); + std::cout << ",\"instructions_analyzed\":" << result.instructions_analyzed + << ",\"memory_events_tracked\":" << result.memory_events_tracked + << ",\"kernels_discovered\":" << result.kernels_discovered + << ",\"kernels_analyzed\":" << result.kernels_analyzed + << ",\"diagnostics_observed\":" << result.diagnostics_observed + << ",\"diagnostics_reported\":" << result.diagnostics_reported + << ",\"passed\":" << (result.passed ? "true" : "false") + << ",\"diagnostics_truncated\":" + << (result.diagnostics_truncated ? "true" : "false") + << ",\"stopped_early\":" << (result.stopped_early ? "true" : "false") + << ",\"analysis_error\":"; + write_json_string(std::cout, analysis_error); + std::cout << ",\"diagnostics\":["; + for (size_t index = 0; index < diagnostics.size(); ++index) { + if (index != 0) + std::cout << ','; + const Diagnostic &value = diagnostics[index]; + std::cout << "{\"code\":"; + write_json_string(std::cout, value.code); + std::cout << ",\"kernel_name\":"; + write_json_string(std::cout, value.kernel_name); + std::cout << ",\"kernel_entry\":" << value.kernel_entry + << ",\"counter\":" << value.counter << ",\"access\":" << value.access + << ",\"register_class\":" << value.register_class + << ",\"register_index\":" << value.register_index + << ",\"register_width\":" << value.register_width + << ",\"section_name\":"; + write_json_string(std::cout, value.section_name); + std::cout << ",\"section_offset\":" << value.section_offset + << ",\"file_offset\":" << value.file_offset << ",\"instruction\":"; + write_json_string(std::cout, value.instruction); + std::cout << ",\"producer_section_offset\":" << value.producer_section_offset + << ",\"producer_file_offset\":" << value.producer_file_offset + << ",\"producer_instruction\":"; + write_json_string(std::cout, value.producer_instruction); + std::cout << ",\"required_count\":" << value.required_count << ",\"message\":"; + write_json_string(std::cout, value.message); + std::cout << '}'; + } + std::cout << "]}\n"; + return status == ROCJITSU_STATUS_SUCCESS ? 0 : 2; +} diff --git a/docker/eval-tools/rocjitsu/Dockerfile b/docker/eval-tools/rocjitsu/Dockerfile new file mode 100644 index 00000000..7a0c2c0a --- /dev/null +++ b/docker/eval-tools/rocjitsu/Dockerfile @@ -0,0 +1,70 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} AS builder + +ARG ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +ARG GOOGLETEST_COMMIT=b514bdc898e2951020cbdca1304b75f5950d1f59 +ARG FLATBUFFERS_COMMIT=595bf0007ab1929570c7671f091313c8fc20644e + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates git software-properties-common \ + && add-apt-repository -y ppa:ubuntu-toolchain-r/test \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + gcc-13=13.4.0-6ubuntu1~22~ppa2 \ + g++-13=13.4.0-6ubuntu1~22~ppa2 \ + ninja-build \ + && rm -rf /var/lib/apt/lists/* + +RUN git init /src/rocm-systems \ + && git -C /src/rocm-systems remote add origin https://github.com/ROCm/rocm-systems.git \ + && git -C /src/rocm-systems fetch --depth 1 origin ${ROCJITSU_COMMIT} \ + && git -C /src/rocm-systems checkout --detach FETCH_HEAD \ + && test "$(git -C /src/rocm-systems rev-parse HEAD)" = "${ROCJITSU_COMMIT}" \ + && git init /src/googletest \ + && git -C /src/googletest remote add origin https://github.com/google/googletest.git \ + && git -C /src/googletest fetch --depth 1 origin ${GOOGLETEST_COMMIT} \ + && git -C /src/googletest checkout --detach FETCH_HEAD \ + && git init /src/flatbuffers \ + && git -C /src/flatbuffers remote add origin https://github.com/google/flatbuffers.git \ + && git -C /src/flatbuffers fetch --depth 1 origin ${FLATBUFFERS_COMMIT} \ + && git -C /src/flatbuffers checkout --detach FETCH_HEAD + +RUN cmake -S /src/rocm-systems/emulation/rocjitsu -B /build/rocjitsu -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc-13 \ + -DCMAKE_CXX_COMPILER=g++-13 \ + -DCMAKE_INSTALL_PREFIX=/opt/rocjitsu \ + -DFETCHCONTENT_SOURCE_DIR_GOOGLETEST=/src/googletest \ + -DFETCHCONTENT_SOURCE_DIR_FLATBUFFERS=/src/flatbuffers \ + -DCMAKE_HIP_ARCHITECTURES=gfx950 \ + -DBUILD_TESTING=ON \ + -DRJ_INSTALL_TESTS=OFF \ + && cmake --build /build/rocjitsu --parallel \ + && cmake --install /build/rocjitsu \ + && mkdir -p /opt/rocjitsu/runtime \ + && cp -a /usr/lib/x86_64-linux-gnu/libstdc++.so.6* /opt/rocjitsu/runtime/ \ + && cp -a /lib/x86_64-linux-gnu/libgcc_s.so.1 /opt/rocjitsu/runtime/ + +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena rocJITsu tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.revision="0bf561a0d8a4a6b88954f2c46bd3a50871cda140" + +COPY --from=builder /opt/rocjitsu /opt/rocjitsu +COPY docker/eval-tools/rocjitsu/rocjitsu-wrapper /usr/local/bin/rocjitsu +RUN chmod 0755 /usr/local/bin/rocjitsu \ + && /usr/local/bin/rocjitsu --help >/dev/null + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools +RUN /opt/venv/bin/python -I \ + /opt/aka-eval-tools/src/eval_tools/adapters/rocjitsu_replay_entrypoint.py \ + --help >/dev/null + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + PATH=/opt/rocjitsu/bin:${PATH} \ + AKA_ROCJITSU_BINARY=/usr/local/bin/rocjitsu \ + AKA_ROCJITSU_CONFIG=/opt/rocjitsu/share/rocjitsu/configs/gfx950_cdna4.json \ + AKA_ROCJITSU_COMMIT=0bf561a0d8a4a6b88954f2c46bd3a50871cda140 diff --git a/docker/eval-tools/rocjitsu/rocjitsu-wrapper b/docker/eval-tools/rocjitsu/rocjitsu-wrapper new file mode 100755 index 00000000..d094b3a8 --- /dev/null +++ b/docker/eval-tools/rocjitsu/rocjitsu-wrapper @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +export LD_LIBRARY_PATH="/opt/rocjitsu/runtime:/opt/rocjitsu/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +# rocJITsu's current gfx950 runtime exposes one simulated agent numbered zero. +# A physical host GPU mask (for example ROCR_VISIBLE_DEVICES=4) would otherwise +# filter that simulated agent out of the launched HIP process. +export ROCR_VISIBLE_DEVICES=0 +export HIP_VISIBLE_DEVICES=0 +export CUDA_VISIBLE_DEVICES=0 +export GPU_DEVICE_ORDINAL=0 +exec /opt/rocjitsu/bin/rocjitsu "$@" diff --git a/docker/eval-tools/rocjitsu/sources.lock.yaml b/docker/eval-tools/rocjitsu/sources.lock.yaml new file mode 100644 index 00000000..2975cc96 --- /dev/null +++ b/docker/eval-tools/rocjitsu/sources.lock.yaml @@ -0,0 +1,4 @@ +rocm_systems: 0bf561a0d8a4a6b88954f2c46bd3a50871cda140 +googletest: b514bdc898e2951020cbdca1304b75f5950d1f59 +flatbuffers: 595bf0007ab1929570c7671f091313c8fc20644e +gcc_13: 13.4.0-6ubuntu1~22~ppa2 diff --git a/docker/eval-tools/triton-fpsan/Dockerfile b/docker/eval-tools/triton-fpsan/Dockerfile new file mode 100644 index 00000000..fa7819cc --- /dev/null +++ b/docker/eval-tools/triton-fpsan/Dockerfile @@ -0,0 +1,21 @@ +ARG BASE_IMAGE=lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +FROM ${BASE_IMAGE} + +LABEL org.opencontainers.image.title="AgentKernelArena Triton FpSan tool runtime" \ + org.opencontainers.image.base.digest="sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78" \ + org.opencontainers.image.version="triton-3.7.0+amd.rocm7.2.0.gitd0d77a509" + +COPY docker/eval-tools/triton-fpsan/requirements.lock /tmp/eval-tool-requirements.lock + +RUN python -m pip uninstall -y \ + triton pytorch-triton pytorch-triton-rocm triton-rocm amd-triton triton-kernels \ + && python -m pip install --no-cache-dir --no-deps --require-hashes \ + -r /tmp/eval-tool-requirements.lock \ + && python -c "import importlib.metadata as m, triton, triton_kernels; assert m.version('triton') == '3.7.0+amd.rocm7.2.0.gitd0d77a509'; assert m.version('triton-kernels') == '1.0.0+amd.rocm7.2.0.gitd0d77a509'" \ + && rm -f /tmp/eval-tool-requirements.lock + +COPY src/eval_tools /opt/aka-eval-tools/src/eval_tools + +ENV AKA_EVAL_TOOL_FRAMEWORK_ROOT=/opt/aka-eval-tools \ + PYTHONPATH=/opt/aka-eval-tools \ + AKA_TRITON_FPSAN_VERSION=3.7.0+amd.rocm7.2.0.gitd0d77a509 diff --git a/docker/eval-tools/triton-fpsan/requirements.lock b/docker/eval-tools/triton-fpsan/requirements.lock new file mode 100644 index 00000000..d8fc5460 --- /dev/null +++ b/docker/eval-tools/triton-fpsan/requirements.lock @@ -0,0 +1,4 @@ +triton @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton/triton-3.7.0+amd.rocm7.2.0.gitd0d77a509-cp310-cp310-linux_x86_64.whl \ + --hash=sha256:3a8acacdfb4723c8bb71844c427f4d3ce047658bf97fbdc23065c06205231687 +triton-kernels @ https://pypi.amd.com/triton/release_/rocm-7.2.0/packages/triton-kernels/triton_kernels-1.0.0+amd.rocm7.2.0.gitd0d77a509-py3-none-any.whl \ + --hash=sha256:df8b42ebf098767c0d31a3916913bd7557a623903aefa26602e7c6818c34b84a diff --git a/docs/README.md b/docs/README.md index 17bd0d0b..a597ed96 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ python -m sphinx -T -b html docs docs/_build/html | `reference/api-reference.md` | Configuration and API reference | Run configuration schema, task `config.yaml` schema, CLI flags, scoring, and the agent registry. | | `reference/benchmark-methodology.md` | Reference | Timing methodology, performance-helper materialization, and speedup interpretation. | | `how-to/run-evaluation.md` | How-to | Choose or create a run configuration, run an experiment through Docker, resume runs, and read results. | +| `how-to/use-evaluation-tools.md` | How-to | Configure isolated sanitizer/analysis sidecars, interpret capability and findings, and understand the strict language/GPU support matrix. | | `how-to/parallel-run.md` | How-to | Run one isolated Docker worker per GPU, use the shared `.parallel/` task queue, resume parallel runs, and parallelize `task_validator`. | | `how-to/agents.md` | How-to | Supported agents, model providers, and A/B testing. | | `how-to/add-task.md` | How-to | Task directory layout, `config.yaml` fields, and task types. | diff --git a/docs/how-to/run-evaluation.md b/docs/how-to/run-evaluation.md index ddf0f36f..0175130e 100644 --- a/docs/how-to/run-evaluation.md +++ b/docs/how-to/run-evaluation.md @@ -15,13 +15,14 @@ resume, and inspect a run. ## Choose or create a run configuration A run configuration selects the agent, tasks, and target GPU. The repository -ships three examples: +ships quickstarts and specialized examples, including: | Configuration | Purpose | | --- | --- | | `example_configs/quickstart_claude_mi300.yaml` | One Claude Code GELU task on MI300/MI300X (`gfx942`). | | `example_configs/quickstart_claude_mi355x.yaml` | One Claude Code GELU task on MI355X (`gfx950`). | | `example_configs/benchmark_cursor_mi355x.yaml` | Curated 60-task Cursor Agent benchmark on MI355X; use only after installing and authenticating Cursor Agent. | +| `example_configs/evaluation_tools_advisory_mi355x.yaml` | Default-disabled template for experimental `gfx950` evaluation-tool sidecars. It still requires task-specific adapters before opt-in. | For a first run, select the quickstart that matches the physical GPU: @@ -78,6 +79,11 @@ select tasks at any level of granularity. See [Configuration and API reference](../reference/api-reference.md) for the full set of run-configuration fields. +Optional sanitizer and analysis sidecars run after ordinary correctness and +before performance. They are disabled by default, verified only on `gfx950`, +and require language/task-specific adapters. See [Check kernels with evaluation +tools](use-evaluation-tools.md) before enabling them. + ## Start a run ```bash diff --git a/docs/how-to/use-evaluation-tools.md b/docs/how-to/use-evaluation-tools.md new file mode 100644 index 00000000..dd7356f3 --- /dev/null +++ b/docs/how-to/use-evaluation-tools.md @@ -0,0 +1,1007 @@ +--- +myst: + html_meta: + "description": "Run Triton FpSan, ROCm GPU AddressSanitizer, rocJITsu Race Detector, Waitcheck, ConSan, and HIP-FpSan as isolated AgentKernelArena evaluation tools." + "keywords": "AgentKernelArena, sanitizer, Triton FpSan, GPU ASan, rocJITsu, Waitcheck, ConSan, HIP-FpSan, ROCm, gfx950" +--- + +# Check kernels with evaluation tools + +AgentKernelArena can run optional kernel-analysis tools after ordinary +compilation and correctness checks and before performance measurement. The +initial tool set is: + +- Triton FpSan for reference-versus-candidate floating-point semantic + comparison. +- ROCm GPU AddressSanitizer (GPU ASan) for invalid device-memory accesses. +- rocJITsu for simulated race detection. +- rocJITsu Waitcheck for static missing or too-weak wait detection on one exact + final code object and kernel entry. +- rocJITsu ConSan for strict dynamic record/replay concurrency checking of one + exact code object launched by a focused native harness. +- HIP-FpSan for explicitly ported HIP/C++ floating-point comparisons. + +This feature is experimental and opt-in. Capability and evidence checks fail +closed; whether an incomplete result blocks performance is controlled by the +`advisory` or `required` policy. It is not a general sanitizer suite: +sanitizer suite: every result is qualified by the kernel language, generated +artifact, adapter, tool image, GPU architecture, and evidence that the intended +kernel was actually instrumented or dispatched. + +> **Current validation boundary:** sidecar build locks, integrated startup +> controls, and end-to-end fixtures exist only for MI355X (`gfx950`). All six +> startup controls passed in the current hardware qualification. Candidate +> readiness still depends on language, artifact, adapter, and attestation. +> `gfx942` is unverified, and the Docker runner currently rejects +> evaluation-tool sidecars on that architecture. Do not interpret normal +> MI300/MI325 task support as sanitizer support. + +## Keep the scoring runtime unchanged + +Evaluation tools do not get installed into the agent or scoring container. +Each enabled tool runs in a separate run- or worker-scoped sidecar, reused +across that worker's tasks, and communicates with the evaluator over a Unix +socket: + +```mermaid +flowchart LR + A["Pinned scoring image
agent + correctness + scoring"] -->|"health and invocation RPC"| B["One tool sidecar"] + B -->|"bounded logs and structured result"| A + C["Repository tree"] -->|"/input read only"| B + D["Per-worker artifact namespace
.eval-tool-artifacts/label"] <-->|"/artifacts read/write"| B + E["Per-tool socket directory"] -->|"one writable UDS path"| B +``` + +The runner creates a narrow repository-root +`.eval-tool-artifacts/` namespace. It mounts only that directory +as writable `/artifacts` in each sidecar and explicitly bind-mounts the same host +directory read/write at `/workspace/.eval-tool-artifacts/` in the +scoring container. The explicit submount keeps reports writable when the broad +repository mount is read-only, as it is in the quality loop, and avoids asking +Docker to create bind sources on a root-squashed NFS home. Task workspaces and +captured source evidence are not reachable through a writable sidecar alias. +In ordinary and parallel runs, the runner also overlays the top-level +`.eval-tool-artifacts` namespace read-only before overlaying only the current +worker child read/write. This prevents the broad writable repository mount from +becoming a second path to sibling workers' reports or bind sources. +All sidecars for that worker still share this artifact namespace. Each sidecar +receives only its own nested writable socket directory, while the scoring +container receives the socket parent read-only. These changes reduce accidental +cross-tool mutation, but do not create an agent/evaluator trust boundary; the +remaining security consequences are described later. + +All six tool images bake the worker, trusted replay helper, and synthetic +probes into the read-only image at `/opt/aka-eval-tools`. Worker startup verifies +that it imported this image-owned tree rather than the repository mounted at +`/input`. This protects the sidecar control-plane code from a task changing the +checkout used by a running worker; candidate-specific commands and inputs remain +separate, explicitly mounted data. + +The verified `gfx950` scoring image remains: + +```text +lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705 +lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78 +``` + +When any evaluation tool is enabled, the runner resolves Docker's immutable +local image ID for both the selected scoring-image reference and the +content-addressed manifest reference above. Startup fails unless the two IDs are +identical, then launches the scoring container by that verified ID rather than +the mutable tag. A different tag that is an exact alias of the same image is +accepted; an upgraded, rebuilt, or retagged scoring image is rejected even if it +looks compatible. The selected reference and verified image ID are recorded under +`plan.source_evidence.metadata.scoring_runtime`, serialized with the report, +and covered by the plan fingerprint. + +The current design deliberately does **not** upgrade that image, FlyDSL 0.2.2, +or AITER `0.1.17.dev110+g9127c94a1`. The Triton FpSan sidecar replaces Triton +only inside its own container; the other tool dependencies are likewise local +to their sidecars. A sidecar is not a replacement scoring image and must not be +used to establish a new performance baseline. + +FlyDSL does not promise that every generated artifact or task remains compatible +across releases. If the scoring image, FlyDSL, AITER, PyTorch, ROCm, or Triton is +upgraded later, treat that as a benchmark-runtime migration: rerun compilation, +correctness, held-out, sanitizer positive/negative fixtures, and performance +baselines rather than assuming backward compatibility. + +The pinned sidecar dependencies are recorded in +`docker/eval-tools/images.lock.yaml`: + +| Sidecar | Isolated dependency change | +| --- | --- | +| `triton_fpsan` | AMD Triton `3.7.0+amd.rocm7.2.0.gitd0d77a509` and matching `triton-kernels` wheels. | +| `gpu_asan` | ROCm 7.2 ASan runtime packages, including `hip-runtime-amd-asan`. | +| `rocjitsu` | rocJITsu from pinned `rocm-systems` commit `0bf561a0...`, built with GCC 13 for `gfx950`. | +| `rocjitsu_waitcheck` | rocJITsu Waitcheck C API and CLI from pinned `rocm-systems` commit `ed35c0b...`; zstd source is separately checksum-locked. | +| `rocjitsu_consan` | rocJITsu ConSan HSA hook from the same pinned `ed35c0b...` source, forced to strict record/replay mode. | +| `hip_fpsan` | HIP-FpSan headers/source from pinned commit `0ac9be8a...`. | + +## Understand support levels + +“The engine can execute a code object” and “the evaluator can safely wrap a +Python submission” are different claims. The report therefore records four +capability dimensions: + +| Dimension | Question | +| --- | --- | +| `engine` | Can this analysis engine reason about the language or generated ISA? | +| `adapter` | Is there a task-specific build, comparison, or replay path that identifies the exact candidate? | +| `runtime` | Is the isolated sidecar reachable, architecture-matched, and carrying the required assets? | +| `effective` | Is the tool ready after combining the other three dimensions? | + +Possible states are `ready`, `adapter_required`, `unsupported`, +`not_applicable`, and `unavailable_runtime`. A non-ready tool is not executed. +Unsupported tooling is also not converted into an ordinary kernel correctness +failure. + +### Use evaluator plugins, not agent skills, for enforcement + +A sanitizer belongs in the evaluator control plane. An agent skill, prompt, or +tool-use recipe may help an optimizer author an adapter or interpret a report, +but it runs on the untrusted optimization side and cannot prove that a required +check happened. It must never be the source of a scoring gate. + +The extension boundary deliberately separates four responsibilities: + +| Layer | Responsibility | +| --- | --- | +| Core manager | Resolve task profiles, combine capability states, enforce policy, fingerprint plans, and serialize one stable report schema. It contains no sanitizer-specific parsing. | +| Tool plugin | Implement deterministic `assess`, `build_invocation`, and `parse` behavior for one tool. It converts raw output into independent execution/finding states. | +| Sidecar runtime provider | Own the pinned image, dependencies, health evidence, runtime-internal paths, architecture guard, and synthetic startup control. | +| Task adapter | Compile or replay the exact candidate and emit candidate-specific build/dispatch attestation. It is selected by language/artifact kind, not merely by tool name. | + +This makes a future analyzer additive: add its typed plugin/parser, isolated +image and lock, worker health/startup control, runner allowlist, task adapters, +support-matrix entry, and positive/negative tests without changing ordinary +correctness or performance code. The current registry is an explicit built-in +allowlist; arbitrary third-party plugin discovery is not implemented. That is +intentional while report provenance and the evaluator/agent isolation boundary +remain experimental. + +### Strict support matrix + +The following table describes the current end-to-end evaluator, not just a +successful standalone experiment. “Ready with adapter” means the task must +provide the dedicated argv/harness and required attestation described later; +it does not mean the ordinary correctness command is automatically reused. + +| Kernel path | Triton FpSan | GPU ASan | rocJITsu | HIP-FpSan | +| --- | --- | --- | --- | --- | +| Editable Triton Python/JIT | Ready with comparison adapter and instrumentation attestation | Ready with dedicated command, fresh JIT cache, XNACK, and build attestation | Trusted `triton_aot` capsule replay is implemented on `gfx950`; whole-Python JIT remains unsupported, and capsule capture/binding to the correctness run is not automatic, so use it only as advisory evidence | Not applicable | +| HIP source controlled by the task | Not applicable | Ready only after recompiling the candidate with `-fsanitize=address -shared-libsan --offload-arch=gfx950:xnack+`, then attesting that artifact | Ready with a dedicated native launcher | Source port and comparison adapter required; both reference and candidate paths must explicitly use `fpsan::Value` | +| FlyDSL 0.2.2 Python/JIT | Unsupported; FlyDSL does not use the Triton FpSan pipeline | Unsupported; the current ROCDL pipeline does not insert AMD GPU ASan instrumentation | Trusted `flydsl_aot` capsule replay is implemented on `gfx950` and detects the seeded LDS race; automatic capsule capture/binding to the correctness run is not ready, so use it only as advisory evidence | Not applicable | +| Editable Triton source inside AITER | Engine may be eligible for the explicitly selected source only, with a dedicated comparison adapter; this does not sanitize AITER library kernels | Unsupported by the current default AITER runtime path | Unsupported by the current Python/AITER runtime | Not applicable | +| AITER or another precompiled HSACO/library kernel | Cannot retrofit instrumentation | Unsupported unless the exact kernel source is rebuilt and attested; preloading the runtime is insufficient | Unsupported by the current evaluator runtime | Cannot retrofit value semantics | +| rocBLAS or RCCL internal kernel | Do not enable; library internals are outside the selected submission | The stock library is not instrumented and is not covered | Not a supported general library-runtime path | Do not enable | + +Waitcheck and ConSan use narrower, explicit final-code-object adapters: + +| Kernel path | rocJITsu Waitcheck | rocJITsu ConSan | +| --- | --- | --- | +| Standalone final `gfx950` HSACO from HIP, Triton, or FlyDSL | Advisory-ready only when `code_object`, `expected_kernel`, and exact `kernel_entry` identify one descriptor in an unbundled final ELF. It is static and receives no GPU device. | Advisory-ready only with `code_object`, a focused native `command` that names and loads that file, and an independent `oracle_command`. Strict record/replay must report the same FNV-1a identity and complete coverage. | +| Whole Python/JIT process | Not automatically captured or bound to the correctness dispatch. Extract and declare the exact final code object first. | Unsupported by this first integration; incidental framework and library code objects make exact candidate attribution ambiguous. | +| AITER, rocBLAS, RCCL, or another broad library runtime | A selected extracted kernel may be inspected statically, but that does not cover the surrounding runtime or prove it was dispatched. | Unsupported; the current adapter deliberately rejects broad library runtimes. | + +Both tools are currently qualified only for `policy: advisory`. The generic +manager can enforce `required` when explicitly configured, but doing so for +these tools is unsupported until the selected task adapter and candidate +provenance have been independently promoted. The evaluator does not +automatically discover optimized kernels or reuse the ordinary correctness +command for either tool. + +This matrix describes engine and adapter support once the corresponding runtime +is qualified. The current `gfx950` startup qualification is stricter: + +| Tool runtime | Current startup qualification | +| --- | --- | +| Triton FpSan | Passing on hardware; eligible task paths can proceed to candidate attestation. | +| HIP-FpSan | Passing on hardware; explicitly ported task paths can proceed to candidate attestation. | +| GPU ASan | Passing on hardware for both HIP and Triton safe/OOB lanes; an applicable candidate still needs its own instrumentation/build attestation. | +| rocJITsu | Passing on hardware with barrier-safe and deliberately racy LDS fixtures; an applicable candidate still needs a native HIP launcher or validated AOT replay capsule. | +| rocJITsu Waitcheck | Passing on hardware: a correct `s_waitcnt lgkmcnt(0)` fixture is clean and a missing-wait fixture produces one exact hazard. Candidate use still requires exact SHA-256, kernel name, and entry attestation. | +| rocJITsu ConSan | Passing on hardware in strict record/replay: a single-wave LDS fixture is clean and a two-wave conflicting fixture produces complete FNV-attributed diagnostics. Candidate use still requires an exact code object, focused loader, and separate oracle. | + +Additional boundaries: + +- Static Triton HSACO fixtures, including dynamic matmul, buffer-async matmul, + and flash-attention fixtures, have executed under the `gfx950` rocJITsu + engine. That evidence does not make `rocjitsu -- python task.py` supported. +- A FlyDSL HSACO with a deliberately missing LDS barrier produced a rocJITsu + race report. Directly wrapping the FlyDSL Python process failed before a + usable dispatch. The implemented adapter therefore extracts that boundary + into a strict, single-dispatch replay capsule rather than wrapping Python. +- For Triton/FlyDSL, the rocJITsu plugin rejects arbitrary launchers, validates + the language-specific capsule and all manifest hashes, requires `gfx950`, and + invokes an image-owned helper that generates and compiles a native launcher. + The parser requires the expected kernel dispatch, capsule/code-object digest + attestation, and a successful replay/golden-output marker. The missing piece + is automatic evaluator-owned capsule capture and proof that the capsule came + from the exact correctness run; a task-supplied capsule is still weak + candidate provenance. +- A deliberately out-of-bounds, **uninstrumented** HIP HSACO exited normally in + a complete GPU ASan runtime. This is why a runtime preload alone is never + accepted as GPU ASan coverage. +- ConSan embeds Waitcheck as a preflight. The ConSan parser records that preflight + as metadata but does not duplicate its diagnostics as ConSan race findings; + enable `rocjitsu_waitcheck` separately when a standalone wait-hazard result is + required. +- `gfx942` has not completed the same image, positive-control, adapter, and + end-to-end validation. Its status is unverified, not unsupported by theory. + +## Build and check the sidecars + +Building requires Docker and network access to the pinned package and source +locations. Runtime sidecars themselves start with networking disabled. + +Build all six local `gfx950` images from the repository root: + +```bash +src/scripts/docker_benchmark.sh build-eval-tool-images +``` + +The default local tags are: + +```text +agent-kernel-arena/eval-tool-triton-fpsan:gfx950 +agent-kernel-arena/eval-tool-gpu-asan:gfx950 +agent-kernel-arena/eval-tool-rocjitsu:gfx950 +agent-kernel-arena/eval-tool-rocjitsu-waitcheck:gfx950 +agent-kernel-arena/eval-tool-rocjitsu-consan:gfx950 +agent-kernel-arena/eval-tool-hip-fpsan:gfx950 +``` + +Check that the workers start and report their pinned assets: + +```bash +src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +To check a subset: + +```bash +AKA_EVAL_TOOLS=gpu_asan,rocjitsu \ + src/scripts/docker_benchmark.sh eval-tools-smoke +``` + +`AKA_EVAL_TOOLS` is a host-side subset override. When it is set for either a +smoke test or a normal run, the runner starts exactly that normalized set and +publishes it through the internal `AKA_EVAL_TOOLS_SELECTED` variable; the +scoring process then plans the same set instead of the YAML `enabled` value. +This prevents a sidecar/plan mismatch. Leave the override unset when YAML should +remain authoritative, and record any override as part of the run invocation. + +Worker startup runs a tool-specific synthetic positive control before the Unix +socket appears. Health output includes its verdict, commands, durations, +bounded log paths/excerpts, and the immutable Docker image ID reported by the +worker: + +| Tool | Startup positive control | +| --- | --- | +| `triton_fpsan` | Compile instrumented reference/candidate kernels and require a known numerical mismatch to produce different digests plus FpSan compiler metadata. | +| `gpu_asan` | Compile and run safe/OOB HIP fixtures and safe/OOB Triton fixtures; the task profile selects the relevant lane. | +| `rocjitsu` | Require a barrier-protected fixture to remain clean and a deliberately racy LDS fixture to report a race. | +| `rocjitsu_waitcheck` | Compile unbundled `gfx950` code objects and run the production entrypoint, inventory, C API, and parser on the correct-wait and missing-wait fixtures; retain a direct CLI hazard check as an independent engine control. | +| `rocjitsu_consan` | Compile raw safe/racy HSACOs plus an image-owned module launcher, run the production entrypoint with separate instrumented/oracle argv, require the oracle environment to be scrubbed, and make the production parser return clean/finding with exact FNV attribution. | +| `hip_fpsan` | Require explicitly ported equivalent expressions to match and a known-wrong expression to produce a different digest. | + +`eval-tools-smoke` prints this evidence and exits nonzero if a worker reports +`degraded`, including a failed nested positive control. Inspect and retain the +JSON summaries before promotion. A normal evaluation with +`positive_control: required` repeats the fail-closed check during the typed +runtime probe. + +As of the current `gfx950` qualification run, all six integrated startup +controls pass on hardware. This qualifies the installed tool runtimes only. It +does not promote a candidate path without the language-specific adapter and +attestation in the strict support matrix. + +The same final image set also passed evaluator-manager-to-sidecar candidate +fixtures on the physical MI355X host: + +| Tool and language | Safe fixture | Seeded bug fixture | +| --- | --- | --- | +| Triton FpSan, Triton | `clean` | Numerical mismatch `found` | +| GPU ASan, HIP | `clean` | Out-of-bounds access `found` | +| GPU ASan, Triton | `clean` | Out-of-bounds access `found` | +| HIP-FpSan, explicitly ported HIP | `clean` | Wrong expression `found` | +| rocJITsu, Triton AOT replay | `clean` | Not exercised in this candidate pair | +| rocJITsu, FlyDSL AOT replay | Not exercised in this candidate pair | Missing-barrier LDS race `found` | + +These are controlled synthetic fixtures that validate the current adapters, +transport, parsing, and attestation paths. They do not qualify any bundled +production task, broaden the strict support matrix, or close the AOT +correctness-dispatch provenance gap. + +## Recommended rollout plan + +Promote one language/tool path at a time. Do not make “all sanitizers enabled” a +global milestone. + +| Phase | Work | Exit criterion | +| --- | --- | --- | +| 0. Freeze baselines | Keep the pinned scoring image, FlyDSL 0.2.2, and AITER version unchanged; build each tool from its lock into a sidecar. | Existing compilation, correctness, held-out, and performance baselines remain unchanged with tools disabled. Sidecar image IDs and the verified scoring-image ID/reference are captured in plans. | +| 1. Qualify installations | Run automatic safe/known-bug startup controls on `gfx950`; repeat the now-passing six-tool qualification on clean hosts. | Both positive and negative lanes pass repeatedly. `eval-tools-smoke` evidence is archived and independently reviewed. | +| 2. Build trusted pilot adapters | Start with one editable Triton task for Triton FpSan, one Triton and one HIP task for GPU ASan, one native HIP task for rocJITsu, one final-HSACO task for Waitcheck, one focused native loader for ConSan, and one explicitly ported HIP-FpSan task. Put harnesses under protected `scripts/` paths and declare all inputs. | Each pilot distinguishes a safe fixture from a seeded bug, identifies the selected candidate, and produces bounded structured artifacts. No precompiled AITER/library kernel is claimed as broadly covered. | +| 3. Finish AOT capture and binding | The trusted `triton_aot`/`flydsl_aot` replay path now validates one-dispatch capsules and generates the launcher. Add evaluator-owned extraction immediately after correctness and bind the capsule to that exact candidate/case. | Safe and racy fixtures pass end to end, malformed capsules fail closed, and a task cannot substitute a different valid capsule for the correctness dispatch. | +| 4. Harden provenance and phase isolation | The runner now uses per-tool writable socket directories, a read-only socket parent in scoring, a narrow per-worker artifact mount, fresh per-invocation artifact directories, a complete serialized plan, and capsule digests in the fingerprint. Next run tools only after the agent exits, freeze the candidate, use evaluator-only/authenticated RPC and evaluator-owned artifacts, strengthen artifact/dispatch binding, and wire resume to plan freshness. | An adversarial task cannot call a worker, overwrite evidence, reach another task's artifacts, spoof a clean result, or reuse a stale report. This phase is required before sanitizer output becomes a reward signal. | +| 5. Advisory campaign | Run qualified paths with `policy: advisory` across representative and private held-out shapes; measure overhead, timeouts, log volume, flakes, false positives, and GPU recovery behavior. | Each task/tool pair has reviewed coverage cases, stable resource limits, and an explicit owner/runbook. Incomplete results remain visible and never score as clean. | +| 6. Narrow required gates | Change only individually qualified task/tool pairs to `required`; leave unsupported and not-yet-qualified paths advisory or disabled. | Required gates block seeded findings and infrastructure failures without changing ordinary correctness semantics or the scoring performance baseline. | +| 7. Add `gfx942` separately | Build architecture-specific images/configs and rerun every startup, adapter, security, and workload fixture on MI300X/MI325X. | Only mark `gfx942` supported after independent qualification; do not infer it from `gfx950`. | + +Phase 0, the six integrated startup controls, complete plan serialization, the +narrower runner mounts, and the trusted AOT replay core exist. No bundled task +has completed production qualification through phases 2–6. Automatic capsule +capture, exact candidate/dispatch provenance, top-level resume freshness, and +the same-phase agent/evaluator boundary remain blockers. + +## Start from the disabled example + +Copy `example_configs/evaluation_tools_advisory_mi355x.yaml`. It is a normal +MI355X run configuration but has `evaluation_tools.enabled: false`, so copying +and running it does not build or start any sidecar when the host +`AKA_EVAL_TOOLS` override is unset: + +```bash +cp example_configs/evaluation_tools_advisory_mi355x.yaml my_sanitized_run.yaml +``` + +After building the images and adding task-specific adapters, opt in to only the +tools that can actually inspect the selected task: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + - rocjitsu + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} + rocjitsu: + options: {} +``` + +Do not enable all six tools merely because all six images exist. On a +heterogeneous task set, irrelevant tools become `not_applicable`, unsupported +paths remain visible as unsupported, and missing adapters remain +`adapter_required`. + +Timeouts are exact YAML/JSON integers from 1 through 3600 seconds; booleans, +floats, numeric strings, and larger values are rejected. A task-level timeout +must also be no larger than the run-level timeout. Explicit +`evaluation_tools` sections and per-tool entries reject unknown fields rather +than silently weakening the requested policy. Supplying both `runtime_ref` and +its `image_digest` alias is rejected when the two assertions differ. + +### Select and attest each tool image separately + +The host runner selects the Docker image using the default local tag or an +environment override. For example: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +make docker-run CONFIG=my_sanitized_run.yaml RUN_ARGS='--run-suffix asan_advisory' +``` + +The override names are `AKA_EVAL_TOOL_IMAGE_TRITON_FPSAN`, +`AKA_EVAL_TOOL_IMAGE_GPU_ASAN`, `AKA_EVAL_TOOL_IMAGE_ROCJITSU`, +`AKA_EVAL_TOOL_IMAGE_ROCJITSU_WAITCHECK`, +`AKA_EVAL_TOOL_IMAGE_ROCJITSU_CONSAN`, and +`AKA_EVAL_TOOL_IMAGE_HIP_FPSAN`. + +After selecting a tool-image reference, the runner resolves its local immutable +image ID with `docker image inspect`, launches the sidecar by that bare +`sha256:...` ID, and injects the same ID as runtime identity evidence. Runtime +health reports that ID, and the typed probe fails with +`RUNTIME_REF_MISMATCH` if the planned and observed values differ. The resolved +ID belongs in the plan fingerprint. Scoring-image verification likewise launches +the scoring container by its verified local ID. A registry reference such as +`name@sha256:...` is not string-equal to Docker's local image ID and must not be +used as a manual substitute. + +YAML `runtime_ref`/`image_digest` fields are identity assertions, not image +selectors. Omit them when using automatic host injection. If supplied manually, +use the exact bare ID returned by: + +```bash +docker image inspect --format '{{.Id}}' +``` + +## Add a task adapter + +Run-level configuration chooses tools, policy, optional image-identity +assertions, and maximum timeout. A task can only add options for an +already-enabled tool and lower its timeout. It cannot enable a tool, change the +image or top-level policy, or raise the run-level timeout. +Reserved framework options are rejected at both run and task level. They are +`positive_control_required`; GPU ASan's `asan_runtime_dir`, +`hip_asan_runtime`, `host_asan_preload`, `host_asan_lib_dir`, and +`normal_rocm_lib_dir`; rocJITsu's `rocjitsu_binary` and `config_path`; and +Waitcheck's `waitcheck_binary` and `waitcheck_capi_wrapper`; ConSan's +`consan_hook`; and HIP-FpSan's `include_dir` and `public_header`. The +host/runtime probe is the only authority for those values. + +Commands must be argv lists, never shell strings. A dedicated tool command is +required because reusing `correctness_command` could instrument the reference, +load a precompiled library kernel, or sanitize the wrong candidate. + +For example, a HIP task can declare the shape of its adapters as follows: + +```yaml +evaluation_profile: + language: hip + artifact_kind: source_aot + framework: standalone + instrumentation_control: recompile + source_available: true + submission_paths: + - optimized_kernel.hip + - scripts/eval_tools/run_gpu_asan.py + - scripts/eval_tools/rocjitsu_launcher + +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] + rocjitsu: + options: + launcher: [scripts/eval_tools/rocjitsu_launcher] + expected_kernel: my_kernel +``` + +These are adapter contracts, not automatically generated files. The task +wrapper must build and launch the optimized candidate, exercise representative +inputs, and emit the required evidence. Sidecar health is the only authority +for container-internal ASan libraries and preload, the rocJITsu binary and +architecture config, the Waitcheck CLI/C API wrapper, the ConSan HSA hook, and +the HIP-FpSan include directory. The runtime probe attests and injects those +values into the plugin context; neither run nor task configuration may supply +or override them. + +Common built-in option keys are: + +| Tool | Required adapter options | Additional evidence/options | +| --- | --- | --- | +| `triton_fpsan` | `comparison_command` or `command` | `attestation_path`; command must emit one `AKA_FPSAN_RESULT` JSON line | +| `gpu_asan` | `command` | Candidate `attestation_path`; a HIP command must use the required compile flags. Runtime/preload/library paths come from health. | +| `rocjitsu` | HIP: `launcher` or `command`. Triton/FlyDSL: `capsule` plus an exact profile adapter of `triton_aot` or `flydsl_aot`; user launchers are forbidden on these AOT paths. | HIP may set `expected_kernel` and `race_report`. AOT capsule path must stay below the task workspace and target `gfx950`; the executable/config and trusted replay helper come from the sidecar image. | +| `rocjitsu_waitcheck` | `code_object`, `expected_kernel`, and integer `kernel_entry` | `code_object` must be an unbundled final AMDGPU ELF below the workspace. The image-owned inventory helper must attest exactly the requested `gfx950` descriptor before the C API runs. | +| `rocjitsu_consan` | `code_object`, `command`, and `oracle_command` | `command` must be a focused native argv that explicitly names and loads `code_object`; `oracle_command` runs separately without the hook. Exact SHA-256 and FNV-1a64 identities are required. | +| `hip_fpsan` | `comparison_command` or `command`, plus `evaluation_profile.fpsan_ported: true` | Candidate `attestation_path`; both paths must be instrumented. The include directory comes from health. | + +The plugin validates these adapter options directly, so both tools may inspect +the same declared candidate in one advisory plan. For example: + +```yaml +evaluation_profile: + language: hip + artifact_kind: hsaco_precompiled + framework: standalone + +evaluation_tools: + tools: + rocjitsu_waitcheck: + options: + code_object: build/optimized.hsaco + expected_kernel: optimized_kernel + kernel_entry: 0 + rocjitsu_consan: + options: + code_object: build/optimized.hsaco + command: [scripts/load_hsaco, build/optimized.hsaco] + oracle_command: [scripts/load_hsaco, build/optimized.hsaco, --check] +``` + +With ROCm 7.2, `hipcc --genco` produces a clang bundle by default; use +`--no-gpu-bundle-output` or explicitly extract the final device ELF before +supplying `code_object`. + +`attestation_path` is resolved below the fresh artifact directory for the +current tool invocation and the same resolved path is used for the injected +environment variable, metadata, and parser. Relative paths are recommended; +an absolute path is accepted only when it resolves below that invocation +directory. Native HIP rocJITsu applies the same containment rule to +`race_report` and requires the report filename to remain `race.log`. AOT replay +does not accept a task-configured race-report path. + +For repository or image-kernel tasks, declare every candidate file whose change +must invalidate evidence with `evaluation_profile.submission_paths`. Paths must +be workspace-relative and cannot contain `..`. If this field is absent, capture +falls back to `source_file_path` and `target_file_path`; silently hashing an +entire multi-gigabyte repository is intentionally avoided. + +Put evaluator-owned adapter code under a harness-protected path such as +`scripts/`, not an arbitrary agent-editable `eval_tools/` directory. Also list +adapter scripts, HSACO files, and input blobs in `submission_paths` when their +contents must affect the general candidate fingerprint. A configured replay +capsule or Waitcheck/ConSan code object receives additional handling: +immediately before plan construction the manager records its SHA-256 and size under +`source_evidence.metadata.option_artifacts`, so that digest is covered by the +plan fingerprint. The validated capsule manifest contains and verifies the +HSACO and blob digests. This binds the plan to the supplied capsule bytes; it +does not prove who captured the capsule or that it came from the same dispatch +as ordinary correctness. + +## Require evidence before accepting “clean” + +Process success is not proof that an analysis ran. The built-in parsers require +tool-specific attestation: + +- GPU ASan requires `build_attestation.json` for the declared artifact. HIP + attestation must include all sanitizer/XNACK flags and `HSA_XNACK=1`; Triton + attestation must include `TRITON_ENABLE_ASAN=1` and `HSA_XNACK=1`. +- Triton FpSan and HIP-FpSan currently require one build attestation whose + evidence contains the self-declared `reference_instrumented` and + `candidate_instrumented` booleans, plus an `AKA_FPSAN_RESULT` payload with the + two digests. Exactly one result marker is required; zero or multiple markers, + a timeout, or any nonzero/unknown process exit is a tool error. They do not + validate two independently attested artifacts. +- Native HIP rocJITsu requires a canonical simulator dispatch in the + evaluator-owned report sink, optionally matched to `expected_kernel`. + Task-launcher stdout/stderr cannot attest a clean dispatch, even if it prints + the complete rocJITsu prefix. The task launcher still inherits the report + configuration, so this lane remains advisory rather than trusted reward + evidence. +- Triton/FlyDSL rocJITsu uses the image-owned replay helper instead of a task + launcher. It revalidates the capsule and manifest, generates the native + launcher, and requires an exact capsule/code-object attestation, expected + kernel dispatch, and `AKA_REPLAY_RESULT pass`. Missing or changed evidence is + inconclusive. This is stronger replay integrity, but the task-supplied capsule + is not yet automatically tied to the correctness run. +- rocJITsu Waitcheck re-hashes the selected final code object, inventories its + descriptors with the image-owned CLI, matches the exact kernel name and entry, + and obtains structured diagnostics through the stable C API. Missing, + duplicate, truncated, or incomplete evidence is inconclusive. +- rocJITsu ConSan re-hashes the code object, requires the configured launcher to + name it, matches the hook's FNV-1a64 identity, enforces strict record/replay + completeness and accounting, and runs a separate correctness oracle without + the hook. Embedded Waitcheck text is preflight metadata, not a duplicate + ConSan finding. +- Build attestation records the compiler, compiler version, and target + architecture. The current validator directly checks tool identity, + `instrumented: true`, required build flags/environment, artifact existence, + and artifact SHA-256. Required flags are matched as complete argv tokens + (including supported split-value forms), not as substrings of unrelated + arguments. The host/runtime `gfx950` guards provide the current + architecture boundary; stricter compiler/version/target comparisons remain + future attestation hardening. + +These checks establish integrity evidence, not complete candidate provenance. +Build-attestation validation does not compare every compiler/version/target +field with an expected build or cryptographically bind the declared artifact to +the actual dispatch. It does require the artifact path to be relative to, and +contained below, the directory holding the attestation. A user-controlled build +JSON and native HIP wrapper output can therefore still satisfy checks without +proving which candidate ran. AOT replay binds its generated launcher to a +validated capsule and digest, but the capsule can still have been supplied for +a different candidate/case. Current `required` policy is suitable for trusted +integration diagnostics, not an adversarial reward boundary. + +The default build-attestation location is the fresh invocation directory below +the external per-task/tool artifact root described later. A wrapper executing +in a sidecar must write it through the writable `/artifacts` mount; the +repository/workspace input mount is read-only. GPU ASan, Triton FpSan, and +HIP-FpSan invocations inject `AKA_BUILD_ATTESTATION_PATH`, and the runtime client +translates that output path into the sidecar namespace. The adapter must place +the built artifact beside or below that JSON file and write `artifact_path` as a +relative path such as `build/candidate.hsaco`. The scoring-side parser resolves +the same relative path below its corresponding artifact directory, rejects +absolute/escaping paths, and checks the declared SHA-256. Do not embed either +the sidecar `/artifacts/...` prefix or a scoring-container absolute path. + +### Positive controls and candidate attestation are both required + +`positive_control: required` is the default. Each sidecar runs its synthetic +known-bug control once at worker startup and writes an audit summary and bounded +logs. Health returns that evidence. For every applicable task, the typed runtime +probe selects the relevant control (HIP or Triton for GPU ASan) and returns +`unavailable_runtime/POSITIVE_CONTROL_FAILED` unless it passed. The requirement +and host-resolved tool runtime identities are covered by the plan inputs. The +separately verified scoring-image ID/reference are recorded in plan source +evidence as well. +Control evidence is retained in runtime capability/reporting, but its startup +artifact content is not itself part of the plan fingerprint. + +This proves that the isolated installation detected its synthetic bug; it does +not prove that the optimized candidate was instrumented or replayed. Candidate +build/dispatch attestation remains a separate requirement. A minimally +meaningful clean diagnostic result needs both: + +1. a passing startup positive control for the applicable language lane; and +2. a task result with the declared candidate's build or dispatch attestation. + +Setting `positive_control: optional`, `disabled`, or `false` records +`positive_control_required: false`. The startup probe still runs and remains +visible in health evidence, but a failure no longer blocks runtime capability. +Use that only for tool bring-up, not trusted benchmark results. + +## Choose the policy + +The policy governs whether performance measurement may proceed. It does not +rewrite `pass_correctness`. + +| Policy | Finding, tool error, inconclusive result, missing adapter, unsupported applicable path, or missing runtime | Performance | +| --- | --- | --- | +| `advisory` | Recorded; `policy_satisfied: false` | Continues (`allowed: true`) | +| `required` | Recorded with a reason | Skipped (`allowed: false`) | + +An explicitly `not_applicable` tool is ignored by the gate. Under `required`, +every other selected tool must have effective capability `ready`, execution +`completed`, and finding status `clean`. Start new integrations in `advisory` +mode. Move to `required` only after task adapters, positive controls, and +architecture-specific fixtures are independently reviewed. + +## Read execution and findings separately + +`execution` answers whether the tool invocation completed. `finding` answers +what the parser concluded about the selected kernel: + +| `execution` | Meaning | +| --- | --- | +| `not_run` | No invocation was made. | +| `completed` | The parser obtained a complete tool outcome; a detected bug can still have this state. | +| `tool_error` | The runtime, wrapper, or parser failed without a valid finding. | +| `timeout` | The isolated process group exceeded its timeout. | + +| `finding` | Meaning | +| --- | --- | +| `not_evaluated` | No supported evaluation ran. | +| `clean` | The intended instrumented/simulated kernel ran and no finding was observed. | +| `found` | One or more structured issues were reported. | +| `inconclusive` | Absence of a finding cannot be trusted, often because attestation is missing. | + +A sanitizer can deliberately terminate with a nonzero process status while +still producing a valid finding. Conversely, an uninstrumented out-of-bounds +kernel can return zero and must remain inconclusive. Never use return code alone +as the sanitizer result. + +The report keeps ordinary scoring fields and adds: + +```yaml +pass_tool_gate: true +tool_policy_satisfied: false +tool_evaluation: + schema_version: 1 + plan_fingerprint: "..." + plan: + schema_version: 1 + policy: advisory + profile: {} + tools: + - tool: gpu_asan + runtime_ref: "sha256:..." + plugin_version: "1" + timeout_s: 600 + options: {positive_control_required: true} + fingerprint: "..." + source_evidence: + metadata: + scoring_runtime: + image_id: "sha256:..." + reference: "lmsysorg/sglang-rocm:v0.5.14-rocm720-mi35x-20260705" + policy: advisory + overall_status: incomplete + resolved_task_profile: {} + source_evidence: {} # legacy mirror of plan.source_evidence (abbreviated) + decision: + allowed: true + policy_satisfied: false + reasons: ["gpu_asan:capability:adapter_required"] + tools: + gpu_asan: + capability: + engine: {state: ready} + adapter: {state: adapter_required} + runtime: {state: ready} + effective: {state: adapter_required} + result: null +``` + +Large stdout/stderr streams are not embedded in `task_result.yaml`; they remain +bounded artifacts. The report serializes the complete immutable `plan`, +including each tool's runtime reference, plugin version, timeout, options, +profile, source evidence, and fingerprint. This makes a result reconstructable +for audit without relying on the digest alone. + +## Artifact and evidence layout + +For a normal run, the relevant files are: + +```text +/ +├── .eval-tool-artifacts/ +│ └── / # dedicated sidecar/scoring RW submount +│ ├── _eval_tool_runtime/ +│ │ └── //positive-control/ +│ │ ├── summary.json # startup verdict and step metadata +│ │ └── .{stdout,stderr}.log +│ └── -/ +│ ├── gpu_asan/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json # when the adapter supplies it +│ │ └── triton-gpu-asan-cache/ # Triton only +│ ├── triton_fpsan/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── build_attestation.json +│ │ └── triton-fpsan-cache/ +│ ├── rocjitsu/// +│ │ ├── stdout.log +│ │ ├── stderr.log +│ │ ├── rocjitsu-report/race.log +│ │ └── rocjitsu-replay/ # generated AOT launcher, when used +│ └── hip_fpsan/// +│ ├── stdout.log +│ ├── stderr.log +│ └── build_attestation.json +└── experiments/workspace__/run_/ + ├── .eval-tool-evidence/ + │ └── / + │ ├── manifest.json + │ └── files/ # captured original declared files + └── / + └── task_result.yaml # nested tool_evaluation summary +``` + +Each tool attempt gets a newly created invocation directory, so a repeated plan +cannot consume an old attestation or race report. Each stdout and stderr file is +limited to 64 MiB by default and records truncation metadata. Tool processes run +in a new process group. The Linux worker also acts as a child subreaper and +tracks the invocation's complete descendant tree through `/proc`, including a +child that starts a new session or process group. Cleanup sends termination and +then kill signals; observing descendants after the command exits makes the +execution fail rather than clean. If survivors cannot be removed, the worker +exits so its container is not reused. Per-sidecar scratch and cache directories +outside `.eval-tool-artifacts` are deleted when the sidecar stops. The report +records absolute scoring-side paths below +`/workspace/.eval-tool-artifacts/`. The runner explicitly mounts +that path read/write even when `/workspace` itself is read-only. The per-worker +namespace prevents a writable sidecar alias to task workspaces but is not a +secret or an adversarial integrity boundary. +Raw parser excerpts are omitted from normal JSON/YAML summaries and remain in +the bounded log artifacts. If either stream is truncated, a result that would +otherwise be clean is changed to `inconclusive`; an already observed finding is +preserved. + +## Resume and plan fingerprints + +Every tool report has a SHA-256 `plan_fingerprint` covering: + +- normalized tool configuration, policy, timeouts, options, and configured + runtime references; +- the selected scoring-image reference and its immutable local image ID, after + the runner resolves the pinned SGLang manifest reference to its local image ID + and verifies that the two local IDs match; +- resolved task profile and explicit overrides; +- enabled plugin versions; +- captured-original and optimized-candidate fingerprints for declared paths, + including both the lexical submission path and its current resolved symlink + target; +- the SHA-256 and size of a configured replay-capsule JSON, whose validated + manifest in turn binds the referenced HSACO and blobs. + +This prevents a report from being considered current after a material input +change **when the caller checks it**. The reporting API exposes an exact +fingerprint check for this purpose. + +Most coverage is declaration-based. A pathname stored in ordinary tool options +contributes only its string; adapter, HSACO, and input contents must be declared +in submission evidence or represented by an explicit digest. Replay `capsule` +is the exception: the manager captures its content digest during plan +construction. The current `has_current_plan` helper checks only the fingerprint +and presence of a tools mapping; it does not prove that every tool completed or +that its result is clean. The serialized `plan` makes the inputs auditable but +does not by itself change resume scheduling. + +The top-level `--resume-run` and `--resume-latest` paths currently skip a task +when its `task_result.yaml` already exists; they do not yet rebuild the tool plan +and call that fingerprint check. Until this is wired into run scheduling: + +- use a new `--run-suffix` after changing tool configuration, plugin code, + sidecar image, adapter, source declaration, or positive-control policy; +- do not change `AKA_EVAL_TOOL_IMAGE_*` while resuming a run; +- do not change the selected scoring-image reference while resuming, even to + another tag for the same image ID, because the reference is plan evidence; +- do not assume a changed `runtime_ref` causes a completed task to rerun; +- if a tool-only rerun is required, archive the old report and start a fresh run + rather than silently combining evidence from two plans. + +## AOT replay capsules for Triton and FlyDSL + +rocJITsu can execute supported generated `gfx950` HSACO, but it cannot safely +wrap the current Triton/FlyDSL Python JIT process. The implemented adapter +boundary is a versioned replay capsule containing the exact HSACO and SHA-256, +kernel symbol, launch geometry, declared lowered ABI, allocation snapshots, +pointer relocations, scratch requirements, target architecture, producer +versions, and case identity. Configure the matching adapter and a +workspace-contained capsule: + +```yaml +evaluation_profile: + language: triton # use flydsl for a FlyDSL artifact + artifact_kind: python_jit + adapter: triton_aot # or flydsl_aot + +evaluation_tools: + tools: + rocjitsu: + options: + capsule: eval_capsule/capsule.json +``` + +The plugin validates the capsule in the scoring process, rejects `launcher` or +`command` on AOT paths, and hashes the capsule JSON into the plan. The +image-owned helper rechecks its digest and manifest inside the sidecar, verifies +the adapter identity and `gfx950` target, generates a native HIP launcher, +compiles it with the sidecar toolchain, and reconstructs exactly one dispatch +inside rocJITsu. Post-execution parsing revalidates the capsule and requires the +expected dispatch, exact capsule/code-object marker, and replay success marker. + +The current validator fails closed for at least: + +- more than one kernel dispatch; +- a capsule with no golden expected-output blob; +- non-integral, boolean, negative, or greater-than-`uint32` launch dimensions; +- launch dimensions, block size, or dynamic shared memory that exceed the + selected device's runtime limits; +- opaque/tensor descriptors; +- empty, misordered, or unsupported ABI arguments and unknown implicit refs; +- invalid relocations and out-of-bounds allocation views; +- missing input blobs, mismatched hashes, or an architecture mismatch. + +Do not reduce a Python cache entry to just “HSACO + kernel name.” Without ABI, +launch, allocation, and framework-version evidence, a clean rocJITsu result may +belong to a different execution than the scored task. The replay adapter itself +is implemented and its safe/racy Triton and FlyDSL capsules have run end to end. +What is not implemented is automatic, evaluator-owned extraction from the +ordinary correctness dispatch and a trusted binding between that capture and +the scored candidate/case. Until that provenance exists, treat a clean AOT +result as advisory diagnostics rather than a production-qualified reward gate. + +## Resource, security, and held-out risks + +### Resource controls + +- The worker is sequential: one sidecar executes one GPU command at a time. + Parallel runs create a separate sidecar set per worker/GPU. +- Do not schedule another benchmark or sanitizer on the same physical GPU. + GPU ASan changes allocation behavior and rocJITsu can be much slower than + native execution; neither runtime is a performance measurement environment. +- Begin with one representative case and a bounded timeout. Expand coverage + only after measuring simulator time, HBM/host memory, and artifact growth. +- A timeout or truncated log is not clean evidence. + +### Isolation is a boundary, not proof of safe untrusted execution + +Runtime sidecars use no network, a read-only root filesystem, dropped Linux +capabilities, `no-new-privileges`, the host UID/GID, a read-only repository +mount, image-owned framework/probe code, and separate writable scratch/artifact +mounts. The scoring container gets the per-worker socket parent read-only, not +Docker access or tool-image credentials. Each sidecar gets only its own nested +writable socket directory, so it cannot replace or connect through a sibling's +mounted UDS path. RPC paths are relative and containment-checked, and normal +commands are argv lists. + +Artifact isolation is narrower but not yet per-task/tool. Every sidecar for one +worker gets the same writable repository-root +`.eval-tool-artifacts/` namespace. It no longer receives a +writable alias of the whole `experiments` tree, task workspaces, or +`.eval-tool-evidence`, but candidate code in one sidecar can still modify another +task/tool report in that worker's diagnostic namespace. The runner explicitly +mounts this namespace writable in the scoring container even when the broad +repository mount is read-only. + +However, a GPU kernel can still hang or reset the device, exploit a driver bug, +consume host/GPU memory, or affect another process sharing the GPU. Run +untrusted submissions on dedicated workers with external time, memory, process, +and machine-recovery controls. Do not mount secrets into a tool sidecar. An +explicit `bash -lc` command should be allowed only in a reviewed, trusted +adapter. + +There is also no evaluator-versus-agent trust boundary inside the current +scoring container. Sidecars start before the agent, and their per-tool Unix +sockets remain mounted for the whole container run without client +authentication. An agent process can therefore observe or call the worker RPC +even though it cannot replace the socket path. The external tool-artifact path +is also writable during optimization, so an adversarial submission could +pre-create evidence unless the adapter independently verifies its provenance. +Filesystem placement and a SHA-256 inside an agent-writable JSON file are +integrity checks, not signatures. + +The pre-agent `.eval-tool-evidence` copy is checked against an in-memory +manifest during an uninterrupted task, which detects ordinary mutation. It is +still stored on the current writable run mount and is not signed. After a crash, +an interrupted-run resume loads that on-disk manifest as its starting point, so +it is not a durable trust anchor against an adversarial process that can rewrite +both the manifest and copied files. + +For adversarial or reward-bearing evaluation, use a separate evaluation phase: +stop the agent container, snapshot the candidate, start fresh sidecars with +evaluator-only sockets, mount the candidate read-only, give each task/tool only +its own evaluator-owned artifact directory, authenticate the RPC client, and +generate attestations and AOT capsules there. Per-tool socket directories and a +narrow per-worker artifact mount are already implemented, but the current +single-container runner does not automate this phase boundary, so its tool +results should be treated as diagnostic evidence rather than tamper-resistant +rewards. + +### Held-out integrity + +Sanitizers exercise only the cases launched by their adapter. A clean visible +case is not proof of safety for all shapes, strides, dtypes, schedules, or race +interleavings, and sanitizer runs do not automatically invoke the separate +held-out evaluator. + +Task-local adapter code, commands, capsules, seeds, input blobs, expected +outputs, and positive-control structure may be visible to the optimizing agent. +The sidecar input mount currently contains the repository root, and the socket +is present during agent execution; merely moving a hidden file to another path +inside that mount does not make it private. +For a trusted held-out campaign: + +1. Keep private cases outside the agent-editable workspace and configuration. +2. Generate replay capsules only after the candidate is frozen. +3. Mount private inputs read-only only for the evaluation phase. +4. Keep private seeds and golden outputs out of committed task files and public + `.eval-tool-artifacts` reports. +5. Run ordinary held-out correctness and sanitizer coverage as separate signals; + neither substitutes for the other. + +## Known limitations checklist + +Before treating evaluation tools as a required scoring gate, account for all of +the following: + +- Only `gfx950` has pinned sidecar locks, passing integrated startup controls, + and end-to-end fixtures; `gfx942` is unverified and rejected by the host + runner. +- Every useful task still needs a reviewed adapter command. Tool installation + alone usually produces `adapter_required`. +- All six startup positive controls pass on the current `gfx950` host. This + qualifies tool installation, not candidate coverage. +- Runtime-internal asset paths are injected from verified sidecar health and + cannot be supplied by task configuration. +- Build-attestation artifact paths must be relative to the attestation file; + absolute paths and paths escaping that per-tool artifact directory are + rejected. +- Build attestations and native-HIP rocJITsu dispatch text are weak, + self-reported integrity evidence. Triton/FlyDSL replay validates and attests + the capsule more strongly, but automatic trusted capture still does not bind + it to the ordinary correctness dispatch. +- Sidecars have isolated writable socket directories, a read-only top-level + artifact namespace in scoring, and a narrow writable mount for only the + current worker. Sockets and that worker's agent-writable report paths remain + visible in the same optimizing/scoring container phase, and its tool sidecars + still share the worker artifact namespace. +- Tool startup resolves both the selected scoring-image reference and the + pinned SGLang content-addressed manifest reference to local image IDs, then + rejects the selected image unless those IDs match. The verified ID and + selected reference are recorded in plan evidence; using an alias does not + authorize a different image build. +- YAML `runtime_ref` does not select the image; it is an assertion compared with + the host-injected, worker-reported local image ID. +- Top-level resume does not yet enforce `plan_fingerprint` freshness. +- Fingerprints cover declared file content plus configured capsule and + Waitcheck/ConSan code-object digests; other option-referenced files need + submission evidence or explicit digests. + The report serializes the complete tool plan. +- Triton/FlyDSL AOT replay is implemented for validated, single-dispatch + `gfx950` capsules and forbids arbitrary launchers. Automatic evaluator-owned + capsule extraction and correctness-run provenance are not implemented. +- FlyDSL GPU ASan and Triton FpSan instrumentation are unavailable. +- Precompiled AITER, rocBLAS, RCCL, and other library kernels are not covered + unless the exact source is rebuilt through a supported and attested path. +- “Clean” means no finding in the executed, attested cases; it is not a proof of + memory safety, race freedom, numerical equivalence for all inputs, or + generalization. + +Use `advisory` while any applicable item above remains unresolved. See the +[configuration and API reference](../reference/api-reference.md#evaluation-tools) +for the canonical field schema. diff --git a/docs/index.rst b/docs/index.rst index d3086ef2..7731218f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -26,6 +26,7 @@ repository. .. grid-item-card:: How to * :doc:`Run an experiment ` + * :doc:`Check kernels with evaluation tools ` * :doc:`Run tasks in parallel across multiple GPUs ` * :doc:`Configure agents and models ` * :doc:`Add a task ` diff --git a/docs/reference/api-reference.md b/docs/reference/api-reference.md index d20154ec..2aaa3932 100644 --- a/docs/reference/api-reference.md +++ b/docs/reference/api-reference.md @@ -50,6 +50,79 @@ log_directory: logs workspace_directory_prefix: workspace ``` +## Evaluation tools + +`evaluation_tools` configures optional, isolated kernel-analysis sidecars. The +section is disabled when it is absent, `null`, or `false`. A configured mapping +with no enabled tools is also disabled unless the host runner supplies +`AKA_EVAL_TOOLS`; for a mapping, that host subset replaces its `enabled` value. +The built-in IDs are `triton_fpsan`, `gpu_asan`, `rocjitsu`, +`rocjitsu_waitcheck`, `rocjitsu_consan`, and `hip_fpsan`. + +Sidecar build locks, integrated positive controls, and end-to-end fixtures +currently exist only for `gfx950`; all six startup controls pass in the current +MI355X qualification. Each applicable candidate still needs a task-specific +adapter and attestation, and enabling an image alone does not imply that a +kernel was analyzed. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md) for the support matrix and operational +requirements. + +When tools are enabled, the selected scoring image must resolve to the same +immutable local Docker image ID as the pinned +`lmsysorg/sglang-rocm@sha256:b435b508b5aa696abb25c909341ce73e41574c4271cf716bed72418dcea86b78` +manifest. The runner rejects a different build, launches by the verified ID, +and records both the selected reference and verified ID in plan source evidence. + +Worker reports live at repository-root +`.eval-tool-artifacts/`. The runner mounts that specific host +directory read/write into both sidecars and the scoring container; the latter +submount remains writable when the quality-loop repository root is read-only. + +| Field | Type | Default | Description | +| --- | --- | --- | --- | +| `evaluation_tools.enabled` | boolean, string, or list of strings | empty | Tool IDs to plan. `true` expands to all six built-ins; `false` disables the feature. A single string is accepted. Hyphens are normalized to underscores. When the host sets `AKA_EVAL_TOOLS`, its normalized subset is authoritative for both started sidecars and the in-container plan. | +| `evaluation_tools.policy` | `advisory` or `required` | `advisory` | `advisory` always permits performance but records an unsatisfied policy. `required` permits performance only when every applicable selected tool is ready, completes, and reports `clean`. | +| `evaluation_tools.positive_control` | `required`, `optional`, `disabled`, or boolean | `required` | Requires the applicable synthetic known-bug startup control to pass before runtime capability is ready. `optional`, `disabled`, and `false` normalize to not required; the worker still runs and reports its control. | +| `evaluation_tools.timeout_s` | exact integer from 1 through 3600 | `3600` | Default maximum execution time for each selected tool. Booleans, floats, and numeric strings are rejected. | +| `evaluation_tools.runtime_profile` | string or `null` | `null` | Fallback runtime-identity assertion for plans whose tool has no `runtime_ref`. It does not select an image and must exactly match worker health when set. | +| `evaluation_tools.tools` | mapping | `{}` | Per-tool configuration keyed by normalized tool ID. Entries do not enable tools. | +| `evaluation_tools.tools..runtime_ref` | string or `null` | automatic host image ID | Exact bare local Docker image ID (`sha256:...`) asserted by the plan and compared with worker health. The `image_digest` key is an alias. This field does not select an image; omit it when using automatic host injection. | +| `evaluation_tools.tools..timeout_s` | exact integer from 1 through 3600 | top-level timeout | Per-tool timeout; it cannot exceed the top-level timeout. | +| `evaluation_tools.tools..options` | mapping | `{}` | Adapter options, including argv lists and candidate-evidence paths. Reserved framework keys are rejected at run and task level: `positive_control_required`; GPU ASan runtime/preload/library keys; rocJITsu binary/config keys; Waitcheck CLI/C API keys; ConSan hook keys; and HIP-FpSan include/header keys. | + +The exact reserved option keys are `positive_control_required` for every tool; +`asan_runtime_dir`, `hip_asan_runtime`, `host_asan_preload`, +`host_asan_lib_dir`, and `normal_rocm_lib_dir` for `gpu_asan`; +`rocjitsu_binary` and `config_path` for `rocjitsu`; `waitcheck_binary` and +`waitcheck_capi_wrapper` for `rocjitsu_waitcheck`; `consan_hook` for +`rocjitsu_consan`; and `include_dir` and `public_header` for `hip_fpsan`. They +are selected and attested by worker health, not YAML. + +An explicit `evaluation_tools` mapping and each per-tool configuration reject +unknown fields. Unknown enabled tool IDs are also rejected. If both +`runtime_ref` and `image_digest` are supplied, they must be identical. + +Example run-level section: + +```yaml +evaluation_tools: + enabled: + - gpu_asan + policy: advisory + positive_control: required + timeout_s: 600 + tools: + gpu_asan: + options: {} +``` + +The image is selected separately with a host override. The runner resolves and +attests its immutable local image ID automatically: + +```bash +export AKA_EVAL_TOOL_IMAGE_GPU_ASAN='registry.example/eval-tool-gpu-asan@sha256:' +``` + ## Command-line flags The in-container `main.py` entrypoint accepts these flags: @@ -157,6 +230,73 @@ For repository-level tasks (`task_type: repository`): See [Add a task](../how-to/add-task.md) for layout and authoring rules. +### Evaluation profile + +The evaluator infers a profile from `task_type`, `repository_language`, source +suffixes, and repository paths. Add `evaluation_profile` only when that +inference is insufficient. Recognized profile overrides, including +`submission_paths`, are recorded in +`resolved_task_profile.explicit_overrides`. Unknown profile fields are currently +ignored, so use only the documented keys. + +| Field | Type | Description | +| --- | --- | --- | +| `evaluation_profile.language` | string | Canonical values are `triton`, `hip`, `flydsl`, and `unknown`. | +| `evaluation_profile.artifact_kind` | string | `source_aot`, `python_jit`, `hsaco_precompiled`, or `unknown`. | +| `evaluation_profile.framework` | string | Framework identity such as `standalone`, `aiter`, `rocblas`, or `rccl`. | +| `evaluation_profile.instrumentation_control` | string | `compiler_controlled`, `recompile`, `none`, or `unknown`. This describes whether the selected candidate can be rebuilt/instrumented. | +| `evaluation_profile.adapter` | string or `null` | Explicit adapter identity, for example `triton_aot`, `flydsl_aot`, or `hip_fpsan_manual`. It is a claim that must still be supported by adapter options/evidence. | +| `evaluation_profile.source_available` | boolean | Whether source for the selected candidate is available to the evaluator. | +| `evaluation_profile.submission_paths` | string or list of strings | Workspace-relative candidate files captured before agent edits and fingerprinted after optimization. Required when repository/image tasks change files beyond the normal source fields. Absolute paths and `..` are rejected. | +| `evaluation_profile.fpsan_ported` | boolean | Explicit evidence that the HIP reference and candidate were manually ported to HIP-FpSan value semantics. | +| `evaluation_profile.rebuilt_from_source` | boolean | Explicit evidence used when a framework/library path is rebuilt from controlled source. It does not replace artifact attestation. | + +### Task-level tool adapters + +A task can add adapter options only for tools enabled by the run. The only +allowed task-level structure is: + +```yaml +evaluation_tools: + tools: + gpu_asan: + timeout_s: 300 + options: + command: [python3, scripts/eval_tools/run_gpu_asan.py] +``` + +`timeout_s` must be between 1 and the run-level value. Task configuration cannot +enable another tool, change the top-level `policy` or `positive_control`, select +another runtime image, increase a timeout, or set any reserved framework option +listed above. Other options are merged over the run-level options. + +Commands are argv lists, not shell strings. The built-in adapter keys are: + +| Tool | Adapter keys | +| --- | --- | +| `triton_fpsan` | `comparison_command` or `command`; optional invocation-artifact-contained `attestation_path`. | +| `gpu_asan` | `command`; optional invocation-artifact-contained candidate `attestation_path`. ASan runtime/preload/library paths come only from verified sidecar health. | +| `rocjitsu` | HIP uses `launcher` or `command`, with optional `expected_kernel` and invocation-artifact-contained `race_report` whose filename must be `race.log`. Triton/FlyDSL requires `capsule` and the exact `triton_aot`/`flydsl_aot` profile adapter; arbitrary launchers and task-configured race-report paths are rejected. The capsule must be workspace-contained, single-dispatch, contain a golden expected output, be manifest-valid, and target `gfx950`. Binary/config and the trusted replay helper come only from the image/health. Automatic trusted capsule capture from correctness is not implemented. | +| `rocjitsu_waitcheck` | `code_object`, `expected_kernel`, and non-negative integer `kernel_entry`. The workspace-contained unbundled final ELF is SHA-256-bound to the plan; an image-owned inventory must match its exact `gfx950` descriptor before structured C API analysis. | +| `rocjitsu_consan` | `code_object`, focused native `command`, and independent `oracle_command`. The instrumented command must explicitly name and load the code object. Strict record/replay, SHA-256/FNV identity, complete accounting, and an oracle pass are required. Broad AITER/rocBLAS/RCCL runtimes are rejected. | +| `hip_fpsan` | `comparison_command` or `command`; optional invocation-artifact-contained candidate `attestation_path`; requires `evaluation_profile.fpsan_ported: true`. The include path comes only from verified sidecar health. | + +Sidecar health attests and injects runtime-internal assets. Candidate/task +configuration cannot override or supply ASan preload/library paths, the +rocJITsu binary/config path, the Waitcheck CLI/C API wrapper, the ConSan hook, +or the HIP-FpSan include path. + +Build-attestation JSON must store `artifact_path` relative to the directory +containing that JSON. The artifact must be beside or below the attestation; +absolute paths, `..`, and symlink resolutions that escape the directory are +rejected. The parser resolves the relative path in the scoring namespace and +verifies the declared SHA-256. + +Each execution uses a fresh +`//` artifact directory. Custom +attestation and race-report paths are resolved against that directory; an +absolute path is accepted only when its resolution remains inside it. + ### Platform support `platform_support.status: skip` excludes a task unconditionally. An active task @@ -175,6 +315,9 @@ Each task produces a `task_result.yaml` in its workspace: | `compilation_error_message` | Error text if compilation failed, else `null` | | `pass_correctness` | Whether correctness passed | | `correctness_error_message` | Error text if correctness failed, else `null` | +| `pass_tool_gate` | Whether the selected evaluation-tool policy allows performance to proceed. Defaults to `true` when tools are disabled. This is independent of `pass_correctness`. | +| `tool_policy_satisfied` | Whether every applicable selected tool was ready and completed with a `clean` finding status. Under `advisory`, this can be `false` while `pass_tool_gate` remains `true`. | +| `tool_evaluation` | Versioned complete plan, plan fingerprint, profile, capability, execution, finding, evidence, and decision data. Omitted when no tool is enabled. | | `base_execution_time` | Baseline runtime in ms | | `best_optimized_execution_time` | Best optimized runtime in ms | | `speedup_ratio` | Speedup over baseline | @@ -187,6 +330,37 @@ Each task produces a `task_result.yaml` in its workspace: | `optimization_summary` | Framework-generated note identifying the optimizing agent and centralized evaluator | | `score` | Computed score (see below) | +`tool_evaluation` uses this high-level shape: + +| Field | Description | +| --- | --- | +| `schema_version` | Evaluation-tool result schema version. | +| `plan_fingerprint` | SHA-256 over normalized configuration, resolved task profile, plugin versions, captured original/candidate evidence for declared paths, verified scoring-image reference/ID, and the content digest of a configured replay capsule. | +| `plan` | Complete immutable plan: schema, policy, profile, ordered tool records (runtime reference, plugin version, timeout, and options), fingerprint, and source evidence. | +| `policy` | `advisory` or `required`. | +| `overall_status` | `clean`, `finding`, `incomplete`, or `not_applicable`. | +| `resolved_task_profile` | Inferred profile plus auditable explicit overrides. | +| `source_evidence` | Captured-original and candidate fingerprints plus manifest metadata, including `metadata.scoring_runtime.image_id` and `.reference` when tools run. | +| `decision` | `allowed`, `policy_satisfied`, and machine-readable reason strings. | +| `tools..capability` | Separate `engine`, `adapter`, `runtime`, and resolved `effective` checks. | +| `tools..result.execution` | `not_run`, `completed`, `tool_error`, or `timeout`. | +| `tools..result.finding` | `not_evaluated`, `clean`, `found`, or `inconclusive`. | +| `tools..result.findings` | Structured finding records. | +| `tools..result.artifacts` | Paths to retained reports/attestations. Raw stdout/stderr is omitted from the YAML summary by default. | + +Execution status and finding status are deliberately independent. A sanitizer +can terminate the candidate while producing a valid finding, and a process can +exit zero without proving that an instrumented kernel ran. + +The complete `plan.tools` records make the selected runtime, plugin version, +timeout, and options reconstructable from the report. Ordinary option path +strings still do not hash referenced adapter, HSACO, or input contents; include +those files in `evaluation_profile.submission_paths` or add explicit digests. A +configured rocJITsu `capsule` is handled specially: its JSON SHA-256/size are +added to source evidence and the fingerprint, while capsule validation verifies +the manifest's HSACO and blob hashes. This does not prove the capsule came from +the ordinary correctness run. + ## Scoring The score is the sum of three components: diff --git a/docs/reference/compatibility-matrix.md b/docs/reference/compatibility-matrix.md index 98f2c829..08f61682 100644 --- a/docs/reference/compatibility-matrix.md +++ b/docs/reference/compatibility-matrix.md @@ -37,6 +37,29 @@ The following software versions are required or verified. | AITER | `0.1.17.dev110+g9127c94a1` in the verified `gfx950` image | Required by AITER-backed task oracles and kernels. | | FlyDSL | `0.2.2` in the verified `gfx950` image (or `make docker-setup-flydsl` when absent) | Required for `flydsl2flydsl`, `torch2flydsl`, and `triton2flydsl` tasks. | +## Evaluation-tool sidecars + +Optional Triton FpSan, GPU ASan, rocJITsu Race Detector, rocJITsu Waitcheck, +rocJITsu ConSan, and HIP-FpSan dependencies are kept out of the scoring image +and installed in one isolated sidecar image per tool. The scoring image, +FlyDSL, and AITER versions in the preceding table remain unchanged. + +| GPU architecture | Sidecar status | Notes | +| --- | --- | --- | +| `gfx950` (MI355X) | Runtime-qualified, candidate-dependent | Pinned image/build locks and all six integrated startup controls pass on the current hardware. End-to-end readiness still depends on language, artifact, adapter, and candidate attestation. Waitcheck and ConSan are qualified only for explicitly configured advisory pilots. Trusted single-dispatch Triton/FlyDSL rocJITsu capsule replay is implemented, but automatic evaluator-owned capsule capture and binding to the correctness run remain advisory-only gaps. | +| `gfx942` (MI300X/MI325X) | Unverified | No equivalent image/adapter/positive-control qualification has completed; the host runner currently rejects evaluation-tool sidecars. | + +The runtime base digest and per-tool package/source locks are recorded in +`docker/eval-tools/images.lock.yaml`. See [Check kernels with evaluation +tools](../how-to/use-evaluation-tools.md#strict-support-matrix) for the strict +Triton, HIP, FlyDSL, AITER, rocBLAS, and RCCL matrix. Normal task compatibility +does not imply sanitizer coverage. Tool startup resolves both the selected +scoring-image reference and the pinned `gfx950` SGLang content-addressed +manifest reference to immutable local image IDs and requires those local IDs to +match. Aliases of that exact image are allowed, but rebuilt, upgraded, or +retagged images are rejected. The scoring container is launched by the verified +image ID. + ## Agents The following templates are selectable in the current `AgentType` registry. See diff --git a/docs/reference/release-notes.md b/docs/reference/release-notes.md index 6d229c38..59de682f 100644 --- a/docs/reference/release-notes.md +++ b/docs/reference/release-notes.md @@ -21,6 +21,11 @@ execution, and RL-ready GPU kernel evaluation. - Added first-class A/B experimentation workflows with labeled baseline and treatment runs. - Exposed compilation, correctness, latency, speedup, and score fields as structured signals for external agent-RL systems. +- Added opt-in, per-tool sidecar plumbing and typed reports for Triton FpSan, + ROCm GPU ASan, rocJITsu Race Detector, rocJITsu Waitcheck, rocJITsu ConSan, + and HIP-FpSan. The initial sidecar locks are verified only for `gfx950`. + Workers automatically execute synthetic startup controls, while useful + candidate runs still require task-specific adapters and attestations. - Added run comparison through `src/tools/compare_runs.py` and the standalone visualization dashboard. - Added held-out evaluation for testing kernel generalization on unseen shapes. - Centralized compilation, correctness, performance measurement, result generation, and scoring outside agent-editable code. @@ -115,6 +120,53 @@ The task validator now includes Codex backend support, repository-task validatio - `cuda2hip` is recognized by the prompt system, but no bundled cuda2hip task suite is currently included. - Local vLLM provider configuration remains specific to the selected agent integration. - GPU task execution requires compatible physical AMD hardware and ROCm driver access. +- Evaluation-tool sidecars are experimental and `gfx950`-only. Startup controls + prove a tool installation can detect its synthetic bug, not that a candidate + was instrumented. No bundled task currently supplies a production-qualified + adapter/attestation. All six integrated startup controls pass on the current + MI355X qualification host. Synthetic manager-to-sidecar candidate pairs also + distinguished clean from seeded-bug Triton FpSan, HIP/Triton GPU ASan, and + HIP-FpSan runs; trusted AOT replay produced a clean Triton result and found the + seeded FlyDSL LDS race. Waitcheck distinguished a correct wait from a missing + `lgkmcnt(0)`, and strict ConSan record/replay distinguished clean single-wave + LDS accesses from seeded cross-wave conflicts. These fixtures do not qualify + a bundled task. Keep the policy advisory until each selected candidate path + is independently qualified. +- Evaluation-tool parsing and execution now fail closed on ambiguous FpSan + markers, process failure, non-finite protocol numbers, stale artifact reuse, + unsupported GPU-library kernels, and replay capsules without a golden output. + Each attempt receives a fresh artifact directory. The worker contains the + complete descendant process tree, including session-detached children, and + treats required cleanup as a failed execution. Replay launch geometry is + validated as exact `uint32` input and checked against runtime device limits. +- Native HIP rocJITsu no longer accepts task-launcher stdout/stderr as clean + dispatch attestation; only canonical records in the evaluator-owned report + sink are eligible. Waitcheck and ConSan startup controls now execute their + production entrypoints and parsers. The Waitcheck control covers inventory + and the C API, while ConSan uses exact raw HSACO identities, separate + instrumented/oracle argv, and an oracle that rejects leaked hook state. +- The current runner exposes evaluation-tool sockets and agent-writable report + paths during the same container run. Per-tool writable socket directories and + a read-only top-level artifact namespace plus one writable + `.eval-tool-artifacts/