Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/scripts/aiter_prebuild_upload.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@ ROCM_VER=`head -n1 "${ROCM_PATH}/.info/version" | cut -d. -f1`

QOLA_DIR="${ROOT_DIR}/3rdparty/QoLA"
AITER_DIR="${QOLA_DIR}/3rdparty/aiter"
QOLA_MANIFEST="${ROOT_DIR}/transformer_engine/common/ck_fused_attn/qola_manifest.toml"
QOLA_MANIFEST="${ROOT_DIR}/transformer_engine/common/qola_manifest.toml"
# This cache feeds aiter_prebuilt.cmake, which only consumes the fused-attn
# kernels. The shared manifest also declares the gfx950-only a4w4 GEMM group,
# so the build must be restricted or the gfx942 leg would fail.
QOLA_GROUP="ck_fused_attn"
GIT_CONFIG_GLOBAL="$(mktemp /tmp/gitconfig.XXXXXX)"
trap 'rm -f "${GIT_CONFIG_GLOBAL}"' EXIT
git config --file "${GIT_CONFIG_GLOBAL}" --add safe.directory "${AITER_DIR}"
Expand Down Expand Up @@ -64,6 +68,7 @@ if [[ "${1:-}" == "--build" ]]; then
--manifest "${QOLA_MANIFEST}" \
--aiter-root "${AITER_DIR}" \
--output-dir "${QOLA_BUILD_DIR}" \
--group "${QOLA_GROUP}" \
"${arch_args[@]}"

# Stage QoLA outputs into the cache layout expected by aiter_prebuilt.cmake.
Expand Down
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
- **Python import flow**: framework selection happens in `transformer_engine/__init__.py` (driven by `NVTE_FRAMEWORK`); native `.so` resolution happens in `transformer_engine/common/__init__.py`. Start tracing from these two files for any import or framework-selection issue — function names inside may change, but the entry points are stable.
- **Build orchestration**: `setup.py` + helpers in `build_tools/` + CMake. ROCm vs CUDA backend detection lives in `build_tools/utils.py` — grep there (and for `NVTE_USE_ROCM`) for current behavior.
- **3rdparty submodules**: read `.gitmodules` for the current set and commit pins to determine current submodules.
- **AITER source**: the tree QoLA/TE actually builds is cloned on-demand to `build/cmake/ck_fused_attn/qola/third_party/aiter/` at the commit pinned in `transformer_engine/common/ck_fused_attn/qola_manifest.toml`. The in-tree `3rdparty/*/aiter` checkouts are stale — read the built tree (and check its HEAD) when reasoning about the linked ABI.
- **AITER source**: the tree QoLA/TE actually builds is cloned on-demand to `build/cmake/ck_fused_attn/qola/third_party/aiter/` at the commit pinned in `transformer_engine/common/qola_manifest.toml`. That single manifest serves every QoLA consumer; each one builds its own `[[modules]]` subset via `qola build --group <name>` (`ck_fused_attn`, `aiter_gemm`). The in-tree `3rdparty/*/aiter` checkouts are stale — read the built tree (and check its HEAD) when reasoning about the linked ABI.

## Hipify convention
The build auto-generates HIP files from CUDA sources via `hipify_torch`. Generated files are marked with `// !!! This is a file automatically generated by hipify!!!` at line 1. **Never edit generated files directly** — edit the CUDA source instead.
Expand Down
7 changes: 7 additions & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ def setup_common_extension() -> CMakeExtension:
elif os.getenv("NVTE_FUSED_ATTN_CK") or os.getenv("NVTE_FUSED_ATTN"):
cmake_flags.append("-DUSE_FUSED_ATTN_CK=ON")

# AITER a4w4 (FP4) GEMM backend (gfx950-only; CMake disables it cleanly
# on other arches).
if int(os.getenv("NVTE_AITER_GEMM", "1"))==0:
cmake_flags.append("-DUSE_AITER_GEMM=OFF")
else:
cmake_flags.append("-DUSE_AITER_GEMM=ON")

if bool(int(os.getenv("NVTE_ENABLE_NVSHMEM", "0"))) and os.getenv("NVTE_ENABLE_ROCSHMEM") is None:
os.environ["NVTE_ENABLE_ROCSHMEM"] = '1'
os.environ["NVTE_ENABLE_NVSHMEM"] = '0'
Expand Down
55 changes: 54 additions & 1 deletion transformer_engine/common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ option(USE_ROCM "Use ROCm" ON)
option(USE_FUSED_ATTN_AOTRITON "Use aotriton backend" ON)
option(USE_FUSED_ATTN_CK "Use ck backend" ON)
option(USE_HIPKITTENS_GEMM "Use HipKittens blockwise FP8 and MXFP8 GEMM kernels" ON)
option(USE_AITER_GEMM "Use AITER a4w4 (FP4) GEMM backend" ON)
set(USE_CUDA OFF)

if (USE_ROCM)
Expand Down Expand Up @@ -384,6 +385,7 @@ if(USE_ROCM)
fused_attn_rocm/fused_attn_aotriton.cpp
fused_attn_rocm/fused_attn_ck.cpp
fused_attn_rocm/utils.cpp
gemm/aiter_a4w4_gemm.cpp
gemm/ck_grouped_gemm/ck_grouped_gemm.cpp
gemm/ck_grouped_gemm/ck_grouped_gemm_fp8.cpp
gemm/ck_grouped_gemm/ck_grouped_gemm_fp16.cpp
Expand Down Expand Up @@ -708,6 +710,34 @@ else() # USE_ROCM
add_subdirectory(gemm/kittens ${CMAKE_CURRENT_BINARY_DIR}/kittens)
endif()

# AITER a4w4 (FP4) GEMM is gfx950-only; skip cleanly on other arches so a
# default-on USE_AITER_GEMM does not break gfx942/gfx1250 builds. The NVTE
# C API in gemm/aiter_a4w4_gemm.cpp keeps stub symbols when disabled.
#
# There is no TE-side wrapper library any more: QoLA exports a C ABI that
# libtransformer_engine.so links against directly, so all this needs is the
# include dir and the built .so files.
list(FIND CMAKE_HIP_ARCHITECTURES "gfx950" _aiter_gemm_gfx950_idx)
if(USE_AITER_GEMM AND NOT _aiter_gemm_gfx950_idx EQUAL -1)
set(__BUILD_AITER_GEMM TRUE)
include(${CMAKE_CURRENT_LIST_DIR}/../../3rdparty/QoLA/cmake/QoLA.cmake)
qola_add_modules(
GROUP aiter_gemm
MANIFEST ${CMAKE_CURRENT_LIST_DIR}/qola_manifest.toml
BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/aiter_gemm/qola
ARCHS gfx950
LIBS te_libgemm_a4w4_blockscale.so te_libgemm_a4w4_asm.so
OUT_INCLUDE_DIR __AITER_GEMM_INCLUDE_DIR
OUT_LIB_DIR __AITER_GEMM_LIB_DIR
OUT_LIBS __AITER_GEMM_LIBS
OUT_CONFIG_DIR __AITER_GEMM_CONFIG_DIR)
else()
set(__BUILD_AITER_GEMM FALSE)
if(USE_AITER_GEMM)
message(STATUS "USE_AITER_GEMM requested but no gfx950 target present; a4w4 GEMM backend disabled.")
endif()
endif()

find_package(hip)
list(APPEND transformer_engine_LINKER_LIBS hip::host hip::device roctx64)
find_package(hiprtc)
Expand All @@ -726,6 +756,26 @@ else() # USE_ROCM
target_compile_definitions(transformer_engine PUBLIC USE_HIPKITTENS_GEMM)
list(APPEND transformer_engine_LINKER_LIBS kittens_gemm)
endif()
if(__BUILD_AITER_GEMM)
target_compile_definitions(transformer_engine PUBLIC USE_AITER_GEMM)
target_include_directories(transformer_engine PRIVATE ${__AITER_GEMM_INCLUDE_DIR})
target_link_directories(transformer_engine PUBLIC ${__AITER_GEMM_LIB_DIR})
foreach(__lib ${__AITER_GEMM_LIBS})
list(APPEND transformer_engine_LINKER_LIBS -l:${__lib})
endforeach()
set(__AITER_GEMM_INSTALL_DIR "${CMAKE_INSTALL_PREFIX}/transformer_engine/lib")
if(NOT "${__AITER_GEMM_LIB_DIR}" STREQUAL "${__AITER_GEMM_INSTALL_DIR}")
foreach(__lib ${__AITER_GEMM_LIBS})
install(FILES ${__AITER_GEMM_LIB_DIR}/${__lib} DESTINATION ${__AITER_GEMM_INSTALL_DIR})
endforeach()
endif()
# Ship the a4w4 tuning table next to the kernels it describes; the Python
# layer reads it instead of importing the `aiter` package.
if(EXISTS "${__AITER_GEMM_CONFIG_DIR}/a4w4_blockscale_tuned_gemm.csv")
install(FILES ${__AITER_GEMM_CONFIG_DIR}/a4w4_blockscale_tuned_gemm.csv
DESTINATION ${__AITER_GEMM_INSTALL_DIR}/configs)
endif()
endif()
target_link_libraries(transformer_engine PUBLIC ${transformer_engine_LINKER_LIBS})
endif()

Expand Down Expand Up @@ -850,7 +900,10 @@ endif()
# Install library
install(TARGETS transformer_engine DESTINATION .)
if (USE_ROCM)
set_target_properties(transformer_engine PROPERTIES INSTALL_RPATH "$ORIGIN/lib;$ORIGIN/transformer_engine/lib")
# "$ORIGIN" covers QoLA-built kernel libraries (e.g. te_libgemm_a4w4_*.so),
# which are installed alongside libtransformer_engine.so rather than under a
# nested lib/ directory.
set_target_properties(transformer_engine PROPERTIES INSTALL_RPATH "$ORIGIN;$ORIGIN/lib;$ORIGIN/transformer_engine/lib")
file(READ "${ROCM_PATH}/.info/version" ROCM_VER)
string(STRIP "${ROCM_VER}" ROCM_VER)
string(REGEX MATCH "^[0-9]+\\.[0-9]+" ROCM_VER "${ROCM_VER}")
Expand Down
12 changes: 10 additions & 2 deletions transformer_engine/common/ck_fused_attn/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,11 @@ endif()
# QoLA-managed AITER source tree to that commit before any consumer reads it
# (header validation below, header includes for the .cpp build later, and
# QoLA's own kernel build if the prebuilt cache misses).
set(__QOLA_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/qola_manifest.toml")
# TE keeps a single QoLA manifest so the whole build shares one AITER commit,
# one patch set and one checkout. Each consumer builds only its own module
# group (see `qola build --group`).
set(__QOLA_MANIFEST "${CMAKE_CURRENT_LIST_DIR}/../qola_manifest.toml")
set(__QOLA_GROUP "ck_fused_attn")
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${__QOLA_MANIFEST}")
file(STRINGS "${__QOLA_MANIFEST}" __AITER_COMMIT_LINES
REGEX "^[ \t]*aiter_commit[ \t]*=[ \t]*\"[^\"]+\"")
Expand Down Expand Up @@ -155,8 +159,11 @@ if(__AITER_CACHE_DIR STREQUAL "")
else()
set(__CK_JIT_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../3rdparty/ck_jit")
endif()
# ck_jit_build.py shells out to `qola build` with a fixed argument list and
# cannot forward --group, so the group is passed through the environment.
execute_process(
COMMAND ${Python_EXECUTABLE} "${__CK_JIT_SOURCE_DIR}/ck_jit_build.py" full
COMMAND ${CMAKE_COMMAND} -E env "QOLA_BUILD_GROUPS=${__QOLA_GROUP}"
${Python_EXECUTABLE} "${__CK_JIT_SOURCE_DIR}/ck_jit_build.py" full
--with-qola
--qola-dir ${__QOLA_DIR}
--qola-manifest ${__QOLA_MANIFEST}
Expand All @@ -182,6 +189,7 @@ if(__AITER_CACHE_DIR STREQUAL "")
--manifest ${__QOLA_MANIFEST}
--aiter-root ${__AITER_SOURCE_DIR}
--output-dir ${__QOLA_BUILD_DIR}
--group ${__QOLA_GROUP}
--arch "${GPU_ARCHS_STR}"
--skip-checkout
RESULT_VARIABLE QOLA_BUILD_RESULT
Expand Down
19 changes: 0 additions & 19 deletions transformer_engine/common/ck_fused_attn/qola_manifest.toml

This file was deleted.

121 changes: 121 additions & 0 deletions transformer_engine/common/gemm/aiter_a4w4_gemm.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*************************************************************************
* Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE for license information.
************************************************************************/

// NVTE C API for AITER's a4w4 (FP4) GEMM. QoLA exports a C ABI over a
// descriptor that is layout-identical to NVTEAiterGemmTensor, so these are
// pure forwards -- the static_asserts below are what keep that true.
//
// When TE is built without the AITER a4w4 backend (USE_AITER_GEMM undefined
// -- e.g. non-gfx950), the symbols still exist but report failure so the
// framework layer degrades gracefully instead of failing to link.

#include "transformer_engine/aiter_gemm.h"

#ifdef USE_AITER_GEMM

#include <hip/hip_runtime.h>

#include <cstddef>

#include "qola_gemm_a4w4.h"

namespace {

static_assert(sizeof(NVTEAiterGemmTensor) == sizeof(qola_tensor_t),
"NVTEAiterGemmTensor must mirror qola_tensor_t");
static_assert(offsetof(NVTEAiterGemmTensor, ptr) == offsetof(qola_tensor_t, ptr),
"NVTEAiterGemmTensor::ptr must mirror qola_tensor_t::ptr");
static_assert(offsetof(NVTEAiterGemmTensor, ndim) == offsetof(qola_tensor_t, ndim),
"NVTEAiterGemmTensor::ndim must mirror qola_tensor_t::ndim");
static_assert(offsetof(NVTEAiterGemmTensor, dtype) == offsetof(qola_tensor_t, dtype),
"NVTEAiterGemmTensor::dtype must mirror qola_tensor_t::dtype");
static_assert(offsetof(NVTEAiterGemmTensor, device_id) == offsetof(qola_tensor_t, device_id),
"NVTEAiterGemmTensor::device_id must mirror qola_tensor_t::device_id");
static_assert(offsetof(NVTEAiterGemmTensor, shape) == offsetof(qola_tensor_t, shape),
"NVTEAiterGemmTensor::shape must mirror qola_tensor_t::shape");
static_assert(offsetof(NVTEAiterGemmTensor, strides) == offsetof(qola_tensor_t, strides),
"NVTEAiterGemmTensor::strides must mirror qola_tensor_t::strides");

// The dtype enumerators are part of the shared ABI too. Compared as ints
// because the two enums are deliberately distinct types.
static_assert(static_cast<int>(kNVTEAiterGemmFP4x2) == static_cast<int>(QOLA_DTYPE_FP4X2),
"a4w4 dtype enum drift");
static_assert(static_cast<int>(kNVTEAiterGemmE8M0) == static_cast<int>(QOLA_DTYPE_E8M0),
"a4w4 dtype enum drift");
static_assert(static_cast<int>(kNVTEAiterGemmBF16) == static_cast<int>(QOLA_DTYPE_BF16),
"a4w4 dtype enum drift");
static_assert(static_cast<int>(kNVTEAiterGemmFP16) == static_cast<int>(QOLA_DTYPE_FP16),
"a4w4 dtype enum drift");
static_assert(static_cast<int>(kNVTEAiterGemmFP32) == static_cast<int>(QOLA_DTYPE_FP32),
"a4w4 dtype enum drift");

inline const qola_tensor_t *as_qola(const NVTEAiterGemmTensor *t) {
return reinterpret_cast<const qola_tensor_t *>(t);
}

} // namespace

extern "C" int nvte_aiter_gemm_a4w4_blockscale(const NVTEAiterGemmTensor *XQ,
const NVTEAiterGemmTensor *WQ,
const NVTEAiterGemmTensor *x_scale,
const NVTEAiterGemmTensor *w_scale,
const NVTEAiterGemmTensor *Y, int split_k,
const char *kernel_name, void *stream, char *err_buf,
size_t err_buf_size) {
return QOLA_C(gemm_a4w4_blockscale)(as_qola(XQ), as_qola(WQ), as_qola(x_scale), as_qola(w_scale),
as_qola(Y), split_k, kernel_name,
static_cast<hipStream_t>(stream), err_buf, err_buf_size);
}

extern "C" int nvte_aiter_gemm_a4w4_asm(const NVTEAiterGemmTensor *A, const NVTEAiterGemmTensor *B,
const NVTEAiterGemmTensor *a_scale,
const NVTEAiterGemmTensor *b_scale,
const NVTEAiterGemmTensor *out,
const NVTEAiterGemmTensor *bias, const char *kernel_name,
float alpha, float beta, int bpreshuffle, int log2_k_split,
void *stream, char *err_buf, size_t err_buf_size) {
return QOLA_C(gemm_a4w4_asm)(as_qola(A), as_qola(B), as_qola(a_scale), as_qola(b_scale),
as_qola(out), as_qola(bias), kernel_name, alpha, beta, bpreshuffle,
log2_k_split, static_cast<hipStream_t>(stream), err_buf,
err_buf_size);
}

#else // !USE_AITER_GEMM

#include <cstring>

namespace {

void report_disabled(char *err_buf, size_t err_buf_size) {
const char *msg = "TransformerEngine was built without the AITER a4w4 GEMM backend";
if (err_buf != nullptr && err_buf_size > 0) {
std::strncpy(err_buf, msg, err_buf_size - 1);
err_buf[err_buf_size - 1] = '\0';
}
}

} // namespace

extern "C" int nvte_aiter_gemm_a4w4_blockscale(const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *, int, const char *,
void *, char *err_buf, size_t err_buf_size) {
report_disabled(err_buf, err_buf_size);
return -1;
}

extern "C" int nvte_aiter_gemm_a4w4_asm(const NVTEAiterGemmTensor *, const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *, const NVTEAiterGemmTensor *,
const NVTEAiterGemmTensor *, const NVTEAiterGemmTensor *,
const char *, float, float, int, int, void *, char *err_buf,
size_t err_buf_size) {
report_disabled(err_buf, err_buf_size);
return -1;
}

Comment on lines +86 to +120

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The "framework layer degrades gracefully" comment at the top of the file (line 11) is a bit optimistic given how these stubs are consumed on the pytorch side: gemm.cpp unconditionally calls NVTE_CHECK(rc == 0, "nvte_aiter_gemm_a4w4_blockscale failed (rc=", rc, ")") and throws. A user building on gfx942 who accidentally exercises the a4w4 path will see a rather cryptic rc=-1 error rather than a "a4w4 GEMM not built for this arch" message.

Two options that would materially improve the experience with almost zero code:

  • use a distinctive negative code (e.g. -2 = "backend not compiled in") plus a matching branch in gemm.cpp's check that produces a targeted error, or
  • expose a build-state query (extern "C" int nvte_aiter_gemm_a4w4_available();) so the pytorch layer can guard the entry points and give the framework a hasattr-like probe from Python.

Not a blocker; landing this as-is is fine so long as the intent is understood.

#endif // USE_AITER_GEMM
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*************************************************************************
* Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE for license information.
************************************************************************/

/*! \file aiter_gemm.h
* \brief C API for AITER a4w4 (FP4 x FP4) GEMM (ROCm only).
*
* Thin executor entry points: kernel selection (tuned-CSV lookup) and
* weight/scale pre-shuffling happen in the framework layer; these functions
* take a resolved kernel name and already-shuffled inputs.
*
* The kernels themselves live in QoLA-built shared objects. This header
* mirrors QoLA's public C ABI so the two descriptors are layout-identical
* and the call needs no translation; the mirroring is asserted at compile
* time in the implementation. It is declared independently (rather than
* including QoLA's header) so that this API stays available on builds where
* the AITER a4w4 backend is disabled.
*/

#ifndef TRANSFORMER_ENGINE_AITER_GEMM_H_
#define TRANSFORMER_ENGINE_AITER_GEMM_H_

#include <stddef.h>
#include <stdint.h>

#ifdef __cplusplus
extern "C" {
#endif

/*! \brief Element type of an a4w4 GEMM operand. Values match QoLA's
* qola_dtype_t. */
typedef enum {
kNVTEAiterGemmFP4x2 = 0, /*!< two packed FP4 (E2M1) values per byte */
kNVTEAiterGemmE8M0 = 1, /*!< 8-bit exponent-only microscale (1 byte) */
kNVTEAiterGemmBF16 = 2,
kNVTEAiterGemmFP16 = 3,
kNVTEAiterGemmFP32 = 4,
kNVTEAiterGemmU8 = 5,
kNVTEAiterGemmI8 = 6,
} NVTEAiterGemmDType;

/*! \brief Lightweight device-tensor descriptor (raw pointer + layout).
* Layout-identical to QoLA's qola_tensor_t. */
typedef struct {
void *ptr;
int32_t ndim;
int32_t dtype; /*!< one of NVTEAiterGemmDType */
int32_t device_id;
int32_t reserved;
int64_t shape[8];
int64_t strides[8];
} NVTEAiterGemmTensor;

/*! \brief CK blockscale a4w4 GEMM: Y = XQ @ WQ^T with per-1x32 microscaling.
*
* \param[out] err_buf Optional buffer receiving a failure message.
* \param[in] err_buf_size Size of \p err_buf in bytes; 0 to discard.
* \return 0 on success, nonzero on failure (or if TE was built without the
* AITER a4w4 backend).
*/
int nvte_aiter_gemm_a4w4_blockscale(const NVTEAiterGemmTensor *XQ, const NVTEAiterGemmTensor *WQ,
const NVTEAiterGemmTensor *x_scale,
const NVTEAiterGemmTensor *w_scale, const NVTEAiterGemmTensor *Y,
int split_k, const char *kernel_name, void *stream,
char *err_buf, size_t err_buf_size);

/*! \brief ASM (f4gemm) a4w4 GEMM: D = alpha*A*B + beta*C. `bias` may be NULL.
* \return 0 on success, nonzero on failure (or if TE was built without the
* AITER a4w4 backend).
*/
int nvte_aiter_gemm_a4w4_asm(const NVTEAiterGemmTensor *A, const NVTEAiterGemmTensor *B,
const NVTEAiterGemmTensor *a_scale, const NVTEAiterGemmTensor *b_scale,
const NVTEAiterGemmTensor *out, const NVTEAiterGemmTensor *bias,
const char *kernel_name, float alpha, float beta, int bpreshuffle,
int log2_k_split, void *stream, char *err_buf, size_t err_buf_size);

#ifdef __cplusplus
} // extern "C"
#endif

#endif // TRANSFORMER_ENGINE_AITER_GEMM_H_
Loading
Loading