Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include "dispatch_registration/common/dispatch-q8-publication.h"
#include "ggml.h"
#include "graph/graph-matcher.h"
#include "hip/hip-dispatches.h"
#include "kernel-corpus/kernel-corpus-catalog-verify.h"

#include <cstdint>
Expand Down Expand Up @@ -69,6 +70,14 @@ static bool same_shape(const Value & lhs, const Value & rhs) {
return true;
}

// Bytes of the route ids a routed kernel reads: token t's first kRoutedFfnRouteCount ids start at t * route_stride.
// llama.cpp passes a [route_count, tokens] view of its [expert_count, tokens] argsort, so the view itself ends after
// the last token's routes; binding the full token_count * route_stride would overrun a view whose source is not the
// whole argsort (a one-token view is route_count ids, not route_stride).
static size_t routed_route_id_bytes(int64_t token_count, int64_t route_stride) {
return static_cast<size_t>((token_count - 1) * route_stride + kRoutedFfnRouteCount) * sizeof(int32_t);
}

static bool is_profile_rms_norm_epsilon(float eps) {
const float expected = kRoutedFfnProfile.rms_norm_epsilon;
return eps >= expected * 0.9f && eps <= expected * 1.1f;
Expand Down Expand Up @@ -994,7 +1003,7 @@ static bool match_decode_routed_ffn_gate_up_swiglu_q4k_q8_dispatch(const Dispatc
to_config_value(kRoutedFfnExpertHiddenSize));
dispatch.kernel.compile_parameters.emplace("qwen3_moe.workload.token_capacity", to_config_value(match.token_count));

const size_t route_id_length = static_cast<size_t>(match.token_count * match.route_stride) * sizeof(int32_t);
const size_t route_id_length = routed_route_id_bytes(match.token_count, match.route_stride);
dispatch.bindings.push_back({ match.input_alternate->alternate_value, 0, match.input_alternate->byte_count });
dispatch.bindings.push_back({ match.route_ids->id, 0, route_id_length });
dispatch.bindings.push_back({ match.gate_weight->id, 0, match.gate_weight->byte_count });
Expand Down Expand Up @@ -1126,7 +1135,7 @@ static bool match_decode_routed_ffn_down_next_q8_dispatch(const DispatchMatchCon
dispatch.kernel.compile_parameters.emplace("qwen3_moe.model.hidden_size", to_config_value(kRoutedFfnInputSize));
dispatch.kernel.compile_parameters.emplace("qwen3_moe.model.rms_epsilon", "0.000001");

const size_t route_id_length = static_cast<size_t>(match.token_count * match.route_stride) * sizeof(int32_t);
const size_t route_id_length = routed_route_id_bytes(match.token_count, match.route_stride);
if (match.input_is_q8) {
dispatch.bindings.push_back({ match.input_alternate->alternate_value, 0, match.input_alternate->byte_count });
} else {
Expand All @@ -1140,6 +1149,11 @@ static bool match_decode_routed_ffn_down_next_q8_dispatch(const DispatchMatchCon
{ match.reduce.next_rmsnorm.norm_weight->id, 0, match.reduce.next_rmsnorm.norm_weight->byte_count });
dispatch.bindings.push_back({ completion_counter_value, 0, sizeof(int32_t) });
dispatch.bindings.push_back({ q8_output, 0, q8_output_bytes });
// A HIP kernel add-on may take the kernel (hip/hip-dispatches.h).
hip_routed_down_dispatch({ match.weight->type, match.input_is_q8, match.token_count, kRoutedFfnExpertHiddenSize,
kRoutedFfnRouteCount, match.route_stride, kRoutedFfnExpertCount, kRoutedFfnInputSize,
kRoutedFfnProfile.rms_norm_epsilon },
dispatch);

dispatch_match.value_aliases.push_back({ match.reduce.residual_input->id, match.output->id });
dispatch_match.completion_counter_requests.push_back({
Expand Down
9 changes: 9 additions & 0 deletions ggml/src/ggml-hrx/hip/hip-dispatches.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ namespace {

HipAttentionSinkHook g_attention_sink_hook = nullptr;
HipSetRowsHook g_set_rows_hook = nullptr;
HipRoutedDownHook g_routed_down_hook = nullptr;

} // namespace

Expand Down Expand Up @@ -52,4 +53,12 @@ bool hip_set_rows_dispatch(const HipSetRowsArgs & args, DispatchMatch & match) {
return g_set_rows_hook != nullptr && g_set_rows_hook(args, match);
}

void set_hip_routed_down_hook(HipRoutedDownHook hook) {
g_routed_down_hook = hook;
}

bool hip_routed_down_dispatch(const HipRoutedDownArgs & args, Dispatch & dispatch) {
return g_routed_down_hook != nullptr && g_routed_down_hook(args, dispatch);
}

} // namespace ggml::hrx
21 changes: 21 additions & 0 deletions ggml/src/ggml-hrx/hip/hip-dispatches.h
Original file line number Diff line number Diff line change
Expand Up @@ -80,4 +80,25 @@ void set_hip_set_rows_hook(HipSetRowsHook hook);
bool hip_set_rows_dispatch(const HipSetRowsArgs & args, DispatchMatch & match);
bool hip_set_rows_hook_installed();

// Routed MoE one-token down projection hook (dispatch_registration/qwen/dispatch-routed-ffn.cpp,
// llm.routed_ffn.decode_down_next_q8). The matcher has built the whole dispatch for its Loom kernel: bindings
// input, route_ids, route_weights, weight, output (residual in, updated in place), norm_weight, completion
// counter (0 on entry and exit), next_q8 (block_q8_1_x4 of the next RMS norm). A hook that returns true has
// replaced dispatch.kernel with a kernel of that same binding contract; false keeps the Loom kernel.
#define GGML_HRX_HIP_ROUTED_DOWN_HOOK 1 // an add-on may test for the hook below
struct HipRoutedDownArgs {
int weight_type; // ggml_type of the expert weights
bool input_is_q8; // input binding is the q8_1_x4 alternate rather than f32
int64_t token_count;
int64_t input_size;
int64_t route_count;
int64_t route_id_stride;
int64_t expert_count;
int64_t output_size;
float eps;
};
using HipRoutedDownHook = bool (*)(const HipRoutedDownArgs & args, Dispatch & dispatch);
void set_hip_routed_down_hook(HipRoutedDownHook hook);
bool hip_routed_down_dispatch(const HipRoutedDownArgs & args, Dispatch & dispatch);

} // namespace ggml::hrx
12 changes: 12 additions & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,18 @@ if (TARGET ggml-hrx)
add_test(NAME test-hrx-fa-masked-v COMMAND test-hrx-fa-masked-v)
set_tests_properties(test-hrx-fa-masked-v PROPERTIES SKIP_RETURN_CODE 77)

# the routed-down hook on llm.routed_ffn.decode_down_next_q8: plan and device kernel names, an fp64 oracle and the
# CPU backend, with the add-on's kernel off (loom) and on (hip, only when a HIP add-on is built in)
add_executable(test-hrx-routed-down-hip test-hrx-routed-down-hip.cpp)
target_link_libraries(test-hrx-routed-down-hip PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc)
target_include_directories(test-hrx-routed-down-hip PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
add_test(NAME test-hrx-routed-down-loom COMMAND test-hrx-routed-down-hip --arm loom)
set_tests_properties(test-hrx-routed-down-loom PROPERTIES SKIP_RETURN_CODE 77)
if (GGML_HRX_HIP_ADDON_DIR)
add_test(NAME test-hrx-routed-down-hip COMMAND test-hrx-routed-down-hip --arm hip)
set_tests_properties(test-hrx-routed-down-hip PROPERTIES SKIP_RETURN_CODE 77)
endif()

add_executable(test-hrx-attention-sink test-hrx-attention-sink.cpp)
target_link_libraries(test-hrx-attention-sink PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc)
target_include_directories(test-hrx-attention-sink PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx)
Expand Down
Loading