diff --git a/HRX-COMPILE-ERRORS.txt b/HRX-COMPILE-ERRORS.txt deleted file mode 100644 index e4b58c7a58a2..000000000000 --- a/HRX-COMPILE-ERRORS.txt +++ /dev/null @@ -1,11 +0,0 @@ -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp -> a function-definition is not allowed here before ‘{’ token -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp -> expected ‘}’ at end of input -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp -> version control conflict marker in file -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat-id.cpp -> ‘kMulMatIdSkinnySwiGLUF32F32Kernel’ was not declared in this scope; did you mean ‘kMulMatIdSwiGLUF32F32WmmaKernel’? -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat-id.cpp -> ‘kMulMatIdTiledSwiGLUF32F32Kernel’ was not declared in this scope; did you mean ‘kMulMatIdSwiGLUF32F32WmmaKernel’? -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp -> cannot convert ‘’ to ‘ggml::hrx::DispatchRegistration’ -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp -> ‘match_get_rows_rmsnorm_binary_dispatch’ was not declared in this scope -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp -> ‘match_get_rows_scale_f32_dispatch’ was not declared in this scope -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp -> ‘struct ggml::hrx::KernelSpecialization’ has no member named ‘workload_specialization’ -ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp -> ‘WorkloadSpecialization’ has not been declared; did you mean ‘KernelSpecialization’? -ggml/src/ggml-hrx/./kernel-corpus/kernel-corpus-catalog-verify.h -> static assertion failed: unknown HRX kernel catalog entry diff --git a/merge-lost/llama-memory-hybrid-idx.ours-block.txt b/merge-lost/llama-memory-hybrid-idx.ours-block.txt deleted file mode 100644 index 744ddf411fbe..000000000000 --- a/merge-lost/llama-memory-hybrid-idx.ours-block.txt +++ /dev/null @@ -1,216 +0,0 @@ - - // a block is keyed on (sequence set, index bucket): a unified cache counts every sequence - // from zero, so the bucket alone would pool two sequences into one block - GGML_ASSERT(r <= 64); - const uint64_t slots_full = r == 64 ? ~uint64_t(0) : ((uint64_t(1) << r) - 1); - - // TODO: this runs per ubatch and is O(n_kv) per stream, about 865 us at 33k context. the cost - // is the per-cell scan rather than these allocations, so hoisting them buys nothing - std::vector blk_of(n_kv); - std::vector cell_grp(n_kv); - std::vector grp_head(n_blocks); - std::vector grp_next; - std::vector grp_first; - std::vector grp_slot0; - std::vector grp_slots; - std::vector grp_bid; - std::vector bid_idx; - std::vector bid_cell; - std::vector bid_slot0; - - std::vector order; - std::vector rank; - - std::fill(dst_blk_pos, dst_blk_pos + 4*n_blocks*n_ns, 0); - - for (int64_t s = 0; s < n_ns; ++s) { - // ubatch index s*n_tps belongs to this stream; ask which cells array it uses - const llama_seq_id seq_of_stream = ubatch->seq_id[s*n_tps][0]; - const auto & cells = get_mem_idx()->get_cells(seq_of_stream); - - int32_t * cur_cell_blk = dst_cell_blk + s*n_kv; - int32_t * cur_blk_cells = dst_blk_cells + s*(r*n_blocks); - - std::fill(cur_blk_cells, cur_blk_cells + r*n_blocks, 0); - - bid_idx .clear(); - bid_cell .clear(); - bid_slot0.clear(); - - int n_seq_present = 0; - - for (int sq = 0; sq < LLAMA_MAX_SEQ && n_seq_present < 2; ++sq) { - if (cells.seq_pos_min(sq) >= 0) { - n_seq_present++; - } - } - - const bool one_seq = n_seq_present <= 1; - - // a cell no block covers needs its own -inf, which a per-block bias cannot carry - // every cache path keeps the position below the cell window, so this stays false - bool oor = false; - - bool dup = false; - - bool ranked = false; - - auto group_cells = [&]() { - // -1 means no usable block: an incomplete or short group cannot be pooled - std::fill(blk_of.begin(), blk_of.end(), -1); - std::fill(cell_grp.begin(), cell_grp.end(), -1); - std::fill(grp_head.begin(), grp_head.end(), -1); - - grp_next .clear(); - grp_first.clear(); - grp_slot0.clear(); - grp_slots.clear(); - grp_bid .clear(); - - oor = false; - dup = false; - - for (int64_t j = 0; j < n_kv; ++j) { - if (cells.is_empty(j)) { - continue; - } - - const int64_t idx = ranked ? rank[j] : cells.pos_get(j); - const int64_t pb = idx/r; - - if (pb >= n_blocks) { - oor = true; - continue; - } - - int32_t g = -1; - - for (int32_t c = grp_head[pb]; c >= 0; c = grp_next[c]) { - if (one_seq || cells.seq_get_all((uint32_t) grp_first[c]) == cells.seq_get_all((uint32_t) j)) { - g = c; - break; - } - } - - if (g < 0) { - g = (int32_t) grp_first.size(); - - grp_next .push_back(grp_head[pb]); - grp_first.push_back((int32_t) j); - grp_slot0.push_back(-1); - grp_slots.push_back(0); - grp_bid .push_back(-1); - - grp_head[pb] = g; - } - - const uint64_t bit = uint64_t(1) << (idx%r); - - dup |= (grp_slots[g] & bit) != 0; - - cell_grp[j] = g; - grp_slots[g] |= bit; - - if (idx%r == 0) { - grp_slot0[g] = (int32_t) j; - } - } - }; - - group_cells(); - - // mrope repeats one position across an image, so rank cells instead of using the position - if (dup && ubatch->is_pos_2d() && one_seq) { - order.clear(); - order.reserve(n_kv); - - for (int64_t j = 0; j < n_kv; ++j) { - if (!cells.is_empty(j)) { - order.push_back((int32_t) j); - } - } - - // same total order the mrope causal mask uses: pos, then ext.y, then ext.x - std::sort(order.begin(), order.end(), [&cells](int32_t a, int32_t b) { - const llama_pos pa = cells.pos_get(a); - const llama_pos pb = cells.pos_get(b); - - if (pa != pb) { - return pa < pb; - } - - const auto & ea = cells.ext_get(a); - - return cells.ext_get(b).is_2d_gt(ea.x, ea.y); - }); - - rank.assign(n_kv, -1); - - for (int64_t k = 0; k < (int64_t) order.size(); ++k) { - rank[order[k]] = (int32_t) k; - } - - ranked = true; - - group_cells(); - } - - GGML_ASSERT((!blk_bias || !oor) && "qsa: cell position runs past the cell window"); - - int32_t n_bid = 0; - - for (int64_t pb = 0; pb < n_blocks; ++pb) { - for (int32_t g = grp_head[pb]; g >= 0; g = grp_next[g]) { - if (grp_slots[g] != slots_full) { - continue; - } - - grp_bid[g] = n_bid++; - - bid_idx .push_back((int32_t) (pb*r)); - bid_cell .push_back(grp_first[g]); - bid_slot0.push_back(grp_slot0[g]); - } - } - - GGML_ASSERT(n_bid <= n_blocks); - - for (int32_t b = 0; b < n_bid; ++b) { - int32_t sec_pos[4] = { bid_idx[b], bid_idx[b], bid_idx[b], bid_idx[b] }; - - if (ranked) { - const int32_t c = bid_slot0[b]; - const llama_pos p = cells.pos_get(c); - const auto & e = cells.ext_get(c); - - sec_pos[0] = p; - sec_pos[1] = e.y; - sec_pos[2] = e.x; - sec_pos[3] = p; - } - - for (int64_t sec = 0; sec < 4; ++sec) { - dst_blk_pos[sec*(n_blocks*n_ns) + s*n_blocks + b] = sec_pos[sec]; - } - } - - // unpooled cells all point at one spare block. a spare block exists only when some - // cell is unpooled: n_bid == n_blocks means every cell sits in a full block. - const bool have_dead = n_bid < n_blocks; - const int32_t dead_bid = have_dead ? n_bid : n_blocks - 1; - - for (int64_t j = 0; j < n_kv; ++j) { - const int32_t g = cell_grp[j]; - - blk_of[j] = g < 0 ? -1 : grp_bid[g]; - - if (blk_of[j] >= 0) { - const int64_t idx = ranked ? rank[j] : cells.pos_get(j); - - cur_blk_cells[blk_of[j]*r + (idx%r)] = (int32_t) j; - } - - cur_cell_blk[j] = blk_of[j] < 0 ? dead_bid : blk_of[j]; - } - - for (int64_t ii = 0; ii < n_tps; ++ii) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7e492864d073..43323bc5e614 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -488,8 +488,14 @@ if (TARGET ggml-hrx) add_executable(hrx-backend-test hrx-backend-test.cpp) target_link_libraries(hrx-backend-test PRIVATE ggml-hrx ggml hrx::hrx loomc::loomc) target_include_directories(hrx-backend-test PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) - add_test(NAME hrx-backend-test COMMAND hrx-backend-test - --hrx-backend-test-jobs 4 --hrx-backend-test-output all --hrx-backend-test-filter ".*") + # our line's hrx-backend-test takes case names as its only arguments (no test_runner::Suite flags) + add_test(NAME hrx-backend-test COMMAND hrx-backend-test) + + # our line's Loom JIT suite (the base's ctest row); AMD's hrx-loom-jit-test is a different file + add_executable(test-hrx-loom-jit test-hrx-loom-jit.cpp) + target_link_libraries(test-hrx-loom-jit PRIVATE ggml-hrx ggml-hrx-kernel-corpus hrx::hrx loomc::loomc) + target_include_directories(test-hrx-loom-jit PRIVATE ../ggml/include ../ggml/src ../ggml/src/ggml-hrx) + add_test(NAME test-hrx-loom-jit COMMAND test-hrx-loom-jit) add_executable(hrx-loom-jit-test hrx-loom-jit-test.cpp) target_link_libraries(hrx-loom-jit-test PRIVATE ggml-hrx ggml-hrx-kernel-corpus hrx::hrx loomc::loomc) diff --git a/tests/hrx-loom-jit-test.cpp b/tests/hrx-loom-jit-test.cpp index 485e5bca9e02..fcbb81ae56d1 100644 --- a/tests/hrx-loom-jit-test.cpp +++ b/tests/hrx-loom-jit-test.cpp @@ -1167,19 +1167,6 @@ static void run_moe_routing_compile_checks() { const auto & partition_table = find_kernel("ggml_moe_build_expert_partition_table"); require_compiled_kernel(compile_kernel(*jit, "moe-expert-partition-table", partition_table, partition_table_workload, moe_routing_config())); - - const auto & weighted_reduce_publish_q8 = - find_kernel("qwen3_moe_routed_down_weighted_reduce_next_rmsnorm_f32_publish_q8"); - REQUIRE(weighted_reduce_publish_q8.bindings.size() == 6); - require_compiled_kernel(compile_kernel(*jit, "moe-weighted-reduce-next-rmsnorm-publish-q8", - weighted_reduce_publish_q8, token_count_workload(2), { - { "qwen3_moe.model.hidden_size", "2048" }, - { "qwen3_moe.model.rms_epsilon", "0.000001" }, - { "qwen3_moe.routed_down.output_size", "2048" }, - { "qwen3_moe.routed_down.route_count", "8" }, - { "qwen3_moe.workload.token_capacity", "2" }, - { "ggml.quantize_q8_1_x4.group_capacity", "32" }, - })); } static void run_prefill_fusion_compile_checks() { diff --git a/tests/test-hrx-loom-jit.cpp b/tests/test-hrx-loom-jit.cpp new file mode 100644 index 000000000000..09f3b8489d75 --- /dev/null +++ b/tests/test-hrx-loom-jit.cpp @@ -0,0 +1,1022 @@ +#include "dispatch/dispatch.h" +#include "hrx-interop-utils.h" +#include "kernel-corpus/kernel-corpus.h" +#include "runtime/kernel-executable-cache.h" +#include "runtime/loom-kernel-jit.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define REQUIRE(condition) \ + do { \ + if (!(condition)) { \ + std::fprintf(stderr, "%s:%d: requirement failed: %s\n", __FILE__, __LINE__, #condition); \ + std::abort(); \ + } \ + } while (false) + +namespace { + +class HrxTestDevice { + public: + ~HrxTestDevice() { + if (device != nullptr) { + hrx_device_release(device); + } + } + + bool open() { + hrx_status_t init_status = hrx_gpu_initialize(0); + if (!hrx_status_is_ok(init_status)) { + if (hrx_status_code(init_status) == HRX_STATUS_ALREADY_EXISTS) { + hrx_status_ignore(init_status); + } else { + hrx_status_ignore(init_status); + return false; + } + } + + int count = 0; + if (ggml::hrx::ErrorResult error = ggml::hrx::take_status(hrx_gpu_device_count(&count))) { + std::fprintf(stderr, "skip executable cache materialization: device count failed: %s\n", error->c_str()); + return false; + } + for (int i = 0; i < count; ++i) { + hrx_device_t candidate = nullptr; + if (ggml::hrx::ErrorResult error = ggml::hrx::take_status(hrx_gpu_device_get(i, &candidate))) { + std::fprintf(stderr, "skip HRX device %d: %s\n", i, error->c_str()); + continue; + } + if (candidate == nullptr) { + continue; + } + hrx_device_retain(candidate); + std::optional candidate_architecture = + read_string_property(candidate, HRX_DEVICE_PROPERTY_ARCHITECTURE); + if (!candidate_architecture) { + hrx_device_release(candidate); + continue; + } + device = candidate; + architecture = *candidate_architecture; + return true; + } + return false; + } + + hrx_device_t device = nullptr; + std::string architecture; + + private: + static std::optional read_string_property(hrx_device_t device, hrx_device_property_t property) { + std::vector buffer(64); + while (buffer.size() <= 4096) { + hrx_status_t status = hrx_device_get_property(device, property, buffer.data(), buffer.size()); + if (hrx_status_is_ok(status)) { + return std::string(buffer.data()); + } + if (hrx_status_code(status) != HRX_STATUS_OUT_OF_RANGE) { + if (ggml::hrx::ErrorResult error = ggml::hrx::take_status(status)) { + std::fprintf(stderr, "HRX property query failed: %s\n", error->c_str()); + } + return std::nullopt; + } + hrx_status_ignore(status); + buffer.resize(buffer.size() * 2); + } + return std::nullopt; + } +}; + +static ggml_hrx_loom_jit_source_format to_jit_source_format(ggml::hrx::KernelSourceFormat format) { + switch (format) { + case ggml::hrx::KERNEL_SOURCE_FORMAT_TEXT: + return GGML_HRX_LOOM_JIT_SOURCE_FORMAT_TEXT; + case ggml::hrx::KERNEL_SOURCE_FORMAT_BINARY: + return GGML_HRX_LOOM_JIT_SOURCE_FORMAT_BYTECODE; + } + return GGML_HRX_LOOM_JIT_SOURCE_FORMAT_TEXT; +} + +static const ggml::hrx::KernelDefinition & find_kernel(const char * name) { + const ggml::hrx::KernelCorpus & corpus = ggml::hrx::get_qwen_kernel_corpus(); + for (const ggml::hrx::KernelDefinition & kernel : corpus.kernels) { + if (std::strcmp(kernel.name, name) == 0) { + return kernel; + } + } + std::fprintf(stderr, "missing test kernel: %s\n", name); + std::abort(); +} + +static const ggml::hrx::KernelDefinition * find_targeted_kernel(const char * name, const char * target) { + const ggml::hrx::KernelCorpus & corpus = ggml::hrx::get_qwen_kernel_corpus(); + for (const ggml::hrx::KernelDefinition & kernel : corpus.kernels) { + if (std::strcmp(kernel.name, name) == 0 && std::strcmp(kernel.target_selector, target) == 0) { + return &kernel; + } + } + return nullptr; +} + +static std::map binary_f32_exact_config(const char * op, int64_t element_count) { + const std::string count = std::to_string(element_count); + return { + { "ggml.binary_f32.op", op }, + { "ggml.binary_f32.ne0", count }, + { "ggml.binary_f32.ne1", "1" }, + { "ggml.binary_f32.ne2", "1" }, + { "ggml.binary_f32.src0_stride1", count }, + { "ggml.binary_f32.src0_stride2", count }, + { "ggml.binary_f32.src0_stride3", count }, + { "ggml.binary_f32.src1_stride1", count }, + { "ggml.binary_f32.src1_stride2", count }, + { "ggml.binary_f32.src1_stride3", count }, + { "ggml.binary_f32.src0_span", count }, + { "ggml.binary_f32.src1_span", count }, + }; +} + +static ggml::hrx::Dispatch make_binary_add_dispatch(const ggml::hrx::KernelDefinition & definition, + int64_t element_count) { + ggml::hrx::Dispatch dispatch; + dispatch.kernel.kernel_id = definition.id; + dispatch.kernel.integer_parameters.emplace("element_count", element_count); + dispatch.kernel.compile_parameters = binary_f32_exact_config("0", element_count); + dispatch.bindings.reserve(definition.bindings.size()); + for (size_t i = 0; i < definition.bindings.size(); ++i) { + dispatch.bindings.push_back({ + ggml::hrx::ValueId(static_cast(i + 1)), + 0, + static_cast(element_count) * sizeof(float), + }); + } + return dispatch; +} + +static std::map binary_f32_exact_workload(int64_t element_count) { + return { + { "element_count", element_count }, + }; +} + +static std::map binary_bc_f32_workload() { + return { + { "element_count", 48 }, + { "ne0", 16 }, + { "ne1", 3 }, + { "ne2", 1 }, + { "ne3", 1 }, + { "src0_element_count", 48 }, + { "src1_element_count", 16 }, + }; +} + +static std::map binary_bc_f32_config(const char * op) { + return { + { "ggml.binary_bc_f32.op", op }, + { "ggml.binary_bc_f32.src0_broadcast_dim0", "0" }, + { "ggml.binary_bc_f32.src0_broadcast_dim1", "0" }, + { "ggml.binary_bc_f32.src0_broadcast_dim2", "0" }, + { "ggml.binary_bc_f32.src0_broadcast_dim3", "0" }, + { "ggml.binary_bc_f32.src1_broadcast_dim0", "0" }, + { "ggml.binary_bc_f32.src1_broadcast_dim1", "1" }, + { "ggml.binary_bc_f32.src1_broadcast_dim2", "0" }, + { "ggml.binary_bc_f32.src1_broadcast_dim3", "0" }, + }; +} + +static std::map token_count_workload(int64_t token_count) { + return { + { "token_count", token_count }, + }; +} + +static std::map flash_attention_workload(int64_t query_token_count, + int64_t key_value_token_count) { + return { + { "query_token_count", query_token_count }, + { "key_value_token_count", key_value_token_count }, + }; +} + +static std::map flash_attention_decode_workload(int64_t key_value_token_count) { + return { + { "key_value_token_count", key_value_token_count }, + }; +} + +static std::map flash_attention_config(const char * query_head_count, + const char * key_value_head_count, + const char * head_size, + const char * attention_scale) { + return { + { "ggml.flash_attention.query_head_count", query_head_count }, + { "ggml.flash_attention.key_value_head_count", key_value_head_count }, + { "ggml.flash_attention.qk_head_size", head_size }, + { "ggml.flash_attention.value_head_size", head_size }, + { "ggml.flash_attention.value_stride", + std::to_string(std::stoll(key_value_head_count) * std::stoll(head_size)) }, + { "ggml.flash_attention.attention_scale", attention_scale }, + { "ggml.flash_attention.apply_gate", "0" }, + { "ggml.flash_attention.gate_stride_head", "1" }, + { "ggml.flash_attention.gate_stride_token", "1" }, + }; +} + +static std::map flash_attention_decode_base_config(const char * query_head_count, + const char * key_value_head_count, + const char * head_size, + const char * attention_scale) { + return { + { "ggml.flash_attention.query_head_count", query_head_count }, + { "ggml.flash_attention.key_value_head_count", key_value_head_count }, + { "ggml.flash_attention.qk_head_size", head_size }, + { "ggml.flash_attention.value_head_size", head_size }, + { "ggml.flash_attention.attention_scale", attention_scale }, + { "ggml.flash_attention.apply_gate", "0" }, + { "ggml.flash_attention.gate_stride_head", head_size }, + { "ggml.flash_attention.gate_stride_token", head_size }, + }; +} + +static std::map flash_attention_decode_config(const char * query_head_count, + const char * key_value_head_count, + const char * head_size, + const char * attention_scale, + const char * key_value_token_capacity) { + std::map config = + flash_attention_decode_base_config(query_head_count, key_value_head_count, head_size, attention_scale); + config["ggml.flash_attention.decode.key_value_token_capacity"] = key_value_token_capacity; + return config; +} + +static std::map mul_mat_id_f16_f16_config(const char * weight_format) { + return { + { "ggml.mul_mat_id_f16_f16.input_size", "768" }, + { "ggml.mul_mat_id_f16_f16.route_count", "8" }, + { "ggml.mul_mat_id_f16_f16.expert_count", "128" }, + { "ggml.mul_mat_id_f16_f16.output_size", "2048" }, + { "ggml.mul_mat_id_f16_f16.weight_format", weight_format }, + { "ggml.workload.token_capacity", "4" }, + }; +} + +static std::map mul_mat_id_swiglu_f16_f16_config(const char * gate_format, + const char * up_format) { + return { + { "ggml.mul_mat_id_swiglu_f16_f16.input_size", "2048" }, + { "ggml.mul_mat_id_swiglu_f16_f16.output_size", "768" }, + { "ggml.mul_mat_id_swiglu_f16_f16.expert_count", "128" }, + { "ggml.mul_mat_id_swiglu_f16_f16.route_count", "8" }, + { "ggml.mul_mat_id_swiglu_f16_f16.gate_weight_format", gate_format }, + { "ggml.mul_mat_id_swiglu_f16_f16.up_weight_format", up_format }, + { "ggml.mul_mat_id_swiglu_f16_f16.descriptor_expert_mask", "127" }, + { "ggml.mul_mat_id_swiglu_f16_f16.descriptor_partition_shift", "7" }, + { "ggml.mul_mat_id_swiglu_f16_f16.descriptor_row_count_shift", "13" }, + }; +} + +static ggml::hrx::LoomKernelCompileRequest make_compile_request( + const ggml::hrx::KernelDefinition & definition, + const std::map & workload, + const std::map & compile_config = {}) { + ggml::hrx::LoomKernelCompileRequest request; + REQUIRE(!definition.compile_recipe.primary_sources.empty()); + + const ggml::hrx::KernelSourceRef & primary_source = definition.compile_recipe.primary_sources.front(); + REQUIRE(primary_source.contents != nullptr); + request.source_data = primary_source.contents->source.data; + request.source_size = primary_source.contents->source.length; + request.source_format = to_jit_source_format(primary_source.contents->source.format); + request.source_identifier = primary_source.path != nullptr ? primary_source.path : ""; + request.symbol = definition.symbol != nullptr ? definition.symbol : ""; + request.launch_config_symbol = definition.name != nullptr ? definition.name : ""; + + request.dependencies.reserve(definition.compile_recipe.library_sources.size()); + for (const ggml::hrx::KernelSourceRef & dependency_ref : definition.compile_recipe.library_sources) { + REQUIRE(dependency_ref.contents != nullptr); + request.dependencies.push_back({ + dependency_ref.contents->source.data, + dependency_ref.contents->source.length, + to_jit_source_format(dependency_ref.contents->source.format), + dependency_ref.path, + }); + } + + std::map merged_config; + for (const ggml::hrx::KernelCompileConfig & config : definition.compile_config) { + merged_config[config.key != nullptr ? config.key : ""] = config.value != nullptr ? config.value : ""; + } + for (const auto & item : compile_config) { + merged_config[item.first] = item.second; + } + request.config_storage.reserve(merged_config.size()); + for (const auto & item : merged_config) { + request.config_storage.push_back(item); + } + + request.workload.reserve(definition.workload_parameters.size()); + for (const ggml::hrx::KernelScalarDefinition & parameter : definition.workload_parameters) { + REQUIRE(parameter.type != nullptr); + REQUIRE(std::strcmp(parameter.type, "index") == 0); + const auto found = workload.find(parameter.name != nullptr ? parameter.name : ""); + REQUIRE(found != workload.end()); + request.workload.push_back(found->second); + } + return request; +} + +static ggml::hrx::LoomCompiledKernelRef compile_kernel(ggml::hrx::LoomJit & jit, + const std::string & key, + const ggml::hrx::KernelDefinition & definition, + const std::map & workload, + const std::map & compile_config = {}) { + return jit.compile(key, make_compile_request(definition, workload, compile_config)); +} + +static bool resolve_with_timeout(const ggml::hrx::LoomCompiledKernelRef & ref, std::chrono::seconds timeout) { + std::atomic done = false; + std::atomic success = false; + std::thread resolver([&] { + success.store(ref->resolve(), std::memory_order_release); + done.store(true, std::memory_order_release); + }); + + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (!done.load(std::memory_order_acquire)) { + if (std::chrono::steady_clock::now() >= deadline) { + std::fprintf(stderr, "timed out waiting for async Loom compile: %s\n", ref->key().c_str()); + std::abort(); + } + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + resolver.join(); + return success.load(std::memory_order_acquire); +} + +static void require_compiled_kernel(const ggml::hrx::LoomCompiledKernelRef & ref) { + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + ggml_hrx_loom_jit_compile_result compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.hsaco_size > 0); + REQUIRE(compiled.launch_config.workgroup_count[0] > 0); + REQUIRE(compiled.launch_config.workgroup_size[0] > 0); + compiled.reset(); +} + +static int64_t elapsed_us(std::chrono::steady_clock::time_point begin, std::chrono::steady_clock::time_point end) { + return std::chrono::duration_cast(end - begin).count(); +} + +static void run_cache_materialize_case(HrxTestDevice & device, + ggml::hrx::LoomJitMode mode, + const ggml::hrx::KernelDefinition & binary) { + ggml::hrx::KernelExecutablePrepareContext context = {}; + context.device = device.device; + context.target = device.architecture.c_str(); + + ggml::hrx::Dispatch dispatch = make_binary_add_dispatch(binary, 64); + std::vector constants; + ggml::hrx::KernelExecutableCache cache(mode); + + ggml::hrx::KernelExecutableRef ref = cache.get_or_compile(context, binary, dispatch, constants); + REQUIRE(ref.valid()); + + std::shared_ptr executable = cache.materialize(context, ref, constants); + REQUIRE(executable != nullptr); + REQUIRE(executable->executable != nullptr); + REQUIRE(executable->export_info.binding_count == dispatch.bindings.size()); + REQUIRE(executable->export_info.constant_byte_length == constants.size()); + REQUIRE(executable->launch.workgroup_count[0] > 0); + REQUIRE(executable->launch.workgroup_size[0] > 0); + + std::shared_ptr loaded_hit = cache.materialize(context, ref, constants); + REQUIRE(loaded_hit == executable); + + std::vector constants_for_prepare; + std::shared_ptr prepared = + cache.prepare(context, binary, dispatch, constants_for_prepare); + REQUIRE(prepared == executable); + + const char * mode_name = mode == ggml::hrx::LoomJitMode::Async ? "async" : "sync"; + std::printf("%s KernelExecutableCache materialized ggml_binary_f32 for %s\n", mode_name, + device.architecture.c_str()); +} + +static void run_targeted_export_materialize_case(HrxTestDevice & device) { + const ggml::hrx::KernelDefinition * definition = + find_targeted_kernel("ggml_linear_q6k_q8_1_x4", device.architecture.c_str()); + if (definition == nullptr) { + return; + } + + ggml::hrx::KernelExecutablePrepareContext context = {}; + context.device = device.device; + context.target = device.architecture.c_str(); + + ggml::hrx::Dispatch dispatch; + dispatch.kernel.kernel_id = definition->id; + dispatch.kernel.integer_parameters.emplace("token_count", 1); + dispatch.kernel.integer_parameters.emplace("input_size", 2048); + dispatch.kernel.integer_parameters.emplace("output_size", 151936); + dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.token_capacity", "1"); + dispatch.kernel.compile_parameters.emplace("ggml.linear_q6k_q8_1_x4.output_capacity", "151936"); + dispatch.bindings.resize(3); + + ggml::hrx::KernelExecutableCache cache(ggml::hrx::LoomJitMode::Sync); + std::vector constants; + std::shared_ptr executable = cache.prepare(context, *definition, dispatch, constants); + REQUIRE(executable != nullptr); + REQUIRE(executable->executable != nullptr); + REQUIRE(executable->launch.workgroup_count[0] == 151936); + std::printf("KernelExecutableCache materialized targeted %s as export %s for %s\n", definition->symbol, + definition->name, device.architecture.c_str()); +} + +} // namespace + +int main() { + static constexpr const char * kTarget = "gfx1100"; + + const ggml::hrx::KernelDefinition & binary = find_kernel("ggml_binary_f32"); + const ggml::hrx::KernelDefinition & binary_bc = find_kernel("ggml_binary_bc_f32"); + const ggml::hrx::KernelDefinition & gather_add = find_kernel("ggml_gather_add_f32"); + const ggml::hrx::KernelDefinition & rmsnorm = find_kernel("qwen3_moe_rmsnorm_f32"); + const ggml::hrx::KernelDefinition & router_top8 = find_kernel("qwen3_moe_router_top8_f32"); + const ggml::hrx::KernelDefinition & expert_table = find_kernel("qwen3_moe_build_expert_table"); + const ggml::hrx::KernelDefinition & partition_table = find_kernel("qwen3_moe_build_expert_partition_table"); + const ggml::hrx::KernelDefinition & mul_mat_id_f16 = find_kernel("ggml_mul_mat_id_f16_f16_wmma"); + const ggml::hrx::KernelDefinition & swiglu_f16 = find_kernel("ggml_mul_mat_id_swiglu_f16_f16_wmma"); + const ggml::hrx::KernelDefinition & flash_prefill = find_kernel("ggml_flash_attention_f32_f16_wmma"); + const ggml::hrx::KernelDefinition & decode_mul_mat = find_kernel("ggml_mul_mat_f32_f32_decode_wave64"); + const ggml::hrx::KernelDefinition & flash_decode = + find_kernel("ggml_flash_attention_decode_split_f32_f16_wmma_next_q8"); + + std::string sync_error; + std::unique_ptr sync_jit = + ggml::hrx::create_loom_jit(kTarget, ggml::hrx::LoomJitMode::Sync, sync_error); + REQUIRE(sync_jit != nullptr); + REQUIRE(!sync_jit->async_enabled()); + + const auto sync_begin = std::chrono::steady_clock::now(); + ggml::hrx::LoomCompiledKernelRef sync_ref = compile_kernel( + *sync_jit, "sync-binary-add-64", binary, binary_f32_exact_workload(64), binary_f32_exact_config("0", 64)); + require_compiled_kernel(sync_ref); + const auto sync_end = std::chrono::steady_clock::now(); + const int64_t sync_compile_us = elapsed_us(sync_begin, sync_end); + std::printf("sync Loom compile completed in %ld us\n", static_cast(sync_compile_us)); + + std::string async_error; + std::unique_ptr async_jit = + ggml::hrx::create_loom_jit(kTarget, ggml::hrx::LoomJitMode::Async, async_error); + REQUIRE(async_jit != nullptr); + REQUIRE(async_jit->async_enabled()); + + std::vector refs; + refs.reserve(31); + const auto enqueue_begin = std::chrono::steady_clock::now(); + refs.push_back(compile_kernel(*async_jit, "async-binary-add-64", binary, binary_f32_exact_workload(64), + binary_f32_exact_config("0", 64))); + refs.push_back(compile_kernel(*async_jit, "async-binary-add-128", binary, binary_f32_exact_workload(128), + binary_f32_exact_config("0", 128))); + refs.push_back(compile_kernel(*async_jit, "async-binary-add-256", binary, binary_f32_exact_workload(256), + binary_f32_exact_config("0", 256))); + refs.push_back(compile_kernel(*async_jit, "async-binary-add-512", binary, binary_f32_exact_workload(512), + binary_f32_exact_config("0", 512))); + refs.push_back(compile_kernel(*async_jit, "async-binary-geglu-256", binary, binary_f32_exact_workload(256), + binary_f32_exact_config("5", 256))); + refs.push_back(compile_kernel(*async_jit, "async-binary-reglu-256", binary, binary_f32_exact_workload(256), + binary_f32_exact_config("6", 256))); + refs.push_back(compile_kernel(*async_jit, "async-binary-geglu-erf-256", binary, binary_f32_exact_workload(256), + binary_f32_exact_config("7", 256))); + refs.push_back(compile_kernel(*async_jit, "async-binary-geglu-quick-256", binary, binary_f32_exact_workload(256), + binary_f32_exact_config("8", 256))); + refs.push_back(compile_kernel(*async_jit, "async-binary-bc-mul-48", binary_bc, binary_bc_f32_workload(), + binary_bc_f32_config("2"))); + refs.push_back(compile_kernel(*async_jit, "async-rmsnorm-1", rmsnorm, + { + { "token_count", 1 } + }, + { + { "qwen3_moe.model.hidden_size", "2048" }, + { "qwen3_moe.model.rms_epsilon", "0.000001" }, + { "qwen3_moe.workload.token_capacity", "1" }, + { "ggml.quantize_q8_1_x4.group_capacity", "256" }, + })); + refs.push_back(compile_kernel(*async_jit, "async-router-top8-1", router_top8, + { + { "token_count", 1 }, + { "route_id_stride", 8 }, + }, + { + { "qwen3_moe.router.expert_count", "128" }, + { "qwen3_moe.router.route_count", "8" }, + { "qwen3_moe.workload.token_capacity", "1" }, + })); + refs.push_back(compile_kernel(*async_jit, "async-expert-table-1", expert_table, + { + { "token_count", 1 }, + { "route_count", 8 }, + { "route_stride", 8 }, + { "expert_count", 128 }, + }, + { + { "qwen3_moe.routed_gate_up.expert_count", "128" }, + { "qwen3_moe.routed_gate_up.route_count", "8" }, + { "qwen3_moe.workload.token_capacity", "1" }, + })); + refs.push_back(compile_kernel(*async_jit, "async-partition-table-1", partition_table, + { + { "token_count", 1 }, + { "route_count", 8 }, + { "expert_count", 128 }, + }, + { + { "qwen3_moe.routed_gate_up.expert_count", "128" }, + { "qwen3_moe.routed_gate_up.route_count", "8" }, + { "qwen3_moe.workload.token_capacity", "1" }, + })); + // Exercise gather-add coverage in the async JIT path. + refs.push_back(compile_kernel(*async_jit, "async-gather-add-2-to-1", gather_add, + { + { "source_token_count", 2 }, + { "output_token_count", 1 }, + { "hidden_size", 2048 }, + })); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-f16-q4", mul_mat_id_f16, token_count_workload(4), + mul_mat_id_f16_f16_config("4"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-f16-q6", mul_mat_id_f16, token_count_workload(4), + mul_mat_id_f16_f16_config("6"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-q4k", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("4", "4"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-q6k", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("6", "6"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-iq4-xs", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("23", "23"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-q3k", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("11", "11"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-iq4-nl", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("20", "20"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-q8-0", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("80", "80"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-q8-1", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("81", "81"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-id-swiglu-f16-f16", swiglu_f16, token_count_workload(4), + mul_mat_id_swiglu_f16_f16_config("16", "16"))); + refs.push_back(compile_kernel(*async_jit, "async-mul-mat-decode-q8-0-gemma-vocab", decode_mul_mat, + { + { "token_count", 1 }, + { "input_size", 3840 }, + { "output_size", 262208 }, + }, + { + { "ggml.mul_mat_f32_f32_decode.token_capacity", "1" }, + { "ggml.mul_mat_f32_f32_decode.output_capacity", "262208" }, + { "ggml.mul_mat_f32_f32_decode.weight_format", "80" }, + })); + refs.push_back(compile_kernel(*async_jit, "async-flash-prefill-128", flash_prefill, + flash_attention_workload(128, 128), + flash_attention_config("32", "4", "128", "0.0883883461"))); + refs.push_back(compile_kernel(*async_jit, "async-flash-prefill-256", flash_prefill, + flash_attention_workload(128, 128), + flash_attention_config("32", "4", "256", "0.0625"))); + refs.push_back(compile_kernel(*async_jit, "async-flash-decode-128", flash_decode, + flash_attention_decode_workload(64), + flash_attention_decode_config("32", "4", "128", "0.0883883461", "64"))); + const auto enqueue_end = std::chrono::steady_clock::now(); + const int64_t enqueue_us = elapsed_us(enqueue_begin, enqueue_end); + std::printf("async Loom enqueue completed in %ld us\n", static_cast(enqueue_us)); + + const int64_t max_expected_enqueue_us = std::max(100000, sync_compile_us / 2); + REQUIRE(enqueue_us < max_expected_enqueue_us); + + for (const ggml::hrx::LoomCompiledKernelRef & ref : refs) { + require_compiled_kernel(ref); + } + + std::printf("async Loom JIT compiled %zu kernels\n", refs.size()); + + const auto & q6_prefill = find_kernel("ggml_mul_mat_q6_k_f16_wmma_prefill_wave32"); + const struct { + int64_t tokens; + int64_t inputs; + int64_t outputs; + int64_t accumulate; + int64_t workgroup_size; + int64_t token_tiles; + int64_t output_tiles; + } q6_launch_cases[] = { + { 128, 256, 4096, 0, 256, 1, 32 }, + { 256, 256, 8192, 0, 512, 1, 64 }, + { 512, 256, 1024, 0, 512, 2, 8 }, + { 512, 256, 3840, 0, 512, 2, 30 }, + { 512, 256, 4096, 0, 1024, 1, 32 }, + { 512, 256, 4224, 0, 1024, 1, 33 }, + { 512, 256, 4096, 1, 256, 4, 32 }, + {1024, 256, 2048, 0, 1024, 2, 16 }, + { 512, 17408, 5120, 0, 512, 1, 80 }, + { 512, 5120, 10240, 0, 1024, 1, 80 }, + { 512, 5120, 1024, 0, 512, 2, 8 }, + { 512, 17408, 4096, 0, 512, 1, 64 }, + { 512, 5120, 1984, 0, 256, 4, 16 }, + { 512, 5120, 2048, 0, 512, 1, 32 }, + {1024, 5120, 1024, 0, 512, 2, 16 }, + {2048, 5120, 1024, 0, 512, 4, 16 }, + }; + for (const auto & test : q6_launch_cases) { + const std::string key = "q6-prefill-" + std::to_string(test.tokens) + "-" + + std::to_string(test.inputs) + "-" + std::to_string(test.outputs) + "-" + + std::to_string(test.accumulate); + auto ref = compile_kernel(*sync_jit, key, q6_prefill, token_count_workload(test.tokens), { + { "ggml.mul_mat_q6_k_packed.input_size", std::to_string(test.inputs) }, + { "ggml.mul_mat_q6_k_packed.output_size", std::to_string(test.outputs) }, + { "ggml.mul_mat_q6_k_packed.output_accumulation", std::to_string(test.accumulate) }, + { "ggml.mul_mat_q6_k_packed.weight_offset", "0" }, + { "ggml.mul_mat_q6_k_packed.token_capacity", std::to_string(test.tokens) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == test.workgroup_size); + REQUIRE(compiled.launch_config.workgroup_count[0] == test.token_tiles); + REQUIRE(compiled.launch_config.workgroup_count[1] == test.output_tiles); + compiled.reset(); + } + + const auto & q4_prefill = find_kernel("ggml_mul_mat_q4_k_f16_wmma_prefill_wave32"); + const struct { + int64_t tokens; + int64_t inputs; + int64_t outputs; + int64_t workgroup_size; + int64_t token_tiles; + int64_t output_tiles; + int64_t input_layout; + } q4_launch_cases[] = { + { 256, 8192, 8192, 512, 1, 64, 0 }, + { 512, 8192, 1024, 512, 2, 8, 0 }, + { 512, 8192, 3840, 512, 2, 30, 0 }, + { 512, 8192, 4096, 512, 1, 64, 1 }, + { 512, 8192, 4224, 512, 1, 66, 1 }, + { 512, 8192, 12288, 512, 1, 192, 1 }, + { 512, 8192, 16384, 512, 1, 256, 1 }, + {1024, 8192, 2048, 1024, 4, 8, 0 }, + { 512, 17408, 5120, 1024, 2, 20, 0 }, + { 512, 5120, 6144, 512, 1, 96, 1 }, + { 512, 6144, 5120, 512, 1, 80, 1 }, + { 512, 5120, 10240, 512, 1, 160, 1 }, + { 512, 5120, 8192, 512, 1, 128, 1 }, + {1024, 5120, 4096, 1024, 4, 16, 0 }, + { 512, 17408, 5120, 512, 1, 80, 1 }, + }; + for (const auto & test : q4_launch_cases) { + const std::string key = "q4-prefill-" + std::to_string(test.tokens) + "-" + + std::to_string(test.inputs) + "-" + std::to_string(test.outputs) + "-" + + std::to_string(test.input_layout); + auto ref = compile_kernel(*sync_jit, key, q4_prefill, token_count_workload(test.tokens), { + { "ggml.mul_mat.input_size", std::to_string(test.inputs) }, + { "ggml.mul_mat.output_size", std::to_string(test.outputs) }, + { "ggml.workload.token_capacity", std::to_string(test.tokens) }, + { "ggml.mul_mat.f16_input_layout", std::to_string(test.input_layout) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == test.workgroup_size); + REQUIRE(compiled.launch_config.workgroup_count[0] == test.token_tiles); + REQUIRE(compiled.launch_config.workgroup_count[1] == test.output_tiles); + compiled.reset(); + } + + const auto & packed_f16 = find_kernel("ggml_copy_f16_k16_major"); + for (const auto * input_is_f16 : { "0", "1" }) { + auto ref = compile_kernel(*sync_jit, std::string("packed-f16-") + input_is_f16, + packed_f16, token_count_workload(512), { + { "ggml.copy_f16_k16_major.input_size", "256" }, + { "ggml.copy_f16_k16_major.token_count", "512" }, + { "ggml.copy_f16_k16_major.input_is_f16", input_is_f16 }, + }); + require_compiled_kernel(ref); + } + + const auto & transpose_f16 = find_kernel("ggml_copy_transpose_f16"); + auto transpose_ref = compile_kernel(*sync_jit, "transpose-f16-8192x1024", transpose_f16, {}, { + { "ggml.copy_transpose_f16.row_count", "8192" }, + { "ggml.copy_transpose_f16.column_count", "1024" }, + }); + require_compiled_kernel(transpose_ref); + auto transposed_attention_config = flash_attention_config("24", "4", "256", "0.0625"); + transposed_attention_config.emplace("ggml.flash_attention.value_layout", "1"); + auto transposed_attention_ref = compile_kernel(*sync_jit, "flash-transposed-value", flash_prefill, + { { "query_token_count", 512 }, { "key_value_token_count", 8192 } }, + transposed_attention_config); + require_compiled_kernel(transposed_attention_ref); + + const auto & paired_q4 = find_kernel("ggml_mul_mat_swiglu_q4_k_f16_wmma_prefill_wave32"); + const struct { + int64_t tokens; + int64_t outputs; + int64_t threads; + int64_t token_tiles; + } paired_q4_cases[] = { + { 256, 4096, 512, 1 }, + { 512, 3840, 512, 2 }, + { 512, 4096, 512, 1 }, + { 768, 8192, 512, 3 }, + {1024, 2048, 512, 2 }, + {2048,17408, 512, 4 }, + }; + for (const auto & test : paired_q4_cases) { + const std::string key = "paired-q4-" + std::to_string(test.tokens) + "-" + std::to_string(test.outputs); + auto ref = compile_kernel(*sync_jit, key, paired_q4, token_count_workload(test.tokens), { + { "ggml.mul_mat_swiglu.input_size", "5120" }, + { "ggml.mul_mat_swiglu.output_size", std::to_string(test.outputs) }, + { "ggml.mul_mat_swiglu.op", "4" }, + { "ggml.mul_mat_swiglu.f16_output_layout", "0" }, + { "ggml.workload.token_capacity", std::to_string(test.tokens) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == test.threads); + REQUIRE(compiled.launch_config.workgroup_count[0] == test.token_tiles); + const bool packed = test.tokens % 512 == 0 && (test.tokens / 512) * (test.outputs / 64) >= 64; + REQUIRE(compiled.launch_config.workgroup_count[1] == test.outputs / (packed ? 32 : 64)); + compiled.reset(); + } + + for (const int64_t tokens : { 256, 512, 1024, 2048 }) { + auto ref = compile_kernel(*sync_jit, "paired-q4-packed-output-" + std::to_string(tokens), + paired_q4, token_count_workload(tokens), { + { "ggml.mul_mat_swiglu.input_size", "5120" }, + { "ggml.mul_mat_swiglu.output_size", "17408" }, + { "ggml.mul_mat_swiglu.op", "4" }, + { "ggml.mul_mat_swiglu.f16_output_layout", "1" }, + { "ggml.workload.token_capacity", std::to_string(tokens) }, + }); + require_compiled_kernel(ref); + } + + + const auto & paired_q4_decode = find_kernel("ggml_mul_mat_swiglu_q4_q8_1_x4_lowtoken_dot"); + const struct { + int64_t tokens; + int64_t inputs; + int64_t outputs; + int64_t threads; + int64_t row_tiles; + } paired_q4_decode_cases[] = { + { 1, 5120, 17408, 128, 2176 }, + { 1, 4096, 8192, 128, 1024 }, + { 1, 4096, 16384, 128, 2048 }, + { 1, 8192, 8192, 128, 1024 }, + { 1, 5376, 17408, 128, 2176 }, + { 2, 5120, 17408, 512, 1088 }, + { 3, 5120, 17408, 512, 1088 }, + { 4, 5120, 17408, 512, 1088 }, + { 5, 5120, 17408, 512, 1088 }, + { 5, 4096, 8192, 512, 256 }, + { 5, 4096, 16384, 512, 1024 }, + { 3, 8192, 8192, 512, 512 }, + { 5, 5632, 12288, 512, 768 }, + { 5, 5376, 17408, 512, 544 }, + { 5, 6144, 17408, 128, 2176 }, + { 5, 256, 4032, 128, 504 }, + { 5, 256, 4096, 512, 128 }, + { 5, 5632, 4096, 512, 128 }, + { 5, 5888, 4096, 128, 512 }, + }; + for (const auto & test : paired_q4_decode_cases) { + const std::string key = "paired-q4-decode-" + std::to_string(test.tokens) + "-" + + std::to_string(test.inputs) + "-" + std::to_string(test.outputs); + auto ref = compile_kernel(*sync_jit, key, paired_q4_decode, token_count_workload(test.tokens), { + { "ggml.mul_mat_swiglu.input_size", std::to_string(test.inputs) }, + { "ggml.mul_mat_swiglu.output_size", std::to_string(test.outputs) }, + { "ggml.mul_mat_swiglu.op", "4" }, + { "ggml.workload.token_capacity", std::to_string(test.tokens) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == test.threads); + REQUIRE(compiled.launch_config.workgroup_count[0] == test.row_tiles); + REQUIRE(compiled.launch_config.workgroup_count[1] == 1); + compiled.reset(); + } + + + + const auto & quant_decode = find_kernel("ggml_mul_mat_f32_f32_wmma"); + const struct { + int64_t tokens; + int64_t inputs; + int64_t outputs; + int64_t format; + int64_t threads; + int64_t row_tiles; + } quant_decode_cases[] = { + { 1, 5120, 48, 4, 256, 12 }, + { 2, 5120, 48, 4, 1024, 12 }, + { 3, 5120, 48, 4, 1024, 12 }, + { 4, 5120, 48, 4, 1024, 12 }, + { 5, 5120, 48, 4, 1024, 12 }, + { 5, 5120, 48, 5, 1024, 12 }, + { 5, 5120, 48, 6, 1024, 12 }, + { 5, 4096, 1, 4, 1024, 1 }, + { 5, 4096, 16, 4, 1024, 4 }, + { 5, 8192, 63, 4, 1024, 16 }, + { 5, 32768, 48, 4, 1024, 12 }, + { 5, 3840, 48, 4, 256, 12 }, + { 5, 4352, 48, 4, 256, 12 }, + { 5, 4096, 64, 4, 128, 8 }, + { 6, 5120, 48, 4, 128, 1 }, + { 1, 17408, 5120, 46, 128, 640 }, + { 4, 17408, 5120, 46, 128, 640 }, + { 5, 17408, 5120, 46, 512, 160 }, + { 5, 8192, 4096, 46, 512, 128 }, + { 5, 8448, 4160, 46, 512, 130 }, + { 5, 7936, 4096, 46, 128, 512 }, + { 5, 8192, 4032, 46, 128, 504 }, + { 5, 8192, 8256, 46, 128, 1032 }, + { 1, 17408, 5120, 44, 128, 640 }, + { 4, 17408, 5120, 44, 128, 640 }, + { 5, 17408, 5120, 44, 1024, 80 }, + { 5, 8192, 4096, 44, 128, 1024 }, + { 1, 6144, 5120, 44, 128, 640 }, + { 4, 6144, 5120, 44, 128, 640 }, + { 5, 6144, 5120, 44, 128, 1280 }, + { 5, 6400, 5120, 44, 128, 640 }, + { 5, 6656, 5120, 44, 128, 1280 }, + { 5, 6144, 4096, 44, 128, 512 }, + { 5, 6144, 6144, 44, 128, 768 }, + { 5, 5120, 6144, 44, 128, 768 }, + { 5, 8448, 4160, 44, 128, 520 }, + { 5, 9216, 9216, 44, 128, 1152 }, + { 5, 12288, 8192, 44, 128, 1024 }, + { 5, 16384, 4096, 44, 128, 512 }, + { 5, 16384, 5056, 44, 128, 632 }, + { 5, 16384, 5120, 44, 1024, 80 }, + { 5, 20480, 4096, 44, 1024, 64 }, + { 5, 20736, 4160, 44, 1024, 65 }, + { 5, 7936, 4096, 44, 128, 512 }, + { 5, 8192, 4032, 44, 128, 504 }, + { 5, 8192, 8256, 44, 128, 1032 }, + }; + for (const auto & test : quant_decode_cases) { + const std::string key = "quant-decode-" + std::to_string(test.tokens) + "-" + + std::to_string(test.inputs) + "-" + std::to_string(test.outputs) + "-" + + std::to_string(test.format); + auto ref = compile_kernel(*sync_jit, key, quant_decode, token_count_workload(test.tokens), { + { "ggml.mul_mat.input_size", std::to_string(test.inputs) }, + { "ggml.mul_mat.output_size", std::to_string(test.outputs) }, + { "ggml.mul_mat.weight_format", std::to_string(test.format) }, + { "ggml.mul_mat.activation_format", test.format == 44 || test.format == 46 ? "9" : "0" }, + { "ggml.mul_mat.output_accumulation", "0" }, + { "ggml.mul_mat.output_unary_op", "23" }, + { "ggml.workload.token_capacity", std::to_string(test.tokens) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == test.threads); + REQUIRE(compiled.launch_config.workgroup_count[0] == test.row_tiles); + REQUIRE(compiled.launch_config.workgroup_count[1] == 1); + compiled.reset(); + } + + + for (const char * name : { "ggml_mul_mat_quantized_f16_wmma_prefill_conv4", + "ggml_mul_mat_quantized_f16_wmma_prefill_conv4_interior" }) { + const auto & quantized_conv4 = find_kernel(name); + REQUIRE(quantized_conv4.bindings.size() == (std::string(name).find("interior") != std::string::npos ? 5 : 6)); + for (const int format : {4, 6}) { + for (const int64_t channels : {2112, 8192, 10240}) { + auto ref = compile_kernel(*sync_jit, std::string(name) + "-" + std::to_string(format) + "-" + + std::to_string(channels), quantized_conv4, token_count_workload(512), { + {"ggml.mul_mat.input_size", "5120"}, + {"ggml.mul_mat.output_size", std::to_string(channels)}, + {"ggml.mul_mat.weight_format", std::to_string(format)}, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == 512); + REQUIRE(compiled.launch_config.workgroup_count[0] == 1); + REQUIRE(compiled.launch_config.workgroup_count[1] == channels / 64); + compiled.reset(); + } + } + } + + const auto & conv_finish = find_kernel("llm_ssm_conv_dconv4_silu_prefill_finish_f32"); + REQUIRE(conv_finish.bindings.size() == 5); + for (const int64_t channels : { 2112, 8192, 8256, 10240 }) { + auto ref = compile_kernel(*sync_jit, "conv-finish-" + std::to_string(channels), conv_finish, {}, { + { "llm.ssm_conv.generic.d_inner", std::to_string(channels) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == 256); + REQUIRE(compiled.launch_config.workgroup_count[0] == (channels + 255) / 256); + } + + const auto & ssm_prefill = find_kernel("llm_ssm_conv_dconv4_silu_prefill_512_wg1024"); + for (const int64_t channels : { 8192, 8224, 8256, 10208, 10240 }) { + auto ref = compile_kernel(*sync_jit, "ssm-prefill-" + std::to_string(channels), ssm_prefill, {}, { + { "llm.ssm_conv.prefill.d_conv", "4" }, + { "llm.ssm_conv.prefill.d_inner", std::to_string(channels) }, + { "llm.ssm_conv.prefill.n_t", "512" }, + { "llm.ssm_conv.prefill.n_s", "1" }, + { "llm.ssm_conv.prefill.state_row_stride", std::to_string(channels) }, + { "llm.ssm_conv.prefill.x_row_stride", std::to_string(channels) }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == 1024); + REQUIRE(compiled.launch_config.workgroup_count[0] == channels / (channels % 64 == 0 ? 64 : 32)); + REQUIRE(compiled.launch_config.workgroup_count[1] == 1); + compiled.reset(); + } + + const struct { + const char * name; + int64_t tokens; + size_t bindings; + } gdn_cases[] = { + { "llm_gated_delta_net_f32_wmma_head128", 17, 7 }, + { "llm_gated_delta_net_f32_wmma_head128_rmsnorm_gate", 17, 11 }, + { "llm_gated_delta_net_f32_wmma_head128_projection_epilogue", 16, 9 }, + { "llm_gated_delta_net_f32_wmma_head128_inplace", 1, 7 }, + { "llm_gated_delta_net_f32_wmma_head128_inplace_projection_epilogue", 1, 9 }, + { "llm_gated_delta_net_f32_wmma_head128_snapshot", 5, 8 }, + { "llm_gated_delta_net_f32_wmma_head128_snapshot_projection_epilogue", 5, 10 }, + }; + for (const auto & test : gdn_cases) { + const auto & definition = find_kernel(test.name); + REQUIRE(definition.bindings.size() == test.bindings); + auto ref = compile_kernel(*sync_jit, test.name, definition, {}, { + { "llm.gated_delta_net.head_width", "128" }, + { "llm.gated_delta_net.head_count", "3" }, + { "llm.gated_delta_net.token_count", std::to_string(test.tokens) }, + { "llm.gated_delta_net.sequence_count", "1" }, + { "llm.gated_delta_net.qk_stride1", "128" }, + { "llm.gated_delta_net.qk_stride2", "640" }, + { "llm.gated_delta_net.qk_stride3", std::to_string(test.tokens * 640) }, + { "llm.gated_delta_net.value_stride1", "128" }, + { "llm.gated_delta_net.value_stride2", "640" }, + { "llm.gated_delta_net.value_stride3", std::to_string(test.tokens * 640) }, + { "llm.gated_delta_net.scalar_stride1", "1" }, + { "llm.gated_delta_net.scalar_stride2", "3" }, + { "llm.gated_delta_net.scalar_stride3", std::to_string(test.tokens * 3) }, + { "llm.gated_delta_net.query_head_count", "1" }, + { "llm.gated_delta_net.query_sequence_ratio", "1" }, + { "llm.gated_delta_net.snapshot_stride", "49216" }, + { "llm.gated_delta_net.l2_epsilon", "0.000001" }, + { "llm.gated_delta_net.additive_epsilon", "0" }, // the merged kernel's new config; 0 = clamp, as before + { "ggml.rmsnorm_gate_f32.rms_epsilon", "0.000001" }, + { "ggml.rmsnorm_gate_f32.gate_op", "15" }, + }); + REQUIRE(ref != nullptr); + REQUIRE(resolve_with_timeout(ref, std::chrono::seconds(120))); + auto compiled = ref->take_result(); + REQUIRE(compiled.hsaco_data != nullptr); + REQUIRE(compiled.launch_config.workgroup_size[0] == 256); + REQUIRE(compiled.launch_config.workgroup_count[0] == 3); + REQUIRE(compiled.launch_config.workgroup_count[1] == 1); + compiled.reset(); + } + + HrxTestDevice device; + if (device.open()) { + run_cache_materialize_case(device, ggml::hrx::LoomJitMode::Sync, binary); + run_cache_materialize_case(device, ggml::hrx::LoomJitMode::Async, binary); + run_targeted_export_materialize_case(device); + } else { + std::printf("skipping KernelExecutableCache materialization checks: no HRX device available\n"); + } + + return 0; +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6fcc8d4a38f5..e15f1f6c1a43 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -31,7 +31,6 @@ else() add_subdirectory(kvshare-tensor-map) # [1bit] target 1: the tensor table a provider writes into (engine#753) add_subdirectory(kvshare-layout-roundtrip) # [1bit] target 1: the provider's import path with no GPU (engine#753) add_subdirectory(kvshare-adopt-check) # [1bit] target 1: a decoder working from NPU-written K/V (engine#753) - add_subdirectory(parser) add_subdirectory(tts) add_subdirectory(mtmd) if (GGML_RPC) diff --git a/tools/parser/CMakeLists.txt b/tools/parser/CMakeLists.txt deleted file mode 100644 index a8df0e7e6e36..000000000000 --- a/tools/parser/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) - # this tool is disabled on Windows when building with shared libraries because it uses internal functions not exported with LLAMA_API - set(TARGET llama-debug-template-parser) - add_executable(${TARGET} debug-template-parser.cpp) - target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) - target_compile_features(${TARGET} PRIVATE cxx_std_17) - - if(LLAMA_TOOLS_INSTALL) - install(TARGETS ${TARGET} RUNTIME) - endif() -endif() - -set(TARGET llama-template-analysis) -add_executable(${TARGET} template-analysis.cpp) -target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) -target_compile_features(${TARGET} PRIVATE cxx_std_17) - -if(LLAMA_TOOLS_INSTALL) - install(TARGETS ${TARGET} RUNTIME) -endif() diff --git a/tools/parser/debug-template-parser.cpp b/tools/parser/debug-template-parser.cpp deleted file mode 100644 index 8a916f79c78e..000000000000 --- a/tools/parser/debug-template-parser.cpp +++ /dev/null @@ -1,469 +0,0 @@ -#include "../src/llama-grammar.h" -#include "chat-auto-parser.h" -#include "chat.h" -#include "common.h" -#include "gguf.h" -#include "jinja/runtime.h" -#include "log.h" -#include "nlohmann/json.hpp" -#include "peg-parser.h" - -#include -#include -#include -#include -#include -#include - -using json = nlohmann::ordered_json; - -enum class output_mode { - ANALYSIS, // Only output analysis results (default) - TEMPLATE, // Only output rendered template - BOTH // Output both -}; - -enum class input_message_type { - NONE, // Don't render any message scenarios (only analysis) - CONTENT_ONLY, // Simple assistant message with content - REASONING_CONTENT, // Message with reasoning_content + content - TOOL_CALL_ONLY, // Message with tool_calls only - CONTENT_TOOL_CALL, // Message with content + tool_calls - REASONING_TOOL_CALL, // Message with reasoning_content + tool_calls - CONTENT_FAKE_TOOL_CALL, // Message with content but no actual tool_calls (for testing) - ALL // Render all scenarios -}; - -struct debug_options { - std::string template_path; - bool with_tools = true; - bool generation_prompt = true; - bool enable_reasoning = true; - bool debug_jinja = false; - bool force_tool_call = false; - bool parallel_tool_calls = true; - output_mode mode = output_mode::BOTH; - input_message_type input_message = input_message_type::NONE; -}; - -static std::string read_file(const std::string & path) { - std::ifstream fin(path, std::ios::binary); - if (!fin.is_open()) { - throw std::runtime_error("Could not open file: " + path); - } - std::ostringstream buf; - buf << fin.rdbuf(); - return buf.str(); -} - -static std::string read_gguf_chat_template(const std::string & path) { - struct gguf_init_params params = { /*no_alloc =*/true, // We only need metadata, not tensor data - /*ctx=*/nullptr }; - - struct gguf_context * ctx = gguf_init_from_file(path.c_str(), params); - if (ctx == nullptr) { - throw std::runtime_error("Could not open GGUF file: " + path); - } - - const char * key = "tokenizer.chat_template"; - int64_t key_id = gguf_find_key(ctx, key); - - if (key_id == -1) { - gguf_free(ctx); - throw std::runtime_error("GGUF file does not contain chat template key: " + std::string(key)); - } - - const char * template_str = gguf_get_val_str(ctx, key_id); - if (template_str == nullptr) { - gguf_free(ctx); - throw std::runtime_error("GGUF file contains chat template key but value is null"); - } - - std::string result = template_str; - gguf_free(ctx); - return result; -} - -static void print_usage(const char * program_name) { - LOG_ERR("Usage: %s [options]\n", program_name); - LOG_ERR("\nOptions:\n"); - LOG_ERR(" --no-tools Disable tool definitions\n"); - LOG_ERR(" --force-tool-call Set tool calls to forced\n"); - LOG_ERR(" --parallel-tool-calls=0|1 Set parallel_tool_calls (default: 1)\n"); - LOG_ERR(" --generation-prompt=0|1 Set add_generation_prompt (default: 1)\n"); - LOG_ERR(" --enable-reasoning=0|1 Enable reasoning parsing (default: 1)\n"); - LOG_ERR(" --output=MODE Output mode: analysis, template, both (default: both)\n"); - LOG_ERR(" --debug-jinja Enable Jinja fine-grained debug\n"); - LOG_ERR(" --input-message=TYPE Message type to render:\n"); - LOG_ERR(" content_only, reasoning_content, tool_call_only,\n"); - LOG_ERR(" content_tool_call, reasoning_tool_call,\n"); - LOG_ERR(" content_fake_tool_call, all\n"); - LOG_ERR("\nExamples:\n"); - LOG_ERR(" %s template.jinja --input-message=all --generation-prompt=1\n", program_name); - LOG_ERR(" %s template.jinja --output=template --input-message=tool_call_only\n", program_name); -} - -static bool parse_bool_option(const std::string & value) { - return value == "1" || value == "true" || value == "yes"; -} - -static bool parse_options(int argc, char ** argv, debug_options & opts) { - if (argc < 2) { - print_usage(argv[0]); - return false; - } - - opts.template_path = argv[1]; - - for (int i = 2; i < argc; ++i) { - std::string arg = argv[i]; - - if (arg == "--force-tool-call") { - opts.force_tool_call = true; - } else if (arg == "--debug-jinja") { - opts.debug_jinja = true; - } else if (arg == "--no-tools") { - opts.with_tools = false; - } else if (arg.rfind("--parallel-tool-calls=", 0) == 0) { - opts.parallel_tool_calls = parse_bool_option(arg.substr(22)); - } else if (arg.rfind("--generation-prompt=", 0) == 0) { - opts.generation_prompt = parse_bool_option(arg.substr(20)); - } else if (arg.rfind("--enable-reasoning=", 0) == 0) { - opts.enable_reasoning = parse_bool_option(arg.substr(19)); - } else if (arg.rfind("--output=", 0) == 0) { - std::string mode = arg.substr(9); - if (mode == "analysis") { - opts.mode = output_mode::ANALYSIS; - } else if (mode == "template") { - opts.mode = output_mode::TEMPLATE; - } else if (mode == "both") { - opts.mode = output_mode::BOTH; - } else { - LOG_ERR("Unknown output mode: %s\n", mode.c_str()); - return false; - } - } else if (arg.rfind("--input-message=", 0) == 0) { - std::string type = arg.substr(16); - if (type == "content_only") { - opts.input_message = input_message_type::CONTENT_ONLY; - } else if (type == "reasoning_content") { - opts.input_message = input_message_type::REASONING_CONTENT; - } else if (type == "tool_call_only") { - opts.input_message = input_message_type::TOOL_CALL_ONLY; - } else if (type == "content_tool_call") { - opts.input_message = input_message_type::CONTENT_TOOL_CALL; - } else if (type == "reasoning_tool_call") { - opts.input_message = input_message_type::REASONING_TOOL_CALL; - } else if (type == "content_fake_tool_call") { - opts.input_message = input_message_type::CONTENT_FAKE_TOOL_CALL; - } else if (type == "all") { - opts.input_message = input_message_type::ALL; - } else { - LOG_ERR("Unknown input message type: %s\n", type.c_str()); - return false; - } - } else { - LOG_ERR("Unknown option: %s\n", arg.c_str()); - print_usage(argv[0]); - return false; - } - } - - return true; -} - -static json build_user_message() { - return json{ - { "role", "user" }, - { "content", "Hello, please help me with a task." } - }; -} - -static json build_content_only_message() { - return json{ - { "role", "assistant" }, - { "content", "Hello! I'm here to help you with your task." } - }; -} - -static json build_reasoning_content_message() { - return json{ - { "role", "assistant" }, - { "content", "Hello! I'm here to help you with your task." }, - { "reasoning_content", "The user is greeting me and asking for help. I should respond politely." } - }; -} - -static json build_tool_call_only_message() { - return json{ - { "role", "assistant" }, - { "content", nullptr }, - { "tool_calls", - json::array({ json{ - { "type", "function" }, - { "function", json{ { "name", "test_function_name" }, - { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } }, - { "id", "123456789" } } }) } - }; -} - -static json build_content_tool_call_message() { - return json{ - { "role", "assistant" }, - { "content", "I'll help you by calling a function." }, - { "tool_calls", - json::array({ json{ - { "type", "function" }, - { "function", - json{ { "name", "test_function_name" }, - { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) } - }; -} - -static json build_reasoning_tool_call_message() { - return json{ - { "role", "assistant" }, - { "content", nullptr }, - { "reasoning_content", "I need to call a function to help with this task." }, - { "tool_calls", - json::array({ json{ - { "type", "function" }, - { "function", - json{ { "name", "test_function_name" }, - { "arguments", json::object({ { "param1", "value1" }, { "param2", "value2" } }) } } } } }) } - }; -} - -static json build_content_fake_tool_call_message() { - // This message has content but NO tool_calls field - // It's used to test if a template renders tool definitions but not tool calls - return json{ - { "role", "assistant" }, - { "content", "I'll help you by calling a function." } - }; -} - -static json build_tools_definition() { - json parameters_schema = json::object(); - parameters_schema["type"] = "object"; - parameters_schema["properties"] = json::object(); - parameters_schema["properties"]["param1"] = json::object({ - { "type", "string" }, - { "description", "First parameter" } - }); - parameters_schema["properties"]["param2"] = json::object({ - { "type", "string" }, - { "description", "Second parameter" } - }); - parameters_schema["required"] = json::array({ "param1" }); - - return json::array({ - json{ { "type", "function" }, - { "function", json{ { "name", "test_function_name" }, - { "description", "A test function for debugging" }, - { "parameters", parameters_schema } } } } - }); -} - -static void render_scenario(const common_chat_template & tmpl, - const std::string & scenario_name, - const json & messages, - const json & tools, - bool add_generation_prompt, - bool enable_thinking) { - LOG_ERR("\n=== Scenario: %s ===\n", scenario_name.c_str()); - LOG_ERR("add_generation_prompt: %s, enable_thinking: %s\n", add_generation_prompt ? "true" : "false", - enable_thinking ? "true" : "false"); - - // When add_generation_prompt is true, add a trailing user message to trigger the prompt - json final_messages = messages; - if (add_generation_prompt && !messages.empty() && messages.back().value("role", "") == "assistant") { - final_messages.push_back(json{ - { "role", "user" }, - { "content", "Now please continue with another response." } - }); - } - - LOG_ERR("Messages:\n%s\n", final_messages.dump(2).c_str()); - - try { - autoparser::generation_params inputs; - inputs.messages = final_messages; - inputs.add_generation_prompt = add_generation_prompt; - inputs.extra_context["enable_thinking"] = enable_thinking; - - if (!tools.is_null() && tools.is_array() && !tools.empty()) { - inputs.tools = tools; - } - - std::string output = common_chat_template_direct_apply(tmpl, inputs); - - LOG_ERR("\n--- Rendered Output ---\n"); - LOG_ERR("%s\n", output.c_str()); - LOG_ERR("--- End Output (length: %zu) ---\n", output.length()); - } catch (const std::exception & e) { - LOG_ERR("Rendering failed: %s\n", e.what()); - } -} - -static void render_all_scenarios(const common_chat_template & tmpl, - const json & tools, - bool add_generation_prompt, - bool enable_thinking, - input_message_type message_type) { - json user_msg = build_user_message(); - - auto render_if = [&](input_message_type type, const std::string & name, const json & assistant_msg) { - if (message_type == input_message_type::ALL || message_type == type) { - json messages = json::array({ user_msg, assistant_msg }); - render_scenario(tmpl, name, messages, tools, add_generation_prompt, enable_thinking); - } - }; - - render_if(input_message_type::CONTENT_ONLY, "content_only", build_content_only_message()); - render_if(input_message_type::REASONING_CONTENT, "reasoning_content", build_reasoning_content_message()); - render_if(input_message_type::TOOL_CALL_ONLY, "tool_call_only", build_tool_call_only_message()); - render_if(input_message_type::CONTENT_TOOL_CALL, "content_tool_call", build_content_tool_call_message()); - render_if(input_message_type::REASONING_TOOL_CALL, "reasoning_tool_call", build_reasoning_tool_call_message()); - render_if(input_message_type::CONTENT_FAKE_TOOL_CALL, "content_fake_tool_call", - build_content_fake_tool_call_message()); - - // Also render with add_generation_prompt=true to show the prompt ending - if (message_type == input_message_type::ALL) { - LOG_ERR("\n\n=== Generation Prompt Scenarios (add_generation_prompt=true) ===\n"); - - json prompt_messages = json::array({ user_msg }); - render_scenario(tmpl, "generation_prompt_only", prompt_messages, tools, true, enable_thinking); - - // With enable_thinking toggled - render_scenario(tmpl, "generation_prompt_thinking_disabled", prompt_messages, tools, true, false); - } -} - -static autoparser::generation_params prepare_params(const debug_options & opts, const json & tools) { - autoparser::generation_params params; - params.messages = json::array({ build_user_message() }); - params.reasoning_format = opts.enable_reasoning ? COMMON_REASONING_FORMAT_DEEPSEEK : COMMON_REASONING_FORMAT_NONE; - params.enable_thinking = opts.enable_reasoning; - params.add_generation_prompt = opts.generation_prompt; - - if (opts.with_tools) { - params.tools = tools; - params.tool_choice = opts.force_tool_call ? COMMON_CHAT_TOOL_CHOICE_REQUIRED : COMMON_CHAT_TOOL_CHOICE_AUTO; - } else { - params.tools = json(); - params.tool_choice = COMMON_CHAT_TOOL_CHOICE_NONE; - } - params.parallel_tool_calls = opts.parallel_tool_calls; - return params; -} - -int main(int argc, char ** argv) { - // Set log level to most verbose to capture all debug output - common_log_set_verbosity_thold(99); - - debug_options opts; - if (!parse_options(argc, argv, opts)) { - return 1; - } - - if (opts.debug_jinja || std::getenv("LLAMA_DEBUG_JINJA") != nullptr) { - jinja::enable_debug(true); - } - - std::string template_source; - try { - // Check if the file is a GGUF file - if (opts.template_path.size() >= 5 && - opts.template_path.compare(opts.template_path.size() - 5, 5, ".gguf") == 0) { - template_source = read_gguf_chat_template(opts.template_path); - } else { - template_source = read_file(opts.template_path); - } - } catch (const std::exception & e) { - LOG_ERR("Error reading template: %s\n", e.what()); - return 1; - } - - LOG_ERR("Analyzing template: %s\n", opts.template_path.c_str()); - LOG_ERR("Options: with_tools=%s, generation_prompt=%s, enable_reasoning=%s\n", opts.with_tools ? "true" : "false", - opts.generation_prompt ? "true" : "false", opts.enable_reasoning ? "true" : "false"); - - try { - common_chat_template chat_template(template_source, "", ""); - - json tools = opts.with_tools ? build_tools_definition() : json(); - - autoparser::generation_params params = prepare_params(opts, tools); - common_chat_params parser_data; - if (std::optional spec_tmpl = - common_chat_try_specialized_template(chat_template, template_source, params)) { - LOG_ERR("\n"); - LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n"); - parser_data = *spec_tmpl; - } else { - // Render template scenarios if requested - if (opts.input_message != input_message_type::NONE && - (opts.mode == output_mode::TEMPLATE || opts.mode == output_mode::BOTH)) { - LOG_ERR("\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE RENDERING OUTPUT\n"); - LOG_ERR("================================================================================\n"); - - render_all_scenarios(chat_template, tools, opts.generation_prompt, opts.enable_reasoning, - opts.input_message); - } - - // Output analysis if requested - if (opts.mode == output_mode::ANALYSIS || opts.mode == output_mode::BOTH) { - LOG_ERR("\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE ANALYSIS\n"); - LOG_ERR("================================================================================\n"); - - autoparser::autoparser analysis; - analysis.analyze_template(chat_template); - - // Generate Parser - parser_data = autoparser::peg_generator::generate_parser(chat_template, params, analysis); - } - } - - if (!std::empty(parser_data.parser)) { - LOG_ERR("\n=== Generated Parser ===\n"); - common_peg_arena arena; - arena.load(parser_data.parser); - LOG_ERR("%s\n", arena.dump(arena.root()).c_str()); - - LOG_ERR("\n=== Generated Grammar ===\n"); - LOG_ERR("%s\n", parser_data.grammar.c_str()); - - LOG_ERR("\n=== Generated Lazy Grammar ===\n"); - LOG_ERR("%d\n", parser_data.grammar_lazy); - - LOG_ERR("\n=== Generated Grammar Triggers ===\n"); - for (const common_grammar_trigger & cgt : parser_data.grammar_triggers) { - LOG_ERR("Token: %d | Type: %d | Value: %s\n", cgt.token, cgt.type, cgt.value.c_str()); - } - - LOG_ERR("\n=== Preserved Tokens ===\n"); - for (const std::string & token : parser_data.preserved_tokens) { - LOG_ERR(" '%s'\n", token.c_str()); - } - - if (!parser_data.grammar.empty()) { - LOG_ERR("\n=== Verifying created grammar ===\n"); - auto * grammar = llama_grammar_init_impl(nullptr, parser_data.grammar.c_str(), "root", - parser_data.grammar_lazy, nullptr, 0, nullptr, 0); - if (grammar != nullptr) { - LOG_ERR("\n=== Grammar successfully created ===\n"); - } - } - } - } catch (const std::exception & e) { - LOG_ERR("Analysis failed: %s\n", e.what()); - return 1; - } - - return 0; -} diff --git a/tools/parser/template-analysis.cpp b/tools/parser/template-analysis.cpp deleted file mode 100644 index bf898a2290f1..000000000000 --- a/tools/parser/template-analysis.cpp +++ /dev/null @@ -1,611 +0,0 @@ -#include "chat-auto-parser.h" -#include "chat-auto-parser-helpers.h" -#include "chat.h" -#include "log.h" -#include "jinja/caps.h" -#include "jinja/runtime.h" - -#include -#include -#include -#include -#include - -#include "nlohmann/json.hpp" - -using json = nlohmann::ordered_json; - -// ANSI color codes - using 256-color palette for brighter colors (all bold) -#define ANSI_RESET "\033[0m" -#define ANSI_PURPLE "\033[1m\x1b[38;5;126m" // Bold bright purple for main headers -#define ANSI_CYAN "\033[1m\x1b[38;5;81m" // Bold bright cyan for section headers -#define ANSI_BLUE "\033[1m\x1b[38;5;12m" // Bold bright blue for labels -#define ANSI_ORANGE "\033[1m\x1b[38;5;209m" // Bold orange for right differences -#define ANSI_GREEN "\033[1m\x1b[38;5;83m" // Bold bright green for left differences -#define ANSI_GRAY "\033[1m\x1b[38;5;240m" // Bold gray (used for "no variables" message) -#define ANSI_BOLD "\033[1m" // Standalone bold -#define ANSI_PREFIX "\033[1m\x1b[38;5;176m" // Bold color for common prefix -#define ANSI_SUFFIX "\033[1m\x1b[38;5;61m" // Bold color for common suffix - -// All template paths extracted from tests/test-chat.cpp -static const std::vector ALL_TEMPLATE_PATHS = { - "models/templates/Apertus-8B-Instruct.jinja", - "models/templates/Apriel-1.6-15b-Thinker-fixed.jinja", - "models/templates/ByteDance-Seed-OSS.jinja", - "models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja", - "models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja", - "models/templates/GLM-4.6.jinja", - "models/templates/GLM-4.7-Flash.jinja", - "models/templates/Kimi-K2-Instruct.jinja", - "models/templates/Kimi-K2-Thinking.jinja", - "models/templates/MiMo-VL.jinja", - "models/templates/MiniMax-M2.jinja", - "models/templates/Mistral-Small-3.2-24B-Instruct-2506.jinja", - "models/templates/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.jinja", - "models/templates/NVIDIA-Nemotron-Nano-v2.jinja", - "models/templates/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use.jinja", - "models/templates/NousResearch-Hermes-3-Llama-3.1-8B-tool_use.jinja", - "models/templates/Qwen-QwQ-32B.jinja", - "models/templates/Qwen-Qwen2.5-7B-Instruct.jinja", - "models/templates/Qwen3-Coder.jinja", - "models/templates/deepseek-ai-DeepSeek-R1-Distill-Llama-8B.jinja", - "models/templates/deepseek-ai-DeepSeek-R1-Distill-Qwen-32B.jinja", - "models/templates/deepseek-ai-DeepSeek-V3.1.jinja", - "models/templates/fireworks-ai-llama-3-firefunction-v2.jinja", - "models/templates/google-gemma-2-2b-it.jinja", - "models/templates/ibm-granite-granite-3.3-2B-Instruct.jinja", - "models/templates/llama-cpp-deepseek-r1.jinja", - "models/templates/meetkai-functionary-medium-v3.1.jinja", - "models/templates/meetkai-functionary-medium-v3.2.jinja", - "models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja", - "models/templates/meta-llama-Llama-3.2-3B-Instruct.jinja", - "models/templates/meta-llama-Llama-3.3-70B-Instruct.jinja", - "models/templates/mistralai-Ministral-3-14B-Reasoning-2512.jinja", - "models/templates/mistralai-Mistral-Nemo-Instruct-2407.jinja", - "models/templates/moonshotai-Kimi-K2.jinja", - "models/templates/openai-gpt-oss-120b.jinja", - "models/templates/unsloth-Apriel-1.5.jinja", - "models/templates/unsloth-mistral-Devstral-Small-2507.jinja", -}; - -struct analysis_options { - std::vector template_paths; - bool analyze_all = false; -}; - -static std::string read_file(const std::string & path) { - std::ifstream fin(path, std::ios::binary); - if (!fin.is_open()) { - throw std::runtime_error("Could not open file: " + path); - } - std::ostringstream buf; - buf << fin.rdbuf(); - return buf.str(); -} - -static void print_usage(const char * program_name) { - LOG_ERR("Usage: %s [options]\n", program_name); - LOG_ERR("\nOptions:\n"); - LOG_ERR(" --template Analyze specific template from test suite (e.g., 'deepseek' or 'DeepSeek-V3.1')\n"); - LOG_ERR(" --template-file Analyze custom template file\n"); - LOG_ERR(" --all Analyze all templates from test suite\n"); - LOG_ERR("\nExamples:\n"); - LOG_ERR(" %s --all\n", program_name); - LOG_ERR(" %s --template deepseek\n", program_name); - LOG_ERR(" %s --template-file my-template.jinja\n", program_name); -} - -static bool parse_options(int argc, char ** argv, analysis_options & opts) { - if (argc < 2) { - print_usage(argv[0]); - return false; - } - - for (int i = 1; i < argc; ++i) { - std::string arg = argv[i]; - - if (arg == "--all") { - opts.analyze_all = true; - } else if (arg == "--template") { - if (i + 1 >= argc) { - LOG_ERR("--template requires an argument\n"); - return false; - } - std::string pattern = argv[++i]; - std::transform(pattern.begin(), pattern.end(), pattern.begin(), ::tolower); - - // Find matching templates - bool found = false; - for (const auto & path : ALL_TEMPLATE_PATHS) { - std::string path_lower = path; - std::transform(path_lower.begin(), path_lower.end(), path_lower.begin(), ::tolower); - if (path_lower.find(pattern) != std::string::npos) { - opts.template_paths.push_back(path); - found = true; - } - } - - if (!found) { - LOG_ERR("No templates found matching: %s\n", pattern.c_str()); - return false; - } - } else if (arg == "--template-file") { - if (i + 1 >= argc) { - LOG_ERR("--template-file requires an argument\n"); - return false; - } - opts.template_paths.push_back(argv[++i]); - } else { - LOG_ERR("Unknown option: %s\n", arg.c_str()); - print_usage(argv[0]); - return false; - } - } - - if (opts.analyze_all) { - opts.template_paths = ALL_TEMPLATE_PATHS; - } - - if (opts.template_paths.empty()) { - LOG_ERR("No templates specified\n"); - print_usage(argv[0]); - return false; - } - - return true; -} - -static json build_tools_definition() { - json parameters_schema = json::object(); - parameters_schema["type"] = "object"; - parameters_schema["properties"] = json::object(); - parameters_schema["properties"]["param1"] = json::object({ - { "type", "string" }, - { "description", "First parameter" } - }); - parameters_schema["properties"]["param2"] = json::object({ - { "type", "string" }, - { "description", "Second parameter" } - }); - parameters_schema["required"] = json::array({ "param1", "param2" }); - - return json::array({ - json{ { "type", "function" }, - { "function", json{ { "name", "test_function_name" }, - { "description", "A test function for debugging" }, - { "parameters", parameters_schema } } } } - }); -} - -// Helper to create a tool call with arguments as JSON object -static json build_tool_call(const std::string & name, const json & args_object, const std::string & id = "call_001") { - return json{ - {"id", id}, - {"type", "function"}, - {"function", json{ - {"name", name}, - {"arguments", args_object} // Pass as JSON object, not serialized string - }} - }; -} - -// Helper functions to create repeating message definitions -static json make_user_msg() { - return json{ - {"role", "user"}, - {"content", "Hello, please help me."} - }; -} - -static json make_user_msg2() { - return json{ - {"role", "user"}, - {"content", "Thank you."} - }; -} - -static json make_user_msg2_continue() { - return json{ - {"role", "user"}, - {"content", "Continue."} - }; -} - -static json make_assistant_no_tool() { - return json{ - {"role", "assistant"}, - {"content", "Let me help you."} - }; -} - -static json make_assistant_one_tool() { - return json{ - {"role", "assistant"}, - {"content", nullptr}, - {"tool_calls", json::array({ - build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})) - })} - }; -} - -static json make_assistant_two_tools() { - return json{ - {"role", "assistant"}, - {"content", nullptr}, - {"tool_calls", json::array({ - build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})), - build_tool_call("test_function_name", json::object({{"param1", "value3"}, {"param2", "value4"}}), "call_002") - })} - }; -} - -static json make_assistant_no_reasoning() { - return json{ - {"role", "assistant"}, - {"content", "I can help you with that."} - }; -} - -static json make_assistant_with_reasoning() { - return json{ - {"role", "assistant"}, - {"content", "I can help you with that."}, - {"reasoning_content", "The user is asking for help. I should respond positively."} - }; -} - -static json make_assistant_one_tool_with_reasoning() { - return json{ - {"role", "assistant"}, - {"content", nullptr}, - {"tool_calls", json::array({ - build_tool_call("test_function_name", json::object({{"param1", "value1"}, {"param2", "value2"}})) - })}, - {"reasoning_content", "I need to call the tool first."} - }; -} - -static void print_diff_split(const std::string & title, const diff_split & diff) { - LOG_ERR("\n%s=== %s ===%s\n", ANSI_CYAN, title.c_str(), ANSI_RESET); - LOG_ERR("%sCommon Prefix:%s '%s'\n", ANSI_PREFIX, ANSI_RESET, diff.prefix.c_str()); - LOG_ERR("%sCommon Suffix:%s '%s'\n", ANSI_SUFFIX, ANSI_RESET, diff.suffix.c_str()); - LOG_ERR("%sLeft (difference):%s '%s'\n", ANSI_GREEN, ANSI_RESET, diff.left.c_str()); - LOG_ERR("%sRight (difference):%s '%s'\n", ANSI_ORANGE, ANSI_RESET, diff.right.c_str()); -} - -static void check_reasoning_variables(const common_chat_template & tmpl) { - LOG_ERR("\n%s=== Checking Reasoning Variables ===%s\n", ANSI_CYAN, ANSI_RESET); - - try { - // Create a list of candidate reasoning/thinking variable names to probe - std::vector candidate_vars = { - "enable_reasoning", - "use_reasoning", - "reasoning_enabled", - "has_reasoning", - "reasoning_mode", - "reasoning_format", - "reasoning_active", - "with_reasoning", - "use_thinking", - "thinking_enabled", - "has_thinking", - "thinking_mode", - "thinking_format", - "thinking_active", - "with_thinking", - "enable_reason", - "reason_enabled", - "enable_think", - "think_enabled", - }; - - jinja::context ctx; - ctx.is_get_stats = true; - - json messages = json::array({ - json{ - {"role", "user"}, - {"content", "Test message"} - }, - json{ - {"role", "assistant"}, - {"content", "Response"}, - {"reasoning_content", "Some reasoning"} - } - }); - - // Set up base context - jinja::global_from_json(ctx, json{ - {"messages", messages}, - {"tools", json::array()}, - {"bos_token", ""}, - {"eos_token", ""}, - {"add_generation_prompt", false}, - {"enable_thinking", true} // Already passed, so we'll exclude this from results - }, true); - - // Add candidate variables as undefined to probe which ones are accessed - for (const auto & var_name : candidate_vars) { - ctx.set_val(var_name, jinja::mk_val(var_name)); - } - - try { - jinja::runtime runtime(ctx); - runtime.execute(tmpl.prog); - } catch (const std::exception & e) { - // Execution may fail, that's okay - we just want to see what variables were accessed - } - - // Check which candidate variables were accessed (stats.used = true) - std::vector accessed_vars; - for (const auto & var_name : candidate_vars) { - auto val = ctx.get_val(var_name); - if (!val->is_undefined()) { - // Variable was overwritten, skip it - continue; - } - if (val->stats.used) { - accessed_vars.push_back(var_name); - } - } - - if (accessed_vars.empty()) { - LOG_ERR("%sNo reasoning/thinking-related variables were queried by the template%s\n", ANSI_GRAY, ANSI_RESET); - } else { - LOG_ERR("Template queries the following reasoning/thinking-related variables:\n"); - for (const auto & var : accessed_vars) { - LOG_ERR(" %s- %s%s\n", ANSI_ORANGE, var.c_str(), ANSI_RESET); - } - } - - } catch (const std::exception & e) { - LOG_ERR("Error checking reasoning variables: %s\n", e.what()); - } -} - -static void analyze_template(const std::string & template_path) { - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_PURPLE); - LOG_ERR("================================================================================\n"); - LOG_ERR(" ANALYZING TEMPLATE: %s\n", template_path.c_str()); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); - - std::string template_source; - try { - template_source = read_file(template_path); - } catch (const std::exception & e) { - LOG_ERR("Error reading template: %s\n", e.what()); - return; - } - - try { - common_chat_template chat_template(template_source, "", ""); - json tools = build_tools_definition(); - - // ===== CAPABILITIES ANALYSIS ===== - LOG_ERR("\n%s=== Template Capabilities (from jinja::caps) ===%s\n", ANSI_CYAN, ANSI_RESET); - auto caps = chat_template.original_caps(); - LOG_ERR("%ssupports_tools:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tools ? "true" : "false"); - LOG_ERR("%ssupports_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_tool_calls ? "true" : "false"); - LOG_ERR("%ssupports_system_role:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_system_role ? "true" : "false"); - LOG_ERR("%ssupports_parallel_tool_calls:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_parallel_tool_calls ? "true" : "false"); - LOG_ERR("%ssupports_typed_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_typed_content ? "true" : "false"); - LOG_ERR("%ssupports_string_content:%s %s\n", ANSI_BLUE, ANSI_RESET, caps.supports_string_content ? "true" : "false"); - - // ===== DIFFERENTIAL ANALYSIS ===== - - // Test 1: With and without tools (single user message) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_no_tools; - params_no_tools.messages = json::array({ user_msg }); - params_no_tools.add_generation_prompt = false; - params_no_tools.tools = json::array(); - - autoparser::generation_params params_with_tools = params_no_tools; - params_with_tools.tools = tools; - - std::string output_no_tools = common_chat_template_direct_apply(chat_template, params_no_tools); - std::string output_with_tools = common_chat_template_direct_apply(chat_template, params_with_tools); - - auto diff = calculate_diff_split(output_no_tools, output_with_tools); - print_diff_split("Diff: With vs Without Tools (single user message)", diff); - } - - // Test 2: With and without add_generation_prompt (single user message) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_no_prompt; - params_no_prompt.messages = json::array({ user_msg }); - params_no_prompt.add_generation_prompt = false; - params_no_prompt.tools = json::array(); - - autoparser::generation_params params_with_prompt = params_no_prompt; - params_with_prompt.add_generation_prompt = true; - - std::string output_no_prompt = common_chat_template_direct_apply(chat_template, params_no_prompt); - std::string output_with_prompt = common_chat_template_direct_apply(chat_template, params_with_prompt); - - auto diff = calculate_diff_split(output_no_prompt, output_with_prompt); - print_diff_split("Diff: With vs Without add_generation_prompt (single user message)", diff); - } - - // Test 3: Assistant with reasoning_content (user, assistant) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_no_reasoning; - params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning() }); - params_no_reasoning.add_generation_prompt = false; - params_no_reasoning.enable_thinking = true; - - autoparser::generation_params params_with_reasoning = params_no_reasoning; - params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning() }); - - std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning); - std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning); - - auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning); - print_diff_split("Diff: With vs Without reasoning_content (user, assistant)", diff); - } - - // Test 4: Assistant with reasoning_content (user, assistant, user) - { - json user_msg = make_user_msg(); - json user_msg2 = make_user_msg2(); - - autoparser::generation_params params_no_reasoning; - params_no_reasoning.messages = json::array({ user_msg, make_assistant_no_reasoning(), user_msg2 }); - params_no_reasoning.add_generation_prompt = false; - params_no_reasoning.enable_thinking = true; - - autoparser::generation_params params_with_reasoning = params_no_reasoning; - params_with_reasoning.messages = json::array({ user_msg, make_assistant_with_reasoning(), user_msg2 }); - - std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning); - std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning); - - auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning); - print_diff_split("Diff: With vs Without reasoning_content (user, assistant, user)", diff); - } - - // Test 5: Tool call in last assistant message (user, assistant) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_no_tool; - params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool() }); - params_no_tool.add_generation_prompt = false; - params_no_tool.tools = tools; - - autoparser::generation_params params_with_tool = params_no_tool; - params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool() }); - - std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool); - std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool); - - auto diff = calculate_diff_split(output_no_tool, output_with_tool); - print_diff_split("Diff: With vs Without tool call (user, assistant)", diff); - } - - // Test 6: Tool call in last assistant message (user, assistant, user) - { - json user_msg = make_user_msg(); - json user_msg2 = make_user_msg2_continue(); - - autoparser::generation_params params_no_tool; - params_no_tool.messages = json::array({ user_msg, make_assistant_no_tool(), user_msg2 }); - params_no_tool.add_generation_prompt = false; - params_no_tool.tools = tools; - - autoparser::generation_params params_with_tool = params_no_tool; - params_with_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 }); - - std::string output_no_tool = common_chat_template_direct_apply(chat_template, params_no_tool); - std::string output_with_tool = common_chat_template_direct_apply(chat_template, params_with_tool); - - auto diff = calculate_diff_split(output_no_tool, output_with_tool); - print_diff_split("Diff: With vs Without tool call (user, assistant, user)", diff); - } - - // Test 7: One vs two tool calls (user, assistant) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_one_tool; - params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool() }); - params_one_tool.add_generation_prompt = false; - params_one_tool.tools = tools; - - autoparser::generation_params params_two_tools = params_one_tool; - params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools() }); - - std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool); - std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools); - - auto diff = calculate_diff_split(output_one_tool, output_two_tools); - print_diff_split("Diff: One vs Two tool calls (user, assistant)", diff); - } - - // Test 8: One vs two tool calls (user, assistant, user) - { - json user_msg = make_user_msg(); - json user_msg2 = make_user_msg2_continue(); - - autoparser::generation_params params_one_tool; - params_one_tool.messages = json::array({ user_msg, make_assistant_one_tool(), user_msg2 }); - params_one_tool.add_generation_prompt = false; - params_one_tool.tools = tools; - - autoparser::generation_params params_two_tools = params_one_tool; - params_two_tools.messages = json::array({ user_msg, make_assistant_two_tools(), user_msg2 }); - - std::string output_one_tool = common_chat_template_direct_apply(chat_template, params_one_tool); - std::string output_two_tools = common_chat_template_direct_apply(chat_template, params_two_tools); - - auto diff = calculate_diff_split(output_one_tool, output_two_tools); - print_diff_split("Diff: One vs Two tool calls (user, assistant, user)", diff); - } - - // Test 9: Tool call with vs without reasoning_content (user, assistant) - { - json user_msg = make_user_msg(); - - autoparser::generation_params params_no_reasoning; - params_no_reasoning.messages = json::array({ user_msg, make_assistant_one_tool() }); - params_no_reasoning.add_generation_prompt = false; - params_no_reasoning.tools = tools; - params_no_reasoning.enable_thinking = true; - - autoparser::generation_params params_with_reasoning = params_no_reasoning; - params_with_reasoning.messages = json::array({ user_msg, make_assistant_one_tool_with_reasoning() }); - - std::string output_no_reasoning = common_chat_template_direct_apply(chat_template, params_no_reasoning); - std::string output_with_reasoning = common_chat_template_direct_apply(chat_template, params_with_reasoning); - - auto diff = calculate_diff_split(output_no_reasoning, output_with_reasoning); - print_diff_split("Diff: Tool call with vs without reasoning_content (user, assistant)", diff); - } - - // Check reasoning variables - check_reasoning_variables(chat_template); - - } catch (const std::exception & e) { - LOG_ERR("Analysis failed: %s\n", e.what()); - } -} - -int main(int argc, char ** argv) { - // Set log level to capture all output - common_log_set_verbosity_thold(99); - - analysis_options opts; - if (!parse_options(argc, argv, opts)) { - return 1; - } - - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_PURPLE); - LOG_ERR("================================================================================\n"); - LOG_ERR(" TEMPLATE ANALYSIS TOOL\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); - LOG_ERR("Analyzing %s%zu%s template(s)\n", ANSI_CYAN, opts.template_paths.size(), ANSI_RESET); - - for (const auto & path : opts.template_paths) { - analyze_template(path); - } - - LOG_ERR("\n"); - LOG_ERR("%s", ANSI_GREEN); - LOG_ERR("================================================================================\n"); - LOG_ERR(" ANALYSIS COMPLETE\n"); - LOG_ERR("================================================================================\n"); - LOG_ERR("%s", ANSI_RESET); - - return 0; -}