From cdce1e030b0c12ab824f17bdee27dd4d7c6a17d2 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 11:14:46 -0400 Subject: [PATCH] hrx: GGML_HRX_DISPATCH_SHAPE_LOG writes the command -> ggml op map hrx2kineto reads (engine #679) The engine's tools/hyperloom/hrx2kineto.py --shapes turns an HRX dispatch profile (HRX_PROFILE_MODE=dispatch) into hrx:: ops with shapes, FLOPs and bytes for TraceLens, from a shape log that nothing produced: every TraceLens run on HRX put 100% of its ops in "other". With GGML_HRX_DISPATCH_SHAPE_LOG= the scheduler keeps, per dispatch match, the nodes it covers and the plan dispatches it emitted, and each built command program appends one hrx_command_program line: every command's phase, ordinal, definition name, export symbol (the profile's event key) and match, and every match's nodes with op, index, desc and {id, name, type, ne, nb} tensors in HRX's input order. The format is the one hrx2kineto.py parses. hrx2kineto matches a command buffer to a program by kernel sequence, and graph-record-order.cpp reorders kernels at record time to save barriers, so with the log on kernels are recorded in program order, as GGML_HRX_GRAPH_ORDER=program does. Off, nothing is recorded or written. LFM2.5-1.2B, 16 tokens, add-on at gpu-kernels d37fa7a: hrx2kineto maps all 17 graph executions to logged programs (was: "matches no logged program" for every command buffer), 3,508 of 3,596 dispatches become hrx:: ops (hrx::mul_mat 1,309, hrx::rms_norm 433, ...). hrx-backend-test dispatch_shape_log_line pins the line's fields. Not yet: hrx_execution lines (layer names per replay); hrx2kineto falls back to build-time node names without them. Co-Authored-By: Claude Opus 5.5 --- ggml/src/ggml-hrx/CMakeLists.txt | 1 + ggml/src/ggml-hrx/dispatch/command-plan.h | 11 + .../ggml-hrx/dispatch/dispatch-scheduler.cpp | 7 + .../ggml-hrx/dispatch/dispatch-shape-log.cpp | 188 ++++++++++++++++++ .../ggml-hrx/dispatch/dispatch-shape-log.h | 59 ++++++ .../ggml-hrx/runtime/graph-program-cache.cpp | 2 + .../ggml-hrx/runtime/graph-record-order.cpp | 5 +- .../src/ggml-hrx/runtime/graph-record-order.h | 2 +- tests/hrx-backend-test.cpp | 61 ++++++ 9 files changed, 334 insertions(+), 2 deletions(-) create mode 100644 ggml/src/ggml-hrx/dispatch/dispatch-shape-log.cpp create mode 100644 ggml/src/ggml-hrx/dispatch/dispatch-shape-log.h diff --git a/ggml/src/ggml-hrx/CMakeLists.txt b/ggml/src/ggml-hrx/CMakeLists.txt index 4ead6f8b94ab..98dd4d2b371e 100644 --- a/ggml/src/ggml-hrx/CMakeLists.txt +++ b/ggml/src/ggml-hrx/CMakeLists.txt @@ -227,6 +227,7 @@ ggml_add_backend_library(ggml-hrx dispatch/command-program-bindings.cpp dispatch/command-program-bindings.h dispatch/command-program-dump.cpp + dispatch/dispatch-shape-log.cpp dispatch/command-program-dump.h dispatch/command-program-diagnostics.cpp dispatch/command-program-diagnostics.h diff --git a/ggml/src/ggml-hrx/dispatch/command-plan.h b/ggml/src/ggml-hrx/dispatch/command-plan.h index 2da1a600fa02..3d0fb9a75c5b 100644 --- a/ggml/src/ggml-hrx/dispatch/command-plan.h +++ b/ggml/src/ggml-hrx/dispatch/command-plan.h @@ -32,6 +32,16 @@ struct CommandPlanCompletionCounterRequest { uint32_t count = 0; }; +// One dispatch match as the scheduler took it: the graph nodes it covers and the range of plan dispatches it emitted. +// Recorded only for GGML_HRX_DISPATCH_SHAPE_LOG (dispatch-shape-log.h), which needs a command -> ggml node mapping. +struct CommandPlanMatchRecord { + std::vector covered_nodes; + size_t first_initialization_dispatch = 0; + size_t initialization_dispatch_count = 0; + size_t first_dispatch = 0; + size_t dispatch_count = 0; +}; + struct CommandPlan { // Initializers prepare device metadata; their writes alone do not mutate caller-owned host inputs. // Caller-visible outputs and in-place updates belong in dispatches. @@ -41,6 +51,7 @@ struct CommandPlan { std::vector constant_initializations; std::vector completion_counter_requests; CommandPlanMetadata metadata; + std::vector match_records; // empty unless the shape log is on Status status; bool valid() const { return status.success(); } diff --git a/ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp b/ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp index 6f00ac7cc57e..95d5c18577f6 100644 --- a/ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp +++ b/ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp @@ -1,6 +1,7 @@ #include #include #include "dispatch-scheduler.h" +#include "dispatch-shape-log.h" #include "ggml.h" #include "graph/graph-traversal.h" @@ -50,6 +51,7 @@ static void clear_plan_results(CommandPlan & plan) { plan.constant_initializations.clear(); plan.completion_counter_requests.clear(); plan.metadata.clear(); + plan.match_records.clear(); } static void append_value_summary(std::ostringstream & stream, const Graph & graph, ValueId value_id) { @@ -328,6 +330,11 @@ bool DispatchScheduler::schedule_graph(Graph & graph, clear_plan_results(plan_); return false; } + if (dispatch_shape_log_enabled()) { + plan_.match_records.push_back({ match.covered_nodes, plan_.initialization_dispatches.size(), + match.initialization_dispatches.size(), plan_.dispatches.size(), + match.dispatches.size() }); + } for (Dispatch & dispatch : match.initialization_dispatches) { plan_.initialization_dispatches.push_back(std::move(dispatch)); } diff --git a/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.cpp b/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.cpp new file mode 100644 index 000000000000..ce7d0a95ccfa --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.cpp @@ -0,0 +1,188 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "dispatch-shape-log.h" + +#include "ggml.h" + +#include +#include +#include +#include +#include +#include + +namespace ggml::hrx { +namespace { + +constexpr const char * kShapeLogEnvironment = "GGML_HRX_DISPATCH_SHAPE_LOG"; + +std::string json_escape(const char * text) { + std::string out; + for (const char * c = text != nullptr ? text : ""; *c != '\0'; ++c) { + switch (*c) { + case '"': + out += "\\\""; + break; + case '\\': + out += "\\\\"; + break; + case '\n': + out += "\\n"; + break; + default: + if (static_cast(*c) < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast(*c)); + out += buf; + } else { + out += *c; + } + } + } + return out; +} + +void append_tensor(std::ostringstream & out, const Graph & graph, ValueId id) { + const Value * value = graph.values().find(id); + if (value == nullptr) { + out << "null"; + return; + } + const char * name = value->tensor != nullptr ? value->tensor->name : ""; + out << "{\"id\":" << value->id.value << ",\"name\":\"" << json_escape(name) << "\",\"type\":\"" + << json_escape(ggml_type_name(value->type)) << "\",\"ne\":["; + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + out << (i ? "," : "") << value->ne[i]; + } + out << "],\"nb\":["; + for (int i = 0; i < GGML_MAX_DIMS; ++i) { + out << (i ? "," : "") << value->nb[i]; + } + out << "]}"; +} + +// The match that emitted plan dispatch |index| of one phase (-1 when none recorded it). +std::vector command_matches(const CommandPlan & plan, size_t command_count, bool initialization) { + std::vector matches(command_count, -1); + for (size_t m = 0; m < plan.match_records.size(); ++m) { + const CommandPlanMatchRecord & record = plan.match_records[m]; + const size_t first = initialization ? record.first_initialization_dispatch : record.first_dispatch; + const size_t count = initialization ? record.initialization_dispatch_count : record.dispatch_count; + for (size_t i = first; i < first + count && i < command_count; ++i) { + matches[i] = static_cast(m); + } + } + return matches; +} + +void append_commands(std::ostringstream & out, + const std::vector & commands, + const std::vector & matches, + const char * phase, + const KernelCorpus & corpus, + const std::string & target, + bool & first) { + for (size_t i = 0; i < commands.size(); ++i) { + const Command & command = commands[i]; + const KernelResolveResult resolved = resolve_kernel_definition(corpus, target, command.kernel.kernel_id); + const std::string kernel = resolved.found() ? kernel_definition_name(*resolved.definition) + : "kernel_id=" + std::to_string(command.kernel.kernel_id); + const char * symbol = resolved.found() && resolved.definition->symbol != nullptr ? resolved.definition->symbol : ""; + out << (first ? "" : ",") << "{\"phase\":\"" << phase << "\",\"ordinal\":" << command.ordinal + << ",\"kernel\":\"" << json_escape(kernel.c_str()) << "\",\"symbol\":\"" << json_escape(symbol) + << "\",\"match\":" << (i < matches.size() ? matches[i] : -1) << "}"; + first = false; + } +} + +} // namespace + +bool dispatch_shape_log_enabled() { + static const bool enabled = [] { + const char * path = std::getenv(kShapeLogEnvironment); + return path != nullptr && path[0] != '\0'; + }(); + return enabled; +} + +std::string format_command_program_shape_line(const Graph & graph, + const CommandPlan & plan, + const CommandProgram & program, + const KernelCorpus & corpus, + const std::string & target, + uint64_t graph_uid, + uint64_t program_id) { + std::ostringstream out; + out << "{\"type\":\"hrx_command_program\",\"program\":" << program_id << ",\"graph_uid\":" << graph_uid + << ",\"commands\":["; + bool first = true; + append_commands(out, program.initialization_commands, + command_matches(plan, program.initialization_commands.size(), true), "init", corpus, target, first); + append_commands(out, program.commands, command_matches(plan, program.commands.size(), false), "main", corpus, + target, first); + out << "],\"matches\":["; + const std::vector & nodes = graph.nodes(); + for (size_t m = 0; m < plan.match_records.size(); ++m) { + out << (m ? "," : "") << "{\"nodes\":["; + const std::vector & covered = plan.match_records[m].covered_nodes; + for (size_t c = 0; c < covered.size(); ++c) { + out << (c ? "," : ""); + if (covered[c] >= nodes.size()) { + out << "null"; + continue; + } + const GraphNode & node = nodes[covered[c]]; + const Value * output = graph.values().find(node.output); + const ggml_tensor * tensor = output != nullptr ? output->tensor : nullptr; + out << "{\"op\":\"" << json_escape(ggml_op_name(node.op)) << "\",\"index\":" << covered[c] + << ",\"desc\":\"" << json_escape(tensor != nullptr ? ggml_op_desc(tensor) : ggml_op_name(node.op)) + << "\",\"out\":"; + append_tensor(out, graph, node.output); + out << ",\"src\":["; + for (size_t s = 0; s < node.inputs.size(); ++s) { + out << (s ? "," : ""); + append_tensor(out, graph, node.inputs[s]); + } + out << "]}"; + } + out << "]}"; + } + out << "]}"; + return out.str(); +} + +void write_command_program_shape_log_if_requested(const Graph & graph, + const CommandPlan & plan, + const CommandProgram & program, + const KernelCorpus & corpus, + const std::string & target, + uint64_t graph_uid) { + if (!dispatch_shape_log_enabled()) { + return; + } + static std::atomic next_program{ 0 }; + static std::mutex mutex; + const std::string line = + format_command_program_shape_line(graph, plan, program, corpus, target, graph_uid, next_program++); + std::lock_guard lock(mutex); + if (FILE * file = std::fopen(std::getenv(kShapeLogEnvironment), "a")) { + std::fputs(line.c_str(), file); + std::fputc('\n', file); + std::fclose(file); + } +} + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.h b/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.h new file mode 100644 index 000000000000..87e5f2ce833e --- /dev/null +++ b/ggml/src/ggml-hrx/dispatch/dispatch-shape-log.h @@ -0,0 +1,59 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GGML_HRX_DISPATCH_SHAPE_LOG=: one JSON line per built HRX command program, mapping every command to the ggml +// dispatch match that emitted it and every match to its ggml nodes and tensors. The engine's +// tools/hyperloom/hrx2kineto.py --shapes reads it next to an HRX dispatch profile (HRX_PROFILE_MODE=dispatch) and turns +// kernel names into hrx:: ops with shapes, FLOPs and bytes for TraceLens (engine #679). Off, the scheduler records +// nothing and no line is written. On, kernels are recorded in program order (graph-record-order.h), so the profile +// keys line up with the commands listed here; compare a profiled arm only against another shape-log or +// GGML_HRX_GRAPH_ORDER=program arm. +// +// Line (type "hrx_command_program"): program (a process-unique, increasing id), graph_uid, commands[] of +// {phase: "init"|"main", ordinal, kernel (definition name), symbol (the code-object export, the profile's event key), +// match (index into matches, -1 when no match emitted it)}, matches[] of {nodes: [{op, index, desc, out, src[]}]}; +// tensors are {id, name, type, ne[4], nb[4]}, src in HRX's own input order (SET_ROWS: rows, ids, destination). + +#pragma once + +#include "command-plan.h" +#include "command-program.h" +#include "graph/graph.h" +#include "kernel-corpus/kernel-corpus.h" + +#include +#include + +namespace ggml::hrx { + +bool dispatch_shape_log_enabled(); + +std::string format_command_program_shape_line(const Graph & graph, + const CommandPlan & plan, + const CommandProgram & program, + const KernelCorpus & corpus, + const std::string & target, + uint64_t graph_uid, + uint64_t program_id); + +// Appends the line for this program to the shape log when it is enabled. +void write_command_program_shape_log_if_requested(const Graph & graph, + const CommandPlan & plan, + const CommandProgram & program, + const KernelCorpus & corpus, + const std::string & target, + uint64_t graph_uid); + +} // namespace ggml::hrx diff --git a/ggml/src/ggml-hrx/runtime/graph-program-cache.cpp b/ggml/src/ggml-hrx/runtime/graph-program-cache.cpp index 25395582737c..e3a501aabaf6 100644 --- a/ggml/src/ggml-hrx/runtime/graph-program-cache.cpp +++ b/ggml/src/ggml-hrx/runtime/graph-program-cache.cpp @@ -2,6 +2,7 @@ #include "backend-buffer-binding.h" #include "dispatch/command-program-dump.h" +#include "dispatch/dispatch-shape-log.h" #include "dispatch/dispatch-scheduler.h" #include "ggml-impl.h" #include "ggml.h" @@ -899,6 +900,7 @@ std::unique_ptr GraphProgramCache::build_program_from_imported(uin return nullptr; } + write_command_program_shape_log_if_requested(imported_graph, scheduler.plan(), commands, corpus, target, uid); std::string command_shape = command_program_shape_key(commands); Status dump_status = dump_command_program_kernels_if_requested(commands, corpus, target, command_shape); for (const std::string & error : dump_status.errors()) { diff --git a/ggml/src/ggml-hrx/runtime/graph-record-order.cpp b/ggml/src/ggml-hrx/runtime/graph-record-order.cpp index 774a1fc017d0..9d8c108c5c7e 100644 --- a/ggml/src/ggml-hrx/runtime/graph-record-order.cpp +++ b/ggml/src/ggml-hrx/runtime/graph-record-order.cpp @@ -15,6 +15,8 @@ #include "runtime/graph-record-order.h" +#include "dispatch/dispatch-shape-log.h" + #include "ggml-impl.h" #include @@ -34,7 +36,8 @@ constexpr size_t kMaxTransientExtent = size_t{ 16 } << 20; bool level_order_enabled() { static const bool enabled = [] { const char * value = std::getenv("GGML_HRX_GRAPH_ORDER"); - return value == nullptr || std::strcmp(value, "program") != 0; + // the shape log lists commands in program order; hrx2kineto matches it against the recorded order + return (value == nullptr || std::strcmp(value, "program") != 0) && !dispatch_shape_log_enabled(); }(); return enabled; } diff --git a/ggml/src/ggml-hrx/runtime/graph-record-order.h b/ggml/src/ggml-hrx/runtime/graph-record-order.h index fc3ae7b6fa60..54ae64626ed8 100644 --- a/ggml/src/ggml-hrx/runtime/graph-record-order.h +++ b/ggml/src/ggml-hrx/runtime/graph-record-order.h @@ -28,7 +28,7 @@ // moving kernels only a few places; full dependency-level order gave the same counts but Qwen3.6-35B // decode read 0.8% lower. Programs where reordering saves no barrier, and prompt programs (transient // arena extent over 16 MiB), keep program order. -// GGML_HRX_GRAPH_ORDER=program records in program order. GGML_HRX_LOG_GRAPH_ORDER=1 logs the barrier +// GGML_HRX_GRAPH_ORDER=program records in program order, and so does GGML_HRX_DISPATCH_SHAPE_LOG (its lines list commands in program order). GGML_HRX_LOG_GRAPH_ORDER=1 logs the barrier // count of both orders for every recorded graph. #pragma once diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index b37d0ac9e3ab..2c9ef5f60606 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -7,6 +7,7 @@ #include "dispatch/command-program-resolver.h" #include "dispatch/command-program.h" #include "dispatch/dispatch-scheduler.h" +#include "dispatch/dispatch-shape-log.h" #include "dispatch_registration/common/dispatch-gather-add.h" #include "dispatch_registration/dispatch-registry.h" #include "ggml-alloc.h" @@ -2026,6 +2027,65 @@ static void run_external_write_retarget_offset_checks() { ggml_free(ctx); } +// GGML_HRX_DISPATCH_SHAPE_LOG line for one scheduled ADD: the command names its export symbol and its match, the match +// names its node with shapes. tools/hyperloom/hrx2kineto.py --shapes reads these fields (engine #679). +static void run_dispatch_shape_log_line_checks() { + ggml_init_params params = {}; + params.mem_size = 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8); + ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, 8); + ggml_tensor * out = ggml_add(ctx, a, b); + REQUIRE(a != nullptr && b != nullptr && out != nullptr); + ggml_set_name(out, "shape_log_sum"); + ggml_cgraph * graph = ggml_new_graph(ctx); + REQUIRE(graph != nullptr); + ggml_build_forward_expand(graph, out); + + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + REQUIRE(imported.graph.nodes().size() == 1); + const ggml::hrx::Value * a_value = imported.graph.values().find_tensor(a); + const ggml::hrx::Value * b_value = imported.graph.values().find_tensor(b); + const ggml::hrx::Value * out_value = imported.graph.values().find_tensor(out); + REQUIRE(a_value != nullptr && b_value != nullptr && out_value != nullptr); + + // the plan the binary f32 matcher builds for this node, and the match record the scheduler keeps with the log on + ggml::hrx::CommandPlan plan; + ggml::hrx::Dispatch dispatch; + dispatch.kernel = + ggml::hrx::make_kernel_specialization(ggml::hrx::kernel_catalog_ref("loom_libs", "ggml_binary_f32")); + add_binary_f32_exact_dispatch_params(dispatch, 8); + dispatch.bindings.push_back({ a_value->id, 0, a_value->byte_count }); + dispatch.bindings.push_back({ b_value->id, 0, b_value->byte_count }); + dispatch.bindings.push_back({ out_value->id, 0, out_value->byte_count }); + plan.dispatches.push_back(std::move(dispatch)); + plan.match_records = { { { 0 }, 0, 0, 0, 1 } }; + + const ggml::hrx::CommandProgram commands = + ggml::hrx::build_command_program(imported.graph, plan, ggml::hrx::get_qwen_kernel_corpus(), "gfx1151"); + REQUIRE(commands.valid()); + REQUIRE(commands.commands.size() == 1); + const std::string line = ggml::hrx::format_command_program_shape_line( + imported.graph, plan, commands, ggml::hrx::get_qwen_kernel_corpus(), "gfx1151", 77, 5); + REQUIRE(line.rfind("{\"type\":\"hrx_command_program\",\"program\":5,\"graph_uid\":77,", 0) == 0); + REQUIRE(string_contains(line, "\"phase\":\"main\",\"ordinal\":0,")); + REQUIRE(string_contains(line, "\"symbol\":\"ggml_binary_f32\",\"match\":0}")); + REQUIRE(string_contains(line, "\"op\":\"ADD\",\"index\":0,")); + REQUIRE(string_contains(line, "\"name\":\"shape_log_sum\",\"type\":\"f32\",\"ne\":[8,1,1,1],\"nb\":[4,32,32,32]")); + REQUIRE(line.back() == '}'); + + // a command no recorded match emitted says so + plan.match_records.clear(); + const std::string unmatched = ggml::hrx::format_command_program_shape_line( + imported.graph, plan, commands, ggml::hrx::get_qwen_kernel_corpus(), "gfx1151", 77, 6); + REQUIRE(string_contains(unmatched, "\"match\":-1}")); + REQUIRE(string_contains(unmatched, "\"matches\":[]}")); + ggml_free(ctx); +} + static void run_completion_counter_plan_checks() { ggml::hrx::Graph graph; ggml::hrx::CommandPlan plan; @@ -15790,6 +15850,7 @@ int main(int argc, char ** argv) { { "graph_snapshot_diagnostics", [] { run_graph_snapshot_diagnostics_checks(); } }, { "unmatched_graph_diagnostics", [] { run_unmatched_graph_diagnostics_checks(); } }, { "completion_counter_plan", [] { run_completion_counter_plan_checks(); } }, + { "dispatch_shape_log_line", [] { run_dispatch_shape_log_line_checks(); } }, { "external_write_retarget_offset", [] { run_external_write_retarget_offset_checks(); } }, { "graph_index", [] { run_graph_index_checks(); } }, { "rope_set_rows_dispatch", [] { run_rope_set_rows_dispatch_checks(); } },