Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ggml/src/ggml-hrx/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ ggml_add_backend_library(ggml-hrx
dispatch/command-program-bindings.cpp
dispatch/command-program-bindings.h
dispatch/command-program-dump.cpp
dispatch/dispatch-shape-log.cpp
dispatch/command-program-dump.h
dispatch/command-program-diagnostics.cpp
dispatch/command-program-diagnostics.h
Expand Down
11 changes: 11 additions & 0 deletions ggml/src/ggml-hrx/dispatch/command-plan.h
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,16 @@ struct CommandPlanCompletionCounterRequest {
uint32_t count = 0;
};

// One dispatch match as the scheduler took it: the graph nodes it covers and the range of plan dispatches it emitted.
// Recorded only for GGML_HRX_DISPATCH_SHAPE_LOG (dispatch-shape-log.h), which needs a command -> ggml node mapping.
struct CommandPlanMatchRecord {
std::vector<size_t> covered_nodes;
size_t first_initialization_dispatch = 0;
size_t initialization_dispatch_count = 0;
size_t first_dispatch = 0;
size_t dispatch_count = 0;
};

struct CommandPlan {
// Initializers prepare device metadata; their writes alone do not mutate caller-owned host inputs.
// Caller-visible outputs and in-place updates belong in dispatches.
Expand All @@ -41,6 +51,7 @@ struct CommandPlan {
std::vector<CommandPlanConstantInitialization> constant_initializations;
std::vector<CommandPlanCompletionCounterRequest> completion_counter_requests;
CommandPlanMetadata metadata;
std::vector<CommandPlanMatchRecord> match_records; // empty unless the shape log is on
Status status;

bool valid() const { return status.success(); }
Expand Down
7 changes: 7 additions & 0 deletions ggml/src/ggml-hrx/dispatch/dispatch-scheduler.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include <cinttypes>
#include <cstdio>
#include "dispatch-scheduler.h"
#include "dispatch-shape-log.h"

#include "ggml.h"
#include "graph/graph-traversal.h"
Expand Down Expand Up @@ -50,6 +51,7 @@ static void clear_plan_results(CommandPlan & plan) {
plan.constant_initializations.clear();
plan.completion_counter_requests.clear();
plan.metadata.clear();
plan.match_records.clear();
}

static void append_value_summary(std::ostringstream & stream, const Graph & graph, ValueId value_id) {
Expand Down Expand Up @@ -328,6 +330,11 @@ bool DispatchScheduler::schedule_graph(Graph & graph,
clear_plan_results(plan_);
return false;
}
if (dispatch_shape_log_enabled()) {
plan_.match_records.push_back({ match.covered_nodes, plan_.initialization_dispatches.size(),
match.initialization_dispatches.size(), plan_.dispatches.size(),
match.dispatches.size() });
}
for (Dispatch & dispatch : match.initialization_dispatches) {
plan_.initialization_dispatches.push_back(std::move(dispatch));
}
Expand Down
188 changes: 188 additions & 0 deletions ggml/src/ggml-hrx/dispatch/dispatch-shape-log.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "dispatch-shape-log.h"

#include "ggml.h"

#include <atomic>
#include <cstdio>
#include <cstdlib>
#include <mutex>
#include <sstream>
#include <vector>

namespace ggml::hrx {
namespace {

constexpr const char * kShapeLogEnvironment = "GGML_HRX_DISPATCH_SHAPE_LOG";

std::string json_escape(const char * text) {
std::string out;
for (const char * c = text != nullptr ? text : ""; *c != '\0'; ++c) {
switch (*c) {
case '"':
out += "\\\"";
break;
case '\\':
out += "\\\\";
break;
case '\n':
out += "\\n";
break;
default:
if (static_cast<unsigned char>(*c) < 0x20) {
char buf[8];
std::snprintf(buf, sizeof(buf), "\\u%04x", static_cast<unsigned char>(*c));
out += buf;
} else {
out += *c;
}
}
}
return out;
}

void append_tensor(std::ostringstream & out, const Graph & graph, ValueId id) {
const Value * value = graph.values().find(id);
if (value == nullptr) {
out << "null";
return;
}
const char * name = value->tensor != nullptr ? value->tensor->name : "";
out << "{\"id\":" << value->id.value << ",\"name\":\"" << json_escape(name) << "\",\"type\":\""
<< json_escape(ggml_type_name(value->type)) << "\",\"ne\":[";
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
out << (i ? "," : "") << value->ne[i];
}
out << "],\"nb\":[";
for (int i = 0; i < GGML_MAX_DIMS; ++i) {
out << (i ? "," : "") << value->nb[i];
}
out << "]}";
}

// The match that emitted plan dispatch |index| of one phase (-1 when none recorded it).
std::vector<int> command_matches(const CommandPlan & plan, size_t command_count, bool initialization) {
std::vector<int> matches(command_count, -1);
for (size_t m = 0; m < plan.match_records.size(); ++m) {
const CommandPlanMatchRecord & record = plan.match_records[m];
const size_t first = initialization ? record.first_initialization_dispatch : record.first_dispatch;
const size_t count = initialization ? record.initialization_dispatch_count : record.dispatch_count;
for (size_t i = first; i < first + count && i < command_count; ++i) {
matches[i] = static_cast<int>(m);
}
}
return matches;
}

void append_commands(std::ostringstream & out,
const std::vector<Command> & commands,
const std::vector<int> & matches,
const char * phase,
const KernelCorpus & corpus,
const std::string & target,
bool & first) {
for (size_t i = 0; i < commands.size(); ++i) {
const Command & command = commands[i];
const KernelResolveResult resolved = resolve_kernel_definition(corpus, target, command.kernel.kernel_id);
const std::string kernel = resolved.found() ? kernel_definition_name(*resolved.definition)
: "kernel_id=" + std::to_string(command.kernel.kernel_id);
const char * symbol = resolved.found() && resolved.definition->symbol != nullptr ? resolved.definition->symbol : "";
out << (first ? "" : ",") << "{\"phase\":\"" << phase << "\",\"ordinal\":" << command.ordinal
<< ",\"kernel\":\"" << json_escape(kernel.c_str()) << "\",\"symbol\":\"" << json_escape(symbol)
<< "\",\"match\":" << (i < matches.size() ? matches[i] : -1) << "}";
first = false;
}
}

} // namespace

bool dispatch_shape_log_enabled() {
static const bool enabled = [] {
const char * path = std::getenv(kShapeLogEnvironment);
return path != nullptr && path[0] != '\0';
}();
return enabled;
}

std::string format_command_program_shape_line(const Graph & graph,
const CommandPlan & plan,
const CommandProgram & program,
const KernelCorpus & corpus,
const std::string & target,
uint64_t graph_uid,
uint64_t program_id) {
std::ostringstream out;
out << "{\"type\":\"hrx_command_program\",\"program\":" << program_id << ",\"graph_uid\":" << graph_uid
<< ",\"commands\":[";
bool first = true;
append_commands(out, program.initialization_commands,
command_matches(plan, program.initialization_commands.size(), true), "init", corpus, target, first);
append_commands(out, program.commands, command_matches(plan, program.commands.size(), false), "main", corpus,
target, first);
out << "],\"matches\":[";
const std::vector<GraphNode> & nodes = graph.nodes();
for (size_t m = 0; m < plan.match_records.size(); ++m) {
out << (m ? "," : "") << "{\"nodes\":[";
const std::vector<size_t> & covered = plan.match_records[m].covered_nodes;
for (size_t c = 0; c < covered.size(); ++c) {
out << (c ? "," : "");
if (covered[c] >= nodes.size()) {
out << "null";
continue;
}
const GraphNode & node = nodes[covered[c]];
const Value * output = graph.values().find(node.output);
const ggml_tensor * tensor = output != nullptr ? output->tensor : nullptr;
out << "{\"op\":\"" << json_escape(ggml_op_name(node.op)) << "\",\"index\":" << covered[c]
<< ",\"desc\":\"" << json_escape(tensor != nullptr ? ggml_op_desc(tensor) : ggml_op_name(node.op))
<< "\",\"out\":";
append_tensor(out, graph, node.output);
out << ",\"src\":[";
for (size_t s = 0; s < node.inputs.size(); ++s) {
out << (s ? "," : "");
append_tensor(out, graph, node.inputs[s]);
}
out << "]}";
}
out << "]}";
}
out << "]}";
return out.str();
}

void write_command_program_shape_log_if_requested(const Graph & graph,
const CommandPlan & plan,
const CommandProgram & program,
const KernelCorpus & corpus,
const std::string & target,
uint64_t graph_uid) {
if (!dispatch_shape_log_enabled()) {
return;
}
static std::atomic<uint64_t> next_program{ 0 };
static std::mutex mutex;
const std::string line =
format_command_program_shape_line(graph, plan, program, corpus, target, graph_uid, next_program++);
std::lock_guard<std::mutex> lock(mutex);
if (FILE * file = std::fopen(std::getenv(kShapeLogEnvironment), "a")) {
std::fputs(line.c_str(), file);
std::fputc('\n', file);
std::fclose(file);
}
}

} // namespace ggml::hrx
59 changes: 59 additions & 0 deletions ggml/src/ggml-hrx/dispatch/dispatch-shape-log.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// GGML_HRX_DISPATCH_SHAPE_LOG=<file>: one JSON line per built HRX command program, mapping every command to the ggml
// dispatch match that emitted it and every match to its ggml nodes and tensors. The engine's
// tools/hyperloom/hrx2kineto.py --shapes reads it next to an HRX dispatch profile (HRX_PROFILE_MODE=dispatch) and turns
// kernel names into hrx:: ops with shapes, FLOPs and bytes for TraceLens (engine #679). Off, the scheduler records
// nothing and no line is written. On, kernels are recorded in program order (graph-record-order.h), so the profile
// keys line up with the commands listed here; compare a profiled arm only against another shape-log or
// GGML_HRX_GRAPH_ORDER=program arm.
//
// Line (type "hrx_command_program"): program (a process-unique, increasing id), graph_uid, commands[] of
// {phase: "init"|"main", ordinal, kernel (definition name), symbol (the code-object export, the profile's event key),
// match (index into matches, -1 when no match emitted it)}, matches[] of {nodes: [{op, index, desc, out, src[]}]};
// tensors are {id, name, type, ne[4], nb[4]}, src in HRX's own input order (SET_ROWS: rows, ids, destination).

#pragma once

#include "command-plan.h"
#include "command-program.h"
#include "graph/graph.h"
#include "kernel-corpus/kernel-corpus.h"

#include <cstdint>
#include <string>

namespace ggml::hrx {

bool dispatch_shape_log_enabled();

std::string format_command_program_shape_line(const Graph & graph,
const CommandPlan & plan,
const CommandProgram & program,
const KernelCorpus & corpus,
const std::string & target,
uint64_t graph_uid,
uint64_t program_id);

// Appends the line for this program to the shape log when it is enabled.
void write_command_program_shape_log_if_requested(const Graph & graph,
const CommandPlan & plan,
const CommandProgram & program,
const KernelCorpus & corpus,
const std::string & target,
uint64_t graph_uid);

} // namespace ggml::hrx
2 changes: 2 additions & 0 deletions ggml/src/ggml-hrx/runtime/graph-program-cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "backend-buffer-binding.h"
#include "dispatch/command-program-dump.h"
#include "dispatch/dispatch-shape-log.h"
#include "dispatch/dispatch-scheduler.h"
#include "ggml-impl.h"
#include "ggml.h"
Expand Down Expand Up @@ -899,6 +900,7 @@ std::unique_ptr<GraphProgram> GraphProgramCache::build_program_from_imported(uin
return nullptr;
}

write_command_program_shape_log_if_requested(imported_graph, scheduler.plan(), commands, corpus, target, uid);
std::string command_shape = command_program_shape_key(commands);
Status dump_status = dump_command_program_kernels_if_requested(commands, corpus, target, command_shape);
for (const std::string & error : dump_status.errors()) {
Expand Down
5 changes: 4 additions & 1 deletion ggml/src/ggml-hrx/runtime/graph-record-order.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

#include "runtime/graph-record-order.h"

#include "dispatch/dispatch-shape-log.h"

#include "ggml-impl.h"

#include <algorithm>
Expand All @@ -34,7 +36,8 @@ constexpr size_t kMaxTransientExtent = size_t{ 16 } << 20;
bool level_order_enabled() {
static const bool enabled = [] {
const char * value = std::getenv("GGML_HRX_GRAPH_ORDER");
return value == nullptr || std::strcmp(value, "program") != 0;
// the shape log lists commands in program order; hrx2kineto matches it against the recorded order
return (value == nullptr || std::strcmp(value, "program") != 0) && !dispatch_shape_log_enabled();
}();
return enabled;
}
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-hrx/runtime/graph-record-order.h
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
// moving kernels only a few places; full dependency-level order gave the same counts but Qwen3.6-35B
// decode read 0.8% lower. Programs where reordering saves no barrier, and prompt programs (transient
// arena extent over 16 MiB), keep program order.
// GGML_HRX_GRAPH_ORDER=program records in program order. GGML_HRX_LOG_GRAPH_ORDER=1 logs the barrier
// GGML_HRX_GRAPH_ORDER=program records in program order, and so does GGML_HRX_DISPATCH_SHAPE_LOG (its lines list commands in program order). GGML_HRX_LOG_GRAPH_ORDER=1 logs the barrier
// count of both orders for every recorded graph.

#pragma once
Expand Down
Loading