From ab87240f38da9416e08a53309cac0ce6cd9aa740 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 20:49:37 -0400 Subject: [PATCH] tools: hrx_dispatch_profile.py, time per HRX dispatch from the backend's serial trace (engine #679) The profiling slice of #679 was blocked on "no binary on this box emits per-dispatch timing". The backend already has a serial mode (GGML_HRX_DEBUG_SERIAL_EXECUTION=1) that waits for the device after every dispatch and logs the launch and the completion with the kernel id, but no time and no name. The tool sets the switch, time-stamps the lines as they arrive, sums per command program and kernel, and names the kernels from the pinned fork's manifests (the catalog id is FNV-1a over family and name). It needs no rebuild and works on the pinned build. docs/hrx.md: how to run it, a measured table (Qwen3-4B, 512 against 8,192 tokens of prompt), and what the numbers are not (the serial mode is slower than a normal run). ctest: hrx_dispatch_profile_self. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 2 + docs/hrx.md | 28 +++++ tools/hrx_dispatch_profile.py | 224 ++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+) create mode 100755 tools/hrx_dispatch_profile.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a0b3ce73b..a478ebcf4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -355,6 +355,8 @@ add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-tes add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) +# engine#679: time per HRX dispatch from the backend's serial trace (the id hash and the per-kernel sums). +add_python_test(hrx_dispatch_profile_self ${CMAKE_SOURCE_DIR}/tools/hrx_dispatch_profile.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/docs/hrx.md b/docs/hrx.md index 184f21fd2..7870a0af9 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -864,6 +864,34 @@ a token threshold still reads the loader's copy. "Not attributed" is mostly runt staging buffer is capped at 64 MiB since [llama.cpp #266](https://github.com/1bit-MONSTER/llama.cpp/pull/266)). It is not weights. This replaces the `bo_trace.py` reproduction in the issue body, which was never committed. +**Which kernel a graph's time is in.** `tools/hrx_dispatch_profile.py` gives the time per HRX dispatch, summed per kernel, +on any HRX build (it needs no rebuild). The backend's serial mode (`GGML_HRX_DEBUG_SERIAL_EXECUTION=1`) waits for the device +after every dispatch and prints a line at launch and at completion; the tool sets the switch, time-stamps those lines and +names the kernels from the pinned fork's manifests. Inside your ticket: + +``` +gpu-hold -l -t 10 -- tools/hrx_dispatch_profile.py run --log prof.log --json prof.json -- \ + llama-bench -m Qwen3-4B-Q4_K_M.gguf -ngl 99 -p 8192 -n 16 -r 1 +tools/hrx_dispatch_profile.py parse prof.log --top 20 # again, from the trace it kept +``` + +It prints one table per command program (a graph shape: prefill batch, decode step). Measured on the pin `2293f95ea`, +Qwen3-4B Q4_K_M, the prefill program of a 512-token prompt against the most frequent prefill program of an 8,192-token +prompt (12 of its 16 batches of 512), one run each, 2026-11-11: + +| kernel | 512-token prompt, ms each | share | 8,192-token prompt, ms each | share | +|---|---:|---:|---:|---:| +| `ggml_flash_attention_f32_f16_wmma` | 0.371 | 7.5% | 2.150 | 49.3% | +| `ggml_mul_mat_swiglu_q4_k_f16_wmma_prefill_wave32` | 1.714 | 33.6% | 1.728 | 19.3% | +| `ggml_mul_mat_q4_k_f16_wmma_prefill_wave32` | 0.425 | 34.2% | 0.393 | 18.0% | +| `ggml_mul_mat_q6_k_f16_wmma_prefill_wave32` | 0.635 | 12.4% | 0.632 | 7.0% | + +The matmuls cost the same at any context; the attention kernel is the part that grows. + +What the numbers are and are not: each is the wall time from launch to device-idle with one dispatch in flight, so it holds +one wake-up of the waiting thread and the run is slower than a normal one (the same command: pp512 2,977 -> 2,500 tok/s, +tg16 67 -> 36, pp8192 1,191 -> 1,167). Use it to rank kernels and to see what grows with the context, not as a rate. + ### Measuring on `HRX0`: what makes a number citable Numbers measured on this device swing far more between *runs* than between *files*, so the harness matters as much as the arm. Everything below is diff --git a/tools/hrx_dispatch_profile.py b/tools/hrx_dispatch_profile.py new file mode 100755 index 000000000..ec7edbb37 --- /dev/null +++ b/tools/hrx_dispatch_profile.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""hrx_dispatch_profile: time per HRX dispatch, from the backend's own serial trace (engine#679). + + hrx_dispatch_profile.py run [--log FILE] [--top N] [--json FILE] -- llama-bench -m M.gguf -ngl 99 ... + hrx_dispatch_profile.py parse LOG [--top N] [--json FILE] + hrx_dispatch_profile.py --self-test + +The HRX backend has a serial mode, GGML_HRX_DEBUG_SERIAL_EXECUTION=1: it waits for the device after every dispatch +and prints one line when the dispatch is launched and one when the device has finished it. The lines carry the +kernel's id but no time and no name. `run` sets that switch, stamps every trace line with a monotonic time as it +arrives, and adds the two up per kernel; the names come from the kernel manifests of the pinned llama.cpp fork. + +What the numbers are: the wall time from "dispatch launched" to "device idle", one dispatch at a time. That is the +kernel's device time plus one wake-up of the waiting thread, so a sum over a graph is larger than the same graph in +a normal run, where dispatches are queued back to back. Use it to see WHICH kernel a graph's time is in and how +that changes with the context length, not as a tok/s figure. Run the command under gpu-hold like any device run. +""" +import argparse +import collections +import glob +import json +import os +import re +import subprocess +import sys +import time + +TRACE_PREFIX = "hrx debug serial: " +DEFAULT_MANIFESTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "third_party", "llama.cpp", "ggml", + "src", "ggml-hrx", "kernel-corpus", "kernels") +# manifests that do not name a family per export use the directory's catalog family +DIRECTORY_FAMILY = {"qwen_moe": "qwen3_moe", "qwen": "qwen", "hrx": "hrx", "loom-libs": "loom_libs"} + + +def kernel_id(family, name): + """kernel_catalog_id() of the fork's kernel-corpus-catalog.h: FNV-1a over family, a zero byte, name.""" + mask, prime = (1 << 64) - 1, 1099511628211 + value = 1469598103934665603 + for byte in family.encode(): + value = ((value ^ byte) * prime) & mask + value = (value * prime) & mask + for byte in name.encode(): + value = ((value ^ byte) * prime) & mask + return value + + +def load_kernel_names(manifest_dir): + names = {} + for path in sorted(glob.glob(os.path.join(manifest_dir, "*", "manifest.json"))): + directory = os.path.basename(os.path.dirname(path)) + with open(path) as handle: + manifest = json.load(handle) + for export in manifest.get("exports", []): + family = export.get("family") or DIRECTORY_FAMILY.get(directory, directory) + names[kernel_id(family, export["name"])] = family + ":" + export["name"] + return names + + +def parse_fields(text): + return dict(re.findall(r"(\w+)=(\S+)", text)) + + +def profile(lines): + """lines: iterable of (time_ns, trace text without the prefix). Returns the per-program, per-kernel totals.""" + programs = collections.OrderedDict() + open_dispatch = {} + for time_ns, text in lines: + event, _, rest = text.partition(" ") + fields = parse_fields(rest) + phase = fields.get("phase") + if "kernel_id" not in fields or "index" not in fields: + continue + key = (fields.get("list"), fields.get("list_commands"), fields["index"]) + if event == "begin" and phase == "command-dispatch": + open_dispatch[key] = [time_ns, None] + elif event == "end" and phase == "command-dispatch" and key in open_dispatch: + open_dispatch[key][1] = time_ns + elif event == "sync-end" and phase == "command-post-dispatch" and key in open_dispatch: + begin_ns, launched_ns = open_dispatch.pop(key) + program = programs.setdefault((fields.get("list"), int(fields.get("list_commands", 0))), {}) + row = program.setdefault(int(fields["kernel_id"]), {"dispatches": 0, "total_ns": 0, "launch_ns": 0}) + row["dispatches"] += 1 + row["total_ns"] += time_ns - begin_ns + row["launch_ns"] += (launched_ns if launched_ns is not None else begin_ns) - begin_ns + return programs + + +def report(programs, names, top, out=sys.stdout): + result = [] + for (list_kind, command_count), kernels in programs.items(): + total_ns = sum(row["total_ns"] for row in kernels.values()) + dispatches = sum(row["dispatches"] for row in kernels.values()) + runs = dispatches / command_count if command_count else 0.0 + print(f"\nprogram list={list_kind} commands={command_count}: {dispatches} dispatches " + f"({runs:.1f} runs), {total_ns / 1e6:.1f} ms in total", file=out) + print(f" {'ms':>10} {'share':>6} {'count':>7} {'ms each':>9} {'launch ms':>10} kernel", file=out) + rows = sorted(kernels.items(), key=lambda item: -item[1]["total_ns"]) + entry = {"list": list_kind, "commands": command_count, "dispatches": dispatches, "total_ms": total_ns / 1e6, + "kernels": []} + for index, (kid, row) in enumerate(rows): + name = names.get(kid, str(kid)) + share = row["total_ns"] / total_ns if total_ns else 0.0 + entry["kernels"].append({"kernel": name, "dispatches": row["dispatches"], "ms": row["total_ns"] / 1e6, + "launch_ms": row["launch_ns"] / 1e6, "share": share}) + if index < top: + print(f" {row['total_ns'] / 1e6:10.2f} {share:6.1%} {row['dispatches']:7d} " + f"{row['total_ns'] / 1e6 / row['dispatches']:9.3f} {row['launch_ns'] / 1e6:10.2f} {name}", + file=out) + if len(rows) > top: + print(f" ... {len(rows) - top} more kernels (--top)", file=out) + result.append(entry) + if not result: + print("no dispatch was traced: is the binary an HRX build, and did the run reach the device?", file=out) + return result + + +def read_log(path): + with open(path) as handle: + for line in handle: + stamp, _, text = line.rstrip("\n").partition(" ") + if stamp.isdigit() and text.startswith(TRACE_PREFIX): + yield int(stamp), text[len(TRACE_PREFIX):] + + +def run_command(command, log_path): + env = dict(os.environ, GGML_HRX_DEBUG_SERIAL_EXECUTION="1") + lines = [] + log = open(log_path, "w") if log_path else None + process = subprocess.Popen(command, env=env, stderr=subprocess.PIPE, text=True, errors="replace", bufsize=1) + try: + for line in process.stderr: + now = time.monotonic_ns() + if line.startswith(TRACE_PREFIX): + lines.append((now, line[len(TRACE_PREFIX):].rstrip("\n"))) + if log: + log.write(f"{now} {line}") + else: + sys.stderr.write(line) + finally: + process.wait() + if log: + log.close() + return lines, process.returncode + + +def self_test(): + assert kernel_id("loom_libs", "ggml_mul_mat_f32_f32_wmma") == 5916100870017593149, "the catalog id hash changed" + detail = "list=main index={i} list_commands=2 ordinal={i} kind=Kernel kernel_id={k} bindings=3" + trace = [] + clock = 1000 + for run in range(2): + for index, (kid, launch, device) in enumerate([(7, 10, 90), (9, 5, 20)]): + text = detail.format(i=index, k=kid) + trace.append((clock, "sync-begin phase=command-pre-dispatch " + text)) + trace.append((clock, "sync-end phase=command-pre-dispatch " + text)) + trace.append((clock, "begin phase=command-dispatch " + text)) + trace.append((clock + launch, "end phase=command-dispatch " + text)) + trace.append((clock + launch, "sync-begin phase=command-post-dispatch " + text)) + trace.append((clock + launch + device, "sync-end phase=command-post-dispatch " + text)) + clock += launch + device + 1000 # host time between dispatches is not charged to a kernel + trace.append((clock, "begin phase=command-list list=main list_commands=2 main_commands=2")) + programs = profile(trace) + assert list(programs) == [("main", 2)], programs + assert programs[("main", 2)][7] == {"dispatches": 2, "total_ns": 200, "launch_ns": 20}, programs + assert programs[("main", 2)][9] == {"dispatches": 2, "total_ns": 50, "launch_ns": 10}, programs + with open(os.devnull, "w") as sink: + result = report(programs, {7: "f:slow", 9: "f:fast"}, 1, out=sink) + assert report({}, {}, 5, out=sink) == [] + assert [k["kernel"] for k in result[0]["kernels"]] == ["f:slow", "f:fast"], result + assert abs(result[0]["kernels"][0]["share"] - 0.8) < 1e-9, result + print("hrx_dispatch_profile self-test: OK") + return 0 + + +def main(): + if "--self-test" in sys.argv[1:]: + return self_test() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="mode", required=True) + for mode in ("run", "parse"): + p = sub.add_parser(mode) + p.add_argument("--top", type=int, default=12, help="kernels printed per program (default 12)") + p.add_argument("--json", help="write the full table here") + p.add_argument("--manifests", default=DEFAULT_MANIFESTS, help="the fork's kernel-corpus/kernels directory") + if mode == "run": + p.add_argument("--log", help="keep the time-stamped trace here (on disk, not /tmp)") + p.add_argument("command", nargs=argparse.REMAINDER, help="-- CMD ARGS...") + else: + p.add_argument("log") + args = parser.parse_args() + names = load_kernel_names(args.manifests) + if not names: + print(f"note: no kernel manifest under {args.manifests}; kernels are shown by id", file=sys.stderr) + code = 0 + if args.mode == "run": + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("run needs a command after --") + lines, code = run_command(command, args.log) + else: + lines = list(read_log(args.log)) + result = report(profile(lines), names, args.top) + if args.json: + with open(args.json, "w") as handle: + json.dump(result, handle, indent=1) + return code if code else (0 if result else 1) + + +if __name__ == "__main__": + sys.exit(main())