diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e18dcbef..728e7efae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -356,6 +356,8 @@ add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-tes add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) # engine#856: one HRX process's GTT split into the loader's copy and HRX's packed second copy, from its own -v log. add_python_test(hrx_gtt_split_self ${CMAKE_SOURCE_DIR}/tools/hrx_gtt_split.py --self-test) +# engine#679: time per HRX dispatch from the backend's serial trace (the id hash and the per-kernel sums). +add_python_test(hrx_dispatch_profile_self ${CMAKE_SOURCE_DIR}/tools/hrx_dispatch_profile.py --self-test) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/docs/hrx.md b/docs/hrx.md index 184f21fd2..7870a0af9 100644 --- a/docs/hrx.md +++ b/docs/hrx.md @@ -864,6 +864,34 @@ a token threshold still reads the loader's copy. "Not attributed" is mostly runt staging buffer is capped at 64 MiB since [llama.cpp #266](https://github.com/1bit-MONSTER/llama.cpp/pull/266)). It is not weights. This replaces the `bo_trace.py` reproduction in the issue body, which was never committed. +**Which kernel a graph's time is in.** `tools/hrx_dispatch_profile.py` gives the time per HRX dispatch, summed per kernel, +on any HRX build (it needs no rebuild). The backend's serial mode (`GGML_HRX_DEBUG_SERIAL_EXECUTION=1`) waits for the device +after every dispatch and prints a line at launch and at completion; the tool sets the switch, time-stamps those lines and +names the kernels from the pinned fork's manifests. Inside your ticket: + +``` +gpu-hold -l -t 10 -- tools/hrx_dispatch_profile.py run --log prof.log --json prof.json -- \ + llama-bench -m Qwen3-4B-Q4_K_M.gguf -ngl 99 -p 8192 -n 16 -r 1 +tools/hrx_dispatch_profile.py parse prof.log --top 20 # again, from the trace it kept +``` + +It prints one table per command program (a graph shape: prefill batch, decode step). Measured on the pin `2293f95ea`, +Qwen3-4B Q4_K_M, the prefill program of a 512-token prompt against the most frequent prefill program of an 8,192-token +prompt (12 of its 16 batches of 512), one run each, 2026-11-11: + +| kernel | 512-token prompt, ms each | share | 8,192-token prompt, ms each | share | +|---|---:|---:|---:|---:| +| `ggml_flash_attention_f32_f16_wmma` | 0.371 | 7.5% | 2.150 | 49.3% | +| `ggml_mul_mat_swiglu_q4_k_f16_wmma_prefill_wave32` | 1.714 | 33.6% | 1.728 | 19.3% | +| `ggml_mul_mat_q4_k_f16_wmma_prefill_wave32` | 0.425 | 34.2% | 0.393 | 18.0% | +| `ggml_mul_mat_q6_k_f16_wmma_prefill_wave32` | 0.635 | 12.4% | 0.632 | 7.0% | + +The matmuls cost the same at any context; the attention kernel is the part that grows. + +What the numbers are and are not: each is the wall time from launch to device-idle with one dispatch in flight, so it holds +one wake-up of the waiting thread and the run is slower than a normal one (the same command: pp512 2,977 -> 2,500 tok/s, +tg16 67 -> 36, pp8192 1,191 -> 1,167). Use it to rank kernels and to see what grows with the context, not as a rate. + ### Measuring on `HRX0`: what makes a number citable Numbers measured on this device swing far more between *runs* than between *files*, so the harness matters as much as the arm. Everything below is diff --git a/tools/hrx_dispatch_profile.py b/tools/hrx_dispatch_profile.py new file mode 100755 index 000000000..ec7edbb37 --- /dev/null +++ b/tools/hrx_dispatch_profile.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""hrx_dispatch_profile: time per HRX dispatch, from the backend's own serial trace (engine#679). + + hrx_dispatch_profile.py run [--log FILE] [--top N] [--json FILE] -- llama-bench -m M.gguf -ngl 99 ... + hrx_dispatch_profile.py parse LOG [--top N] [--json FILE] + hrx_dispatch_profile.py --self-test + +The HRX backend has a serial mode, GGML_HRX_DEBUG_SERIAL_EXECUTION=1: it waits for the device after every dispatch +and prints one line when the dispatch is launched and one when the device has finished it. The lines carry the +kernel's id but no time and no name. `run` sets that switch, stamps every trace line with a monotonic time as it +arrives, and adds the two up per kernel; the names come from the kernel manifests of the pinned llama.cpp fork. + +What the numbers are: the wall time from "dispatch launched" to "device idle", one dispatch at a time. That is the +kernel's device time plus one wake-up of the waiting thread, so a sum over a graph is larger than the same graph in +a normal run, where dispatches are queued back to back. Use it to see WHICH kernel a graph's time is in and how +that changes with the context length, not as a tok/s figure. Run the command under gpu-hold like any device run. +""" +import argparse +import collections +import glob +import json +import os +import re +import subprocess +import sys +import time + +TRACE_PREFIX = "hrx debug serial: " +DEFAULT_MANIFESTS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "third_party", "llama.cpp", "ggml", + "src", "ggml-hrx", "kernel-corpus", "kernels") +# manifests that do not name a family per export use the directory's catalog family +DIRECTORY_FAMILY = {"qwen_moe": "qwen3_moe", "qwen": "qwen", "hrx": "hrx", "loom-libs": "loom_libs"} + + +def kernel_id(family, name): + """kernel_catalog_id() of the fork's kernel-corpus-catalog.h: FNV-1a over family, a zero byte, name.""" + mask, prime = (1 << 64) - 1, 1099511628211 + value = 1469598103934665603 + for byte in family.encode(): + value = ((value ^ byte) * prime) & mask + value = (value * prime) & mask + for byte in name.encode(): + value = ((value ^ byte) * prime) & mask + return value + + +def load_kernel_names(manifest_dir): + names = {} + for path in sorted(glob.glob(os.path.join(manifest_dir, "*", "manifest.json"))): + directory = os.path.basename(os.path.dirname(path)) + with open(path) as handle: + manifest = json.load(handle) + for export in manifest.get("exports", []): + family = export.get("family") or DIRECTORY_FAMILY.get(directory, directory) + names[kernel_id(family, export["name"])] = family + ":" + export["name"] + return names + + +def parse_fields(text): + return dict(re.findall(r"(\w+)=(\S+)", text)) + + +def profile(lines): + """lines: iterable of (time_ns, trace text without the prefix). Returns the per-program, per-kernel totals.""" + programs = collections.OrderedDict() + open_dispatch = {} + for time_ns, text in lines: + event, _, rest = text.partition(" ") + fields = parse_fields(rest) + phase = fields.get("phase") + if "kernel_id" not in fields or "index" not in fields: + continue + key = (fields.get("list"), fields.get("list_commands"), fields["index"]) + if event == "begin" and phase == "command-dispatch": + open_dispatch[key] = [time_ns, None] + elif event == "end" and phase == "command-dispatch" and key in open_dispatch: + open_dispatch[key][1] = time_ns + elif event == "sync-end" and phase == "command-post-dispatch" and key in open_dispatch: + begin_ns, launched_ns = open_dispatch.pop(key) + program = programs.setdefault((fields.get("list"), int(fields.get("list_commands", 0))), {}) + row = program.setdefault(int(fields["kernel_id"]), {"dispatches": 0, "total_ns": 0, "launch_ns": 0}) + row["dispatches"] += 1 + row["total_ns"] += time_ns - begin_ns + row["launch_ns"] += (launched_ns if launched_ns is not None else begin_ns) - begin_ns + return programs + + +def report(programs, names, top, out=sys.stdout): + result = [] + for (list_kind, command_count), kernels in programs.items(): + total_ns = sum(row["total_ns"] for row in kernels.values()) + dispatches = sum(row["dispatches"] for row in kernels.values()) + runs = dispatches / command_count if command_count else 0.0 + print(f"\nprogram list={list_kind} commands={command_count}: {dispatches} dispatches " + f"({runs:.1f} runs), {total_ns / 1e6:.1f} ms in total", file=out) + print(f" {'ms':>10} {'share':>6} {'count':>7} {'ms each':>9} {'launch ms':>10} kernel", file=out) + rows = sorted(kernels.items(), key=lambda item: -item[1]["total_ns"]) + entry = {"list": list_kind, "commands": command_count, "dispatches": dispatches, "total_ms": total_ns / 1e6, + "kernels": []} + for index, (kid, row) in enumerate(rows): + name = names.get(kid, str(kid)) + share = row["total_ns"] / total_ns if total_ns else 0.0 + entry["kernels"].append({"kernel": name, "dispatches": row["dispatches"], "ms": row["total_ns"] / 1e6, + "launch_ms": row["launch_ns"] / 1e6, "share": share}) + if index < top: + print(f" {row['total_ns'] / 1e6:10.2f} {share:6.1%} {row['dispatches']:7d} " + f"{row['total_ns'] / 1e6 / row['dispatches']:9.3f} {row['launch_ns'] / 1e6:10.2f} {name}", + file=out) + if len(rows) > top: + print(f" ... {len(rows) - top} more kernels (--top)", file=out) + result.append(entry) + if not result: + print("no dispatch was traced: is the binary an HRX build, and did the run reach the device?", file=out) + return result + + +def read_log(path): + with open(path) as handle: + for line in handle: + stamp, _, text = line.rstrip("\n").partition(" ") + if stamp.isdigit() and text.startswith(TRACE_PREFIX): + yield int(stamp), text[len(TRACE_PREFIX):] + + +def run_command(command, log_path): + env = dict(os.environ, GGML_HRX_DEBUG_SERIAL_EXECUTION="1") + lines = [] + log = open(log_path, "w") if log_path else None + process = subprocess.Popen(command, env=env, stderr=subprocess.PIPE, text=True, errors="replace", bufsize=1) + try: + for line in process.stderr: + now = time.monotonic_ns() + if line.startswith(TRACE_PREFIX): + lines.append((now, line[len(TRACE_PREFIX):].rstrip("\n"))) + if log: + log.write(f"{now} {line}") + else: + sys.stderr.write(line) + finally: + process.wait() + if log: + log.close() + return lines, process.returncode + + +def self_test(): + assert kernel_id("loom_libs", "ggml_mul_mat_f32_f32_wmma") == 5916100870017593149, "the catalog id hash changed" + detail = "list=main index={i} list_commands=2 ordinal={i} kind=Kernel kernel_id={k} bindings=3" + trace = [] + clock = 1000 + for run in range(2): + for index, (kid, launch, device) in enumerate([(7, 10, 90), (9, 5, 20)]): + text = detail.format(i=index, k=kid) + trace.append((clock, "sync-begin phase=command-pre-dispatch " + text)) + trace.append((clock, "sync-end phase=command-pre-dispatch " + text)) + trace.append((clock, "begin phase=command-dispatch " + text)) + trace.append((clock + launch, "end phase=command-dispatch " + text)) + trace.append((clock + launch, "sync-begin phase=command-post-dispatch " + text)) + trace.append((clock + launch + device, "sync-end phase=command-post-dispatch " + text)) + clock += launch + device + 1000 # host time between dispatches is not charged to a kernel + trace.append((clock, "begin phase=command-list list=main list_commands=2 main_commands=2")) + programs = profile(trace) + assert list(programs) == [("main", 2)], programs + assert programs[("main", 2)][7] == {"dispatches": 2, "total_ns": 200, "launch_ns": 20}, programs + assert programs[("main", 2)][9] == {"dispatches": 2, "total_ns": 50, "launch_ns": 10}, programs + with open(os.devnull, "w") as sink: + result = report(programs, {7: "f:slow", 9: "f:fast"}, 1, out=sink) + assert report({}, {}, 5, out=sink) == [] + assert [k["kernel"] for k in result[0]["kernels"]] == ["f:slow", "f:fast"], result + assert abs(result[0]["kernels"][0]["share"] - 0.8) < 1e-9, result + print("hrx_dispatch_profile self-test: OK") + return 0 + + +def main(): + if "--self-test" in sys.argv[1:]: + return self_test() + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="mode", required=True) + for mode in ("run", "parse"): + p = sub.add_parser(mode) + p.add_argument("--top", type=int, default=12, help="kernels printed per program (default 12)") + p.add_argument("--json", help="write the full table here") + p.add_argument("--manifests", default=DEFAULT_MANIFESTS, help="the fork's kernel-corpus/kernels directory") + if mode == "run": + p.add_argument("--log", help="keep the time-stamped trace here (on disk, not /tmp)") + p.add_argument("command", nargs=argparse.REMAINDER, help="-- CMD ARGS...") + else: + p.add_argument("log") + args = parser.parse_args() + names = load_kernel_names(args.manifests) + if not names: + print(f"note: no kernel manifest under {args.manifests}; kernels are shown by id", file=sys.stderr) + code = 0 + if args.mode == "run": + command = args.command[1:] if args.command[:1] == ["--"] else args.command + if not command: + parser.error("run needs a command after --") + lines, code = run_command(command, args.log) + else: + lines = list(read_log(args.log)) + result = report(profile(lines), names, args.top) + if args.json: + with open(args.json, "w") as handle: + json.dump(result, handle, indent=1) + return code if code else (0 if result else 1) + + +if __name__ == "__main__": + sys.exit(main())