From c541bfdc8227bcd645aee810c05cf5432a8c0f04 Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 16:31:19 -0400 Subject: [PATCH 1/2] diag: sample the IOMMU fault rate, because dmesg can neither be counted nor classified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#2542 quoted three populations for one boot (20, 78, 41 faults) and all three were true of the dmesg buffer when read: `printk_ratelimited` dropped 5611 fault callbacks and the ring buffer then rotated. Neither a census nor a `grep -c` can say how fast a stream is running, and only two ordered samples can. `tools/diag/iommu_fault_sampler.py` appends ` ` per interval. It imports `suppressed()` from the existing census rather than re-deriving the regex, and it decides an interval by the DROPPED delta: a frozen printed count with a growing counter is the case a dmesg-only reading calls silence. Because the total sums per-message counters, it can also go backwards when the buffer forgets an old line, which is reported as `rotated … re-baselined` instead of a negative fault count. Live on this host: `41 5611 baseline`, then three `41 5611 quiet` intervals 3 s apart -- i.e. the 08:52-08:53 stream is not still running, which a single census could not have told us. The self-test caught one wrong assumption of mine while writing it: two suppression lines for the same caller add rather than replace, so the sampled total is not monotonic unless the buffer stops rotating. That case is now a fixture and an assertion. Adds a two-sentence pointer in docs/nvme-iommu-fault-triage.md where the census is introduced, so the sampler is discoverable from the runbook that already says to split by (device, domain) first. --- docs/nvme-iommu-fault-triage.md | 5 +- tools/diag/iommu_fault_sampler.py | 155 ++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 1 deletion(-) create mode 100755 tools/diag/iommu_fault_sampler.py diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 225ab6de6..8e981c578 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -224,7 +224,10 @@ Four things generalise, and each one was a decision rather than a discovery: 1. **Split by `(device, domain)` before counting.** The NVMe stream is domain `0x0015`, the NPU's is `0x0003`; a raw `grep -c IO_PAGE_FAULT` mixes them and inflates whichever you were looking for. `tools/diag/iommu_fault_census.py` - exists for exactly this. + exists for exactly this. To measure the **rate** rather than take one census of a buffer that both throttles and + rotates, `tools/diag/iommu_fault_sampler.py` appends ` ` per interval: the + dropped-callback delta is the only count `printk_ratelimited` cannot hide, and a falling total means the ring + buffer forgot (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read). 2. **`flags` is not decoded here, and must not be until it is cited.** The streams carry **three** values, not one: ``` diff --git a/tools/diag/iommu_fault_sampler.py b/tools/diag/iommu_fault_sampler.py new file mode 100755 index 000000000..ea003b77e --- /dev/null +++ b/tools/diag/iommu_fault_sampler.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Record the IOMMU fault RATE, because `dmesg` cannot be counted. + +engine#2542 quoted three different populations for one boot -- 20, 78, 41 faults -- and all three were +true of the buffer at the moment it was read, because `printk_ratelimited` drops most of the stream and +the ring buffer discards the rest. A census taken once cannot say how fast it is happening, and a +throttled log cannot say how many times it happened; only two samples in order can. + +So this appends one line per interval: + + + + # one interval every 5s, forever (root: reading dmesg) + sudo tools/diag/iommu_fault_sampler.py --interval 5 --out /var/log/iommu-fault-rate.log + + # a bounded sample, for a window run + tools/diag/iommu_fault_sampler.py --interval 2 --count 10 --dmesg "cat fixture.txt" + +`suppressed_total` is `sum(io.census.suppressed(text).values())`: the counters grow monotonically, so +the DELTA across two lines is the number of faults the kernel dropped in that interval. A rising +suppressed_total with a flat printed count is the normal case under load and is not a quiet system -- +that is the whole point of keeping it. + +Nothing here talks to a device, and `--dmesg` takes any command, so the self-test runs on fixtures. +""" + +import argparse +import importlib.util +import os +import subprocess +import sys +import time + +_HERE = os.path.dirname(os.path.abspath(__file__)) + + +def _load_census(): + """Import the sibling census module, sharing its `suppressed()` instead of re-deriving the regex.""" + spec = importlib.util.spec_from_file_location("iommu_fault_census", + os.path.join(_HERE, "iommu_fault_census.py")) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def sample(census, text): + """(printed fault lines, dropped-callback total) for one dmesg snapshot.""" + printed = sum(1 for line in text.splitlines() if "IO_PAGE_FAULT" in line) + return printed, sum(census.suppressed(text).values()) + + +def describe(prev, cur): + """One interval's meaning, so the log does not need a human to interpret it later.""" + if prev is None: + return "baseline" + d_printed = cur[0] - prev[0] + d_dropped = cur[1] - prev[1] + if d_dropped < 0: + # The ring buffer discarded earlier suppression lines, so the total fell. That is not a negative + # rate; it is the buffer forgetting. Re-baseline instead of reporting a number that never happened. + return f"rotated ({d_dropped} from total), re-baselined" + if d_dropped > 0: + # The dropped count is the only quantity that cannot be throttled away. + return f"FAULTING +{d_dropped} dropped, +{d_printed} printed" + if d_printed > 0: + return f"faulting +{d_printed} printed, none dropped" + return "quiet" + + +def self_test(): + census = _load_census() + F = ("kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 " + "address=0x{addr} flags=0x0000]\n") + quiet = "kernel: amdxdna 0000:c6:00.1: nothing to report\n" + ratelimited = "kernel: amd_iommu_report_page_fault: 5611 callbacks suppressed\n" + + # A quiet snapshot has no lines and no counter. + assert sample(census, quiet) == (0, 0) + + # Printed lines are counted only from fault lines, and the suppressed total is the counter value. + burst = quiet + F.format(addr="4128000") + F.format(addr="4129000") + ratelimited + assert sample(census, burst) == (2, 5611), sample(census, burst) + + # The interval's meaning is decided by the DROPPED delta, not the printed delta: a frozen printed + # count with a growing counter is the case dmesg-only analysis reads as silence. + assert describe(None, (2, 5611)) == "baseline" + assert describe((2, 5611), (2, 5611)) == "quiet" + assert describe((2, 5611), (2, 9000)) == "FAULTING +3389 dropped, +0 printed" + assert describe((2, 5611), (9, 5611)) == "faulting +7 printed, none dropped" + + # Two suppression lines for the same caller SUM, because each message reports the count dropped + # since the previous one -- `census.suppressed()` adds per caller on purpose. I wrote this assertion + # the other way round first, and the self-test caught it. + two = ratelimited + "kernel: amd_iommu_report_page_fault: 5700 callbacks suppressed\n" + assert sample(census, two) == (0, 11311), sample(census, two) + + # Because the total SUMS, it can go backwards when the buffer drops an old suppression line. An + # interval must not be rendered as a negative fault count; it means the buffer forgot. + assert describe((5, 11311), (5, 5611)) == "rotated (-5700 from total), re-baselined" + + # The fixture above is the shape that misled engine#2542: two devices, one log format, so the + # printed count mixes populations. The sampler does not classify by device -- the census does -- + # it only measures rate, which is why it can stay this small. + print("self-test ok") + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--interval", type=float, default=5.0, help="seconds between samples") + ap.add_argument("--count", type=int, default=0, help="samples to take (0 = forever)") + ap.add_argument("--out", default="-", help="log file to append to ('-' = stdout)") + ap.add_argument("--dmesg", default="dmesg", help="command whose stdout is the dmesg text") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args() + + if args.selftest: + self_test() + return 0 + + census = _load_census() + out = sys.stdout if args.out == "-" else open(args.out, "a", buffering=1) + prev, n = None, 0 + try: + while not args.count or n < args.count: + text = subprocess.run(args.dmesg, shell=True, capture_output=True, text=True).stdout + cur = sample(census, text) + print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}", file=out) + prev, n = cur, n + 1 + if args.count and n >= args.count: + break + time.sleep(args.interval) + except KeyboardInterrupt: + pass + finally: + if out is not sys.stdout: + out.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From bca3c21bf643a22e3f69e397fc1fc3e2500b09ec Mon Sep 17 00:00:00 2001 From: pi-hrx-compat Date: Sun, 15 Nov 2026 17:31:40 -0400 Subject: [PATCH 2/2] diag: give the sampler's self-test a ctest row, and use the repo's --self-test spelling The build check failed on tests_referenced, and it was right: tests_referenced: tools/diag/iommu_fault_sampler.py declares a self-test and no runner names it -- add a ctest row That is exactly the rule this PR exists for, one level up: a self-test no runner names is a self-test nobody runs. The gate caught my own omission on the PR that added the tool, which is the check doing its job (CMakeLists.txt already records it catching engine#2429's). Two changes: * `--selftest` becomes `--self-test`, the spelling every other tool in the tree uses and the one the gate names, so a future reader greps one word rather than two. * `add_python_test(iommu_fault_sampler_self ... --self-test)` beside the other tool self-test rows. Verified both directions, because a check that cannot fail is the same defect: $ sed -i '/iommu_fault_sampler_self/d' CMakeLists.txt $ python3 tools/ci/tests_referenced_check.py --check --root . tests_referenced: tools/diag/iommu_fault_sampler.py declares a self-test and no runner names it rc=1 $ (row restored) rc=0 My first reading of those exit codes was wrong: `... | tail; echo $?` reports tail's status, so both runs printed rc=0 while one of them was failing. Re-measured without the pipe, which is how the 1 above was obtained. --- CMakeLists.txt | 5 +++++ tools/diag/iommu_fault_sampler.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 061e3f38e..00ab02832 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -448,6 +448,11 @@ add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py # the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a # uniform shift stays 0, and an asymmetric file set is refused by name). add_python_test(logits_kld_self ${CMAKE_SOURCE_DIR}/tools/logits_kld.py --self-test) +# engine#2542: the sampler's rate arithmetic (a frozen printed count with a growing dropped count is FAULTING, +# and a falling total is the ring buffer rotating, not a negative rate) is why the file is not a two-line shell +# loop. `tests_referenced` requires a row for any tools/**/*.py that declares --self-test, and it caught this +# omission on this PR -- which is the gate doing its job, not an obstacle. +add_python_test(iommu_fault_sampler_self ${CMAKE_SOURCE_DIR}/tools/diag/iommu_fault_sampler.py --self-test) # engine#2018: this one was in tools/ci/unreferenced_py_tests.txt as a baseline entry, and it should not have been: the # file is portable and passes (worst-case |sum| 11.18e6 inside int32 at K=11008, 4 arms), it just never had a runner. A # baseline records a test as never-run, which for a passing test is the wrong outcome -- so it is registered and the line diff --git a/tools/diag/iommu_fault_sampler.py b/tools/diag/iommu_fault_sampler.py index ea003b77e..ea7096f08 100755 --- a/tools/diag/iommu_fault_sampler.py +++ b/tools/diag/iommu_fault_sampler.py @@ -124,10 +124,10 @@ def main(): ap.add_argument("--count", type=int, default=0, help="samples to take (0 = forever)") ap.add_argument("--out", default="-", help="log file to append to ('-' = stdout)") ap.add_argument("--dmesg", default="dmesg", help="command whose stdout is the dmesg text") - ap.add_argument("--selftest", action="store_true") + ap.add_argument("--self-test", action="store_true") args = ap.parse_args() - if args.selftest: + if args.self_test: self_test() return 0