Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,11 @@ add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py
# the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a
# uniform shift stays 0, and an asymmetric file set is refused by name).
add_python_test(logits_kld_self ${CMAKE_SOURCE_DIR}/tools/logits_kld.py --self-test)
# engine#2542: the sampler's rate arithmetic (a frozen printed count with a growing dropped count is FAULTING,
# and a falling total is the ring buffer rotating, not a negative rate) is why the file is not a two-line shell
# loop. `tests_referenced` requires a row for any tools/**/*.py that declares --self-test, and it caught this
# omission on this PR -- which is the gate doing its job, not an obstacle.
add_python_test(iommu_fault_sampler_self ${CMAKE_SOURCE_DIR}/tools/diag/iommu_fault_sampler.py --self-test)
# engine#2018: this one was in tools/ci/unreferenced_py_tests.txt as a baseline entry, and it should not have been: the
# file is portable and passes (worst-case |sum| 11.18e6 inside int32 at K=11008, 4 arms), it just never had a runner. A
# baseline records a test as never-run, which for a passing test is the wrong outcome -- so it is registered and the line
Expand Down
5 changes: 4 additions & 1 deletion docs/nvme-iommu-fault-triage.md
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,10 @@ Four things generalise, and each one was a decision rather than a discovery:

1. **Split by `(device, domain)` before counting.** The NVMe stream is domain `0x0015`, the NPU's is `0x0003`; a raw
`grep -c IO_PAGE_FAULT` mixes them and inflates whichever you were looking for. `tools/diag/iommu_fault_census.py`
exists for exactly this.
exists for exactly this. To measure the **rate** rather than take one census of a buffer that both throttles and
rotates, `tools/diag/iommu_fault_sampler.py` appends `<epoch> <printed> <suppressed_total>` per interval: the
dropped-callback delta is the only count `printk_ratelimited` cannot hide, and a falling total means the ring
buffer forgot (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read).
2. **`flags` is not decoded here, and must not be until it is cited.** The streams carry **three** values, not one:

```
Expand Down
155 changes: 155 additions & 0 deletions tools/diag/iommu_fault_sampler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Record the IOMMU fault RATE, because `dmesg` cannot be counted.

engine#2542 quoted three different populations for one boot -- 20, 78, 41 faults -- and all three were
true of the buffer at the moment it was read, because `printk_ratelimited` drops most of the stream and
the ring buffer discards the rest. A census taken once cannot say how fast it is happening, and a
throttled log cannot say how many times it happened; only two samples in order can.

So this appends one line per interval:

<epoch> <printed_fault_lines> <suppressed_total>

# one interval every 5s, forever (root: reading dmesg)
sudo tools/diag/iommu_fault_sampler.py --interval 5 --out /var/log/iommu-fault-rate.log

# a bounded sample, for a window run
tools/diag/iommu_fault_sampler.py --interval 2 --count 10 --dmesg "cat fixture.txt"

`suppressed_total` is `sum(io.census.suppressed(text).values())`: the counters grow monotonically, so
the DELTA across two lines is the number of faults the kernel dropped in that interval. A rising
suppressed_total with a flat printed count is the normal case under load and is not a quiet system --
that is the whole point of keeping it.

Nothing here talks to a device, and `--dmesg` takes any command, so the self-test runs on fixtures.
"""

import argparse
import importlib.util
import os
import subprocess
import sys
import time

_HERE = os.path.dirname(os.path.abspath(__file__))


def _load_census():
"""Import the sibling census module, sharing its `suppressed()` instead of re-deriving the regex."""
spec = importlib.util.spec_from_file_location("iommu_fault_census",
os.path.join(_HERE, "iommu_fault_census.py"))
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod


def sample(census, text):
"""(printed fault lines, dropped-callback total) for one dmesg snapshot."""
printed = sum(1 for line in text.splitlines() if "IO_PAGE_FAULT" in line)
return printed, sum(census.suppressed(text).values())


def describe(prev, cur):
"""One interval's meaning, so the log does not need a human to interpret it later."""
if prev is None:
return "baseline"
d_printed = cur[0] - prev[0]
d_dropped = cur[1] - prev[1]
if d_dropped < 0:
# The ring buffer discarded earlier suppression lines, so the total fell. That is not a negative
# rate; it is the buffer forgetting. Re-baseline instead of reporting a number that never happened.
return f"rotated ({d_dropped} from total), re-baselined"
if d_dropped > 0:
# The dropped count is the only quantity that cannot be throttled away.
return f"FAULTING +{d_dropped} dropped, +{d_printed} printed"
if d_printed > 0:
return f"faulting +{d_printed} printed, none dropped"
return "quiet"


def self_test():
census = _load_census()
F = ("kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged [IO_PAGE_FAULT domain=0x0015 "
"address=0x{addr} flags=0x0000]\n")
quiet = "kernel: amdxdna 0000:c6:00.1: nothing to report\n"
ratelimited = "kernel: amd_iommu_report_page_fault: 5611 callbacks suppressed\n"

# A quiet snapshot has no lines and no counter.
assert sample(census, quiet) == (0, 0)

# Printed lines are counted only from fault lines, and the suppressed total is the counter value.
burst = quiet + F.format(addr="4128000") + F.format(addr="4129000") + ratelimited
assert sample(census, burst) == (2, 5611), sample(census, burst)

# The interval's meaning is decided by the DROPPED delta, not the printed delta: a frozen printed
# count with a growing counter is the case dmesg-only analysis reads as silence.
assert describe(None, (2, 5611)) == "baseline"
assert describe((2, 5611), (2, 5611)) == "quiet"
assert describe((2, 5611), (2, 9000)) == "FAULTING +3389 dropped, +0 printed"
assert describe((2, 5611), (9, 5611)) == "faulting +7 printed, none dropped"

# Two suppression lines for the same caller SUM, because each message reports the count dropped
# since the previous one -- `census.suppressed()` adds per caller on purpose. I wrote this assertion
# the other way round first, and the self-test caught it.
two = ratelimited + "kernel: amd_iommu_report_page_fault: 5700 callbacks suppressed\n"
assert sample(census, two) == (0, 11311), sample(census, two)

# Because the total SUMS, it can go backwards when the buffer drops an old suppression line. An
# interval must not be rendered as a negative fault count; it means the buffer forgot.
assert describe((5, 11311), (5, 5611)) == "rotated (-5700 from total), re-baselined"

# The fixture above is the shape that misled engine#2542: two devices, one log format, so the
# printed count mixes populations. The sampler does not classify by device -- the census does --
# it only measures rate, which is why it can stay this small.
print("self-test ok")


def main():
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--interval", type=float, default=5.0, help="seconds between samples")
ap.add_argument("--count", type=int, default=0, help="samples to take (0 = forever)")
ap.add_argument("--out", default="-", help="log file to append to ('-' = stdout)")
ap.add_argument("--dmesg", default="dmesg", help="command whose stdout is the dmesg text")
ap.add_argument("--self-test", action="store_true")
args = ap.parse_args()

if args.self_test:
self_test()
return 0

census = _load_census()
out = sys.stdout if args.out == "-" else open(args.out, "a", buffering=1)
prev, n = None, 0
try:
while not args.count or n < args.count:
text = subprocess.run(args.dmesg, shell=True, capture_output=True, text=True).stdout
cur = sample(census, text)
print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}", file=out)
prev, n = cur, n + 1
if args.count and n >= args.count:
break
time.sleep(args.interval)
except KeyboardInterrupt:
pass
finally:
if out is not sys.stdout:
out.close()
return 0


if __name__ == "__main__":
sys.exit(main())
Loading