From 9f45ac8171e424ce8b200f238cf6eb9bd6df0773 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 20:25:10 -0400 Subject: [PATCH 1/2] diag: the IOMMU fault sampler counts each kernel line once, so a rotation cannot hide drops (Refs #2542) The sampler counted the whole dmesg buffer on every snapshot and took the difference. When an old suppression line left the ring buffer in the same interval that new ones arrived, the interval read "rotated (-2611), re-baselined" for 3000 new drops, or "+389" for 6000 (review of #2580). It now keeps cumulative totals over the lines each snapshot ADDS: a line counts only if its dmesg stamp `[ s.us ]` is later than the newest one already counted. Whatever leaves the buffer is simply not seen again, so the totals never fall and an interval is exactly what was printed and dropped in it. The "rotated" case is gone. Lines without a stamp (dmesg -T, journalctl's default) cannot be ordered: they count only in the baseline, and a later interval holding them says so. Self-test: the two rotation shapes now read FAULTING +3000 / +6000; a repeated snapshot is not counted twice; a fault line rotating out never lowers the printed count; an unstamped fault line after the baseline is reported, not counted. Mutants: counting every stamped line each time fails (3, 16833); the old whole-buffer snapshot fails ((1, 5611), (0, 3000)). Live, this boot's `journalctl -k -o short-monotonic`: 740 printed / 5699 dropped, then quiet. Co-Authored-By: Claude Opus 5.5 --- docs/nvme-iommu-fault-triage.md | 5 +- tools/diag/iommu_fault_sampler.py | 94 ++++++++++++++++++++++++++----- 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 1f56e9bf3..578736722 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -239,8 +239,9 @@ Four things generalise, and each one was a decision rather than a discovery: `grep -c IO_PAGE_FAULT` mixes them and inflates whichever you were looking for. `tools/diag/iommu_fault_census.py` exists for exactly this. To measure the **rate** rather than take one census of a buffer that both throttles and rotates, `tools/diag/iommu_fault_sampler.py` appends ` ` per interval: the - dropped-callback delta is the only count `printk_ratelimited` cannot hide, and a falling total means the ring - buffer forgot (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read). + dropped-callback delta is the only count `printk_ratelimited` cannot hide. Both counts are cumulative over the + lines each snapshot adds (by `dmesg`'s `[ s.us ]` stamp), so the buffer rotating cannot hide or shrink an + interval (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read). 2. **`flags` is not decoded here, and must not be until it is cited.** The streams carry **three** values, not one: ``` diff --git a/tools/diag/iommu_fault_sampler.py b/tools/diag/iommu_fault_sampler.py index d4a429089..ee92be425 100755 --- a/tools/diag/iommu_fault_sampler.py +++ b/tools/diag/iommu_fault_sampler.py @@ -22,7 +22,15 @@ So this appends one line per interval: - + + +Both counts are cumulative since the sampler started, built from the lines that are NEW in each snapshot +(kernel stamp `[ s.us ]` later than the last one seen), so the ring buffer rotating cannot change them: the +delta between two lines is exactly what the kernel printed, and dropped, in that interval. Counting the +whole buffer each time instead let a rotation hide new drops (an old suppression line leaving while new +ones arrive read as "rotated", or as a smaller rise -- review of engine#2580). That needs `dmesg`'s +default `[ s.us ]` prefix; `dmesg -T` and `journalctl` lines carry none and cannot be ordered, so they are +counted only in the baseline, and a later interval that holds such fault lines says so. # one interval every 5s, forever (root: reading dmesg) sudo tools/diag/iommu_fault_sampler.py --interval 5 --out /var/log/iommu-fault-rate.log @@ -30,12 +38,12 @@ # a bounded sample, for a window run tools/diag/iommu_fault_sampler.py --interval 2 --count 10 --dmesg "cat fixture.txt" -`suppressed_total` is `census.suppressed(text)["amd_iommu_report_page_fault"]`, the IOMMU fault +`suppressed_total` sums `census.suppressed(new_lines)["amd_iommu_report_page_fault"]`, the IOMMU fault reporter's own counter and no other caller's: the same journal carries `kauditd_printk_skb` and `set_capacity_and_notify` suppression lines (267 and 2 on the 11-15 boot, beside the IOMMU's 5699), and -summing every caller counted an audit burst as dropped faults. `--caller` names another reporter. The -counters grow monotonically, so the DELTA across two lines is the number of faults the kernel dropped -in that interval. A rising +summing every caller counted an audit burst as dropped faults. `--caller` names another reporter. Each +suppression message reports what was dropped since the previous one, so the new lines' counts SUM to the +interval's drops. A rising suppressed_total with a flat printed count is the normal case under load and is not a quiet system -- that is the whole point of keeping it. @@ -45,6 +53,7 @@ import argparse import importlib.util import os +import re import subprocess import sys import time @@ -72,16 +81,51 @@ def sample(census, text, caller=IOMMU_CALLER): return printed, census.suppressed(text).get(caller, 0) +# dmesg's default prefix, `[ 1234.567890] `: the only order the lines carry +STAMP = re.compile(r"^\[\s*(\d+\.\d+)\]") + + +class Tally: + """Cumulative (printed, dropped) since the first snapshot, from the lines each snapshot adds. + + A line counts once: in the first snapshot (the baseline) every line counts; after that only lines + stamped later than the newest stamp seen so far. Lines that fall out of the buffer are simply not + seen again, so a rotation can neither subtract from the totals nor mask what arrived meanwhile.""" + + def __init__(self, census, caller=IOMMU_CALLER): + self.census, self.caller = census, caller + self.last = None # newest kernel stamp counted + self.printed = self.dropped = 0 + self.unstamped = 0 # fault/suppression lines after the baseline that could not be ordered + + def add(self, text): + lines = text.splitlines() + new, last = [], self.last + for line in lines: + m = STAMP.match(line) + if m is None: + if self.last is None: + new.append(line) # baseline: nothing to order against yet + elif "IO_PAGE_FAULT" in line or "callbacks suppressed" in line: + self.unstamped += 1 + continue + t = float(m.group(1)) + if self.last is None or t > self.last: + new.append(line) + last = t if last is None or t > last else last + self.last = last if last is not None else (self.last if self.last is not None else float("-inf")) + p, d = sample(self.census, "\n".join(new), self.caller) + self.printed += p + self.dropped += d + return self.printed, self.dropped + + def describe(prev, cur): """One interval's meaning, so the log does not need a human to interpret it later.""" if prev is None: return "baseline" d_printed = cur[0] - prev[0] d_dropped = cur[1] - prev[1] - if d_dropped < 0: - # The ring buffer discarded earlier suppression lines, so the total fell. That is not a negative - # rate; it is the buffer forgetting. Re-baseline instead of reporting a number that never happened. - return f"rotated ({d_dropped} from total), re-baselined" if d_dropped > 0: # The dropped count is the only quantity that cannot be throttled away. return f"FAULTING +{d_dropped} dropped, +{d_printed} printed" @@ -117,9 +161,28 @@ def self_test(): two = ratelimited + "kernel: amd_iommu_report_page_fault: 5700 callbacks suppressed\n" assert sample(census, two) == (0, 11311), sample(census, two) - # Because the total SUMS, it can go backwards when the buffer drops an old suppression line. An - # interval must not be rendered as a negative fault count; it means the buffer forgot. - assert describe((5, 11311), (5, 5611)) == "rotated (-5700 from total), re-baselined" + # Rotation (review of engine#2580): counting the whole buffer each time, an old suppression line + # leaving in the same interval as new ones arrive read as "rotated" (3000 new drops shown as none) or as + # a smaller rise (6000 shown as 389). The tally counts only lines stamped after the last one seen, so + # the interval is the new lines alone, whatever left the buffer. + S = "[%11.6f] " + b0 = S % 1.0 + F.format(addr="4128000") + S % 2.0 + ratelimited + for new_drops in (3000, 6000): + t = Tally(census) + before = t.add(b0) + b1 = S % 6.0 + "kernel: amd_iommu_report_page_fault: %d callbacks suppressed\n" % new_drops # b0 rotated out + after = t.add(b1) + assert (before, after) == ((1, 5611), (1, 5611 + new_drops)), (before, after) + assert describe(before, after) == "FAULTING +%d dropped, +0 printed" % new_drops, describe(before, after) + # A snapshot that still holds the old lines does not count them twice, and a fault line that rotates + # out never makes the printed count fall. + t = Tally(census) + t.add(b0) + assert t.add(b0) == (1, 5611), t.add(b0) + assert t.add(S % 7.0 + quiet) == (1, 5611) + # After the baseline an unstamped fault line cannot be ordered: it is not counted, and it is reported. + t.add(F.format(addr="4130000")) + assert (t.printed, t.unstamped) == (1, 1), (t.printed, t.unstamped) # Only the IOMMU reporter's counter is a dropped fault. The 11-15 journal had kauditd (267) and # set_capacity (2) suppression lines beside it; summing every caller turned an audit burst into @@ -152,13 +215,16 @@ def main(): return 0 census = _load_census() + tally = Tally(census, args.caller) out = sys.stdout if args.out == "-" else open(args.out, "a", buffering=1) prev, n = None, 0 try: while not args.count or n < args.count: text = subprocess.run(args.dmesg, shell=True, capture_output=True, text=True).stdout - cur = sample(census, text, args.caller) - print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}", file=out) + seen = tally.unstamped + cur = tally.add(text) + note = "" if tally.unstamped == seen else f"; {tally.unstamped - seen} unstamped line(s) not counted" + print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}{note}", file=out) prev, n = cur, n + 1 if args.count and n >= args.count: break From 38df53000391bf5fadf23402eae7b99dc6b1600d Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 20:47:29 -0400 Subject: [PATCH 2/2] diag: the sampler picks its mode from the source, and a clock restart re-baselines (review of #2603) Two CHANGES on #2603, both right: - journalctl -k's default format carries no `[ s.us ]` on any line (0 of 6140 here), so the cursor never advanced and every interval read "quiet". The first snapshot now picks the mode: a stamped source uses the cursor; a source with no stamps is counted whole each time, as before #2603, with "rotated ... drops in this interval unknown" when the total falls. The baseline line names the mode. - A snapshot whose stamps are all older than the newest seen (a clock restart under a persistent source) is counted as a new buffer and the interval says "stamps went backwards (reboot?)", instead of freezing the totals. Self-test adds both shapes: a journalctl-format pair (buffer mode, FAULTING +150, +2) and a stamp restart (counted, named). Mutants: forcing cursor mode fails ('cursor', (1, 5611), (4, 11372)); removing the restart branch fails ((1, 5611), ''); removing the cursor fails (3, 16833). The reviewer's growing journal-format file now reads FAULTING +150 dropped, +2 printed (was quiet). Co-Authored-By: Claude Opus 5.5 --- tools/diag/iommu_fault_sampler.py | 103 ++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/tools/diag/iommu_fault_sampler.py b/tools/diag/iommu_fault_sampler.py index ee92be425..354fcb840 100755 --- a/tools/diag/iommu_fault_sampler.py +++ b/tools/diag/iommu_fault_sampler.py @@ -29,8 +29,9 @@ delta between two lines is exactly what the kernel printed, and dropped, in that interval. Counting the whole buffer each time instead let a rotation hide new drops (an old suppression line leaving while new ones arrive read as "rotated", or as a smaller rise -- review of engine#2580). That needs `dmesg`'s -default `[ s.us ]` prefix; `dmesg -T` and `journalctl` lines carry none and cannot be ordered, so they are -counted only in the baseline, and a later interval that holds such fault lines says so. +default `[ s.us ]` prefix (`dmesg`, `journalctl -k -o short-monotonic`). A source with no stamps at all +(`journalctl -k`'s default format, `dmesg -T`) cannot be ordered, so it is counted whole each time as +before, and the baseline line says which mode is in use (see Tally). # one interval every 5s, forever (root: reading dmesg) sudo tools/diag/iommu_fault_sampler.py --interval 5 --out /var/log/iommu-fault-rate.log @@ -86,34 +87,51 @@ def sample(census, text, caller=IOMMU_CALLER): class Tally: - """Cumulative (printed, dropped) since the first snapshot, from the lines each snapshot adds. - - A line counts once: in the first snapshot (the baseline) every line counts; after that only lines - stamped later than the newest stamp seen so far. Lines that fall out of the buffer are simply not - seen again, so a rotation can neither subtract from the totals nor mask what arrived meanwhile.""" + """Cumulative (printed, dropped) since the first snapshot. + + The first snapshot picks the mode, because the two sources on this box differ exactly here: + * "cursor" -- the source carries dmesg's `[ s.us ]` stamp (`dmesg`, `journalctl -k -o short-monotonic`). + A line counts once: every line in the baseline, after that only lines stamped later than the newest + stamp seen. Lines that leave the buffer are not seen again, so a rotation can neither subtract from the + totals nor mask what arrived meanwhile. If every stamp in a snapshot is older than the newest one seen, + the clock restarted (a reboot under a persistent source): that snapshot is counted as a new buffer, and + the interval says so instead of freezing at the old totals. + * "buffer" -- no line carries a stamp (`journalctl -k` in its default format, `dmesg -T`). Nothing can be + ordered, so each snapshot is counted whole, as before engine#2603: a rotation can hide drops, and the + interval says `rotated` when the total falls. The mode is announced on the baseline line, because a + cursor that never advances would read `quiet` for ever (review of engine#2603). + """ def __init__(self, census, caller=IOMMU_CALLER): self.census, self.caller = census, caller - self.last = None # newest kernel stamp counted + self.mode = None # "cursor" or "buffer", set by the first snapshot + self.last = None # newest kernel stamp counted (cursor mode) self.printed = self.dropped = 0 - self.unstamped = 0 # fault/suppression lines after the baseline that could not be ordered + self.unstamped = 0 # cursor mode: fault/suppression lines in the last snapshot with no stamp + self.event = "" # what the last add() noticed, for the interval's meaning def add(self, text): lines = text.splitlines() - new, last = [], self.last - for line in lines: - m = STAMP.match(line) - if m is None: - if self.last is None: - new.append(line) # baseline: nothing to order against yet - elif "IO_PAGE_FAULT" in line or "callbacks suppressed" in line: - self.unstamped += 1 - continue - t = float(m.group(1)) - if self.last is None or t > self.last: - new.append(line) - last = t if last is None or t > last else last - self.last = last if last is not None else (self.last if self.last is not None else float("-inf")) + stamped = [(float(m.group(1)), line) for line in lines for m in [STAMP.match(line)] if m] + self.event = "" + if self.mode is None: + self.mode = "cursor" if stamped else "buffer" + if self.mode == "buffer": + self.printed, self.dropped = sample(self.census, text, self.caller) + return self.printed, self.dropped + if self.last is None: + new = lines # baseline: nothing to order against yet + elif stamped and max(t for t, _ in stamped) < self.last: + self.event = "stamps went backwards (reboot?): counted the new buffer" + new = [line for _, line in stamped] + self.last = None + else: + new = [line for t, line in stamped if t > self.last] + self.unstamped = 0 if self.last is None else sum( + 1 for line in lines if not STAMP.match(line) and ("IO_PAGE_FAULT" in line or "callbacks suppressed" in line)) + if stamped: + newest = max(t for t, _ in stamped) + self.last = newest if self.last is None else max(self.last, newest) p, d = sample(self.census, "\n".join(new), self.caller) self.printed += p self.dropped += d @@ -126,6 +144,10 @@ def describe(prev, cur): return "baseline" d_printed = cur[0] - prev[0] d_dropped = cur[1] - prev[1] + if d_dropped < 0: + # buffer mode only (no stamps to order by): the ring buffer discarded earlier suppression lines, so the + # total fell. That is not a negative rate; it is the buffer forgetting, and drops in it are unknown. + return f"rotated ({d_dropped} from total), re-baselined; drops in this interval unknown" if d_dropped > 0: # The dropped count is the only quantity that cannot be throttled away. return f"FAULTING +{d_dropped} dropped, +{d_printed} printed" @@ -182,7 +204,27 @@ def self_test(): assert t.add(S % 7.0 + quiet) == (1, 5611) # After the baseline an unstamped fault line cannot be ordered: it is not counted, and it is reported. t.add(F.format(addr="4130000")) - assert (t.printed, t.unstamped) == (1, 1), (t.printed, t.unstamped) + assert (t.mode, t.printed, t.unstamped) == ("cursor", 1, 1), (t.mode, t.printed, t.unstamped) + + # journalctl's default format carries no `[ s.us ]` on any line (0 of 6140 on 11-16). A cursor would never + # advance and read "quiet" for ever (review of engine#2603), so a source with no stamps counts the buffer whole. + J = "Nov 15 10:00:0%d h kernel: " + j0 = J % 0 + F.format(addr="4128000") + J % 1 + ratelimited + j1 = j0 + J % 3 + F.format(addr="4129000") + J % 4 + F.format(addr="4129000") + \ + J % 5 + "kernel: amd_iommu_report_page_fault: 150 callbacks suppressed\n" + t = Tally(census) + before, after = t.add(j0), t.add(j1) + assert t.mode == "buffer" and (before, after) == ((1, 5611), (3, 5761)), (t.mode, before, after) + assert describe(before, after) == "FAULTING +150 dropped, +2 printed", describe(before, after) + # ... and in that mode a rotation is named, never a negative count or "quiet" + assert describe((3, 5761), (2, 150)).startswith("rotated (-5611 from total)"), describe((3, 5761), (2, 150)) + + # A clock restart (every stamp older than the newest seen) is counted as a new buffer and named, instead of + # freezing the totals at their old values while the new boot faults. + t = Tally(census) + t.add(S % 500.0 + F.format(addr="4128000") + S % 501.0 + ratelimited) + after = t.add(S % 3.0 + F.format(addr="4129000") + S % 4.0 + "kernel: amd_iommu_report_page_fault: 7 callbacks suppressed\n") + assert after == (2, 5618) and t.event.startswith("stamps went backwards"), (after, t.event) # Only the IOMMU reporter's counter is a dropped fault. The 11-15 journal had kauditd (267) and # set_capacity (2) suppression lines beside it; summing every caller turned an audit burst into @@ -221,10 +263,17 @@ def main(): try: while not args.count or n < args.count: text = subprocess.run(args.dmesg, shell=True, capture_output=True, text=True).stdout - seen = tally.unstamped cur = tally.add(text) - note = "" if tally.unstamped == seen else f"; {tally.unstamped - seen} unstamped line(s) not counted" - print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}{note}", file=out) + meaning = describe(prev, cur) + if prev is None: + meaning += (" (mode: cursor on kernel stamps)" if tally.mode == "cursor" else + " (mode: whole buffer -- no kernel stamps in this source, so a rotation can hide drops;" + " dmesg or journalctl -k -o short-monotonic carry them)") + if tally.event: + meaning += "; " + tally.event + if tally.mode == "cursor" and tally.unstamped: + meaning += f"; {tally.unstamped} unstamped line(s) in the buffer not counted" + print(f"{int(time.time())} {cur[0]} {cur[1]} {meaning}", file=out) prev, n = cur, n + 1 if args.count and n >= args.count: break