diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 1f56e9bf3..578736722 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -239,8 +239,9 @@ Four things generalise, and each one was a decision rather than a discovery: `grep -c IO_PAGE_FAULT` mixes them and inflates whichever you were looking for. `tools/diag/iommu_fault_census.py` exists for exactly this. To measure the **rate** rather than take one census of a buffer that both throttles and rotates, `tools/diag/iommu_fault_sampler.py` appends ` ` per interval: the - dropped-callback delta is the only count `printk_ratelimited` cannot hide, and a falling total means the ring - buffer forgot (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read). + dropped-callback delta is the only count `printk_ratelimited` cannot hide. Both counts are cumulative over the + lines each snapshot adds (by `dmesg`'s `[ s.us ]` stamp), so the buffer rotating cannot hide or shrink an + interval (engine#2542 quoted three different populations for one boot, 20 / 78 / 41, all true when read). 2. **`flags` is not decoded here, and must not be until it is cited.** The streams carry **three** values, not one: ``` diff --git a/tools/diag/iommu_fault_sampler.py b/tools/diag/iommu_fault_sampler.py index d4a429089..354fcb840 100755 --- a/tools/diag/iommu_fault_sampler.py +++ b/tools/diag/iommu_fault_sampler.py @@ -22,7 +22,16 @@ So this appends one line per interval: - + + +Both counts are cumulative since the sampler started, built from the lines that are NEW in each snapshot +(kernel stamp `[ s.us ]` later than the last one seen), so the ring buffer rotating cannot change them: the +delta between two lines is exactly what the kernel printed, and dropped, in that interval. Counting the +whole buffer each time instead let a rotation hide new drops (an old suppression line leaving while new +ones arrive read as "rotated", or as a smaller rise -- review of engine#2580). That needs `dmesg`'s +default `[ s.us ]` prefix (`dmesg`, `journalctl -k -o short-monotonic`). A source with no stamps at all +(`journalctl -k`'s default format, `dmesg -T`) cannot be ordered, so it is counted whole each time as +before, and the baseline line says which mode is in use (see Tally). # one interval every 5s, forever (root: reading dmesg) sudo tools/diag/iommu_fault_sampler.py --interval 5 --out /var/log/iommu-fault-rate.log @@ -30,12 +39,12 @@ # a bounded sample, for a window run tools/diag/iommu_fault_sampler.py --interval 2 --count 10 --dmesg "cat fixture.txt" -`suppressed_total` is `census.suppressed(text)["amd_iommu_report_page_fault"]`, the IOMMU fault +`suppressed_total` sums `census.suppressed(new_lines)["amd_iommu_report_page_fault"]`, the IOMMU fault reporter's own counter and no other caller's: the same journal carries `kauditd_printk_skb` and `set_capacity_and_notify` suppression lines (267 and 2 on the 11-15 boot, beside the IOMMU's 5699), and -summing every caller counted an audit burst as dropped faults. `--caller` names another reporter. The -counters grow monotonically, so the DELTA across two lines is the number of faults the kernel dropped -in that interval. A rising +summing every caller counted an audit burst as dropped faults. `--caller` names another reporter. Each +suppression message reports what was dropped since the previous one, so the new lines' counts SUM to the +interval's drops. A rising suppressed_total with a flat printed count is the normal case under load and is not a quiet system -- that is the whole point of keeping it. @@ -45,6 +54,7 @@ import argparse import importlib.util import os +import re import subprocess import sys import time @@ -72,6 +82,62 @@ def sample(census, text, caller=IOMMU_CALLER): return printed, census.suppressed(text).get(caller, 0) +# dmesg's default prefix, `[ 1234.567890] `: the only order the lines carry +STAMP = re.compile(r"^\[\s*(\d+\.\d+)\]") + + +class Tally: + """Cumulative (printed, dropped) since the first snapshot. + + The first snapshot picks the mode, because the two sources on this box differ exactly here: + * "cursor" -- the source carries dmesg's `[ s.us ]` stamp (`dmesg`, `journalctl -k -o short-monotonic`). + A line counts once: every line in the baseline, after that only lines stamped later than the newest + stamp seen. Lines that leave the buffer are not seen again, so a rotation can neither subtract from the + totals nor mask what arrived meanwhile. If every stamp in a snapshot is older than the newest one seen, + the clock restarted (a reboot under a persistent source): that snapshot is counted as a new buffer, and + the interval says so instead of freezing at the old totals. + * "buffer" -- no line carries a stamp (`journalctl -k` in its default format, `dmesg -T`). Nothing can be + ordered, so each snapshot is counted whole, as before engine#2603: a rotation can hide drops, and the + interval says `rotated` when the total falls. The mode is announced on the baseline line, because a + cursor that never advances would read `quiet` for ever (review of engine#2603). + """ + + def __init__(self, census, caller=IOMMU_CALLER): + self.census, self.caller = census, caller + self.mode = None # "cursor" or "buffer", set by the first snapshot + self.last = None # newest kernel stamp counted (cursor mode) + self.printed = self.dropped = 0 + self.unstamped = 0 # cursor mode: fault/suppression lines in the last snapshot with no stamp + self.event = "" # what the last add() noticed, for the interval's meaning + + def add(self, text): + lines = text.splitlines() + stamped = [(float(m.group(1)), line) for line in lines for m in [STAMP.match(line)] if m] + self.event = "" + if self.mode is None: + self.mode = "cursor" if stamped else "buffer" + if self.mode == "buffer": + self.printed, self.dropped = sample(self.census, text, self.caller) + return self.printed, self.dropped + if self.last is None: + new = lines # baseline: nothing to order against yet + elif stamped and max(t for t, _ in stamped) < self.last: + self.event = "stamps went backwards (reboot?): counted the new buffer" + new = [line for _, line in stamped] + self.last = None + else: + new = [line for t, line in stamped if t > self.last] + self.unstamped = 0 if self.last is None else sum( + 1 for line in lines if not STAMP.match(line) and ("IO_PAGE_FAULT" in line or "callbacks suppressed" in line)) + if stamped: + newest = max(t for t, _ in stamped) + self.last = newest if self.last is None else max(self.last, newest) + p, d = sample(self.census, "\n".join(new), self.caller) + self.printed += p + self.dropped += d + return self.printed, self.dropped + + def describe(prev, cur): """One interval's meaning, so the log does not need a human to interpret it later.""" if prev is None: @@ -79,9 +145,9 @@ def describe(prev, cur): d_printed = cur[0] - prev[0] d_dropped = cur[1] - prev[1] if d_dropped < 0: - # The ring buffer discarded earlier suppression lines, so the total fell. That is not a negative - # rate; it is the buffer forgetting. Re-baseline instead of reporting a number that never happened. - return f"rotated ({d_dropped} from total), re-baselined" + # buffer mode only (no stamps to order by): the ring buffer discarded earlier suppression lines, so the + # total fell. That is not a negative rate; it is the buffer forgetting, and drops in it are unknown. + return f"rotated ({d_dropped} from total), re-baselined; drops in this interval unknown" if d_dropped > 0: # The dropped count is the only quantity that cannot be throttled away. return f"FAULTING +{d_dropped} dropped, +{d_printed} printed" @@ -117,9 +183,48 @@ def self_test(): two = ratelimited + "kernel: amd_iommu_report_page_fault: 5700 callbacks suppressed\n" assert sample(census, two) == (0, 11311), sample(census, two) - # Because the total SUMS, it can go backwards when the buffer drops an old suppression line. An - # interval must not be rendered as a negative fault count; it means the buffer forgot. - assert describe((5, 11311), (5, 5611)) == "rotated (-5700 from total), re-baselined" + # Rotation (review of engine#2580): counting the whole buffer each time, an old suppression line + # leaving in the same interval as new ones arrive read as "rotated" (3000 new drops shown as none) or as + # a smaller rise (6000 shown as 389). The tally counts only lines stamped after the last one seen, so + # the interval is the new lines alone, whatever left the buffer. + S = "[%11.6f] " + b0 = S % 1.0 + F.format(addr="4128000") + S % 2.0 + ratelimited + for new_drops in (3000, 6000): + t = Tally(census) + before = t.add(b0) + b1 = S % 6.0 + "kernel: amd_iommu_report_page_fault: %d callbacks suppressed\n" % new_drops # b0 rotated out + after = t.add(b1) + assert (before, after) == ((1, 5611), (1, 5611 + new_drops)), (before, after) + assert describe(before, after) == "FAULTING +%d dropped, +0 printed" % new_drops, describe(before, after) + # A snapshot that still holds the old lines does not count them twice, and a fault line that rotates + # out never makes the printed count fall. + t = Tally(census) + t.add(b0) + assert t.add(b0) == (1, 5611), t.add(b0) + assert t.add(S % 7.0 + quiet) == (1, 5611) + # After the baseline an unstamped fault line cannot be ordered: it is not counted, and it is reported. + t.add(F.format(addr="4130000")) + assert (t.mode, t.printed, t.unstamped) == ("cursor", 1, 1), (t.mode, t.printed, t.unstamped) + + # journalctl's default format carries no `[ s.us ]` on any line (0 of 6140 on 11-16). A cursor would never + # advance and read "quiet" for ever (review of engine#2603), so a source with no stamps counts the buffer whole. + J = "Nov 15 10:00:0%d h kernel: " + j0 = J % 0 + F.format(addr="4128000") + J % 1 + ratelimited + j1 = j0 + J % 3 + F.format(addr="4129000") + J % 4 + F.format(addr="4129000") + \ + J % 5 + "kernel: amd_iommu_report_page_fault: 150 callbacks suppressed\n" + t = Tally(census) + before, after = t.add(j0), t.add(j1) + assert t.mode == "buffer" and (before, after) == ((1, 5611), (3, 5761)), (t.mode, before, after) + assert describe(before, after) == "FAULTING +150 dropped, +2 printed", describe(before, after) + # ... and in that mode a rotation is named, never a negative count or "quiet" + assert describe((3, 5761), (2, 150)).startswith("rotated (-5611 from total)"), describe((3, 5761), (2, 150)) + + # A clock restart (every stamp older than the newest seen) is counted as a new buffer and named, instead of + # freezing the totals at their old values while the new boot faults. + t = Tally(census) + t.add(S % 500.0 + F.format(addr="4128000") + S % 501.0 + ratelimited) + after = t.add(S % 3.0 + F.format(addr="4129000") + S % 4.0 + "kernel: amd_iommu_report_page_fault: 7 callbacks suppressed\n") + assert after == (2, 5618) and t.event.startswith("stamps went backwards"), (after, t.event) # Only the IOMMU reporter's counter is a dropped fault. The 11-15 journal had kauditd (267) and # set_capacity (2) suppression lines beside it; summing every caller turned an audit burst into @@ -152,13 +257,23 @@ def main(): return 0 census = _load_census() + tally = Tally(census, args.caller) out = sys.stdout if args.out == "-" else open(args.out, "a", buffering=1) prev, n = None, 0 try: while not args.count or n < args.count: text = subprocess.run(args.dmesg, shell=True, capture_output=True, text=True).stdout - cur = sample(census, text, args.caller) - print(f"{int(time.time())} {cur[0]} {cur[1]} {describe(prev, cur)}", file=out) + cur = tally.add(text) + meaning = describe(prev, cur) + if prev is None: + meaning += (" (mode: cursor on kernel stamps)" if tally.mode == "cursor" else + " (mode: whole buffer -- no kernel stamps in this source, so a rotation can hide drops;" + " dmesg or journalctl -k -o short-monotonic carry them)") + if tally.event: + meaning += "; " + tally.event + if tally.mode == "cursor" and tally.unstamped: + meaning += f"; {tally.unstamped} unstamped line(s) in the buffer not counted" + print(f"{int(time.time())} {cur[0]} {cur[1]} {meaning}", file=out) prev, n = cur, n + 1 if args.count and n >= args.count: break