diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 868b2050e..f587a1fdc 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -113,6 +113,27 @@ only if you want to confirm that, not as a discriminator. ```bash # 1. pick the top faulting IOVA from the census +<<<<<<< HEAD +======= +**The interval is seconds, not milliseconds (measured 2026-11-16).** A filtered probe on one faulting IOVA +(`0xe1e18000`), captured over a boot's first 28 minutes, found **60 unmaps of that page, every one a single 4 KiB +page, 59 of them from `` (interrupt context)**: the NVMe completion path repeatedly maps and tears down one +pooled page. **Three of the four faults on it** followed an unmap of that same page by **5.65 s, 24.66 s and +24.71 s**, and a correlation window of ±5 ms found none of them. The fourth (boot t=1605.6 s) has **no unmap of that +page within 60 s** -- the nearest is **+74 s**, after the fault -- so this capture correlates three faults and shows +the ceiling on the fourth; a wider window does not help it, and the earlier "19.60 s" I quoted for it came from a +different capture and does not reproduce on this one. So `tools/diag/unmap_fault_order.py` defaults to **30 s** and +the page's own numbers are **5-25 s**: a millisecond window reports "nothing correlated" on a capture that contains +the correlation. Because the same sentence is printed whether the window is too narrow or the fault genuinely has no +nearby unmap, **the negative now names the window it used and the measured interval** -- a default alone does not fix +the failure, only a negative that says *at what scale* it found nothing does. + +**One probe owns the shared buffer.** A kprobe on `iommu_unmap_fast` with **no filter** wrote 579,000 events and left +a filtered probe on the same function with **zero** -- the trace ring buffer is shared, so the *narrow* probe (one +IOVA, one call site) is the one that should be armed, and checking `grep -c ':' trace` before trusting a +capture is the same "prove the plumbing" rule two paragraphs up. + +>>>>>>> 2b1faf1b2 (diag: a negative names the window it used, and the window reads in seconds) # 2. kprobe iommu_unmap_fast on that iova alone (filter), stacktrace on, capture for minutes in the background # 3. in the same capture window, note the fault timestamps for that iova ``` diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py index 056f3317d..3ad4f007a 100755 --- a/tools/diag/unmap_fault_order.py +++ b/tools/diag/unmap_fault_order.py @@ -53,10 +53,6 @@ # and a `dmesg`/trace-style capture writes `[1234.567890]` seconds-since-boot. A fault line is only correlatable # when its clock is the trace's, so the seconds form is preferred when both appear. ISO_RE = re.compile(r"(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?)") -# The kernel pads the stamp to a fixed width: `[ 321.308031]`, two spaces here and one for values >= 1000 s. -# Written without the inner `\s*` the regex only matched the unpadded shape a doc example uses, so EVERY real -# boot-stamped line parsed as untimed and the tool could not correlate on a capture it produced itself -- -# measured 2026-11-16 on `journalctl -k -o short-monotonic`: `[ 321.308031] … IO_PAGE_FAULT …` -> t=None. BOOT_RE = re.compile(r"^\s*\[\s*(\d+\.\d+)\]") @@ -130,18 +126,27 @@ def pointer_shaped(unmaps): return bool(unmaps) and all(u[1] >= IOVA_MAX for u in unmaps) +def _span(window_us): + """The window as a reader thinks of it: seconds once it is a second or more. `+-30000000 us` is not readable.""" + return f"{window_us / 1e6:g} s" if window_us >= 1e6 else f"{window_us:.0f} us" + + def render(rows, unmaps, window_us): lines = [f"unmap events parsed: {len(unmaps)} over {len({u[1] for u in unmaps})} distinct iova(s)"] if unmaps: lines.append(f" sizes: {sorted({u[2] for u in unmaps})[0]:#x} .. {sorted({u[2] for u in unmaps})[-1]:#x}") hit = [r for r in rows if r[3] is not None] - lines.append(f"faults parsed: {len(rows)}; with an unmap of the same iova within +-{window_us:.0f} us: {len(hit)}") + lines.append(f"faults parsed: {len(rows)}; with an unmap of the same iova within +-{_span(window_us)}: {len(hit)}") for t, iova, _text, best in hit[:20]: (ut, _u, size), d = best verdict = "unmap BEFORE fault (translation gone)" if d < 0 else "unmap AFTER fault (stale mapping or other cause)" lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}") if rows and not hit: - lines.append(" no fault shares an iova with an unmap inside the window: ordering not shown by this capture") + # A negative must name its own condition. The same sentence at a 5 ms window and at 30 s read alike, and the + # 5 ms one was simply wrong, so the scale is in the line and the remedy is named with it. + lines.append(f" no fault shares an iova with an unmap inside +-{_span(window_us)}: ordering not shown by " + f"this capture AT THIS WINDOW -- the measured unmap-to-fault interval on this box is 5-25 s, " + f"so widen --window before reading this as 'no unmap preceded the fault'") return lines @@ -176,26 +181,27 @@ def chk(cond, msg): "iova=0xffff8e83841c3a00 size=0x45ec0"]) chk(pointer_shaped(ptr), "a pointer-shaped iova column is detected, not read as uncorrelated") chk(not pointer_shaped(got), "a real IOVA column is not mistaken for one") - # 7. the REAL kernel stamp is PADDED (`[ 321.308031]`, two spaces; `[ 0.000000]` early in a dmesg). The doc - # example above writes the unpadded `[1234.567890]`, and an arm built only from it passed while every real - # capture parsed as untimed -- the tool then reported "nothing correlated" on data it had just been given. - padded = "[ 321.308031] strixhalo kernel: nvme 0000:c4:00.0: AMD-Vi: Event logged " \ - "[IO_PAGE_FAULT domain=0x0015 address=0x4128000]" - pf = parse_faults([padded], lambda _s: None) - chk(pf and pf[0][0] == 321.308031, "a padded boot stamp parses (measure with journalctl -k -o short-monotonic)") - ptr_tr = parse_trace([" bash-1 [002] .... 321.308000: unm: (iommu_unmap_fast) iova=0x4128000 size=0x1000"]) - hits = correlate(ptr_tr, pf, 5000) - chk(hits and hits[0][3] and hits[0][3][1] < 0, - "a padded fault correlates with an unmap 31 us earlier (BEFORE, not an uncorrelated negative)") + # 7. the window reads in seconds once it is seconds, and a negative names the window it used. A default alone + # does not fix the failure I am fixing: 30 s finding nothing printed the same sentence 5 ms did. + # The fault is 100 s from any unmap: at 30 s that is still a miss, which is the shape a too-narrow window gives. + out = " ".join(render(correlate(unm, [(300.000, 0x4128000, "f5")], 30_000_000), unm, 30_000_000)) + chk("+-30 s" in out and "30000000 us" not in out, "a 30 s window reads as '+-30 s', not '+-30000000 us'") + chk("AT THIS WINDOW" in out and "5-25 s" in out, + "a negative names the window it used and the measured interval, not 'nothing correlated' bare") + chk("+-5000 us" in " ".join(render(correlate(unm, [(100.500, 0x4128000, "f3")], 5000), unm, 5000)), + "a sub-second window still reads in microseconds") return 0 if ok else 1 - def main(): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--trace", help="a tracefs capture with a kprobe on the unmap path") ap.add_argument("--journal", action="append", help="kernel-log lines (repeatable; default stdin)") - ap.add_argument("--window", type=float, default=5000.0, help="window in microseconds (default 5000)") + # 30 s, not 5 ms: measured on this box 2026-11-16, three of four faults had an unmap of the SAME 4 KiB page + # 5.6 s / 24.7 s / 24.7 s earlier, and a 5 ms default answered "no fault shares an iova with an unmap" on the + # very capture that contains them. The interval is seconds here; a small default hides the finding. + ap.add_argument("--window", type=float, default=30_000_000.0, + help="window in microseconds (default 30,000,000 = 30 s; measured interval on this box is 5-25 s)") ap.add_argument("--epoch-offset", type=float, default=0.0, help="seconds to add to journal stamps to reach the trace's boot clock") ap.add_argument("--self-test", action="store_true")