From 63440f423bc0a0355e042e17cbe20a4d3d2611d1 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 01:51:36 -0400 Subject: [PATCH 1/4] diag: a pairing whose window held many unmaps is not discriminating, and says so A positive needs the same treatment the negative just got. Measured at 05:40Z (150 s, three probes filtered to one IOVA): the NVMe paths map and unmap ONE pool page ~470 times a second -- 27644 iommu_map <- __iommu_dma_map <- dma_map_phys <- nvme_prep_rq 35907 iommu_unmap_fast <- __iommu_dma_unmap <- dma_unmap_phys <- nvme_pci_complete_batch 33141 iommu_unmap_fast <- dma_iova_unlink <- dma_iova_destroy <- nvme_unmap_data so a +/-30 s window holds thousands of unmaps of the same IOVA and "the nearest one" proves nothing. correlate() now counts the unmaps inside the window and render() flags any pairing with more than three: [1 of 7 unmaps of this iova in the window -- NOT discriminating, narrow --window] which fires on the burst capture's own -5.65 s pairing (1 of 7). Two arms, including one asserting a lone in-window unmap is NOT flagged. (cherry picked from commit 20a882ac1f0d34c784181e4e067827c89193b7ba) --- docs/nvme-iommu-fault-triage.md | 9 +++++++++ tools/diag/unmap_fault_order.py | 29 +++++++++++++++++++++++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 95ada4752..79b8e5fef 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -149,6 +149,15 @@ the correlation. Because the same sentence is printed whether the window is too nearby unmap, **the negative now names the window it used and the measured interval** -- a default alone does not fix the failure, only a negative that says *at what scale* it found nothing does. +**And the reverse is just as wrong: a positive must say whether it discriminates.** The same page's cadence is +workload-dependent -- a later capture on this box (05:40Z, 150 s, one IOVA filtered) found the NVMe paths mapping it +from `nvme_prep_rq` through `dma_map_phys` and unmapping it from `nvme_pci_complete_batch` through +`iommu_unmap_fast`, **about 470 times a second**. At that rate a ±30 s window holds ~14,000 unmaps of the page, so +"the nearest one was 31 us before the fault" is not evidence of anything. **The tool now carries the count**: a +pairing whose window held more than three unmaps of the same IOVA is printed with +`[1 of N unmaps of this iova in the window -- NOT discriminating, narrow --window]`. On the burst capture above that +fires immediately -- the -5.65 s pairing is `1 of 7` -- which is the honest reading of it. + **One probe owns the shared buffer.** A kprobe on `iommu_unmap_fast` with **no filter** wrote 579,000 events and left a filtered probe on the same function with **zero** -- the trace ring buffer is shared, so the *narrow* probe (one IOVA, one call site) is the one that should be armed, and checking `grep -c ':' trace` before trusting a diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py index d51d660c7..f7a0b4866 100755 --- a/tools/diag/unmap_fault_order.py +++ b/tools/diag/unmap_fault_order.py @@ -105,19 +105,24 @@ def parse_faults(lines, journal_epoch): def correlate(unmaps, faults, window_us): - """[(fault, delta_us, unmap)] -- the nearest unmap of the same iova within the window, or None.""" + """[(fault, delta_us, unmap, n)] -- the nearest unmap of the same iova within the window, or None, and how many + unmaps of that iova the window held. `n` is the discriminator: a page recycled ~470 times a second puts thousands + of unmaps inside a 30 s window, and then "the nearest one" is not evidence of anything.""" by_iova = {} for t, iova, size in unmaps: by_iova.setdefault(iova, []).append((t, size)) rows = [] for t, iova, text in faults: best = None + n = 0 if t is not None: for ut, size in by_iova.get(iova, ()): d = (ut - t) * 1e6 # negative = the unmap came first - if abs(d) <= window_us and (best is None or abs(d) < abs(best[1])): - best = ((ut, iova, size), d) - rows.append((t, iova, text, best)) + if abs(d) <= window_us: + n += 1 + if best is None or abs(d) < abs(best[1]): + best = ((ut, iova, size), d) + rows.append((t, iova, text, best, n)) return rows @@ -146,10 +151,13 @@ def render(rows, unmaps, window_us): lines.append(f" sizes: {sorted({u[2] for u in unmaps})[0]:#x} .. {sorted({u[2] for u in unmaps})[-1]:#x}") hit = [r for r in rows if r[3] is not None] lines.append(f"faults parsed: {len(rows)}; with an unmap of the same iova within +-{_span(window_us)}: {len(hit)}") - for t, iova, _text, best in hit[:20]: + for t, iova, _text, best, n in hit[:20]: (ut, _u, size), d = best verdict = "unmap BEFORE fault (translation gone)" if d < 0 else "unmap AFTER fault (stale mapping or other cause)" - lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}") + # A positive must say whether it discriminates. Measured 2026-11-16: one NVMe pool page is mapped and unmapped + # ~470 times a second, so a 30 s window holds thousands of unmaps of it and "the nearest" means nothing. + disc = "" if n <= 3 else f" [1 of {n} unmaps of this iova in the window -- NOT discriminating, narrow --window]" + lines.append(f" iova {iova:#x} size {size:#x}: unmap {d:+.0f} us, {verdict}{disc}") if rows and not hit: # A negative must name its own condition. The same sentence at a 5 ms window and at 30 s read alike, and the # 5 ms one was simply wrong, so the scale is in the line and the remedy is named with it. @@ -221,6 +229,15 @@ def chk(cond, msg): hit = correlate(near, pf[:1], 30_000_000) chk(hit and hit[0][3] is not None and hit[0][3][1] < 0, "a padded fault correlates with an unmap 31 us earlier (BEFORE), not an uncorrelated negative") + # 9. a POSITIVE must say whether it discriminates: a page recycled hundreds of times a second puts thousands of + # unmaps in a wide window, and then "the nearest one" is not evidence. Measured 2026-11-16: one NVMe pool page + # is mapped and unmapped ~470 times a second (nvme_prep_rq maps it, nvme_pci_complete_batch unmaps it). + hot = [(100.0 + i * 0.001, 0x4128000, 0x1000) for i in range(500)] # 500 unmaps over 0.5 s + hr = correlate(hot, [(100.4, 0x4128000, "f6")], 30_000_000) + chk(hr[0][4] == 500 and "NOT discriminating" in " ".join(render(hr, hot, 30_000_000)), + "a page recycled hundreds of times a second is flagged as not discriminating") + chk("NOT discriminating" not in " ".join(render(correlate(unm, [(100.0005, 0x4128000, "f1")], 5000), unm, 5000)), + "a lone unmap in the window is not flagged") return 0 if ok else 1 From fc05c5f967c54bc876a8a8d7d20c1be3c3a51cb0 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 02:39:47 -0400 Subject: [PATCH 2/4] diag: the churn rate, and three ways a filtered kprobe silently measures nothing Follow-up to #2625, whose last commit (20a882ac1) was pushed after the merge and is not in main -- the exact loss llama.cpp#406 describes. Cherry-picked with -x, and the detector it proposes confirms it: git diff origin/main 20a882ac1 -- tools/diag/unmap_fault_order.py (non-empty) The '5-25 s' in the negative and in --help was the rate of THAT capture, not of this stream. Measured again on a busier boot, one NVMe pool page is mapped and unmapped ~470 times a second (nvme_prep_rq -> dma_map_phys -> iommu_map; nvme_pci_complete_batch -> dma_unmap_phys and dma_iova_unlink), so a +/-30 s window holds ~14,000 unmaps of it. Both messages now say 'seconds on a sparse stream, sub-millisecond where the page is recycled' instead of a constant. Plus the three instrument traps that cost runs while taking those counts: tracing_on=0 under enable=1; a stacktrace trigger hiding the event line (and blocking event deletion until !stacktrace:0); and tracefs rejecting range filters in a way that leaves the probe unfiltered. --- docs/nvme-iommu-fault-triage.md | 34 +++++++++++++++++++++++++++++---- tools/diag/unmap_fault_order.py | 9 ++++++--- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index 79b8e5fef..da8eabc36 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -144,10 +144,36 @@ pooled page. **Three of the four faults on it** followed an unmap of that same p page within 60 s** -- the nearest is **+74 s**, after the fault -- so this capture correlates three faults and shows the ceiling on the fourth; a wider window does not help it, and the earlier "19.60 s" I quoted for it came from a different capture and does not reproduce on this one. So `tools/diag/unmap_fault_order.py` defaults to **30 s** and -the page's own numbers are **5-25 s**: a millisecond window reports "nothing correlated" on a capture that contains -the correlation. Because the same sentence is printed whether the window is too narrow or the fault genuinely has no -nearby unmap, **the negative now names the window it used and the measured interval** -- a default alone does not fix -the failure, only a negative that says *at what scale* it found nothing does. +the page's own numbers are **5-25 s on that sparse stream**: a millisecond window reports "nothing correlated" on a +capture that contains the correlation. Because the same sentence is printed whether the window is too narrow or the +fault genuinely has no nearby unmap, **the negative names the window it used and the measured interval** -- a default +alone does not fix the failure, only a negative that says *at what scale* it found nothing does. + +**But "seconds" is not the rate of this stream -- it is the rate of that capture.** The same page measured again on a +busier boot is mapped and unmapped **about 470 times a second**: `nvme_prep_rq` maps it through `dma_map_phys` -> +`iommu_map`, and `nvme_pci_complete_batch` unmaps it from the completion IRQ through both `dma_unmap_phys` and +`dma_iova_unlink`/`dma_iova_destroy` (150 s, one IOVA, filtered, counts 27644 / 35907 / 33141). **So a wide window is +the wrong instrument in the other direction**: at that rate a +/-30 s window holds ~14,000 unmaps of the page and "the +nearest one was 31 us before the fault" is not evidence of anything. The tool therefore **carries the count** and flags +a pairing whose window held more than three unmaps of the same IOVA: +`[1 of N unmaps of this iova in the window -- NOT discriminating, narrow --window]`. On the burst capture above it +fires on the -5.65 s pairing immediately (`1 of 7`), which is the honest reading: **the interval is real, its precision +is not.** + +**Three ways a filtered kprobe silently measures nothing**, all hit while taking the counts above: + +1. **`tracing_on=0` with `enable=1`** records nothing, and the empty buffer reads exactly like "0 events". Check + `tracing_on` and `entries-written` on every capture, and run a **positive control** first (30 s of events, abort if + zero). +2. **A `stacktrace` trigger suppresses the event's own line.** The dump is `` blocks with no `iova=` text + at all, so a filter that is working perfectly looks like it never applied. Count events *without* the trigger first. + And `stacktrace:unlimited` must be cleared (`echo '!stacktrace:0' > .../trigger`) before `-:kprobes/x` will delete + the event -- otherwise the delete fails with `echo: I/O error` and the probe stays armed. +3. **tracefs rejects range filters, and the failed write leaves the probe UNFILTERED.** + `(iova + size > 0xX && iova <= 0xX)` gives `parse_error: Invalid operator`; the event then records **every** unmap + (1,947 in 20 s, against 18 in 30 s for the working exact filter). **Read the filter file back after writing it**, and + keep in mind that an exact-iova filter cannot see a *larger* mapping that covers the page -- for this driver that is + tolerable because its unmaps are all 4 KiB (60 of 60 measured), but it is a measured caveat, not a proof. **And the reverse is just as wrong: a positive must say whether it discriminates.** The same page's cadence is workload-dependent -- a later capture on this box (05:40Z, 150 s, one IOVA filtered) found the NVMe paths mapping it diff --git a/tools/diag/unmap_fault_order.py b/tools/diag/unmap_fault_order.py index f7a0b4866..f1acf072b 100755 --- a/tools/diag/unmap_fault_order.py +++ b/tools/diag/unmap_fault_order.py @@ -162,7 +162,8 @@ def render(rows, unmaps, window_us): # A negative must name its own condition. The same sentence at a 5 ms window and at 30 s read alike, and the # 5 ms one was simply wrong, so the scale is in the line and the remedy is named with it. lines.append(f" no fault shares an iova with an unmap inside +-{_span(window_us)}: ordering not shown by " - f"this capture AT THIS WINDOW -- the measured unmap-to-fault interval on this box is 5-25 s, " + f"this capture AT THIS WINDOW -- the interval is seconds on a sparse stream and sub-millisecond " + f"where the page is recycled (measured 470/s on one NVMe pool page), " f"so widen --window before reading this as 'no unmap preceded the fault'") return lines @@ -211,7 +212,7 @@ def chk(cond, msg): # The fault is 100 s from any unmap: at 30 s that is still a miss, which is the shape a too-narrow window gives. out = " ".join(render(correlate(unm, [(300.000, 0x4128000, "f5")], 30_000_000), unm, 30_000_000)) chk("+-30 s" in out and "30000000 us" not in out, "a 30 s window reads as '+-30 s', not '+-30000000 us'") - chk("AT THIS WINDOW" in out and "5-25 s" in out, + chk("AT THIS WINDOW" in out and "sparse stream" in out, "a negative names the window it used and the measured interval, not 'nothing correlated' bare") chk("+-5000 us" in " ".join(render(correlate(unm, [(100.500, 0x4128000, "f3")], 5000), unm, 5000)), "a sub-second window still reads in microseconds") @@ -249,7 +250,9 @@ def main(): # 5.6 s / 24.7 s / 24.7 s earlier, and a 5 ms default answered "no fault shares an iova with an unmap" on the # very capture that contains them. The interval is seconds here; a small default hides the finding. ap.add_argument("--window", type=float, default=30_000_000.0, - help="window in microseconds (default 30,000,000 = 30 s; measured interval on this box is 5-25 s)") + help="window in microseconds (default 30,000,000 = 30 s). The interval is seconds on a sparse stream " + "and sub-millisecond on a recycled page (470/s measured), so the count of unmaps in the " + "window is reported: a pairing with many candidates is not evidence") ap.add_argument("--epoch-offset", type=float, default=0.0, help="seconds to add to journal stamps to reach the trace's boot clock") ap.add_argument("--self-test", action="store_true") From 2736e882e4734e86c81fcda23260d903c234b9cb Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 02:58:23 -0400 Subject: [PATCH 3/4] docs: repair the conflict markers #2625 left on main, and record the map-side result #2625 merged with its own conflict markers still in docs/nvme-iommu-fault-triage.md (line 137 <<<<<<< HEAD, line 157 >>>>>>>), and they were INSIDE a bash code block: the numbered recipe was split by several paragraphs of prose and the page no longer rendered. The prose moves after the fence, the marker lines go, and the recipe reads #1/#2/#3 again. Everything else here is the map side of #2478, measured 06:42Z with the same page: a filtered capture on five addresses chosen from the fault census recorded ZERO events in 30 s and aborted on its own positive control, because the fault target moves -- the last three faults were on an address the list did not have. Unfiltered for 13 min (252,704 map/unmap events, 795 s covered): 0xebf50000 x4 faults NO map or unmap of this iova anywhere in the window 0x6925f000 x1 nearest map +230 ms AFTER the fault 0xe407f000 x2 nearest -58 s 0xe1d00000 x1 nearest -347 s Four of eight faults hit an IOVA never mapped and never unmapped in 795 s, so the translation is absent and nothing re-creates it: a descriptor carrying an address the driver has finished with, not a stale TLB entry. Two of my own instrument errors, corrected in the page: iommu_map's size is %cx (%dx is paddr, and the probe printed size=0x138ec90000 for an 84 GB 'size'), and a regex anchored on the task name dropped 2,176 of 115,941 map lines because 'next-server (v1-8386' contains a space. --- docs/nvme-iommu-fault-triage.md | 42 ++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index da8eabc36..cc96d97d8 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -134,8 +134,10 @@ only if you want to confirm that, not as a discriminator. ```bash # 1. pick the top faulting IOVA from the census -<<<<<<< HEAD -======= +# 2. kprobe iommu_unmap_fast on that iova alone (filter), stacktrace on, capture for minutes in the background +# 3. in the same capture window, note the fault timestamps for that iova +``` + **The interval is seconds, not milliseconds (measured 2026-11-16).** A filtered probe on one faulting IOVA (`0xe1e18000`), captured over a boot's first 28 minutes, found **60 unmaps of that page, every one a single 4 KiB page, 59 of them from `` (interrupt context)**: the NVMe completion path repeatedly maps and tears down one @@ -175,6 +177,38 @@ is not.** keep in mind that an exact-iova filter cannot see a *larger* mapping that covers the page -- for this driver that is tolerable because its unmaps are all 4 KiB (60 of 60 measured), but it is a measured caveat, not a proof. +**Arm the map side too, unfiltered, and the argument for one address stops being needed (measured 11-16 06:42Z).** +A filtered capture on five addresses chosen from the fault census recorded **zero events in 30 s** and aborted on its +own positive control -- because **the fault target moves**: the last three faults were all on an address the list did +not contain. The capture that worked armed `iommu_map` and `iommu_unmap_fast` with **no filter** for 13 minutes +(44 unmaps/s is what a quiet minute looks like; a busy one is ~1,000/s, and the per-CPU ring still covered all 795 s). +Of the 8 faults in that window: + +``` +0xebf50000 x4 faults NO map or unmap of this iova anywhere in the 795 s +0x6925f000 x1 nearest map +230 ms AFTER the fault +0xe407f000 x2 nearest map/unmap -58 s +0xe1d00000 x1 nearest -347 s +``` + +**Four of eight faults hit an IOVA that was never mapped and never unmapped at any point in 795 seconds of a capture +holding 252,704 map/unmap events** -- so the translation is not stale, it is **absent, and nothing re-creates it**. That +moves the question to the descriptor: the device writes to an address the driver has already finished with, which is a +DMA-mapping lifetime bug, not a TLB or lazy-invalidation one. `0x6925f000` is the exception and worth separating: it is +**re-mapped 230 ms after** the fault, the profile of a live pool page faulting before its re-map. + +The kprobe recipes, corrected against the kernel's own argument order -- the first one was wrong in this page's tool: + +``` +iommu_map(domain, iova, paddr, size, prot) -> iova=%si size=%cx # %dx is PADDR, and a probe + # using it printed size=0x138ec90000 +iommu_unmap_fast(domain, iova, size, ...) -> iova=%si size=%dx +``` + +**A task name can contain a space** (`next-server (v1-8386`), so a regex anchored with `^\s+\S+\s+\[` silently dropped +**2,176 of 115,941 map lines** -- 1.9 %, which is exactly the fraction that can contain the event being looked for. +Anchor on the timestamp instead, and reconcile the parse count against `grep -c` before believing a negative. + **And the reverse is just as wrong: a positive must say whether it discriminates.** The same page's cadence is workload-dependent -- a later capture on this box (05:40Z, 150 s, one IOVA filtered) found the NVMe paths mapping it from `nvme_prep_rq` through `dma_map_phys` and unmapping it from `nvme_pci_complete_batch` through @@ -189,10 +223,6 @@ a filtered probe on the same function with **zero** -- the trace ring buffer is IOVA, one call site) is the one that should be armed, and checking `grep -c ':' trace` before trusting a capture is the same "prove the plumbing" rule two paragraphs up. ->>>>>>> 2b1faf1b2 (diag: a negative names the window it used, and the window reads in seconds) -# 2. kprobe iommu_unmap_fast on that iova alone (filter), stacktrace on, capture for minutes in the background -# 3. in the same capture window, note the fault timestamps for that iova -``` The question the ordering answers: does a fault follow an unmap of the *same* address, or does it arrive while the address is still mapped? The first is a write-after-unmap — the device dirtying a buffer the CPU released, or the driver From e903849645a1ce730e5e49cd4bce332cbe12184c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Mon, 16 Nov 2026 03:09:21 -0400 Subject: [PATCH 4/4] ci: a conflict marker committed at the start of a line fails the pre-PR checks #2625 merged with its own markers still in docs/nvme-iommu-fault-triage.md -- '<<<<<<< HEAD' at 137 and '>>>>>>> 2b1faf1b2' at 157, both INSIDE a bash block, so the numbered recipe was split by prose, the fence closed in the wrong place and the page other lanes reason from did not render. Nothing read the file back: the merge kept what a hand-resolved conflict should have removed. tools/ci/conflict_marker_check.py, stateful on purpose: '<<<<<<<', '|||||||' and '>>>>>>>' are git's own, and '=======' counts ONLY inside an unterminated block -- the first version treated a bare '=======' as a separator and flagged 10 Markdown underlines and shell rules across 4 files on a clean tree, which is the shape of check nobody keeps. The real tree now reports 0. Registered where the tree needs it: a ctest --check row and its --self-test (tests_referenced_check requires the invocation, not the mention: 72 tool self-tests, 52 python tests), the pre-pr-checks runner, and its page -- the runner refuses to run when the page and the array disagree, which is how the missing page row surfaced rather than silently. Self-test: 7 arms (mid-line quotes and setext underlines are not conflicts; the #2625 shape yields start+separator+end; build/ and .bin skipped; rc 0/1/2). pre-pr-checks: all 11 green, 3 skipped. --- CMakeLists.txt | 6 ++ docs/pre-pr-checks.md | 1 + scripts/pre-pr-checks.sh | 1 + tools/ci/conflict_marker_check.py | 166 ++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 tools/ci/conflict_marker_check.py diff --git a/CMakeLists.txt b/CMakeLists.txt index da5cecc42..4a40aeafd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -460,6 +460,12 @@ add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_or add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test) add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR}) add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test) +# engine#2641: #2625 merged with its own conflict markers still in docs/nvme-iommu-fault-triage.md, inside a ```bash +# block, so the page did not render and nothing noticed -- the merge kept them because the file was never read back. +# The check is stateful on purpose: a bare `=======` is a Markdown underline or a shell rule (the first version +# flagged 10 of those on a clean tree), so a separator counts only inside an unterminated conflict block. +add_python_test(conflict_marker ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --check --root ${CMAKE_SOURCE_DIR}) +add_python_test(conflict_marker_self ${CMAKE_SOURCE_DIR}/tools/ci/conflict_marker_check.py --self-test) # engine#803 gate, second half: the comparator that turns two --dump-logits directories into the KLD verdict the # P4 gate asks for. No device and no model, so the gate is one command when a ticket is free; the self-test pins # the four cases a KLD implementation gets wrong (identical -> exactly 0, a shift is positive and directional, a diff --git a/docs/pre-pr-checks.md b/docs/pre-pr-checks.md index f03fda321..0182d6702 100644 --- a/docs/pre-pr-checks.md +++ b/docs/pre-pr-checks.md @@ -32,6 +32,7 @@ below says why. The table's order is the runner's order: cheapest and most commo | `python3 tools/ci/tests_referenced_check.py` | a test, tool self-test or python test exists that nothing runs | | `python3 tools/ci/ci_targets_check.py --check` | a CI row names a target or path that does not exist | | `python3 tools/ci/tmp_path_check.py --check` | a new `/tmp` or `/dev/shm` path, where builds and dumps belong under `~/wt` or `~/.cache/scratch-*` | +| `python3 tools/ci/conflict_marker_check.py --check` | a merge-conflict marker committed at the start of a line -- #2625 shipped `<<<<<<< HEAD` inside a ```bash block and the page stopped rendering, and nothing read it back | | `python3 tools/ci/submodule_pin_check.py --check` | a submodule's checkout is not the commit the tree pins | | `python3 tools/ci/cmake_configure_check.py --check` | `CMakeLists.txt` no longer configures -- CI's Configure step would fail, and no text check can see a line that starts correctly and ends wrong | | `python3 tools/ci/registration_loss_check.py --check --base origin/main` | your branch would drop rows `main` has | diff --git a/scripts/pre-pr-checks.sh b/scripts/pre-pr-checks.sh index 23b107889..27ab399c9 100755 --- a/scripts/pre-pr-checks.sh +++ b/scripts/pre-pr-checks.sh @@ -63,6 +63,7 @@ checks=( "tools/ci/tests_referenced_check.py" "tools/ci/ci_targets_check.py --check" "tools/ci/tmp_path_check.py --check" + "tools/ci/conflict_marker_check.py --check" "tools/ci/submodule_pin_check.py --check" # engine#2445: the tree must CONFIGURE, not just parse as text. A rebase resolution left one line # ending `…_test.py)ms)` and every check above stayed green -- they read files, run Python and diff diff --git a/tools/ci/conflict_marker_check.py b/tools/ci/conflict_marker_check.py new file mode 100644 index 000000000..5b085d599 --- /dev/null +++ b/tools/ci/conflict_marker_check.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Merge-conflict markers committed to the tree. + +Engine#2625 merged with its own conflict markers still in `docs/nvme-iommu-fault-triage.md` -- `<<<<<<< HEAD` at +line 137 and `>>>>>>> 2b1faf1b2 …` at line 157, both **inside a ```bash block**, so the numbered recipe was split by +paragraphs of prose, the fence closed in the wrong place, and the page other lanes reason from did not render. The +merge kept the markers because the conflict was resolved by hand in the wrong place and the file was never read back: +the same shape as engine#2622 (a parser returning `None` for every real line), #2630 (`0 unmaps` indistinguishable +from a dead probe) and llama.cpp#406 (counters that do not sum) -- an output nobody checked. + +The check is deliberately narrow. It looks for a marker at the **start of a line** (`<<<<<<<`, `=======`, `>>>>>>>`, +`|||||||`), which is what git writes; prose is free to quote markers mid-line, and this file is such prose. + +Usage: + tools/ci/conflict_marker_check.py --check [--root .] # rc 1 on any marker, 0 when clean + tools/ci/conflict_marker_check.py --self-test # rc 1 unless the arms below behave +""" +from __future__ import annotations + +import argparse +import os +import pathlib +import re +import sys + +# `=======` is only a conflict separator INSIDE a conflict block. On its own it is a Markdown setext underline or a +# shell rule, and the first version of this check flagged 10 of those across 4 files on a tree with no conflicts at +# all -- a check that fails on clean input is one nobody keeps. So the scan is stateful: `open` counts the markers +# git writes, and `=======` is reported only while a `<<<<<<<` is still open. +MARKERS = { + "<<<<<<<": "conflict start", + "|||||||": "conflict base", + ">>>>>>>": "conflict end", +} +EQUALS = "=======" + +SKIP_DIRS = {".git", "build", "build-host", "build-shp", "__pycache__", "third_party", "vendor"} +SKIP_SUFFIX = {".onnx", ".gguf", ".q4nx", ".fconst", ".bin", ".so", ".a", ".o", ".hsaco", ".png", ".jpg", ".pdf"} + + +def scan(root: pathlib.Path) -> list[tuple[str, int, str]]: + """[(relpath, line number, marker kind)] for every marker at the start of a line in a text file.""" + hits: list[tuple[str, int, str]] = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for fn in filenames: + p = pathlib.Path(dirpath) / fn + if p.suffix.lower() in SKIP_SUFFIX: + continue + try: + text = p.read_text(errors="replace") + except OSError: + continue + if "\0" in text[:4096]: # binary that slipped past the suffix list + continue + open_block = False + for i, line in enumerate(text.splitlines(), 1): + head = line[:8] + if head.startswith("<<<<<<<"): + open_block = True + hits.append((str(p.relative_to(root)), i, "conflict start")) + elif head.startswith("|||||||"): + hits.append((str(p.relative_to(root)), i, "conflict base")) + elif head.startswith(">>>>>>>"): + open_block = False + hits.append((str(p.relative_to(root)), i, "conflict end")) + elif open_block and head.startswith(EQUALS): + hits.append((str(p.relative_to(root)), i, "conflict separator")) + return hits + + +def main(argv: list[str]) -> int: + g = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + g.add_argument("--check", action="store_true") + g.add_argument("--self-test", action="store_true") + g.add_argument("--root", default=".") + a = g.parse_args(argv) + if a.self_test: + return self_test() + if not a.check: + g.print_help() + return 2 + root = pathlib.Path(a.root) + if not root.is_dir(): + print(f"conflict_marker: no such root {root}") + return 2 + hits = scan(root) + if hits: + for rel, n, what in hits[:40]: + print(f" {rel}:{n}: {what}") + if len(hits) > 40: + print(f" ... and {len(hits) - 40} more") + print(f"conflict_marker: {len(hits)} marker(s) committed in {len({h[0] for h in hits})} file(s)") + return 1 + print("conflict_marker: 0 marker(s) at the start of any line") + return 0 + + +def self_test() -> int: + import tempfile + + ok = True + + def chk(cond: bool, what: str) -> None: + nonlocal ok + print(("ok " if cond else "FAIL ") + what) + ok = ok and cond + + with tempfile.TemporaryDirectory() as td: + root = pathlib.Path(td) + (root / "docs").mkdir() + (root / "build").mkdir() + + clean = ( + "# Title\n\n" + "A paragraph that QUOTES the markers mid-line, as this tool's own docstring does: `<<<<<<< HEAD` is what\n" + "git writes, and a `=======` rule inside a sentence is not a conflict.\n\n" + "```\n# not a conflict either\n=======\n```\n" + ) + # 1. a document that only mentions the markers mid-line is clean -- otherwise this check would fail on its + # own docstring, and a check that cannot pass on a clean tree is a check nobody keeps. + (root / "docs" / "clean.md").write_text(clean) + chk(scan(root) == [], "markers quoted mid-line, and a bare ======= line with no open block, are not conflicts") + (root / "docs" / "rule.md").write_text("Title\n=====\n\nsome prose\n\n=======\n\nmore prose\n") + chk(scan(root) == [], "a setext underline and a rule that look like markers are not conflicts either") + (root / "docs" / "rule.md").unlink() + + # 2. the shape engine#2625 shipped: markers at the start of a line, inside a fence + (root / "docs" / "broken.md").write_text("```bash\n# 1. pick the IOVA\n<<<<<<< HEAD\n=======\nprose\n>>>>>>> 2b1faf1b2 (msg)\n# 2. arm it\n```\n") + hits = scan(root) + kinds = {h[2] for h in hits} + chk(len(hits) == 3 and kinds == {"conflict start", "conflict separator", "conflict end"}, + "the #2625 shape yields start, separator and end (3 markers)") + chk(any(h[0].endswith("broken.md") and h[1] == 3 for h in hits), "... and the line number is the marker's own") + + # 3. build dirs and binaries are not scanned: a build tree holds no committed text, and a binary can carry + # bytes that look like a marker. + (root / "build" / "x.md").write_text("<<<<<<< HEAD\n") + (root / "blob.bin").write_bytes(b"<<<<<<< HEAD\n") + hits = scan(root) + chk(len(hits) == 3, "build/ and .bin are skipped") + + # 4. rc follows: 0 clean, 1 dirty, 2 no root + chk(main(["--check", "--root", str(root)]) == 1, "--check returns 1 when a marker is committed") + (root / "docs" / "broken.md").unlink() + chk(main(["--check", "--root", str(root)]) == 0, "--check returns 0 on the clean tree") + chk(main(["--check", "--root", str(root / "nope")]) == 2, "--check returns 2 for a root that is not a directory") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))