From dcc0086df3e12d86fd8f198023eed12be41c5683 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 02:26:25 -0400 Subject: [PATCH 1/4] server_launch_check: a group kill is the FIX, not the defect (and a waiver for the residue) Ran the merged checker (b9a04cc3) on the two correct remedies for its own bug: setsid "$BIN" serve -p $PORT & PID=$! ... kill -- -$PID -> rc 1, 2 findings setsid "$BIN" serve -p $PORT & ... kill -TERM -- -$PGID -> rc 1, 1 finding Both are right: `setsid cmd &` makes the child a session leader, so its pgid IS the leader's pid -- what $! holds -- and a group kill reaches the server even though the leader exited. It is the form gpu-hold's kill_tree uses. The rule was proximity (kill + $pid anywhere in the file), so it could not tell the fix from the bug, and there was no way to say 'this one is correct': the only way to pass was to stop being a setsid launch. Now: a group kill anywhere in the file decides it (clean); an explicit '# server-launch-check: ok ' within three lines above the target line is honoured; everything else reports exactly as before. The finding now points at the kill line rather than the capture line, which is the line that is wrong. 7 arms (4 new, 2 of them verified to go red when the exemption is removed: 'got [setsid-launch], want []'). Real tree: rc 0, no findings; a real offender file: rc 1, both findings. --- tools/ci/server_launch_check.py | 51 ++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/tools/ci/server_launch_check.py b/tools/ci/server_launch_check.py index a0b8bd04e..c7e76a4a7 100755 --- a/tools/ci/server_launch_check.py +++ b/tools/ci/server_launch_check.py @@ -42,31 +42,44 @@ _LAUNCH = re.compile(r"\b(serve|forward-serve|llama-server|llama-cli)\b") _SETSID = re.compile(r"\bsetsid\b") _AMP = re.compile(r"&\s*$") +# A kill that targets a whole PROCESS GROUP is the correct remedy, not the defect: `setsid cmd &` makes the +# child a session leader, so its pgid IS the leader's pid -- which is what `$!` holds -- and a group kill +# reaches the server even though the leader has exited. Without this the checker reds the fix (2026-11-15). +_GROUP_KILL = re.compile(r"\bkill\s+(-[A-Za-z]+\s+)*--?\s*-") +# An explicit waiver, for the residue a text scan cannot decide: `# server-launch-check: ok ` on or +# just above the kill line. +_SUPPRESS = re.compile(r"server-launch-check:\s*ok\b") def check_text(text, path=""): """-> list of (lineno, line, kind) findings. kind is 'setsid-launch' or 'kill-after-setsid'.""" lines = text.splitlines() - findings = [] - setsid_launch = False + launches = [] for i, line in enumerate(lines, 1): stripped = line.strip() if stripped.startswith("#"): continue # a comment naming the pattern is not the pattern if _SETSID.search(stripped) and _LAUNCH.search(stripped) and _AMP.search(stripped): - findings.append((i, stripped, "setsid-launch")) - setsid_launch = True + launches.append((i, stripped)) + if not launches: + return [] + # A group kill anywhere in the file is the remedy, and it decides the whole file: `kill -- -$PID` signals + # the session the server lives in, so the cleanup cannot miss it even though the leader has exited. + body = [(i, l.strip()) for i, l in enumerate(lines, 1) + if not l.strip().startswith("#") and re.match(r"kill\b", l.strip())] + if any(_GROUP_KILL.search(t) for _, t in body): + return [] # The second half of the defect: `$!` captured from a setsid launch is the wrong process. Report it # once, because that is what turns "setsid is unusual" into "the cleanup provably misses the server". - if setsid_launch: - for i, line in enumerate(lines, 1): - stripped = line.strip() - if stripped.startswith("#"): - continue - if re.search(r"\bkill\b.*\$(PID|\!|pid)\b", stripped) or "$!" == stripped[-2:]: - findings.append((i, stripped, "kill-after-setsid")) - break - return findings + targets = [(i, t) for i, t in body if re.search(r"\bkill\b.*\$(PID|\!|pid)\b", t)] + if not targets: # no kill: fall back to the capture line itself + targets = [(i, l.strip()) for i, l in enumerate(lines, 1) + if not l.strip().startswith("#") and l.strip().endswith("$!")] + for i, stripped in targets: + if any(_SUPPRESS.search(c) for c in lines[max(0, i - 4):i - 1]): + return [] # explicitly waived, with a stated reason + return [(n, t, "setsid-launch") for n, t in launches] + [(i, stripped, "kill-after-setsid")] + return [(n, t, "setsid-launch") for n, t in launches] def self_test(): @@ -79,6 +92,18 @@ def self_test(): '"$BIN" forward-serve -m "$MODEL" -p "$PORT" &\nPID=$!\nkill $PID\n', []), ("a COMMENT naming setsid is not the defect", '# do not use setsid here: it makes $! the wrong pid\nnohup "$BIN" serve -p $PORT &\n', []), + # 2026-11-15: the correct remedy must be GREEN, or the only way to satisfy the gate is to avoid + # setsid rather than to fix the cleanup. + ("kill -- -$PID (group kill by the captured setsid pid) is the fix, not the defect", + 'setsid "$BIN" forward-serve -p "$PORT" &\nPID=$!\nsleep 3\nkill -- -"$PID"\n', []), + ("kill -TERM -- -$PGID is the same fix, named group", + 'setsid "$BIN" serve -p $PORT &\nsleep 3\nkill -TERM -- -$PGID\n', []), + ("an explicit waiver just above the kill line is honoured", + 'setsid "$BIN" serve -p $PORT &\nPID=$!\n# server-launch-check: ok pid re-resolved below\nkill $PID\n', + []), + ("the un-waived defect itself is still the defect", + 'setsid "$BIN" serve -p $PORT &\nPID=$!\nkill $PID\n', + ["setsid-launch", "kill-after-setsid"]), ] bad = 0 for name, text, want in arms: From e5d52977eb7535473734fdf7bd26d5feb3b823ae Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 04:05:48 -0400 Subject: [PATCH 2/4] Refs #2478: the runbook for its two root-only tests, so the next person does not repeat the week MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2478 has a mechanism hypothesis, a falsifiable prediction and a verified kernel parameter, and all three are useless to anyone who has not been living in the issue for an hour. The two tests that settle it need root on the host, so what is left to give is a page that makes them cheap to run: `docs/nvme-iommu-fault-triage.md`. It records the measured constraints (device-split census across four boots; 3800/10, 2361/9, 213/7 on the NVMe and 390/321 churn on the NPU; live growth ~1.7/min; the 64 MiB HMB on a DRAM-less PC41Q), the kernel's own settings (`lazy mode` invalidation, SWIOTLB), and then the two tests with what each result *means*: 1. **Disable the HMB** (`nvme set-feature /dev/nvme0 -f 0x0d -v 0`, no reboot). Stream stops ⇒ the faulting region is the HMB. Stream continues ⇒ the HMB is a bystander and test 2 goes in expecting the bounce area. 2. **`iommu.strict=1`**, one reboot. Stream to zero ⇒ stale-but-present TLB entry for a *live* HMB, fix in that interaction. Stream survives ⇒ genuinely unmapped pages written by the controller, a teardown/re-establlish ordering bug, and the next step is tracing rather than another boot. Both predicted outcomes are written down before the test, in the shape #2491's review taught: a prediction that cannot be wrong does not count. Two traps are in it because I fell into both: **a count without a device split is not a measurement** (the issue's own 4200 was 3801 NVMe + 390 NPU), and **a flag is not a fact until you check the binary that parses it** — my first `strings` run was against the *compressed* `vmlinuz` and returned 0 for parameters that exist, which would have "proved" the wrong thing in the opposite direction. Linked from `tools/diag/iommu_fault_census.py`'s own docstring, because a page nothing links to is a page nobody runs. Written for whoever has root on strixhalo, since this lane does not. --- docs/nvme-iommu-fault-triage.md | 99 ++++++++++++++++++++++++++++++++ tools/diag/iommu_fault_census.py | 1 + 2 files changed, 100 insertions(+) create mode 100644 docs/nvme-iommu-fault-triage.md diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md new file mode 100644 index 000000000..d6f91a8bc --- /dev/null +++ b/docs/nvme-iommu-fault-triage.md @@ -0,0 +1,99 @@ + + +# Triaging an `IO_PAGE_FAULT` stream on a DRAM-less NVMe + +*A runbook for the two tests in engine#2478 that need root on the host: 2361 `AMD-Vi: Event logged [IO_PAGE_FAULT]` +lines on the root SSD in the boot that died at 23:47:35, still accumulating at ~1.7/min. Everything below is what an +unprivileged lane could establish, plus the two commands that settle it.* + +## What is already measured + +``` +tools/diag/iommu_fault_census.py --journal boot-2.txt --journal boot-1.txt --journal boot-0.txt + +boot -2 0000:c4:00.0 domain=0x0015: 3800 faults, 10 distinct x688 top +boot -1 0000:c4:00.0 domain=0x0015: 2361 faults, 9 distinct x301 top +boot 0 0000:c4:00.0 domain=0x0015: 213 faults, 7 distinct x81 top + -> few live mappings faulting over and over -> a stale translation: find the buffer + +boot -2 0000:c6:00.1 domain=0x0002: 390 faults, 321 distinct max 6 each + -> many distinct pages about once each -> re-allocation: find the race +``` + +Four facts that constrain the answer, each of which cost a measurement to learn: + +1. **Split by device before counting.** A raw `journalctl -k -b N | grep -c IO_PAGE_FAULT` reported 4200 for one boot; + the real NVMe figure was 3801 and the rest belonged to the NPU on a different PCI function. Every count in the + original report mixed two unrelated faults. `distinct` is the tell: one device cannot have 4200 faults over 341 + addresses and also 3800 over 10. +2. **The stream is older than the report's table says.** Boot −6 (Nov 11 15:30) already has 112 faults, before the boot + the table called the start. The window was the limit, not the fault. +3. **It is live, not historical.** 127 faults at 06:4xZ → 213 later the same morning, ~1.7/min on a box up three hours. +4. **The buffer has a name.** `nvme nvme0: allocated 64 MiB host memory buffer (1 segment)` on a DRAM-less YMTC PC41Q. + An HMB is host DRAM the controller writes into on its own initiative, which is why the stream needs no driver action + and never trips an NVMe timeout, reset or abort. + +And the settings the kernel reports about itself: + +``` +iommu: Default domain type: Translated +iommu: DMA domain TLB invalidation policy: lazy mode <- the default here, nothing asked for +PCI-DMA: Using software bounce buffering for IO (SWIOTLB) +software IO TLB: mapped [mem 0x5927e000-0x5d27e000] (64MB) +``` + +## Test 1 — disable the HMB (no reboot, decides whether the buffer is the HMB at all) + +```bash +nvme get-feature /dev/nvme0 -f 0x0d -H # Host Memory Buffer: current preference +nvme set-feature /dev/nvme0 -f 0x0d -v 0 # ask for no HMB +journalctl -k -f | grep -c IO_PAGE_FAULT # watch one burst interval (~2-5 min under load) +``` + +**Prediction:** the `c4:00.0` stream stops within one burst interval. +**If it stops** the faulting region is the HMB, and the remaining question is only why its mapping goes stale. +**If it continues** the HMB is a bystander and the faulting region is the SWIOTLB bounce area or another map — go to +test 2 with that expectation written down first. + +## Test 2 — strict IOMMU invalidation (one reboot, separates "stale" from "unmapped") + +```bash +# verify the parameter against the kernel that will parse it, not from memory: +/usr/src/linux-headers-$(uname -r)/scripts/extract-vmlinux /boot/vmlinuz-$(uname -r) > /tmp/vmlinux +strings /tmp/vmlinux | grep -c 'iommu.strict' # 3 on 7.4.0-rc1-1bit +strings /tmp/vmlinux | grep -c 'amd_iommu=strict' # 0 -- this spelling does NOT exist here +``` + +Add to the kernel command line: + +``` +iommu.strict=1 +``` + +**Prediction:** the `c4:00.0`/`0x0015` stream goes to **zero for a full boot**; the NPU `c6:00.1` churn is unchanged. +**If it goes to zero** the mapping is *stale but still present* in a lazily-invalidated TLB for a **live** HMB — the fix +belongs in how the HMB mapping interacts with lazy invalidation, not in the NVMe driver. +**If it survives strict mode** the HMB pages are genuinely unmapped while the controller still writes them: a +teardown/re-establish ordering bug on the NVMe side, and the next step is tracing that ordering, not another boot. + +## Two traps this runbook exists to stop + +* **A count without a device split is not a measurement.** `distinct` exposes the mixing. +* **A flag is not a fact until you check the binary that parses it.** The first `strings` attempt here ran against the + *compressed* `vmlinuz` and returned 0 for every pattern, including ones that exist — which would have "proved" the + parameter absent. Decompress first (`extract-vmlinux`), then count. diff --git a/tools/diag/iommu_fault_census.py b/tools/diag/iommu_fault_census.py index 2f38b3d91..2de0fdeba 100755 --- a/tools/diag/iommu_fault_census.py +++ b/tools/diag/iommu_fault_census.py @@ -27,6 +27,7 @@ synthetic fixture in the self-test. journalctl -k -b -1 | python3 tools/diag/iommu_fault_census.py --label -1 + # the runbook that reads this output, and the two root-only tests it decides: docs/nvme-iommu-fault-triage.md journalctl -k -b -2 | python3 tools/diag/iommu_fault_census.py --label -2 --json python3 tools/diag/iommu_fault_census.py --journal a.txt --journal b.txt # intersects the files From ca90123d0f6b34117827f924735135caca10278e Mon Sep 17 00:00:00 2001 From: t Date: Sun, 15 Nov 2026 04:06:18 -0400 Subject: [PATCH 3/4] server_launch_check: the group-kill exemption is PER LAUNCH, not per file Review on #2499, measured: one correct pair exempted the whole file, so setsid serve -p 1111 & PA=$! kill -- -$PA # correct setsid forward-serve -p 2222 & PB=$! kill "$PB" # BROKEN came back rc 0. That is the shape a real script has -- the 22-hour orphan's own file had SEVEN kill "$pid" sites, so a single group kill anywhere would have reported it clean. Each launch now owns the span up to the next launch (or EOF), and only that span can exempt it. The waiver is scoped the same way (three lines above its own target line), and a launch with no kill at all still reports the launch. Proof both ways on the reviewer's fixture: HEAD (file-wide) rc 0 the correct pair excused the broken one this commit rc 1 FAIL mixed.sh:5 [setsid-launch] + :6 [kill-after-setsid] 8 arms: the 7 that were there all still pass, plus 'a correct pair does not excuse a broken pair in the same file'. --- tools/ci/server_launch_check.py | 48 +++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/tools/ci/server_launch_check.py b/tools/ci/server_launch_check.py index c7e76a4a7..33aa67853 100755 --- a/tools/ci/server_launch_check.py +++ b/tools/ci/server_launch_check.py @@ -63,23 +63,32 @@ def check_text(text, path=""): launches.append((i, stripped)) if not launches: return [] - # A group kill anywhere in the file is the remedy, and it decides the whole file: `kill -- -$PID` signals - # the session the server lives in, so the cleanup cannot miss it even though the leader has exited. - body = [(i, l.strip()) for i, l in enumerate(lines, 1) - if not l.strip().startswith("#") and re.match(r"kill\b", l.strip())] - if any(_GROUP_KILL.search(t) for _, t in body): - return [] - # The second half of the defect: `$!` captured from a setsid launch is the wrong process. Report it - # once, because that is what turns "setsid is unusual" into "the cleanup provably misses the server". - targets = [(i, t) for i, t in body if re.search(r"\bkill\b.*\$(PID|\!|pid)\b", t)] - if not targets: # no kill: fall back to the capture line itself - targets = [(i, l.strip()) for i, l in enumerate(lines, 1) - if not l.strip().startswith("#") and l.strip().endswith("$!")] - for i, stripped in targets: - if any(_SUPPRESS.search(c) for c in lines[max(0, i - 4):i - 1]): - return [] # explicitly waived, with a stated reason - return [(n, t, "setsid-launch") for n, t in launches] + [(i, stripped, "kill-after-setsid")] - return [(n, t, "setsid-launch") for n, t in launches] + # The exemption is PER LAUNCH, not per file (review on #2499, measured): a group kill is the remedy for the + # launch it cleans up, and one correct pair must not excuse a broken pair elsewhere in the same file. The + # 22-hour orphan's own file had seven `kill "$pid"` sites -- a file-wide rule called it clean if one of + # them had been a group kill. + findings = [] + for idx, (li, ltext) in enumerate(launches): + end = launches[idx + 1][0] - 1 if idx + 1 < len(launches) else len(lines) + span = [(n, lines[n - 1].strip()) for n in range(li, end + 1) + if not lines[n - 1].strip().startswith("#")] + kills = [(n, t) for n, t in span if re.match(r"kill\b", t)] + if any(_GROUP_KILL.search(t) for _, t in kills): + continue # this launch is cleaned up by a group kill + # The defect's second half: `$!` captured from a setsid launch is the wrong process. Report it once, + # because that is what turns "setsid is unusual" into "the cleanup provably misses the server". + targets = [(n, t) for n, t in kills if re.search(r"\bkill\b.*\$(PID|\!|pid)\b", t)] + if not targets: # no kill: fall back to the capture line itself + targets = [(n, t) for n, t in span if t.endswith("$!")] + if not targets: + findings.append((li, ltext, "setsid-launch")) + continue + n, t = targets[0] + if any(_SUPPRESS.search(c) for c in lines[max(0, n - 4):n - 1]): + continue # explicitly waived, with a stated reason + findings.append((li, ltext, "setsid-launch")) + findings.append((n, t, "kill-after-setsid")) + return findings def self_test(): @@ -101,6 +110,11 @@ def self_test(): ("an explicit waiver just above the kill line is honoured", 'setsid "$BIN" serve -p $PORT &\nPID=$!\n# server-launch-check: ok pid re-resolved below\nkill $PID\n', []), + # The reviewer of #2499 measured this one: the exemption was file-wide, so a correct pair excused + # a BROKEN pair later in the same file. The 22-hour orphan had seven kill sites in one file. + ("a correct pair does not excuse a broken pair in the same file", + 'setsid "$BIN" serve -p 1111 &\nPA=$!\nkill -- -$PA\nsetsid "$BIN" forward-serve -p 2222 &\nPB=$!\nkill "$PB"\n', + ["setsid-launch", "kill-after-setsid"]), ("the un-waived defect itself is still the defect", 'setsid "$BIN" serve -p $PORT &\nPID=$!\nkill $PID\n', ["setsid-launch", "kill-after-setsid"]), From 970c77f11efc094d3ea7487d02ccc1042b90fbff Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 04:15:31 -0400 Subject: [PATCH 4/4] The runbook tells someone to run a command this box does not have `command -v nvme` returns nothing on strixhalo, so test 1's first line would have failed with 'command not found' and read like the *test* failing rather than a missing package. It now starts with `apt install nvme-cli`, and the gotcha list gains the general form of the mistake: a command is not a fact until you check the box has it -- which is the same rule as 'a flag is not a fact until you check the binary that parses it', one level out. Found by applying this page's own rule to this page. --- docs/nvme-iommu-fault-triage.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/nvme-iommu-fault-triage.md b/docs/nvme-iommu-fault-triage.md index d6f91a8bc..43984a4ab 100644 --- a/docs/nvme-iommu-fault-triage.md +++ b/docs/nvme-iommu-fault-triage.md @@ -59,7 +59,12 @@ software IO TLB: mapped [mem 0x5927e000-0x5d27e000] (64MB) ## Test 1 — disable the HMB (no reboot, decides whether the buffer is the HMB at all) +**`nvme-cli` is not installed on this host** (checked: `command -v nvme` → nothing), so the first command below is +`apt install nvme-cli` on this box, not `nvme …`. Whoever takes this on should expect that step and not treat +"command not found" as the test failing: + ```bash +sudo apt install nvme-cli # not present as of 11-15 08:1xZ nvme get-feature /dev/nvme0 -f 0x0d -H # Host Memory Buffer: current preference nvme set-feature /dev/nvme0 -f 0x0d -v 0 # ask for no HMB journalctl -k -f | grep -c IO_PAGE_FAULT # watch one burst interval (~2-5 min under load) @@ -94,6 +99,8 @@ teardown/re-establish ordering bug on the NVMe side, and the next step is tracin ## Two traps this runbook exists to stop * **A count without a device split is not a measurement.** `distinct` exposes the mixing. +* **A command is not a fact until you check the box has it.** This page's first draft opened with `nvme get-feature`, + and `nvme-cli` is not installed on the very host it is written for -- found by applying this page's own rule to itself. * **A flag is not a fact until you check the binary that parses it.** The first `strings` attempt here ran against the *compressed* `vmlinuz` and returned 0 for every pattern, including ones that exist — which would have "proved" the parameter absent. Decompress first (`extract-vmlinux`), then count.