From e0ebe68f959e938a3b79320e9f190d460f3e216a Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 12:37:11 -0300 Subject: [PATCH 1/3] boot: login-lane's diagnostic rows, sshd and logind logging SPIN_LANE_DIAG=1 replaces the rows with sshd at LogLevel DEBUG1 and logind at debug, and prints the first boots' journal lines with their offsets from accept: which step the ~55 ms before authentication and the ~29 ms before logind's new session are made of. Logging inflates what it logs; these say what a hop is, not what it costs. Co-Authored-By: Claude Opus 5.5 --- boot/login_lane_test.go | 68 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/boot/login_lane_test.go b/boot/login_lane_test.go index 670a447..ebb90a0 100644 --- a/boot/login_lane_test.go +++ b/boot/login_lane_test.go @@ -6,6 +6,7 @@ import ( "bufio" "encoding/json" "fmt" + "maps" "math/rand/v2" "os" "path/filepath" @@ -41,6 +42,11 @@ import ( // // SPIN_LOGIN_LANE=1 run at all // REPS= boots of each row (default 10) +// SPIN_LANE_DIAG=1 instead of boot:ssh-first's rows, two that log what the lane does: sshd at +// LogLevel DEBUG1 and logind at debug, and the first RAW boots' journal lines +// with their offsets from accept. Logging inflates what it logs, so these say +// which step a hop is made of, not what it costs. +// RAW= boots whose timeline is printed (default 3 with SPIN_LANE_DIAG, else 0) func TestLoginLane(t *testing.T) { if os.Getenv("SPIN_LOGIN_LANE") == "" { t.Skip("set SPIN_LOGIN_LANE=1: boots VMs with a vsock device and logs into them") @@ -54,17 +60,32 @@ func TestLoginLane(t *testing.T) { } reps := envInt(t, "REPS", 10) rows, signer := firstLoginRows(t) + raw := 0 + if os.Getenv("SPIN_LANE_DIAG") != "" { + base := rows[len(rows)-1] + sshd, logind := base, base + sshd.label, logind.label = "sshd debug", "logind debug" + sshd.files, logind.files = maps.Clone(base.files), maps.Clone(base.files) + sshd.files["/etc/ssh/sshd_config.d/99-lane-debug.conf"] = "LogLevel DEBUG1\n" + logind.files["/etc/systemd/system/systemd-logind.service.d/99-lane-debug.conf"] = "[Service]\nEnvironment=SYSTEMD_LOG_LEVEL=debug\n" + rows = []variant{sshd, logind} + raw = 3 + } + raw = envInt(t, "RAW", raw) marks := []string{"accept", "authenticated", "session open", "logind", "scope active", "command"} got := map[string][][]time.Duration{} // row -> boot -> mark offsets from accept - for range reps { + for i := range reps { for _, v := range rows { - m, err := loginLane(t, out, v, signer) + m, timeline, err := loginLane(t, out, v, signer) if err != nil { t.Logf("%s: %v", v.label, err) continue } got[v.label] = append(got[v.label], m) + if i < raw { + t.Logf("%s, boot %d, ms after accept:\n%s", v.label, i+1, timeline) + } } } @@ -96,7 +117,7 @@ func TestLoginLane(t *testing.T) { // loginLane boots v once, logs in with the probe as the first login's command, reads the journal // in a second login, and returns the marks' times in TestLoginLane's order. -func loginLane(t *testing.T, out string, v variant, signer xssh.Signer) ([]time.Duration, error) { +func loginLane(t *testing.T, out string, v variant, signer xssh.Signer) ([]time.Duration, string, error) { t.Helper() socks, err := os.MkdirTemp("/tmp", "ll") if err != nil { @@ -120,7 +141,7 @@ systemctl show -p Id -p InactiveExitTimestampMonotonic -p ActiveEnterTimestampMo deadline := time.Now().Add(60 * time.Second) for !vsockAccepts(r.cid, 22) { if time.Now().After(deadline) { - return nil, fmt.Errorf("sshd never accepted on vsock") + return nil, "", fmt.Errorf("sshd never accepted on vsock") } time.Sleep(2 * time.Millisecond) } @@ -132,16 +153,49 @@ systemctl show -p Id -p InactiveExitTimestampMonotonic -p ActiveEnterTimestampMo break } if time.Now().After(deadline) { - return nil, fmt.Errorf("never logged in: %w", err) + return nil, "", fmt.Errorf("never logged in: %w", err) } time.Sleep(5 * time.Millisecond) } _, journal, err := sshRun(r.cid, signer, xssh.KeyExchangeMLKEM768X25519, "sudo journalctl -b -o json --output-fields=MESSAGE,SYSLOG_IDENTIFIER,__MONOTONIC_TIMESTAMP SYSLOG_IDENTIFIER=sshd-session SYSLOG_IDENTIFIER=sshd SYSLOG_IDENTIFIER=systemd-logind SYSLOG_IDENTIFIER=spin-lane") if err != nil { - return nil, fmt.Errorf("reading the journal: %w", err) + return nil, "", fmt.Errorf("reading the journal: %w", err) + } + m, err := laneMarks(units, journal) + if err != nil { + return nil, "", err + } + return m, laneTimeline(m[0], journal), nil +} + +// laneTimeline is the journal's lines from accept until a second after, each with its offset in +// ms, its identifier and its message cut to a line. +func laneTimeline(accept time.Duration, journal string) string { + var b strings.Builder + sc := bufio.NewScanner(strings.NewReader(journal)) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + var e struct { + Message string `json:"MESSAGE"` + Ident string `json:"SYSLOG_IDENTIFIER"` + Mono string `json:"__MONOTONIC_TIMESTAMP"` + } + if json.Unmarshal(sc.Bytes(), &e) != nil { + continue + } + n, _ := strconv.ParseInt(e.Mono, 10, 64) + at := time.Duration(n)*time.Microsecond - accept + if at < -5*time.Millisecond || at > time.Second { + continue + } + msg := e.Message + if len(msg) > 110 { + msg = msg[:110] + } + fmt.Fprintf(&b, " %8.1f %-15s %s\n", float64(at)/float64(time.Millisecond), e.Ident, msg) } - return laneMarks(units, journal) + return b.String() } // laneMarks reads the marks out of systemctl show's output and the journal's: the first of each From e2608817b608735184f4f3a005e6855cff114a0c Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 12:43:30 -0300 Subject: [PATCH 2/3] boot: rows with logind's varlink socket before sysinit (experiment) The lane's drop-in took logind's default dependencies away, not its varlink socket's: logind was ready ~230 ms into the boot, behind sysinit.target and udevd (lab 44557605832), and the first login's pam_systemd waited ~32 ms for it (lab 44557162195). SPIN_LOGIND_EARLY=1 adds the socket without default dependencies, and that with modprobe@drm masked, which logind is ordered after. Co-Authored-By: Claude Opus 5.5 --- boot/ssh_first_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 8a47dbb..4f4abf1 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -7,11 +7,13 @@ import ( "cmp" "crypto/ed25519" "fmt" + "maps" "math/rand/v2" "net" "os" "os/exec" "path/filepath" + "slices" "strings" "sync" "syscall" @@ -175,6 +177,24 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} + // EXPERIMENT (branch lane-diag): logind before sysinit. The lane's drop-in took logind's + // default dependencies away, not its varlink socket's, and the service waits for its socket: + // on the bench logind was ready ~230 ms into the boot, behind sysinit.target and udevd + // (boot:waits TARGET=systemd-logind.service, lab 44557605832), and the first login's + // pam_systemd waited ~32 ms for it (login-lane, lab 44557162195). logind is also ordered after + // modprobe@drm, which finds no module on a kernel that has none. + if os.Getenv("SPIN_LOGIND_EARLY") != "" { + early := asBuilt + early.label = "logind early" + early.files = maps.Clone(asBuilt.files) + early.links = maps.Clone(asBuilt.links) + early.files["/etc/systemd/system/systemd-logind-varlink.socket.d/10-lane.conf"] = + "[Unit]\nDefaultDependencies=no\nConflicts=shutdown.target\nBefore=shutdown.target\n" + noDRM := early + noDRM.label = "logind early, no drm" + noDRM.mask = append(slices.Clone(asBuilt.mask), "modprobe@drm.service") + rows = append(rows, early, noDRM) + } return rows, signer } From b7fdb6a38586dc3869560c390826d760800b8832 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 12:52:17 -0300 Subject: [PATCH 3/3] image: logind's varlink socket early, and the lane's diagnostics The lane took logind's default dependencies away, not its Varlink socket's, and the service waits for its socket: logind was ready ~230 ms into the boot behind udevd, and the first login's pam_systemd waited ~32 ms for it. Without them, 20 boots of each on the bench: that hop 24.5/41.8 -> 15.8/22.7 ms, the first login 332/355 -> 328/337. build.sh asserts the drop-in like the lane's others; the experiment rows go. Co-Authored-By: Claude Opus 5.5 --- boot/ssh_first_test.go | 23 ++----------------- docs/log.md | 12 ++++++++++ image/build.sh | 8 ++++--- .../20-lane.conf | 16 +++++++++++++ 4 files changed, 35 insertions(+), 24 deletions(-) create mode 100644 image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind-varlink.socket.d/20-lane.conf diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 4f4abf1..9cc2190 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -7,13 +7,11 @@ import ( "cmp" "crypto/ed25519" "fmt" - "maps" "math/rand/v2" "net" "os" "os/exec" "path/filepath" - "slices" "strings" "sync" "syscall" @@ -161,7 +159,8 @@ install -d "$MNT/var/lib/systemd/linger" && touch "$MNT/var/lib/systemd/linger/s // edits done the same way. Measured against the release as it was published, interleaved. rootfs := filepath.Join("..", "image", "mkosi.extra", "usr", "local", "lib", "spin-base", "rootfs") for _, d := range []string{"sshd-vsock.socket.d/10-lane.conf", "sshd@.service.d/10-lane.conf", - "systemd-logind.service.d/10-lane.conf", "user-runtime-dir@.service.d/10-lane.conf", + "systemd-logind.service.d/10-lane.conf", "systemd-logind-varlink.socket.d/20-lane.conf", + "user-runtime-dir@.service.d/10-lane.conf", "user@.service.d/10-lane.conf", "systemd-udev-trigger.service.d/10-coldplug.conf"} { b, err := os.ReadFile(filepath.Join(rootfs, "etc", "systemd", "system", d)) if err != nil { @@ -177,24 +176,6 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} - // EXPERIMENT (branch lane-diag): logind before sysinit. The lane's drop-in took logind's - // default dependencies away, not its varlink socket's, and the service waits for its socket: - // on the bench logind was ready ~230 ms into the boot, behind sysinit.target and udevd - // (boot:waits TARGET=systemd-logind.service, lab 44557605832), and the first login's - // pam_systemd waited ~32 ms for it (login-lane, lab 44557162195). logind is also ordered after - // modprobe@drm, which finds no module on a kernel that has none. - if os.Getenv("SPIN_LOGIND_EARLY") != "" { - early := asBuilt - early.label = "logind early" - early.files = maps.Clone(asBuilt.files) - early.links = maps.Clone(asBuilt.links) - early.files["/etc/systemd/system/systemd-logind-varlink.socket.d/10-lane.conf"] = - "[Unit]\nDefaultDependencies=no\nConflicts=shutdown.target\nBefore=shutdown.target\n" - noDRM := early - noDRM.label = "logind early, no drm" - noDRM.mask = append(slices.Clone(asBuilt.mask), "modprobe@drm.service") - rows = append(rows, early, noDRM) - } return rows, signer } diff --git a/docs/log.md b/docs/log.md index ff62c96..e30b122 100644 --- a/docs/log.md +++ b/docs/log.md @@ -363,6 +363,17 @@ Measured and not shipped: - `boot:firmware-stages`: QEMU start 30.5 against 28.1, the PCI walk 5.00 against 2.22 (173 exits, ~29 against 13 µs each), kernel and init 108.7 against 69.5 (44535161371, 44535156851). An r7i.metal-24xl (Sapphire Rapids, slower per core) read 296 ms (44534028478), faster than the c8i.metal. Unexplained; repeat before quoting it. Every number in §1 is nested. + +**11-19 · boot/image · where the first login's own time goes, and logind's socket early (this change).** `boot:login-lane` marks the guest's journal through the first login: accept → authenticated 55.5, PAM's session → logind's new session 29.0, → scope 4.3, → command 8.4 ms (v20261119.01, lab 44555905383). Its diagnostic rows (`SPIN_LANE_DIAG`, sshd at DEBUG1 and logind at debug, lab 44557162195) say what they are: +- **Before authentication.** PID 1 starting `sshd@` takes ~11 ms, and OpenSSH 10's three execs (`sshd` → `sshd-session` → `sshd-auth`) ~12. Loading PAM's modules cold takes 8.3 ms, against 0.9 at the second login. ML-KEM is 3.5. +- **The hop to logind** is logind not running yet. Its Varlink socket kept its default dependencies, so logind waited behind `sysinit.target` and udevd, ready ~230 ms into the boot (`boot:waits TARGET=systemd-logind.service`, lab 44557605832). The first login waited ~32 ms for it, the second 1. + +The socket without default dependencies (`systemd-logind-varlink.socket.d/20-lane.conf`), 20 boots of each (labs 44558035927, 44558053261): +- the hop: 24.5/41.8 → 15.8/22.7; +- accept → login: 131/137 → 120/128; +- the first login: 332/355 → 328/337. The p50 moves little because logind starting sooner takes the CPU sshd was starting on. + +Masking `modprobe@drm`, which logind is ordered after, added nothing (327/345). What is left of the login is CPU contention on 2 vCPUs, `user@1000` (38 ms) and the host key generation (21 ms) starting in the same window, rather than waits. --- ## 3. Rework cycles @@ -540,6 +551,7 @@ Also the reusable builder, added and reverted on 10-13. | CPUWeight/IOWeight/Nice on the first login's lane | −2 ms over coldplug filtered; second login 63 vs 45 | lab 43518915618 | 11-13 | | One sshd for every connection (vsock Accept=no, `sshd -D` early) | First login 382 vs 381; second 40 vs 48 ms | lab 43524721490 | 11-13 | | Login without pam_systemd (diagnostic bound) | 380 vs 436, but no session, no XDG_RUNTIME_DIR | lab 43518044150 | 11-13 | +| Mask `modprobe@drm` (logind is ordered after it) | First login 327/345 vs 328/337 with logind's socket early | lab 44558035927 | 11-19 | | Mask `systemd-udev-load-credentials` | First login −8 then −6 ms, p95 unchanged: not separable from noise at 20 boots | lab 44537549305, 44538877783 | 11-19 | | systemd with `-Db_lto=true` | Ubuntu's dpkg-buildflags already pass `-flto=auto`: the same packages to within a few hundred bytes | image/systemd/Dockerfile | 11-19 | | Lower the vCPU ceiling, drop the balloon or ACPI for boot time | Not tried: hotplug and dynamic memory are an invariant | AGENTS.md | 11-13 | diff --git a/image/build.sh b/image/build.sh index 1b74ebc..b036776 100755 --- a/image/build.sh +++ b/image/build.sh @@ -389,10 +389,12 @@ stat_in_image /etc/systemd/system/sysinit.target.wants/spin-machine-sshd-keygen. # The first SSH login's lane (configure-system.sh and the units' 10-lane.conf), read back from # the finished filesystem: each piece alone gains little and leaves the login waiting on the # next, so a missing one costs the lane without failing anything. -for u in sshd-vsock.socket sshd@.service systemd-logind.service user-runtime-dir@.service user@.service; do - "$bin/debugfs" -R "cat /etc/systemd/system/$u.d/10-lane.conf" /work/base.raw 2>/dev/null | +for f in sshd-vsock.socket.d/10-lane.conf sshd@.service.d/10-lane.conf systemd-logind.service.d/10-lane.conf \ + systemd-logind-varlink.socket.d/20-lane.conf user-runtime-dir@.service.d/10-lane.conf \ + user@.service.d/10-lane.conf; do + "$bin/debugfs" -R "cat /etc/systemd/system/$f" /work/base.raw 2>/dev/null | grep -qx 'DefaultDependencies=no' || { - echo "ERROR: $u is not in the first login's lane (10-lane.conf)" >&2 + echo "ERROR: ${f%%.d/*} is not in the first login's lane ($f)" >&2 exit 1; } done dbus=$("$bin/debugfs" -R "cat /etc/systemd/system/dbus.socket" /work/base.raw 2>/dev/null) diff --git a/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind-varlink.socket.d/20-lane.conf b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind-varlink.socket.d/20-lane.conf new file mode 100644 index 0000000..8cd494e --- /dev/null +++ b/image/mkosi.extra/usr/local/lib/spin-base/rootfs/etc/systemd/system/systemd-logind-varlink.socket.d/20-lane.conf @@ -0,0 +1,16 @@ +# The first SSH login's lane: logind's Varlink socket, without systemd's default dependencies, +# as logind itself is (systemd-logind.service.d/10-lane.conf). +# +# The service waits for its socket, and the socket kept After=sysinit.target: logind was ready +# ~230 ms into the boot, behind udevd (boot:waits TARGET=systemd-logind.service, lab run +# 44557605832), and the first login's pam_systemd waited for it to start - ~32 ms from PAM's +# session open to logind's new session, against 1 ms for the second login (login-lane with +# SPIN_LANE_DIAG, lab run 44557162195). Without them, on the bench fleet, 20 boots of each +# interleaved (lab runs 44558035927, 44558053261, 2026-11-19): that hop 24.5/41.8 -> 15.8/22.7 +# ms, accept to login 131/137 -> 120/128, the first login 332/355 -> 328/337 - the p50 moves +# little because logind starting sooner takes the CPU sshd was starting on. Ordering logind +# after modprobe@drm cost nothing measurable either way (masked: 327/345). +[Unit] +DefaultDependencies=no +Conflicts=shutdown.target +Before=shutdown.target