Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 61 additions & 7 deletions boot/login_lane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"bufio"
"encoding/json"
"fmt"
"maps"
"math/rand/v2"
"os"
"path/filepath"
Expand Down Expand Up @@ -41,6 +42,11 @@ import (
//
// SPIN_LOGIN_LANE=1 run at all
// REPS=<n> boots of each row (default 10)
// SPIN_LANE_DIAG=1 instead of boot:ssh-first's rows, two that log what the lane does: sshd at
// LogLevel DEBUG1 and logind at debug, and the first RAW boots' journal lines
// with their offsets from accept. Logging inflates what it logs, so these say
// which step a hop is made of, not what it costs.
// RAW=<n> boots whose timeline is printed (default 3 with SPIN_LANE_DIAG, else 0)
func TestLoginLane(t *testing.T) {
if os.Getenv("SPIN_LOGIN_LANE") == "" {
t.Skip("set SPIN_LOGIN_LANE=1: boots VMs with a vsock device and logs into them")
Expand All @@ -54,17 +60,32 @@ func TestLoginLane(t *testing.T) {
}
reps := envInt(t, "REPS", 10)
rows, signer := firstLoginRows(t)
raw := 0
if os.Getenv("SPIN_LANE_DIAG") != "" {
base := rows[len(rows)-1]
sshd, logind := base, base
sshd.label, logind.label = "sshd debug", "logind debug"
sshd.files, logind.files = maps.Clone(base.files), maps.Clone(base.files)
sshd.files["/etc/ssh/sshd_config.d/99-lane-debug.conf"] = "LogLevel DEBUG1\n"
logind.files["/etc/systemd/system/systemd-logind.service.d/99-lane-debug.conf"] = "[Service]\nEnvironment=SYSTEMD_LOG_LEVEL=debug\n"
rows = []variant{sshd, logind}
raw = 3
}
raw = envInt(t, "RAW", raw)

marks := []string{"accept", "authenticated", "session open", "logind", "scope active", "command"}
got := map[string][][]time.Duration{} // row -> boot -> mark offsets from accept
for range reps {
for i := range reps {
for _, v := range rows {
m, err := loginLane(t, out, v, signer)
m, timeline, err := loginLane(t, out, v, signer)
if err != nil {
t.Logf("%s: %v", v.label, err)
continue
}
got[v.label] = append(got[v.label], m)
if i < raw {
t.Logf("%s, boot %d, ms after accept:\n%s", v.label, i+1, timeline)
}
}
}

Expand Down Expand Up @@ -96,7 +117,7 @@ func TestLoginLane(t *testing.T) {

// loginLane boots v once, logs in with the probe as the first login's command, reads the journal
// in a second login, and returns the marks' times in TestLoginLane's order.
func loginLane(t *testing.T, out string, v variant, signer xssh.Signer) ([]time.Duration, error) {
func loginLane(t *testing.T, out string, v variant, signer xssh.Signer) ([]time.Duration, string, error) {
t.Helper()
socks, err := os.MkdirTemp("/tmp", "ll")
if err != nil {
Expand All @@ -120,7 +141,7 @@ systemctl show -p Id -p InactiveExitTimestampMonotonic -p ActiveEnterTimestampMo
deadline := time.Now().Add(60 * time.Second)
for !vsockAccepts(r.cid, 22) {
if time.Now().After(deadline) {
return nil, fmt.Errorf("sshd never accepted on vsock")
return nil, "", fmt.Errorf("sshd never accepted on vsock")
}
time.Sleep(2 * time.Millisecond)
}
Expand All @@ -132,16 +153,49 @@ systemctl show -p Id -p InactiveExitTimestampMonotonic -p ActiveEnterTimestampMo
break
}
if time.Now().After(deadline) {
return nil, fmt.Errorf("never logged in: %w", err)
return nil, "", fmt.Errorf("never logged in: %w", err)
}
time.Sleep(5 * time.Millisecond)
}
_, journal, err := sshRun(r.cid, signer, xssh.KeyExchangeMLKEM768X25519,
"sudo journalctl -b -o json --output-fields=MESSAGE,SYSLOG_IDENTIFIER,__MONOTONIC_TIMESTAMP SYSLOG_IDENTIFIER=sshd-session SYSLOG_IDENTIFIER=sshd SYSLOG_IDENTIFIER=systemd-logind SYSLOG_IDENTIFIER=spin-lane")
if err != nil {
return nil, fmt.Errorf("reading the journal: %w", err)
return nil, "", fmt.Errorf("reading the journal: %w", err)
}
m, err := laneMarks(units, journal)
if err != nil {
return nil, "", err
}
return m, laneTimeline(m[0], journal), nil
}

// laneTimeline is the journal's lines from accept until a second after, each with its offset in
// ms, its identifier and its message cut to a line.
func laneTimeline(accept time.Duration, journal string) string {
var b strings.Builder
sc := bufio.NewScanner(strings.NewReader(journal))
sc.Buffer(make([]byte, 1<<20), 1<<20)
for sc.Scan() {
var e struct {
Message string `json:"MESSAGE"`
Ident string `json:"SYSLOG_IDENTIFIER"`
Mono string `json:"__MONOTONIC_TIMESTAMP"`
}
if json.Unmarshal(sc.Bytes(), &e) != nil {
continue
}
n, _ := strconv.ParseInt(e.Mono, 10, 64)
at := time.Duration(n)*time.Microsecond - accept
if at < -5*time.Millisecond || at > time.Second {
continue
}
msg := e.Message
if len(msg) > 110 {
msg = msg[:110]
}
fmt.Fprintf(&b, " %8.1f %-15s %s\n", float64(at)/float64(time.Millisecond), e.Ident, msg)
}
return laneMarks(units, journal)
return b.String()
}

// laneMarks reads the marks out of systemctl show's output and the journal's: the first of each
Expand Down
3 changes: 2 additions & 1 deletion boot/ssh_first_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ install -d "$MNT/var/lib/systemd/linger" && touch "$MNT/var/lib/systemd/linger/s
// edits done the same way. Measured against the release as it was published, interleaved.
rootfs := filepath.Join("..", "image", "mkosi.extra", "usr", "local", "lib", "spin-base", "rootfs")
for _, d := range []string{"sshd-vsock.socket.d/10-lane.conf", "sshd@.service.d/10-lane.conf",
"systemd-logind.service.d/10-lane.conf", "user-runtime-dir@.service.d/10-lane.conf",
"systemd-logind.service.d/10-lane.conf", "systemd-logind-varlink.socket.d/20-lane.conf",
"user-runtime-dir@.service.d/10-lane.conf",
"user@.service.d/10-lane.conf", "systemd-udev-trigger.service.d/10-coldplug.conf"} {
b, err := os.ReadFile(filepath.Join(rootfs, "etc", "systemd", "system", d))
if err != nil {
Expand Down
12 changes: 12 additions & 0 deletions docs/log.md
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,17 @@ Measured and not shipped:
- `boot:firmware-stages`: QEMU start 30.5 against 28.1, the PCI walk 5.00 against 2.22 (173 exits, ~29 against 13 µs each), kernel and init 108.7 against 69.5 (44535161371, 44535156851).

An r7i.metal-24xl (Sapphire Rapids, slower per core) read 296 ms (44534028478), faster than the c8i.metal. Unexplained; repeat before quoting it. Every number in §1 is nested.

**11-19 · boot/image · where the first login's own time goes, and logind's socket early (this change).** `boot:login-lane` marks the guest's journal through the first login: accept → authenticated 55.5, PAM's session → logind's new session 29.0, → scope 4.3, → command 8.4 ms (v20261119.01, lab 44555905383). Its diagnostic rows (`SPIN_LANE_DIAG`, sshd at DEBUG1 and logind at debug, lab 44557162195) say what they are:
- **Before authentication.** PID 1 starting `sshd@` takes ~11 ms, and OpenSSH 10's three execs (`sshd` → `sshd-session` → `sshd-auth`) ~12. Loading PAM's modules cold takes 8.3 ms, against 0.9 at the second login. ML-KEM is 3.5.
- **The hop to logind** is logind not running yet. Its Varlink socket kept its default dependencies, so logind waited behind `sysinit.target` and udevd, ready ~230 ms into the boot (`boot:waits TARGET=systemd-logind.service`, lab 44557605832). The first login waited ~32 ms for it, the second 1.

The socket without default dependencies (`systemd-logind-varlink.socket.d/20-lane.conf`), 20 boots of each (labs 44558035927, 44558053261):
- the hop: 24.5/41.8 → 15.8/22.7;
- accept → login: 131/137 → 120/128;
- the first login: 332/355 → 328/337. The p50 moves little because logind starting sooner takes the CPU sshd was starting on.

Masking `modprobe@drm`, which logind is ordered after, added nothing (327/345). What is left of the login is CPU contention on 2 vCPUs, `user@1000` (38 ms) and the host key generation (21 ms) starting in the same window, rather than waits.
---

## 3. Rework cycles
Expand Down Expand Up @@ -540,6 +551,7 @@ Also the reusable builder, added and reverted on 10-13.
| CPUWeight/IOWeight/Nice on the first login's lane | −2 ms over coldplug filtered; second login 63 vs 45 | lab 43518915618 | 11-13 |
| One sshd for every connection (vsock Accept=no, `sshd -D` early) | First login 382 vs 381; second 40 vs 48 ms | lab 43524721490 | 11-13 |
| Login without pam_systemd (diagnostic bound) | 380 vs 436, but no session, no XDG_RUNTIME_DIR | lab 43518044150 | 11-13 |
| Mask `modprobe@drm` (logind is ordered after it) | First login 327/345 vs 328/337 with logind's socket early | lab 44558035927 | 11-19 |
| Mask `systemd-udev-load-credentials` | First login −8 then −6 ms, p95 unchanged: not separable from noise at 20 boots | lab 44537549305, 44538877783 | 11-19 |
| systemd with `-Db_lto=true` | Ubuntu's dpkg-buildflags already pass `-flto=auto`: the same packages to within a few hundred bytes | image/systemd/Dockerfile | 11-19 |
| Lower the vCPU ceiling, drop the balloon or ACPI for boot time | Not tried: hotplug and dynamic memory are an invariant | AGENTS.md | 11-13 |
Expand Down
8 changes: 5 additions & 3 deletions image/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -389,10 +389,12 @@ stat_in_image /etc/systemd/system/sysinit.target.wants/spin-machine-sshd-keygen.
# The first SSH login's lane (configure-system.sh and the units' 10-lane.conf), read back from
# the finished filesystem: each piece alone gains little and leaves the login waiting on the
# next, so a missing one costs the lane without failing anything.
for u in sshd-vsock.socket sshd@.service systemd-logind.service user-runtime-dir@.service user@.service; do
"$bin/debugfs" -R "cat /etc/systemd/system/$u.d/10-lane.conf" /work/base.raw 2>/dev/null |
for f in sshd-vsock.socket.d/10-lane.conf sshd@.service.d/10-lane.conf systemd-logind.service.d/10-lane.conf \
systemd-logind-varlink.socket.d/20-lane.conf user-runtime-dir@.service.d/10-lane.conf \
user@.service.d/10-lane.conf; do
"$bin/debugfs" -R "cat /etc/systemd/system/$f" /work/base.raw 2>/dev/null |
grep -qx 'DefaultDependencies=no' || {
echo "ERROR: $u is not in the first login's lane (10-lane.conf)" >&2
echo "ERROR: ${f%%.d/*} is not in the first login's lane ($f)" >&2
exit 1; }
done
dbus=$("$bin/debugfs" -R "cat /etc/systemd/system/dbus.socket" /work/base.raw 2>/dev/null)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# The first SSH login's lane: logind's Varlink socket, without systemd's default dependencies,
# as logind itself is (systemd-logind.service.d/10-lane.conf).
#
# The service waits for its socket, and the socket kept After=sysinit.target: logind was ready
# ~230 ms into the boot, behind udevd (boot:waits TARGET=systemd-logind.service, lab run
# 44557605832), and the first login's pam_systemd waited for it to start - ~32 ms from PAM's
# session open to logind's new session, against 1 ms for the second login (login-lane with
# SPIN_LANE_DIAG, lab run 44557162195). Without them, on the bench fleet, 20 boots of each
# interleaved (lab runs 44558035927, 44558053261, 2026-11-19): that hop 24.5/41.8 -> 15.8/22.7
# ms, accept to login 131/137 -> 120/128, the first login 332/355 -> 328/337 - the p50 moves
# little because logind starting sooner takes the CPU sshd was starting on. Ordering logind
# after modprobe@drm cost nothing measurable either way (masked: 327/345).
[Unit]
DefaultDependencies=no
Conflicts=shutdown.target
Before=shutdown.target