diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 1e4f973..51612ec 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -35,6 +35,7 @@ on: - boot:waits - boot:ssh-first - boot:restore-first + - boot:login-lane - boot:page-cache - boot:firmware - boot:firmware-stages diff --git a/Taskfile.yml b/Taskfile.yml index 199e18a..ac5222a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -634,11 +634,27 @@ tasks: mkdir -p _output/lab/tmp gh release download "$RELEASE" -R "$GITHUB_REPOSITORY" -p '*.tar.gz' -D _output/lab tar -xzf _output/lab/*.tar.gz -C _output/lab + # On disk: the overlays and memory files go here, gigabytes a tmpfs /tmp holds in RAM. On + # the host's local NVMe when it has one, which is where spin's hosts keep checkpoints: a + # restore read its state from gp3 at ~300 MB/s on a c8i, ~680 of its ~840 ms (lab + # 44542990433). An instance store nothing has mounted is formatted and mounted here; the + # summary says which disk it was, so no restore number is read without it. + tmp=$PWD/_output/lab/tmp + store=$(lsblk -dnpo NAME,MODEL | awk '/Instance Storage/ {print $1; exit}') + if [ -n "$store" ]; then + mnt=$(findmnt -nro TARGET -S "$store" | head -1) + if [ -z "$mnt" ]; then + mnt=/mnt/instance-store + sudo mkfs.ext4 -q -F "$store" + sudo mkdir -p "$mnt" && sudo mount "$store" "$mnt" + fi + sudo install -d -o "$(id -u)" -g "$(id -g)" "$mnt/spin-lab-tmp" + tmp=$mnt/spin-lab-tmp + fi { echo "RELEASE=$RELEASE" echo "SPIN_MACHINE_OUTPUT=$PWD/_output/lab/spin-machine-${RELEASE}-linux-x86_64/usr/share/spin-stack" - # On disk: the overlays and memory files go here, gigabytes a tmpfs /tmp holds in RAM. - echo "TMPDIR=$PWD/_output/lab/tmp" + echo "TMPDIR=$tmp" } >> "${GITHUB_ENV:-/dev/stdout}" lab:fetch: @@ -726,6 +742,7 @@ tasks: if [ -n "$KERNEL_RUN$KERNEL_RUN_C$QEMU_RUN" ]; then echo "- booted: $VARIANT the release"; fi echo "- vars: ${VARS:-none}" echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs" + echo "- TMPDIR: ${TMPDIR:-/tmp} on $(df --output=source "${TMPDIR:-/tmp}" | tail -1) ($(lsblk -dno MODEL "$(df --output=source "${TMPDIR:-/tmp}" | tail -1)" 2>/dev/null | sed 's/ *$//' || echo unknown))" echo "- load: $(cut -d' ' -f1-3 /proc/loadavg)" if sudo -n true 2>/dev/null; then echo "- sudo -n: yes"; else echo "- sudo -n: **no** (targets that edit an overlay skip)"; fi } | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" @@ -781,6 +798,7 @@ tasks: fi echo "- gh: $(command -v gh || echo '**missing**')" echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs" + echo "- TMPDIR: ${TMPDIR:-/tmp} on $(df --output=source "${TMPDIR:-/tmp}" | tail -1) ($(lsblk -dno MODEL "$(df --output=source "${TMPDIR:-/tmp}" | tail -1)" 2>/dev/null | sed 's/ *$//' || echo unknown))" } | tee -a "${GITHUB_STEP_SUMMARY:-/dev/null}" [ -r /dev/kvm ] && [ -w /dev/kvm ] || { echo "::error::no usable /dev/kvm: nothing here can be measured"; exit 1; } diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 679e382..76ab8bc 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -174,6 +174,16 @@ tasks: cmds: - SPIN_SSH_FIRST=1 go test ./boot/ -run '^TestFirstSSH$' -count=1 -v -timeout 30m + login-lane: + desc: >- + Where the first SSH login's own time goes between sshd's accept and the command running, on + the guest's clock: authenticated, PAM's session open, logind's new session, the scope active, + the command, for each of boot:ssh-first's rows. Needs KVM, /dev/vhost-vsock (task vsock), a + built release and sudo. REPS= boots. + deps: [':tools'] + cmds: + - SPIN_LOGIN_LANE=1 go test ./boot/ -run '^TestLoginLane$' -count=1 -v -timeout 60m + restore-first: desc: >- The goal's second number: a checkpoint restored to the first SSH login over vsock, beside diff --git a/boot/login_lane_test.go b/boot/login_lane_test.go new file mode 100644 index 0000000..670a447 --- /dev/null +++ b/boot/login_lane_test.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "bufio" + "encoding/json" + "fmt" + "math/rand/v2" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + xssh "golang.org/x/crypto/ssh" + + "github.com/spin-stack/spin-machine/boot" +) + +// Where the first SSH login's own time goes, between sshd's accept and the command running, on +// the guest's monotonic clock: boot:ssh-first says how long accept to login is (~125-145 ms on the +// bench fleet, 2026-11-19), and docs/log.md puts ~56 of it in pam_systemd (lab 43518044150, a +// login without it) without saying which hop. +// +// The first login's command is the probe: it marks the journal the moment it runs, and prints +// the start of its own connection's sshd@ unit and of its session's scope. A second login, as +// root through sudo, reads this boot's journal. The marks, in order: +// +// accept sshd@'s InactiveExit: PID 1 started the connection's sshd +// authenticated sshd: "Accepted publickey" - the key exchange and the key +// session open pam_unix(sshd:session): "session opened" - pam_systemd comes after it in +// common-session, so from here to the command is pam_systemd and what follows +// logind systemd-logind: "New session" - logind has the request and asked for the scope +// scope active the session scope's ActiveEnter: PID 1 ran the scope's job; logind answers then +// command the command's own journal mark: the shell ran it +// +// Journal times are when journald received a line, so each mark is a little late by the same +// kind of amount; the scope's is PID 1's own. Every row of boot:ssh-first, interleaved. +// +// SPIN_LOGIN_LANE=1 run at all +// REPS= boots of each row (default 10) +func TestLoginLane(t *testing.T) { + if os.Getenv("SPIN_LOGIN_LANE") == "" { + t.Skip("set SPIN_LOGIN_LANE=1: boots VMs with a vsock device and logs into them") + } + out := releaseDir(t) + if !canSudo() { + t.Skip("this writes a key into the boot's own overlay through qemu-nbd, which needs sudo") + } + if _, err := os.Stat("/dev/vhost-vsock"); err != nil { + t.Skip("no /dev/vhost-vsock: run task vsock") + } + reps := envInt(t, "REPS", 10) + rows, signer := firstLoginRows(t) + + marks := []string{"accept", "authenticated", "session open", "logind", "scope active", "command"} + got := map[string][][]time.Duration{} // row -> boot -> mark offsets from accept + for range reps { + for _, v := range rows { + m, err := loginLane(t, out, v, signer) + if err != nil { + t.Logf("%s: %v", v.label, err) + continue + } + got[v.label] = append(got[v.label], m) + } + } + + var b strings.Builder + fmt.Fprintf(&b, "\nms on the guest's clock, p50/p95 over the boots of each row, interleaved\n\n %-16s", "") + for _, m := range marks[1:] { + fmt.Fprintf(&b, " %17s", "to "+m) + } + b.WriteString("\n") + for _, v := range rows { + fmt.Fprintf(&b, " %-16s", v.label) + for i := 1; i < len(marks); i++ { + var ds []time.Duration + for _, m := range got[v.label] { + ds = append(ds, m[i]-m[i-1]) + } + p50, ok := boot.Percentile(ds, .5) + p95, _ := boot.Percentile(ds, .95) + if !ok { + fmt.Fprintf(&b, " %17s", "-") + continue + } + fmt.Fprintf(&b, " %8.1f/%-8.1f", float64(p50)/float64(time.Millisecond), float64(p95)/float64(time.Millisecond)) + } + fmt.Fprintf(&b, " (%d boots)\n", len(got[v.label])) + } + t.Log(b.String()) +} + +// loginLane boots v once, logs in with the probe as the first login's command, reads the journal +// in a second login, and returns the marks' times in TestLoginLane's order. +func loginLane(t *testing.T, out string, v variant, signer xssh.Signer) ([]time.Duration, error) { + t.Helper() + socks, err := os.MkdirTemp("/tmp", "ll") + if err != nil { + t.Fatal(err) + } + defer func() { _ = os.RemoveAll(socks) }() + r := &checkpointRun{v: v, label: v.label, out: out, signer: signer, dir: t.TempDir(), socks: socks} + mustRun(t, filepath.Join(out, "bin", "qemu-img"), "create", "-f", "qcow2", + "-F", "qcow2", "-b", filepath.Join(out, "image", "rootfs.qcow2"), r.overlay()) + editOverlay(t, r.overlay(), v) + r.cid = uint32(100000 + rand.IntN(1<<20)) + _, stop := r.launch(t, filepath.Join(socks, "q")) + defer stop() + + // The mark first, before anything the command does costs time; then the starts PID 1 keeps, + // read while the connection's sshd@ is still running and its scope still exists. + const probe = `logger -t spin-lane command +unit=$(systemctl list-units 'sshd@*' --state=active --plain --no-legend | awk 'NR==1{print $1}') +scope=$(sed -n 's|.*/\(session-[0-9]*\.scope\)$|\1|p' /proc/self/cgroup) +systemctl show -p Id -p InactiveExitTimestampMonotonic -p ActiveEnterTimestampMonotonic "$unit" "$scope"` + deadline := time.Now().Add(60 * time.Second) + for !vsockAccepts(r.cid, 22) { + if time.Now().After(deadline) { + return nil, fmt.Errorf("sshd never accepted on vsock") + } + time.Sleep(2 * time.Millisecond) + } + var units string + for { + _, o, err := sshRun(r.cid, signer, xssh.KeyExchangeMLKEM768X25519, probe) + if err == nil { + units = o + break + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("never logged in: %w", err) + } + time.Sleep(5 * time.Millisecond) + } + _, journal, err := sshRun(r.cid, signer, xssh.KeyExchangeMLKEM768X25519, + "sudo journalctl -b -o json --output-fields=MESSAGE,SYSLOG_IDENTIFIER,__MONOTONIC_TIMESTAMP SYSLOG_IDENTIFIER=sshd-session SYSLOG_IDENTIFIER=sshd SYSLOG_IDENTIFIER=systemd-logind SYSLOG_IDENTIFIER=spin-lane") + if err != nil { + return nil, fmt.Errorf("reading the journal: %w", err) + } + return laneMarks(units, journal) +} + +// laneMarks reads the marks out of systemctl show's output and the journal's: the first of each +// in the boot, which is the first login's. +func laneMarks(units, journal string) ([]time.Duration, error) { + us := func(s string) time.Duration { + n, _ := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + return time.Duration(n) * time.Microsecond + } + var accept, scope time.Duration + for _, block := range strings.Split(strings.TrimSpace(units), "\n\n") { + p := map[string]string{} + for _, line := range strings.Split(block, "\n") { + k, v, _ := strings.Cut(line, "=") + p[k] = v + } + switch { + case strings.HasPrefix(p["Id"], "sshd@"): + accept = us(p["InactiveExitTimestampMonotonic"]) + case strings.HasPrefix(p["Id"], "session-"): + scope = us(p["ActiveEnterTimestampMonotonic"]) + } + } + first := map[string]time.Duration{} + sc := bufio.NewScanner(strings.NewReader(journal)) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + var e struct { + Message string `json:"MESSAGE"` + Ident string `json:"SYSLOG_IDENTIFIER"` + Mono string `json:"__MONOTONIC_TIMESTAMP"` + } + if json.Unmarshal(sc.Bytes(), &e) != nil { + continue + } + var mark string + switch { + case strings.HasPrefix(e.Message, "Accepted publickey for spin"): + mark = "authenticated" + case strings.Contains(e.Message, "pam_unix(sshd:session): session opened for user spin"): + mark = "session open" + case e.Ident == "systemd-logind" && strings.HasPrefix(e.Message, "New session"): + mark = "logind" + case e.Ident == "spin-lane": + mark = "command" + default: + continue + } + if _, seen := first[mark]; !seen { + first[mark] = us(e.Mono) + } + } + m := []time.Duration{accept, first["authenticated"], first["session open"], first["logind"], scope, first["command"]} + names := []string{"accept", "authenticated", "session open", "logind", "scope active", "command"} + for i, d := range m { + if d == 0 { + return nil, fmt.Errorf("no %s mark (units %q)", names[i], units) + } + } + return m, nil +} diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go index 6a04ca3..ab99bbe 100644 --- a/boot/restore_first_test.go +++ b/boot/restore_first_test.go @@ -4,6 +4,7 @@ package boot_test import ( "fmt" + "io" "math/rand/v2" "net" "os" @@ -33,6 +34,12 @@ import ( // Each state is evicted from the host's page cache before its restore, and the cache is asked // whether it was: a checkpoint kept for later is read from the disk. // +// Each row is also restored a second way, interleaved with the first: its state evicted, read +// back whole in one sequential pass - which times what the disk gives, READ - and restored from +// the page cache. Evicted against cached is what the disk costs a restore; READ against that +// difference says whether QEMU reads the file as fast as the disk can (2026-11-19: a restore took +// ~790 ms to running on nested and on metal alike, and nothing said which part was the disk). +// // SPIN_RESTORE_FIRST=1 run at all // REPS= repetitions of each row (default 10) // CPUS= vCPUs (default 2) @@ -50,31 +57,39 @@ func TestRestoreFirstSSH(t *testing.T) { reps := envInt(t, "REPS", 10) rows, signer := firstLoginRows(t) + type mode struct { + label string + cached bool + } + var modes []mode + for _, v := range rows { + modes = append(modes, mode{v.label, false}, mode{v.label + ", cached", true}) + } var runs []*checkpointRun for i := range reps { - for _, v := range rows { + for j, m := range modes { socks, err := os.MkdirTemp("/tmp", "rf") if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = os.RemoveAll(socks) }) - r := &checkpointRun{v: v, out: out, signer: signer, dir: t.TempDir(), socks: socks} + r := &checkpointRun{v: rows[j/2], label: m.label, cached: m.cached, out: out, signer: signer, dir: t.TempDir(), socks: socks} r.cold(t) - t.Logf("%s, repetition %d: cold login %v, saved in %v", v.label, i+1, - r.coldLogin.Round(time.Millisecond), r.save.Round(time.Millisecond)) + t.Logf("%s, repetition %d: cold login %v, saved %d MiB in %v", m.label, i+1, + r.coldLogin.Round(time.Millisecond), r.size>>20, r.save.Round(time.Millisecond)) runs = append(runs, r) } } for i, r := range runs { r.restore(t) - t.Logf("%s, repetition %d: restored up %v, running %v, login %v", r.v.label, i/len(rows)+1, - r.up.Round(time.Millisecond), r.running.Round(time.Millisecond), r.login.Round(time.Millisecond)) + t.Logf("%s, repetition %d: read %v, restored up %v, running %v, login %v", r.label, i/len(modes)+1, + r.read.Round(time.Millisecond), r.up.Round(time.Millisecond), r.running.Round(time.Millisecond), r.login.Round(time.Millisecond)) } ms := func(label string, f func(*checkpointRun) time.Duration) string { var ds []time.Duration for _, r := range runs { - if r.v.label == label { + if r.label == label { ds = append(ds, f(r)) } } @@ -85,27 +100,41 @@ func TestRestoreFirstSSH(t *testing.T) { var b strings.Builder fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-30s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") - for _, v := range rows { - fmt.Fprintf(&b, " %-30s %s %s %s %s %s\n", v.label, - ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), - ms(v.label, func(r *checkpointRun) time.Duration { return r.save }), - ms(v.label, func(r *checkpointRun) time.Duration { return r.up }), - ms(v.label, func(r *checkpointRun) time.Duration { return r.running }), - ms(v.label, func(r *checkpointRun) time.Duration { return r.login })) + fmt.Fprintf(&b, " %-30s %9s %13s %13s %13s %13s %13s %13s\n", "", "STATE MiB", "COLD LOGIN", "SAVE", "READ", "RESTORE: UP", "RUNNING", "LOGIN") + for _, m := range modes { + var size int64 + for _, r := range runs { + if r.label == m.label { + size = max(size, r.size) + } + } + read := " - " + if m.cached { + read = ms(m.label, func(r *checkpointRun) time.Duration { return r.read }) + } + fmt.Fprintf(&b, " %-30s %9d %s %s %s %s %s %s\n", m.label, size>>20, + ms(m.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), + ms(m.label, func(r *checkpointRun) time.Duration { return r.save }), + read, + ms(m.label, func(r *checkpointRun) time.Duration { return r.up }), + ms(m.label, func(r *checkpointRun) time.Duration { return r.running }), + ms(m.label, func(r *checkpointRun) time.Duration { return r.login })) } t.Log(b.String()) } type checkpointRun struct { v variant + label string // the row's, and ", cached" when its state is restored from the page cache + cached bool out string signer xssh.Signer dir string // the overlay and the state, on the disk TMPDIR names socks string // the QMP sockets: a Unix socket's path has to fit in 108 bytes, and dir's need not cid uint32 - coldLogin, save, up, running, login time.Duration + size int64 // the state's bytes + coldLogin, save, read, up, running, login time.Duration } func (r *checkpointRun) overlay() string { return filepath.Join(r.dir, "overlay.qcow2") } @@ -182,6 +211,11 @@ func (r *checkpointRun) cold(t *testing.T) { t.Fatalf("%s: QEMU was still running 30s after its state was saved", r.v.label) } evict(t, r.state()) + st, err := os.Stat(r.state()) + if err != nil { + t.Fatal(err) + } + r.size = st.Size() } // restore resumes the saved state read from the disk, logs in, and kills the machine: a restored @@ -189,6 +223,9 @@ func (r *checkpointRun) cold(t *testing.T) { func (r *checkpointRun) restore(t *testing.T) { t.Helper() evict(t, r.state()) + if r.cached { + r.read = readWhole(t, r.state()) + } qmp := filepath.Join(r.socks, "restored.qmp") t0 := time.Now() _, stop := r.launch(t, qmp, "--incoming", "defer") @@ -209,6 +246,28 @@ func (r *checkpointRun) restore(t *testing.T) { stop() } +// readWhole reads path once, start to end, in 1 MiB reads, and says how long that took: what the +// disk gives a sequential reader, and the state in the page cache after it. +func readWhole(t *testing.T, path string) time.Duration { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + buf := make([]byte, 1<<20) + t0 := time.Now() + for { + _, err := f.Read(buf) + if err == io.EOF { + return time.Since(t0) + } + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + } +} + // evict drops path from the host's page cache, and asks the cache whether it did: a restore of a // state still in memory measures the cache and not the disk, and fadvise says nothing about what // it dropped. diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index e53f7d3..8a47dbb 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -324,14 +324,21 @@ func vsockAccepts(cid, port uint32) bool { // sshLogin runs `true` as spin over SSH on cid's port 22, offering only the key exchange kex, // and says how long it took from the connect. func sshLogin(cid uint32, signer xssh.Signer, kex string) (time.Duration, error) { + d, _, err := sshRun(cid, signer, kex, "true") + return d, err +} + +// sshRun runs command as spin over SSH on cid's port 22, offering only the key exchange kex, and +// returns how long it took from the connect and what it wrote to stdout. +func sshRun(cid uint32, signer xssh.Signer, kex, command string) (time.Duration, string, error) { t0 := time.Now() fd, err := unix.Socket(unix.AF_VSOCK, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0) if err != nil { - return 0, fmt.Errorf("opening a vsock socket: %w", err) + return 0, "", fmt.Errorf("opening a vsock socket: %w", err) } if err := unix.Connect(fd, &unix.SockaddrVM{CID: cid, Port: 22}); err != nil { _ = unix.Close(fd) - return 0, fmt.Errorf("connecting to vsock %d:22: %w", cid, err) + return 0, "", fmt.Errorf("connecting to vsock %d:22: %w", cid, err) } conn := vsockConn{os.NewFile(uintptr(fd), "vsock")} defer conn.Close() @@ -342,19 +349,20 @@ func sshLogin(cid uint32, signer xssh.Signer, kex string) (time.Duration, error) Config: xssh.Config{KeyExchanges: []string{kex}}, }) if err != nil { - return 0, fmt.Errorf("ssh to vsock %d:22: %w", cid, err) + return 0, "", fmt.Errorf("ssh to vsock %d:22: %w", cid, err) } client := xssh.NewClient(c, chans, reqs) defer client.Close() s, err := client.NewSession() if err != nil { - return 0, fmt.Errorf("opening a session on vsock %d:22: %w", cid, err) + return 0, "", fmt.Errorf("opening a session on vsock %d:22: %w", cid, err) } defer s.Close() - if err := s.Run("true"); err != nil { - return 0, fmt.Errorf("running true on vsock %d:22: %w", cid, err) + out, err := s.Output(command) + if err != nil { + return 0, "", fmt.Errorf("running %q on vsock %d:22: %w", command, cid, err) } - return time.Since(t0), nil + return time.Since(t0), string(out), nil } // vsockConn is a connected vsock socket as the net.Conn x/crypto/ssh takes: net.FileConn diff --git a/kernel/README.md b/kernel/README.md index 66c2720..6caf51f 100644 --- a/kernel/README.md +++ b/kernel/README.md @@ -86,13 +86,6 @@ image built with it (#261). What that costs is the directives the framework impl and nothing here uses them. Confidentiality is kept on purpose: integrity would give the helper back by letting root read the kernel's memory again. Logins are unaffected (`task boot:logind`). -The fix belongs in systemd, not here (#261): systemd 259 tries to load the program at every boot, -whether or not a unit sets `RestrictFileSystems=`, so no unit can silence it. The program can -read `s_magic` by walking its `file` argument through BTF instead of through the helper; that -version loaded under this kernel's lockdown where the shipped one fails (2026-11-19). When a -systemd with it reaches Ubuntu 26.04, both lines go and the boot test that they stay gone -belongs beside `TestTheGuestKernelIsLockedDown`. - Measured 2026-10-01, `task boot:initcalls` with `SPIN_KERNEL_B` set to the kernel without these options: no cost that can be told from noise. Over 20 interleaved boots each, kernel start to `Freeing unused kernel image` had a p50 of 196.3 ms with them and 208.3 ms without, on a host at