Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ jobs:
id: touches
env:
BASE: ${{ github.event.pull_request.base.sha || github.event.before }}
run: go tool task touches -- alpine e2fsprogs ubuntu mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml
run: go tool task touches -- alpine e2fsprogs ubuntu systemd-source mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml

build:
needs: changed
Expand Down Expand Up @@ -130,3 +130,4 @@ jobs:
# Not uploaded as an artifact: it is ~860 MB and the release workflow rebuilds it
# from the same inputs. What this job is for is finding out that it still builds and
# still passes its own checks.

1 change: 1 addition & 0 deletions .github/workflows/lab.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ on:
- boot:initcalls
- boot:waits
- boot:ssh-first
- boot:restore-first
- boot:page-cache
- boot:firmware
- boot:firmware-stages
Expand Down
49 changes: 48 additions & 1 deletion .github/workflows/refresh.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
# checkpoints keep resuming, so a stable release is one versions.yaml line. kernel.yml builds it
# on the pull request; a person boots it and merges it. Nothing here publishes.
#
# And the systemd source package the image rebuilds (task refresh:systemd), which the archive
# stops serving the day Ubuntu replaces it.
#
# And the apk locks of QEMU's and e2fsprogs' builders (task refresh:apk), because the pinned
# Alpine's mirror drops the builds they name and a stale lock fails the next build - a release
# found that three times. qemu.yml and image.yml build the pull request.
Expand Down Expand Up @@ -70,7 +73,51 @@ jobs:
GH_TOKEN: ${{ steps.app.outputs.token }}
run: go tool task refresh:kernel

# Its own job: refresh:kernel leaves the checkout on the branch it proposed.
# Its own job, as apk's: the refresh leaves the checkout on the branch it proposed. Ubuntu's
# archive serves only the systemd source resolute publishes now, so image/systemd fails to build
# from the day it moves until the pin follows; this proposes the move that day. image.yml builds
# the pull request.
systemd:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0
id: app
with:
client-id: Iv23licJKNgsPIM00SD0 # spin-stack-release's client ID, public: app-id is deprecated
private-key: ${{ secrets.RELEASE_APP_KEY }}
owner: spin-stack
repositories: spin-machine

- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
# The branch is pushed as the app, below, and not with this run's token.
persist-credentials: false

- name: Set up Go
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
cache: true

- name: Commit as the app
env:
GH_TOKEN: ${{ steps.app.outputs.token }}
APP_SLUG: ${{ steps.app.outputs.app-slug }}
run: go tool task git:app

- name: Propose the systemd resolute publishes now
env:
# versions refresh opens the pull request as GITHUB_TOKEN before GH_TOKEN, and the run's
# own, set above, can neither open it nor start the checks on it; git pushes through gh,
# as GH_TOKEN.
GITHUB_TOKEN: ${{ steps.app.outputs.token }}
GH_TOKEN: ${{ steps.app.outputs.token }}
run: go tool task refresh:systemd

# Its own job: each refresh above leaves the checkout on the branch it proposed.
apk:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-24.04
Expand Down
5 changes: 5 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,11 @@ something, ends up in `boot/` as a Task target, so the next person measures the
- **Driving a guest.** `--init /bin/sh` gives a shell with nothing mounted; its console is
on stdin/stdout, or on a socket with `--console unix:PATH,server=on,wait=off`. `--qmp
PATH` opens the socket that `attach`, `detach` and `save` talk to.
- **A socket's path fits in 108 bytes.** `sun_path` is that long, and a QMP or console socket
under `t.TempDir()`, a scratch directory or `_output/` passes it easily. QEMU then does not
start, and what the probe reports is the guest never answering, not the path. Put sockets in a
short directory of their own (`os.MkdirTemp("/tmp", …)`) and keep overlays and states where
they belong; it has been hit more than once, again in boot:restore-first (2026-11-19).

```sh
spin-machine boot --hotplug-disks 2 --qmp /tmp/q.sock --init /bin/sh &
Expand Down
7 changes: 7 additions & 0 deletions Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ vars:
KERNEL_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:kernel,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/kernel,mode=max,compression=zstd{{end}}{{end}}'
E2FSPROGS_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/e2fsprogs{{end}}'
E2FSPROGS_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/e2fsprogs,mode=max,compression=zstd{{end}}{{end}}'
SYSTEMD_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/systemd{{end}}'
SYSTEMD_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/systemd,mode=max,compression=zstd{{end}}{{end}}'
# The container that builds the base image. Small — mkosi, qemu-utils — and cached like
# the rest: uncached, every CI run reinstalls it from the archive before any image work
# starts.
Expand Down Expand Up @@ -877,6 +879,11 @@ tasks:
cmds:
- '{{.VERSIONS}} refresh -propose -label kernel kernel'

refresh:systemd:
desc: Propose the systemd source package resolute now publishes as a pull request (refresh/systemd-source-<sum>); nothing when the pin is current or that version was proposed already.
cmds:
- '{{.VERSIONS}} refresh -propose -label systemd systemd-source'

# The apk locks go stale without anyone touching them: the pinned Alpine's mirror serves only the
# newest build of each package, so a lock that resolved yesterday fails a build today (`unable to
# select packages`), and a release found it three times (6e7ba47, 8d495c5, release run
Expand Down
11 changes: 11 additions & 0 deletions boot/Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,17 @@ tasks:
cmds:
- SPIN_SSH_FIRST=1 go test ./boot/ -run '^TestFirstSSH$' -count=1 -v -timeout 30m

restore-first:
desc: >-
The goal's second number: a checkpoint restored to the first SSH login over vsock, beside
the same machine booted cold, for each of boot:ssh-first's rows. Every cold boot runs and is
saved before any restore, and each state is read from the disk, not the page cache. Needs
KVM, /dev/vhost-vsock (task vsock), a built release, sudo and a TMPDIR on a disk. REPS=
repetitions.
deps: [':tools']
cmds:
- SPIN_RESTORE_FIRST=1 go test ./boot/ -run '^TestRestoreFirstSSH$' -count=1 -v -timeout 60m

firmware-stages:
desc: >-
Where the time before the kernel goes on the machine a workspace boots: QEMU's start and
Expand Down
251 changes: 251 additions & 0 deletions boot/restore_first_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,251 @@
// SPDX-License-Identifier: Apache-2.0

package boot_test

import (
"fmt"
"math/rand/v2"
"net"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"testing"
"time"
"unsafe"

xssh "golang.org/x/crypto/ssh"
"golang.org/x/sys/unix"

"github.com/spin-stack/spin-machine/boot"
)

// The second of the two numbers in CLAUDE.md's goal: a checkpoint restored to the same first
// login. Each row of boot:ssh-first (firstLoginRows) is booted cold, logged into, saved with
// `spin-machine save` - which ends it - and later restored with `boot --incoming defer` and
// `spin-machine restore`, and logged into again, from the moment before QEMU is exec'd.
//
// Two phases, as #246's boot:score had them: every cold boot first, of every row and repetition
// interleaved, then every restore in the same order. On the bench fleet, whose hosts run KVM
// nested in a cloud VM, a cold boot run after a restored guest had run was ~1.8x slower in every
// phase alike (2026-11-13), so interleaving boot-save-restore measured that and not the machine.
// Each state is evicted from the host's page cache before its restore, and the cache is asked
// whether it was: a checkpoint kept for later is read from the disk.
//
// SPIN_RESTORE_FIRST=1 run at all
// REPS=<n> repetitions of each row (default 10)
// CPUS=<n> vCPUs (default 2)
func TestRestoreFirstSSH(t *testing.T) {
if os.Getenv("SPIN_RESTORE_FIRST") == "" {
t.Skip("set SPIN_RESTORE_FIRST=1: boots, saves and restores VMs with a vsock device")
}
out := releaseDir(t)
if !canSudo() {
t.Skip("this writes a key into the boot's own overlay through qemu-nbd, which needs sudo")
}
if _, err := os.Stat("/dev/vhost-vsock"); err != nil {
t.Skip("no /dev/vhost-vsock: run task vsock")
}
reps := envInt(t, "REPS", 10)
rows, signer := firstLoginRows(t)

var runs []*checkpointRun
for i := range reps {
for _, v := range rows {
socks, err := os.MkdirTemp("/tmp", "rf")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.RemoveAll(socks) })
r := &checkpointRun{v: v, out: out, signer: signer, dir: t.TempDir(), socks: socks}
r.cold(t)
t.Logf("%s, repetition %d: cold login %v, saved in %v", v.label, i+1,
r.coldLogin.Round(time.Millisecond), r.save.Round(time.Millisecond))
runs = append(runs, r)
}
}
for i, r := range runs {
r.restore(t)
t.Logf("%s, repetition %d: restored up %v, running %v, login %v", r.v.label, i/len(rows)+1,
r.up.Round(time.Millisecond), r.running.Round(time.Millisecond), r.login.Round(time.Millisecond))
}

ms := func(label string, f func(*checkpointRun) time.Duration) string {
var ds []time.Duration
for _, r := range runs {
if r.v.label == label {
ds = append(ds, f(r))
}
}
p50, _ := boot.Percentile(ds, .5)
p95, _ := boot.Percentile(ds, .95)
return fmt.Sprintf("%6d/%-6d", p50.Milliseconds(), p95.Milliseconds())
}
var b strings.Builder
fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n",
reps, rows[0].cpus)
fmt.Fprintf(&b, " %-30s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN")
for _, v := range rows {
fmt.Fprintf(&b, " %-30s %s %s %s %s %s\n", v.label,
ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }),
ms(v.label, func(r *checkpointRun) time.Duration { return r.save }),
ms(v.label, func(r *checkpointRun) time.Duration { return r.up }),
ms(v.label, func(r *checkpointRun) time.Duration { return r.running }),
ms(v.label, func(r *checkpointRun) time.Duration { return r.login }))
}
t.Log(b.String())
}

type checkpointRun struct {
v variant
out string
signer xssh.Signer
dir string // the overlay and the state, on the disk TMPDIR names
socks string // the QMP sockets: a Unix socket's path has to fit in 108 bytes, and dir's need not
cid uint32

coldLogin, save, up, running, login time.Duration
}

func (r *checkpointRun) overlay() string { return filepath.Join(r.dir, "overlay.qcow2") }
func (r *checkpointRun) state() string { return filepath.Join(r.dir, "state") }

// launch starts the machine with its monitor on qmp and returns a channel closed when it has
// exited, and a stop that kills its process group and waits. The test's end stops it if nothing
// has.
func (r *checkpointRun) launch(t *testing.T, qmp string, extra ...string) (exited <-chan struct{}, stop func()) {
t.Helper()
args := append([]string{"boot", "--release", r.out, "--disk", r.overlay(), "--memory", r.v.memory,
"--cpus", r.v.cpus, "--vsock-cid", fmt.Sprint(r.cid), "--qmp", qmp, "--console", "file:" + qmp + ".console",
"--append", "init=/sbin/init"}, extra...)
cmd := exec.Command(filepath.Join(r.out, "bin", "spin-machine"), args...)
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
if err := cmd.Start(); err != nil {
t.Fatalf("%s: launching the machine: %v", r.v.label, err)
}
done := make(chan struct{})
go func() { _ = cmd.Wait(); close(done) }()
stop = func() {
select {
case <-done:
default:
_ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL)
<-done
}
}
t.Cleanup(stop)
return done, stop
}

// loggedIn waits for sshd to accept on vsock, then logs in, and returns when the login was done.
func (r *checkpointRun) loggedIn(t *testing.T, t0 time.Time, what string) time.Duration {
t.Helper()
deadline := t0.Add(60 * time.Second)
for !vsockAccepts(r.cid, 22) {
if time.Now().After(deadline) {
t.Fatalf("%s, %s: sshd never accepted on vsock", r.v.label, what)
}
time.Sleep(2 * time.Millisecond)
}
var last error
for time.Now().Before(deadline) {
if _, last = sshLogin(r.cid, r.signer, xssh.KeyExchangeMLKEM768X25519); last == nil {
return time.Since(t0)
}
time.Sleep(5 * time.Millisecond)
}
t.Fatalf("%s, %s: sshd accepted but was never logged into: %v", r.v.label, what, last)
return 0
}

// cold boots a fresh overlay of the release with the row's edits, logs in, saves - which ends the
// machine - and evicts the state the save has just put in the host's page cache.
func (r *checkpointRun) cold(t *testing.T) {
t.Helper()
mustRun(t, filepath.Join(r.out, "bin", "qemu-img"), "create", "-f", "qcow2",
"-F", "qcow2", "-b", filepath.Join(r.out, "image", "rootfs.qcow2"), r.overlay())
editOverlay(t, r.overlay(), r.v)
r.cid = uint32(100000 + rand.IntN(1<<20))

qmp := filepath.Join(r.socks, "cold.qmp")
t0 := time.Now()
exited, _ := r.launch(t, qmp)
r.coldLogin = r.loggedIn(t, t0, "cold")

t0 = time.Now()
mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "save", "--qmp", qmp, "--to", r.state())
r.save = time.Since(t0)
select { // save quits QEMU, and the restore opens the same overlay
case <-exited:
case <-time.After(30 * time.Second):
t.Fatalf("%s: QEMU was still running 30s after its state was saved", r.v.label)
}
evict(t, r.state())
}

// restore resumes the saved state read from the disk, logs in, and kills the machine: a restored
// guest left running would be what the restores after it measured.
func (r *checkpointRun) restore(t *testing.T) {
t.Helper()
evict(t, r.state())
qmp := filepath.Join(r.socks, "restored.qmp")
t0 := time.Now()
_, stop := r.launch(t, qmp, "--incoming", "defer")
for {
if c, err := net.Dial("unix", qmp); err == nil {
_ = c.Close()
break
}
if time.Since(t0) > 30*time.Second {
t.Fatalf("%s: the restoring QEMU never opened its QMP socket", r.v.label)
}
time.Sleep(time.Millisecond)
}
r.up = time.Since(t0)
mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "restore", "--qmp", qmp, "--from", r.state())
r.running = time.Since(t0)
r.login = r.loggedIn(t, t0, "restored")
stop()
}

// evict drops path from the host's page cache, and asks the cache whether it did: a restore of a
// state still in memory measures the cache and not the disk, and fadvise says nothing about what
// it dropped.
func evict(t *testing.T, path string) {
t.Helper()
f, err := os.Open(path)
if err != nil {
t.Fatal(err)
}
defer func() { _ = f.Close() }()
fd := int(f.Fd())
if err := unix.Fsync(fd); err != nil {
t.Fatalf("syncing %s: %v", path, err)
}
if err := unix.Fadvise(fd, 0, 0, unix.FADV_DONTNEED); err != nil {
t.Fatalf("evicting %s from the page cache: %v", path, err)
}
st, err := f.Stat()
if err != nil {
t.Fatal(err)
}
m, err := unix.Mmap(fd, 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED)
if err != nil {
t.Fatalf("mapping %s: %v", path, err)
}
defer func() { _ = unix.Munmap(m) }()
page := os.Getpagesize()
vec := make([]byte, (len(m)+page-1)/page)
// x/sys has no wrapper for mincore(2).
if _, _, errno := unix.Syscall(unix.SYS_MINCORE, uintptr(unsafe.Pointer(&m[0])), uintptr(len(m)), uintptr(unsafe.Pointer(&vec[0]))); errno != 0 {
t.Fatalf("asking which pages of %s are cached: %v", path, errno)
}
cached := 0
for _, v := range vec {
cached += int(v & 1)
}
if cached > 0 {
t.Fatalf("%s: %d KiB still in the page cache after evicting it", path, cached*page/1024)
}
}
Loading