diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 8f66dc8..1546b81 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -67,7 +67,7 @@ jobs: id: touches env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} - run: go tool task touches -- alpine e2fsprogs ubuntu mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml + run: go tool task touches -- alpine e2fsprogs ubuntu systemd-source mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml build: needs: changed @@ -130,3 +130,4 @@ jobs: # Not uploaded as an artifact: it is ~860 MB and the release workflow rebuilds it # from the same inputs. What this job is for is finding out that it still builds and # still passes its own checks. + diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 034215a..36d0a72 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -34,6 +34,7 @@ on: - boot:initcalls - boot:waits - boot:ssh-first + - boot:restore-first - boot:page-cache - boot:firmware - boot:firmware-stages diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml index 9d34537..3750f68 100644 --- a/.github/workflows/refresh.yml +++ b/.github/workflows/refresh.yml @@ -5,6 +5,9 @@ # checkpoints keep resuming, so a stable release is one versions.yaml line. kernel.yml builds it # on the pull request; a person boots it and merges it. Nothing here publishes. # +# And the systemd source package the image rebuilds (task refresh:systemd), which the archive +# stops serving the day Ubuntu replaces it. +# # And the apk locks of QEMU's and e2fsprogs' builders (task refresh:apk), because the pinned # Alpine's mirror drops the builds they name and a stale lock fails the next build - a release # found that three times. qemu.yml and image.yml build the pull request. @@ -70,7 +73,51 @@ jobs: GH_TOKEN: ${{ steps.app.outputs.token }} run: go tool task refresh:kernel - # Its own job: refresh:kernel leaves the checkout on the branch it proposed. + # Its own job, as apk's: the refresh leaves the checkout on the branch it proposed. Ubuntu's + # archive serves only the systemd source resolute publishes now, so image/systemd fails to build + # from the day it moves until the pin follows; this proposes the move that day. image.yml builds + # the pull request. + systemd: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + id: app + with: + client-id: Iv23licJKNgsPIM00SD0 # spin-stack-release's client ID, public: app-id is deprecated + private-key: ${{ secrets.RELEASE_APP_KEY }} + owner: spin-stack + repositories: spin-machine + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The branch is pushed as the app, below, and not with this run's token. + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Commit as the app + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + APP_SLUG: ${{ steps.app.outputs.app-slug }} + run: go tool task git:app + + - name: Propose the systemd resolute publishes now + env: + # versions refresh opens the pull request as GITHUB_TOKEN before GH_TOKEN, and the run's + # own, set above, can neither open it nor start the checks on it; git pushes through gh, + # as GH_TOKEN. + GITHUB_TOKEN: ${{ steps.app.outputs.token }} + GH_TOKEN: ${{ steps.app.outputs.token }} + run: go tool task refresh:systemd + + # Its own job: each refresh above leaves the checkout on the branch it proposed. apk: if: github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 diff --git a/AGENTS.md b/AGENTS.md index 3622b3f..a719c86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,6 +170,11 @@ something, ends up in `boot/` as a Task target, so the next person measures the - **Driving a guest.** `--init /bin/sh` gives a shell with nothing mounted; its console is on stdin/stdout, or on a socket with `--console unix:PATH,server=on,wait=off`. `--qmp PATH` opens the socket that `attach`, `detach` and `save` talk to. +- **A socket's path fits in 108 bytes.** `sun_path` is that long, and a QMP or console socket + under `t.TempDir()`, a scratch directory or `_output/` passes it easily. QEMU then does not + start, and what the probe reports is the guest never answering, not the path. Put sockets in a + short directory of their own (`os.MkdirTemp("/tmp", …)`) and keep overlays and states where + they belong; it has been hit more than once, again in boot:restore-first (2026-11-19). ```sh spin-machine boot --hotplug-disks 2 --qmp /tmp/q.sock --init /bin/sh & diff --git a/Taskfile.yml b/Taskfile.yml index 680d651..199e18a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -107,6 +107,8 @@ vars: KERNEL_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:kernel,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/kernel,mode=max,compression=zstd{{end}}{{end}}' E2FSPROGS_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/e2fsprogs{{end}}' E2FSPROGS_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/e2fsprogs,mode=max,compression=zstd{{end}}{{end}}' + SYSTEMD_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/systemd{{end}}' + SYSTEMD_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/systemd,mode=max,compression=zstd{{end}}{{end}}' # The container that builds the base image. Small — mkosi, qemu-utils — and cached like # the rest: uncached, every CI run reinstalls it from the archive before any image work # starts. @@ -877,6 +879,11 @@ tasks: cmds: - '{{.VERSIONS}} refresh -propose -label kernel kernel' + refresh:systemd: + desc: Propose the systemd source package resolute now publishes as a pull request (refresh/systemd-source-); nothing when the pin is current or that version was proposed already. + cmds: + - '{{.VERSIONS}} refresh -propose -label systemd systemd-source' + # The apk locks go stale without anyone touching them: the pinned Alpine's mirror serves only the # newest build of each package, so a lock that resolved yesterday fails a build today (`unable to # select packages`), and a release found it three times (6e7ba47, 8d495c5, release run diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 467f42b..679e382 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -174,6 +174,17 @@ tasks: cmds: - SPIN_SSH_FIRST=1 go test ./boot/ -run '^TestFirstSSH$' -count=1 -v -timeout 30m + restore-first: + desc: >- + The goal's second number: a checkpoint restored to the first SSH login over vsock, beside + the same machine booted cold, for each of boot:ssh-first's rows. Every cold boot runs and is + saved before any restore, and each state is read from the disk, not the page cache. Needs + KVM, /dev/vhost-vsock (task vsock), a built release, sudo and a TMPDIR on a disk. REPS= + repetitions. + deps: [':tools'] + cmds: + - SPIN_RESTORE_FIRST=1 go test ./boot/ -run '^TestRestoreFirstSSH$' -count=1 -v -timeout 60m + firmware-stages: desc: >- Where the time before the kernel goes on the machine a workspace boots: QEMU's start and diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go new file mode 100644 index 0000000..6a04ca3 --- /dev/null +++ b/boot/restore_first_test.go @@ -0,0 +1,251 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "fmt" + "math/rand/v2" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + "unsafe" + + xssh "golang.org/x/crypto/ssh" + "golang.org/x/sys/unix" + + "github.com/spin-stack/spin-machine/boot" +) + +// The second of the two numbers in CLAUDE.md's goal: a checkpoint restored to the same first +// login. Each row of boot:ssh-first (firstLoginRows) is booted cold, logged into, saved with +// `spin-machine save` - which ends it - and later restored with `boot --incoming defer` and +// `spin-machine restore`, and logged into again, from the moment before QEMU is exec'd. +// +// Two phases, as #246's boot:score had them: every cold boot first, of every row and repetition +// interleaved, then every restore in the same order. On the bench fleet, whose hosts run KVM +// nested in a cloud VM, a cold boot run after a restored guest had run was ~1.8x slower in every +// phase alike (2026-11-13), so interleaving boot-save-restore measured that and not the machine. +// Each state is evicted from the host's page cache before its restore, and the cache is asked +// whether it was: a checkpoint kept for later is read from the disk. +// +// SPIN_RESTORE_FIRST=1 run at all +// REPS= repetitions of each row (default 10) +// CPUS= vCPUs (default 2) +func TestRestoreFirstSSH(t *testing.T) { + if os.Getenv("SPIN_RESTORE_FIRST") == "" { + t.Skip("set SPIN_RESTORE_FIRST=1: boots, saves and restores VMs with a vsock device") + } + out := releaseDir(t) + if !canSudo() { + t.Skip("this writes a key into the boot's own overlay through qemu-nbd, which needs sudo") + } + if _, err := os.Stat("/dev/vhost-vsock"); err != nil { + t.Skip("no /dev/vhost-vsock: run task vsock") + } + reps := envInt(t, "REPS", 10) + rows, signer := firstLoginRows(t) + + var runs []*checkpointRun + for i := range reps { + for _, v := range rows { + socks, err := os.MkdirTemp("/tmp", "rf") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(socks) }) + r := &checkpointRun{v: v, out: out, signer: signer, dir: t.TempDir(), socks: socks} + r.cold(t) + t.Logf("%s, repetition %d: cold login %v, saved in %v", v.label, i+1, + r.coldLogin.Round(time.Millisecond), r.save.Round(time.Millisecond)) + runs = append(runs, r) + } + } + for i, r := range runs { + r.restore(t) + t.Logf("%s, repetition %d: restored up %v, running %v, login %v", r.v.label, i/len(rows)+1, + r.up.Round(time.Millisecond), r.running.Round(time.Millisecond), r.login.Round(time.Millisecond)) + } + + ms := func(label string, f func(*checkpointRun) time.Duration) string { + var ds []time.Duration + for _, r := range runs { + if r.v.label == label { + ds = append(ds, f(r)) + } + } + p50, _ := boot.Percentile(ds, .5) + p95, _ := boot.Percentile(ds, .95) + return fmt.Sprintf("%6d/%-6d", p50.Milliseconds(), p95.Milliseconds()) + } + var b strings.Builder + fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n", + reps, rows[0].cpus) + fmt.Fprintf(&b, " %-30s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") + for _, v := range rows { + fmt.Fprintf(&b, " %-30s %s %s %s %s %s\n", v.label, + ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.save }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.up }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.running }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.login })) + } + t.Log(b.String()) +} + +type checkpointRun struct { + v variant + out string + signer xssh.Signer + dir string // the overlay and the state, on the disk TMPDIR names + socks string // the QMP sockets: a Unix socket's path has to fit in 108 bytes, and dir's need not + cid uint32 + + coldLogin, save, up, running, login time.Duration +} + +func (r *checkpointRun) overlay() string { return filepath.Join(r.dir, "overlay.qcow2") } +func (r *checkpointRun) state() string { return filepath.Join(r.dir, "state") } + +// launch starts the machine with its monitor on qmp and returns a channel closed when it has +// exited, and a stop that kills its process group and waits. The test's end stops it if nothing +// has. +func (r *checkpointRun) launch(t *testing.T, qmp string, extra ...string) (exited <-chan struct{}, stop func()) { + t.Helper() + args := append([]string{"boot", "--release", r.out, "--disk", r.overlay(), "--memory", r.v.memory, + "--cpus", r.v.cpus, "--vsock-cid", fmt.Sprint(r.cid), "--qmp", qmp, "--console", "file:" + qmp + ".console", + "--append", "init=/sbin/init"}, extra...) + cmd := exec.Command(filepath.Join(r.out, "bin", "spin-machine"), args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("%s: launching the machine: %v", r.v.label, err) + } + done := make(chan struct{}) + go func() { _ = cmd.Wait(); close(done) }() + stop = func() { + select { + case <-done: + default: + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-done + } + } + t.Cleanup(stop) + return done, stop +} + +// loggedIn waits for sshd to accept on vsock, then logs in, and returns when the login was done. +func (r *checkpointRun) loggedIn(t *testing.T, t0 time.Time, what string) time.Duration { + t.Helper() + deadline := t0.Add(60 * time.Second) + for !vsockAccepts(r.cid, 22) { + if time.Now().After(deadline) { + t.Fatalf("%s, %s: sshd never accepted on vsock", r.v.label, what) + } + time.Sleep(2 * time.Millisecond) + } + var last error + for time.Now().Before(deadline) { + if _, last = sshLogin(r.cid, r.signer, xssh.KeyExchangeMLKEM768X25519); last == nil { + return time.Since(t0) + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("%s, %s: sshd accepted but was never logged into: %v", r.v.label, what, last) + return 0 +} + +// cold boots a fresh overlay of the release with the row's edits, logs in, saves - which ends the +// machine - and evicts the state the save has just put in the host's page cache. +func (r *checkpointRun) cold(t *testing.T) { + t.Helper() + mustRun(t, filepath.Join(r.out, "bin", "qemu-img"), "create", "-f", "qcow2", + "-F", "qcow2", "-b", filepath.Join(r.out, "image", "rootfs.qcow2"), r.overlay()) + editOverlay(t, r.overlay(), r.v) + r.cid = uint32(100000 + rand.IntN(1<<20)) + + qmp := filepath.Join(r.socks, "cold.qmp") + t0 := time.Now() + exited, _ := r.launch(t, qmp) + r.coldLogin = r.loggedIn(t, t0, "cold") + + t0 = time.Now() + mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "save", "--qmp", qmp, "--to", r.state()) + r.save = time.Since(t0) + select { // save quits QEMU, and the restore opens the same overlay + case <-exited: + case <-time.After(30 * time.Second): + t.Fatalf("%s: QEMU was still running 30s after its state was saved", r.v.label) + } + evict(t, r.state()) +} + +// restore resumes the saved state read from the disk, logs in, and kills the machine: a restored +// guest left running would be what the restores after it measured. +func (r *checkpointRun) restore(t *testing.T) { + t.Helper() + evict(t, r.state()) + qmp := filepath.Join(r.socks, "restored.qmp") + t0 := time.Now() + _, stop := r.launch(t, qmp, "--incoming", "defer") + for { + if c, err := net.Dial("unix", qmp); err == nil { + _ = c.Close() + break + } + if time.Since(t0) > 30*time.Second { + t.Fatalf("%s: the restoring QEMU never opened its QMP socket", r.v.label) + } + time.Sleep(time.Millisecond) + } + r.up = time.Since(t0) + mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "restore", "--qmp", qmp, "--from", r.state()) + r.running = time.Since(t0) + r.login = r.loggedIn(t, t0, "restored") + stop() +} + +// evict drops path from the host's page cache, and asks the cache whether it did: a restore of a +// state still in memory measures the cache and not the disk, and fadvise says nothing about what +// it dropped. +func evict(t *testing.T, path string) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + fd := int(f.Fd()) + if err := unix.Fsync(fd); err != nil { + t.Fatalf("syncing %s: %v", path, err) + } + if err := unix.Fadvise(fd, 0, 0, unix.FADV_DONTNEED); err != nil { + t.Fatalf("evicting %s from the page cache: %v", path, err) + } + st, err := f.Stat() + if err != nil { + t.Fatal(err) + } + m, err := unix.Mmap(fd, 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + t.Fatalf("mapping %s: %v", path, err) + } + defer func() { _ = unix.Munmap(m) }() + page := os.Getpagesize() + vec := make([]byte, (len(m)+page-1)/page) + // x/sys has no wrapper for mincore(2). + if _, _, errno := unix.Syscall(unix.SYS_MINCORE, uintptr(unsafe.Pointer(&m[0])), uintptr(len(m)), uintptr(unsafe.Pointer(&vec[0]))); errno != 0 { + t.Fatalf("asking which pages of %s are cached: %v", path, errno) + } + cached := 0 + for _, v := range vec { + cached += int(v & 1) + } + if cached > 0 { + t.Fatalf("%s: %d KiB still in the page cache after evicting it", path, cached*page/1024) + } +} diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 058b4c6..e53f7d3 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -55,6 +55,57 @@ func TestFirstSSH(t *testing.T) { t.Skip("no /dev/vhost-vsock: run task vsock") } reps := envInt(t, "REPS", 10) + rows, signer := firstLoginRows(t) + + type samples struct{ listen, login, accepted, again, prompt []time.Duration } + got := map[string]*samples{} + failed, classical := map[string]int{}, map[string]int{} + for range reps { + for _, v := range rows { + r := firstSSH(t, out, v, signer) + if got[v.label] == nil { + got[v.label] = &samples{} + } + if r.listen == 0 || r.login == 0 { + failed[v.label]++ + t.Logf("%s: a boot with no login: listen %v, login %v; %s", v.label, r.listen, r.login, r.why) + continue + } + if r.classical { + classical[v.label]++ + } + g := got[v.label] + g.listen, g.login, g.again, g.prompt = append(g.listen, r.listen), append(g.login, r.login), append(g.again, r.again), append(g.prompt, r.prompt) + g.accepted = append(g.accepted, r.login-r.listen) + } + } + ms := func(ds []time.Duration, q float64) string { + d, ok := boot.Percentile(ds, q) + if !ok { + return "—" + } + return fmt.Sprintf("%.0f", float64(d)/float64(time.Millisecond)) + } + var b strings.Builder + fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) + fmt.Fprintf(&b, " %-30s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") + for _, v := range rows { + g := got[v.label] + if g == nil { + g = &samples{} + } + fmt.Fprintf(&b, " %-30s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, + ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), + ms(g.again, .5), ms(g.again, .95), + ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) + } + t.Log(b.String()) +} + +// firstLoginRows are the rows boot:ssh-first and boot:restore-first measure, and the key both log +// in with: the release as published, and the release with this branch's image put into it. +func firstLoginRows(t *testing.T) ([]variant, xssh.Signer) { + t.Helper() row, signer := sshRow(t) // Rows measured against "as built" and dropped once they had answered, on the bench fleet, // 10 boots of each interleaved, lab run 43308825692 (2026-11-12), p50/p95 ms of the first @@ -124,50 +175,7 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} - - type samples struct{ listen, login, accepted, again, prompt []time.Duration } - got := map[string]*samples{} - failed, classical := map[string]int{}, map[string]int{} - for range reps { - for _, v := range rows { - r := firstSSH(t, out, v, signer) - if got[v.label] == nil { - got[v.label] = &samples{} - } - if r.listen == 0 || r.login == 0 { - failed[v.label]++ - t.Logf("%s: a boot with no login: listen %v, login %v; %s", v.label, r.listen, r.login, r.why) - continue - } - if r.classical { - classical[v.label]++ - } - g := got[v.label] - g.listen, g.login, g.again, g.prompt = append(g.listen, r.listen), append(g.login, r.login), append(g.again, r.again), append(g.prompt, r.prompt) - g.accepted = append(g.accepted, r.login-r.listen) - } - } - ms := func(ds []time.Duration, q float64) string { - d, ok := boot.Percentile(ds, q) - if !ok { - return "—" - } - return fmt.Sprintf("%.0f", float64(d)/float64(time.Millisecond)) - } - var b strings.Builder - fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-16s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") - for _, v := range rows { - g := got[v.label] - if g == nil { - g = &samples{} - } - fmt.Fprintf(&b, " %-16s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, - ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), - ms(g.again, .5), ms(g.again, .95), - ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) - } - t.Log(b.String()) + return rows, signer } // sshRow makes a key to log in with and returns it with a function that makes a row of the diff --git a/boot/testdata/lockdown-check.sh b/boot/testdata/lockdown-check.sh index 29c9b03..7c668a7 100644 --- a/boot/testdata/lockdown-check.sh +++ b/boot/testdata/lockdown-check.sh @@ -20,5 +20,29 @@ if echo none > /sys/kernel/security/lockdown 2>/dev/null; then fi grep -q '\[confidentiality\]' /sys/kernel/security/lockdown +# The boot asks for nothing lockdown refuses (#261). The archive's systemd loads its restrict-fs +# BPF program at every start and lockdown refuses it the bpf_probe_read_kernel it uses; the +# systemd image/systemd builds has no BPF framework. Which one PID 1 is, the installed systemd +# says: PID 1 runs that file. The journal has to hold this boot's kernel start before an absent +# line means anything - PID 1 logs at err (optimize-systemd.sh), so its own banner is not there. +# compat.yml boots the release before's image, which until a release carries image/systemd's is +# the archive's. +version=$(systemctl --version | head -1) +echo "systemd: $version" +journalctl -b -k --no-pager -o cat | grep -q '^Linux version' +case "$version" in +*+spin[0-9]*) + if journalctl -b --no-pager -o cat | grep -q 'bpf-restrict-fs'; then + echo "systemd tried restrict-fs: $(journalctl -b --no-pager -o cat | grep -m1 'bpf-restrict-fs')" + exit 1 + fi + if dmesg | grep -q 'use of bpf to read kernel RAM'; then + echo "lockdown refused a BPF read: $(dmesg | grep -m1 'use of bpf to read kernel RAM')" + exit 1 + fi + ;; +*) echo "the archive's systemd: its restrict-fs line is expected (#261)" ;; +esac + trap - EXIT echo LOCKDOWN_CHECK_OK diff --git a/docs/log.md b/docs/log.md index f2b11a4..5f11d58 100644 --- a/docs/log.md +++ b/docs/log.md @@ -343,6 +343,25 @@ Measured and not shipped: **11-13 · image · udev's coldplug keeps cpu (this change).** The coldplug filtered in `630fa2c` left out cpu, and a core attached before systemd starts - spin gives a workspace its cores while its own init is PID 1 - is onlined by that replay reaching `40-spin-hotadd.rules` and by nothing else: with v20261113.03 every workspace came up with one core whatever it was sized to (spin's kvm run 43527839765, four tests "still waiting for the guest to have its floor's two cores"). `task boot:hotadd` attaches a core before systemd and reads `online` after: `0` on v20261113.03's image, `0-1` with cpu in the filter. What cpu costs the first login was not measured; it is one device per core. + +### Week 11: 2026-11-14 to 11-20 + +**11-19 · image · systemd from Ubuntu's source package, stage1 (this change).** The image installs Ubuntu's own systemd 259.9-0ubuntu0.1, rebuilt from its .dsc (versions.yaml `systemd-source`) by `image/systemd/Dockerfile`. The build uses Debian's `stage1` build profile (no BPF framework, audit, AppArmor, SELinux, OpenSSL, TPM2, homed, cryptsetup…) and links systemd-executor, udevd, systemctl and journalctl statically against libsystemd-shared. The packages are versioned `+spin1` and held. *Why:* PID 1 spent ~25 ms in the BPF verifier on socket-bind at every boot (#240, "measured, nothing done"), and restrict-fs failed to load under lockdown on every boot (#261). *Measured:* +- `boot:ssh-first` on the bench fleet, 20 boots of each, interleaved, first login p50 against the archive's systemd: + - stage1: −47 (lab 44531842773), −38 (44537549305) and −47 ms (44538877783); + - static over stage1: −10 and −1 ms, within noise. Accept → login was −7 ms in both runs. +- `boot:waits` on a laptop: PID 1's setup before the generators goes from 25.8 to 5.6 ms. +- `boot:restore-first` (new, below): unchanged. The guest's part, from running to login, is ~50 ms with either systemd (44533227886). + +*Left out:* masking `systemd-udev-load-credentials` (−8, −6 ms) and `-Db_lto=true` (Ubuntu already builds with `-flto=auto`); see §4. The patch that would make restrict-fs load under lockdown, reading `s_magic` through BTF, loaded in this guest. It is for systemd upstream and is not needed here. *Cost:* a rebuild whenever Ubuntu publishes a systemd. The archive serves only the current source, so the build fails the day it moves, and `refresh:systemd` proposes the move. + +**11-19 · boot · `boot:restore-first` (this change).** The goal's second number had had no probe in the tree since #246 closed. Every row of `boot:ssh-first` is booted cold, logged into and saved, then restored with its state evicted from the page cache (checked with mincore) and logged into again. Every cold boot runs before every restore. On the bench fleet a restore reaches the first login at 841/913 ms, ~790 of it before the guest runs, reading the state from the runner's gp3 volume (44533227886). #246 read 386 ms on 11-13; what moved is not known. On a c8i.metal-48xl it is 905/942 (44535167653), so the restore is the host's disk, not the virtualization. + +**11-19 · lab · nested against metal.** `bench-metal` ran for the first time (spin-stack/ci#20 keeps it to c8i.metal-48xl, on-demand when spot has none; spin-machine#272 caps the job at 60 minutes). The same CPU (Xeon 6975P-C) on both, p50: +- first login: 378-392 ms nested (two runs) against 332 metal (44537498507), so **nesting costs ~50-60 ms**; +- `boot:firmware-stages`: QEMU start 30.5 against 28.1, the PCI walk 5.00 against 2.22 (173 exits, ~29 against 13 µs each), kernel and init 108.7 against 69.5 (44535161371, 44535156851). + +An r7i.metal-24xl (Sapphire Rapids, slower per core) read 296 ms (44534028478), faster than the c8i.metal. Unexplained; repeat before quoting it. Every number in §1 is nested. --- ## 3. Rework cycles @@ -520,6 +539,8 @@ Also the reusable builder, added and reverted on 10-13. | CPUWeight/IOWeight/Nice on the first login's lane | −2 ms over coldplug filtered; second login 63 vs 45 | lab 43518915618 | 11-13 | | One sshd for every connection (vsock Accept=no, `sshd -D` early) | First login 382 vs 381; second 40 vs 48 ms | lab 43524721490 | 11-13 | | Login without pam_systemd (diagnostic bound) | 380 vs 436, but no session, no XDG_RUNTIME_DIR | lab 43518044150 | 11-13 | +| Mask `systemd-udev-load-credentials` | First login −8 then −6 ms, p95 unchanged: not separable from noise at 20 boots | lab 44537549305, 44538877783 | 11-19 | +| systemd with `-Db_lto=true` | Ubuntu's dpkg-buildflags already pass `-flto=auto`: the same packages to within a few hundred bytes | image/systemd/Dockerfile | 11-19 | | Lower the vCPU ceiling, drop the balloon or ACPI for boot time | Not tried: hotplug and dynamic memory are an invariant | AGENTS.md | 11-13 | --- @@ -583,7 +604,7 @@ Also the reusable builder, added and reverted on 10-13. These are #242's checklist. - **Is `10-seats.conf` still needed?** The control that should fail passes on systemd 259 (#240, "Not settled here"). - **virtio-rng removal.** -13.6 ms to the prompt (lab 43321809471), a fingerprint change, not taken (#240). -- **PID 1 time.** ~25 ms in the BPF verifier for systemd's socket-bind program, "measured, nothing done" (#240). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`). +- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`); with systemd built here, it can now be instrumented. - **musl under long-running load.** Never measured (`31a57e7`). - **`page_reporting_order`.** Revisit "only if memory per host becomes the limit" (docs/memory-and-disk.md). - **Moving `MachineType` off pc-q35-11.1.** "its own change" (CLAUDE.md rule 1, `hack/machinetype`). diff --git a/hack/release b/hack/release index fd2b17a..3e367c7 100755 --- a/hack/release +++ b/hack/release @@ -16,8 +16,9 @@ VERSION="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo d # The versions and the upstream tarballs' SHA-256s this release was compiled from, read from # versions.yaml - the same values the builds were handed and checked the tarballs against - so # SOURCES states them without anybody reading a build file. Nothing passes these in, so there -# is no second copy to disagree: QEMU_VERSION, QEMU_SHA256, KERNEL_*, E2FSPROGS_*, QBOOT_*. -pins=$(versions env qemu kernel e2fsprogs qboot) +# is no second copy to disagree: QEMU_VERSION, QEMU_SHA256, KERNEL_*, E2FSPROGS_*, QBOOT_*, +# SYSTEMD_SOURCE_*. +pins=$(versions env qemu kernel e2fsprogs qboot systemd-source) while IFS='=' read -r key value; do declare "${key}=${value}" done <<< "${pins}" @@ -26,6 +27,7 @@ done <<< "${pins}" QEMU_URL=$(versions url qemu) KERNEL_URL=$(versions url kernel) E2FSPROGS_URL=$(versions url e2fsprogs) +SYSTEMD_SOURCE_URL=$(versions url systemd-source) RELEASE_NAME="spin-machine-${VERSION}-linux-${KERNEL_ARCH}" RELEASE_DIR="${OUTPUT_DIR}/release/${RELEASE_NAME}" @@ -209,10 +211,22 @@ Ubuntu userland image: usr/share/spin-stack/image/rootfs.qcow2 contents: usr/share/spin-stack/packages.txt — every package and its exact version licences: usr/share/spin-stack/licenses.txt — the copyright and licence text of each - source: the Ubuntu archive. For any package listed there: + source: the Ubuntu archive. For any package listed there whose version does not end + in +spinN: apt-get source = built by: image/mkosi.conf and the scripts beside it +systemd ${SYSTEMD_SOURCE_VERSION}+spin1 + packages: those in usr/share/spin-stack/packages.txt at ${SYSTEMD_SOURCE_VERSION}+spin1 + (systemd, systemd-sysv, udev, libsystemd0, libsystemd-shared, libudev1, + libpam-systemd and the rest of that source's binary packages the image installs) + source: Ubuntu's source package, unpatched: ${SYSTEMD_SOURCE_URL} + and the tarballs it names beside it + sha256: ${SYSTEMD_SOURCE_SHA256} (the .dsc, which holds the tarballs' checksums) + built by: image/systemd/Dockerfile: Debian's stage1 build profile, SELinux disabled and + systemd-executor, udevd, systemctl and journalctl linked statically against + libsystemd-shared, with a changelog entry for the +spin1 version + Written offer For three years from the date of this release, the copyright holders will provide, to anyone who asks, the complete corresponding machine-readable source for the GPL and @@ -250,7 +264,7 @@ sbom dpkg -name base-image -version "${VERSION}" -distro ubuntu -out "${PARTS}/b # pedigree, so the SBOM does not say upstream's source is what was compiled. sbom pins -name sources -version "${VERSION}" -out "${PARTS}/sources.cdx.json" \ -patches qemu=qemu/patches -patches qboot=qemu/qboot -patches kernel="${OUTPUT_DIR}/kernel/patches" \ - qemu qboot e2fsprogs kernel ubuntu task git-lfs buildkit + qemu qboot e2fsprogs kernel ubuntu systemd-source task git-lfs buildkit # The files themselves, by their hashes: the firmware is the QEMU tarball's own build of it. for f in "${SHARE}"/bin/qemu-system-x86_64 "${SHARE}"/bin/qemu-system-x86_64-tcg "${SHARE}"/bin/qemu-img \ "${SHARE}"/qemu/bios.bin "${SHARE}"/qemu/bios-256k.bin "${SHARE}"/qemu/pvh.bin "${SHARE}"/qemu/kvmvapic.bin "${SHARE}"/qemu/efi-virtio.rom; do diff --git a/image/Dockerfile b/image/Dockerfile index 7495d02..a7c6bad 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -27,7 +27,9 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ # What mkosi itself runs: apt and dpkg to assemble the tree, ubuntu-keyring for the archive # it comes from (gpgv, which checks the signatures, arrives with apt). curl fetches mkosi -# itself, below. qemu-utils wraps the filesystem in a qcow2. +# itself, below. qemu-utils wraps the filesystem in a qcow2. apt-utils for apt-ftparchive, with +# which mkosi makes the systemd packages image/systemd builds into a local repository +# (--package-directory, image/build.sh): without it mkosi stops at "apt-ftparchive not found". # # No e2fsprogs. The filesystem is made and checked with the static e2fsprogs this repository # builds (e2fsprogs/Dockerfile), which image/build.sh takes from the mounted _output: an ext4's @@ -45,6 +47,7 @@ RUN --mount=type=cache,sharing=locked,id=image-aptlib,target=/var/lib/apt \ ubuntu-keyring \ curl \ apt \ + apt-utils \ dpkg \ ca-certificates \ qemu-utils \ diff --git a/image/README.md b/image/README.md index 46c3acf..e7b2656 100644 --- a/image/README.md +++ b/image/README.md @@ -13,6 +13,14 @@ The contents are an Ubuntu 26.04 userland with systemd as the container's first the tools a workspace expects, and the boot optimizations in `mkosi.extra/usr/local/lib/spin-base/optimize-systemd.sh`. +Every package comes from the Ubuntu archive except systemd's. Those are Ubuntu's own source +package rebuilt here (`systemd/Dockerfile`, which says what is built out and what it measured), +versioned `+spin1` and held, so an `apt upgrade` in a workspace cannot bring back the archive's +build. `build.sh` fails an image whose systemd is not that build or whose packages are not held. +The other side of that: another package of the same source the image does not carry - +`systemd-container`, `systemd-resolved` - asks for the archive's systemd at exactly its version, +so `apt install` of it in a workspace is refused rather than mixing the two builds. + **ext4**, because the guest kernel has `EXT4_FS` and `OVERLAY_FS` and explicitly not `XFS_FS`, `BTRFS_FS`, `SQUASHFS` or `EROFS_FS`. Anything else starts with a kernel config change one directory over — which `kernel/Dockerfile` asserts. diff --git a/image/Taskfile.yml b/image/Taskfile.yml index f1c5cdd..c145605 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -8,6 +8,7 @@ tasks: build: desc: Build rootfs.qcow2 (Ubuntu 26.04 userland, ext4, partitionless) into _output/. cmds: + - task: systemd - task: builder - mkdir -p {{.OUTPUT_ABS}} {{.MKOSI_CACHE_DIR}} # `docker run` and not `docker buildx build`, and the reason is in image/Dockerfile: @@ -21,6 +22,7 @@ tasks: --cap-add SYS_ADMIN \ -v {{.OUTPUT_ABS}}:/out \ -v {{.MKOSI_CACHE_DIR}}:/cache \ + -v {{.OUTPUT_ABS}}/systemd:/debs:ro \ {{.IMAGE_BUILDER_TAG}} summary: @@ -31,6 +33,27 @@ tasks: ls -la {{.OUTPUT_ABS}}/image/rootfs.qcow2 echo "sha256: $(sha256sum {{.OUTPUT_ABS}}/image/rootfs.qcow2 | cut -d' ' -f1)" + systemd: + desc: >- + Build the systemd the base image installs, Ubuntu's rebuilt from its source package + (image/systemd/Dockerfile), into _output/systemd/. + cmds: + # Emptied first, as qemu:build's output is: --output writes files and removes none, and a + # package this build stopped producing would still be installed by the image. + - rm -rf {{.OUTPUT_ABS}}/systemd + - mkdir -p {{.OUTPUT_ABS}}/systemd + - | + docker buildx build \ + --file image/systemd/Dockerfile \ + --target debs \ + --platform linux/amd64 \ + --cache-from {{.SYSTEMD_CACHE_FROM}} \ + {{.SYSTEMD_CACHE_TO}} \ + $({{.VERSIONS}} args ubuntu systemd-source) \ + --build-arg JOBS={{.QEMU_JOBS}} \ + --output type=local,dest={{.OUTPUT_ABS}}/systemd \ + . + builder: desc: Build the container that builds the base image (mkosi, qemu-utils). internal: true diff --git a/image/build.sh b/image/build.sh index e5d743e..1b74ebc 100755 --- a/image/build.sh +++ b/image/build.sh @@ -51,7 +51,11 @@ mkdir -p "$(dirname "$OUT")" "$SHARE" /work/out /cache cd /work echo "==> mkosi: assembling the userland" -mkosi --force build +# The systemd this repository builds (image/systemd, task image:systemd), mounted at /debs: a +# local repository apt prefers by version (+spin1) for everything the tree installs from it. +compgen -G '/debs/systemd_*+spin[0-9]*_*.deb' > /dev/null || + { echo "ERROR: /debs has no systemd package of ours - task image:systemd" >&2; exit 1; } +mkosi --force --package-directory=/debs build tree=/work/out/rootfs test -d "$tree" || { echo "ERROR: mkosi produced no tree at $tree" >&2; exit 1; } @@ -106,6 +110,18 @@ dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${Package} ${Version} ${Archit 2>/dev/null | sort > "$SHARE/packages.txt" echo "==> $(wc -l < "$SHARE/packages.txt") packages recorded" +# What came out of /debs: systemd is ours, and every package of ours is held +# (mkosi.postinst.chroot), by dpkg's own status and not apt-mark's word. A tree that took the +# archive's systemd boots, and pays again for everything image/systemd/Dockerfile says it built out. +grep -qE '^systemd [^ ]+\+spin[0-9]+ ' "$SHARE/packages.txt" || + { echo "ERROR: the image's systemd is the archive's, not image/systemd's" >&2; exit 1; } +while read -r pkg version _; do + case "$version" in *+spin[0-9]*) ;; *) continue ;; esac + want=$(dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${db:Status-Want}' "$pkg") + [ "$want" = hold ] || { echo "ERROR: $pkg $version is ours and not held (it is '$want')" >&2; exit 1; } + echo "==> held: $pkg $version" +done < "$SHARE/packages.txt" + # The filesystem, sized to what the tree needs, plus a small margin, and nothing else. Free space in the base # is free space in every overlay built on it, whether or not that overlay's disk was meant to # have it: a disk larger than the image, with the filesystem grown to fill it, has the space it diff --git a/image/mkosi.conf b/image/mkosi.conf index 75a29f3..6597f15 100644 --- a/image/mkosi.conf +++ b/image/mkosi.conf @@ -100,6 +100,11 @@ Packages= systemd systemd-sysv + # Named, not left to arrive: it came in with docker's dbus-user-session, asking for a + # logind, when the scripts run apt in the chroot - where image/systemd's repository is + # not, and the archive's libpam-systemd wants the archive's systemd exactly. Without it + # nothing registers a login with logind (configure-system.sh's lane checks the line). + libpam-systemd dbus-broker udev diff --git a/image/mkosi.postinst.chroot b/image/mkosi.postinst.chroot index 63d57ac..9c4d614 100755 --- a/image/mkosi.postinst.chroot +++ b/image/mkosi.postinst.chroot @@ -34,6 +34,11 @@ lib=/usr/local/lib/spin-base done } > /var/lib/spin-machine-licenses.txt +# A package this repository built (+spinN, image/systemd) is held, so an `apt upgrade` in a workspace cannot replace it with the archive's build of the same +# source - which would bring back what was built out, the BPF framework among it - nor half of +# it: libsystemd-shared and systemd come from one build or the boot fails. build.sh checks it. +dpkg-query -W -f='${Package} ${Version}\n' | awk '$2 ~ /\+spin[0-9]+$/ {print $1}' | xargs -r apt-mark hold + "$lib/optimize-systemd.sh" # The configuration files, copied *after* the scripts and not delivered by ExtraTrees diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile new file mode 100644 index 0000000..6d4936c --- /dev/null +++ b/image/systemd/Dockerfile @@ -0,0 +1,88 @@ +# The systemd the base image installs: Ubuntu's own, rebuilt from its source package with less +# in it, because what PID 1, udevd and the units' executor do at every boot is the first login's +# largest share after the kernel. +# +# The source is Ubuntu's source package of the version the image would otherwise install, by +# its .dsc: its patches, and so its security fixes, are Ubuntu's. What changes is the build: +# +# stage1 Debian's own reduced build, maintained in debian/rules and debian/control: no +# audit, cryptsetup, coredump, AppArmor, OpenSSL, TPM2, p11-kit, FIDO2, PCRE2, homed, +# userdb, importd, curl, sysupdate, boot loader or ukify, and no BPF framework. That +# last one is where the time was: PID 1's setup before the generators went from 25.8 +# to 5.6 ms (boot:waits), the BPF verifier loading socket-bind at every boot and +# restrict-fs failing to under lockdown (#261). So no unit here can use +# RestrictFileSystems=, SocketBind*= or RestrictNetworkInterfaces=; none does, and +# spin's guard loads its own programs. None of the packages the image installs is +# left out by the profile. +# SELinux off as well: the guest kernel's LSMs are lockdown, capability and bpf. +# static systemd-executor - exec'd once for every unit PID 1 starts - udevd and its +# helpers, systemctl and journalctl link libsystemd-shared in instead of loading it. +# PID 1, journald and logind have no such option. +# +# Measured on the bench fleet, 20 boots of each interleaved, p50 ms of the first SSH login: stage1 +# −47, −38 and −47 against the archive's systemd (lab runs 44531842773, 44537549305, 44538877783); +# static over stage1 −10 and −1, within noise, with accept-to-login −7 in both runs. A checkpoint +# restored to the first login: unchanged, its guest part ~50 ms either way (44533227886). Not +# taken: -Db_lto=true (Ubuntu's dpkg-buildflags already compile with -flto=auto, and it built the +# same packages to within a few hundred bytes). +# +# The version gets a +spin1 suffix, so apt prefers it over the archive's, a reader of +# packages.txt can tell the two apart, and image/mkosi.postinst.chroot can hold it. +ARG UBUNTU_IMAGE +FROM ${UBUNTU_IMAGE} AS build + +# versions.yaml's systemd-source: the .dsc, by sha256. dpkg-source checks the tarballs it names +# against the checksums inside it, so the pin holds the whole source. +ARG SYSTEMD_SOURCE_VERSION +ARG SYSTEMD_SOURCE_SHA256 +ARG JOBS=8 +ENV DEBIAN_FRONTEND=noninteractive \ + DEB_BUILD_PROFILES="stage1 nocheck noinsttest nodoc" \ + DEB_BUILD_OPTIONS="nocheck nodoc noautodbgsym" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# From the archive's deb-src, which apt checks against the signed Sources index, and then the .dsc +# against the pin. The archive keeps only the version resolute publishes now: once Ubuntu moves +# systemd, this fails at the download, which is when the pin has to move with it (task +# refresh:systemd proposes it). Launchpad keeps every upload, versions.yaml's source, but its +# librarian answered 502 to a whole build here (2026-11-19). +RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \ + apt-get update && \ + apt-get install -y --no-install-recommends dpkg-dev devscripts + +WORKDIR /build +RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \ + echo "${SYSTEMD_SOURCE_SHA256} systemd_${SYSTEMD_SOURCE_VERSION}.dsc" | sha256sum -c - && \ + dpkg-source -x "systemd_${SYSTEMD_SOURCE_VERSION}.dsc" src && test -f src/debian/rules && \ + apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" + +# The options after the packaging's own, which meson takes over them: debian/rules hands +# $(CONFFLAGS_UPSTREAM) last and never sets it. meson-took.py then asks the configured build +# whether it took each, so a misspelt option is a failed build and not stage1 under another name. +COPY image/systemd/meson-took.py /build/meson-took.py +WORKDIR /build/src +RUN extra="-Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false" && \ + grep -q -- '-Dselinux=enabled' debian/rules && \ + sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \ + DEBFULLNAME="spin-machine" DEBEMAIL="noreply@spin-stack.invalid" \ + dch --newversion "${SYSTEMD_SOURCE_VERSION}+spin1" --distribution resolute \ + "Rebuilt by spin-machine: the stage1 profile, SELinux off, ${extra}." && \ + CONFFLAGS_UPSTREAM="${extra}" dpkg-buildpackage -b -uc -us -j"${JOBS}" && \ + meson introspect --buildoptions obj-* | python3 /build/meson-took.py ${extra} + +# What came out: every package the image installs, at the new version; libsystemd-shared without +# SELinux, AppArmor, audit or libcrypto; and an executor that loads no libsystemd at all. +RUN mkdir -p /debs && cp /build/*.deb /debs/ && \ + for p in systemd systemd-sysv udev libsystemd0 libsystemd-shared libudev1 libpam-systemd; do \ + ls /debs/${p}_*+spin1_*.deb > /dev/null || { echo "ERROR: no ${p} built"; exit 1; }; \ + done && \ + dpkg-deb -x /debs/libsystemd-shared_*.deb /tmp/shared && \ + so=$(find /tmp/shared -name 'libsystemd-shared-*.so') && test -n "$so" && \ + ! ldd "$so" | grep -E 'libselinux|libapparmor|libaudit|libcrypto' && \ + dpkg-deb -x /debs/systemd_*.deb /tmp/systemd && \ + ! ldd /tmp/systemd/usr/lib/systemd/systemd-executor | grep libsystemd && \ + echo "OK: $(ls /debs | wc -l) packages, libsystemd-shared without the LSMs it was built without, the executor static" + +FROM scratch AS debs +COPY --from=build /debs/ / diff --git a/image/systemd/meson-took.py b/image/systemd/meson-took.py new file mode 100644 index 0000000..6f9082e --- /dev/null +++ b/image/systemd/meson-took.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fail unless meson configured the build with every option it was handed. + +Reads `meson introspect --buildoptions` on stdin and the -Dname=value arguments from +argv: an option misspelt, or one the packaging set after ours, would otherwise build the +stage1 packages without it and say nothing. +""" +import json +import sys + +opts = {o["name"]: o["value"] for o in json.load(sys.stdin)} +for arg in sys.argv[1:]: + name, want = arg.removeprefix("-D").split("=", 1) + got = opts.get(name) + if str(got).lower() != want.lower(): + sys.exit(f"ERROR: meson has {name}={got}, it was asked for {want}") + print(f"meson took {name}={got}") diff --git a/kernel/README.md b/kernel/README.md index 7a4837a..6caf51f 100644 --- a/kernel/README.md +++ b/kernel/README.md @@ -69,20 +69,22 @@ release's kernel, and says so. They trace a kernel B built with of `Dockerfile`'s check, on a branch or in a scratch worktree, never in a release. The rest of the boot probes read the ring buffer and systemd, which lockdown leaves alone. -systemd notices at boot, and the console shows two lines that read like missing protection -and are lockdown working: +The archive's systemd noticed at boot, and the console showed two lines that read like missing +protection and are lockdown working: ``` bpf-restrict-fs: Failed to load BPF object: Invalid argument Lockdown: init: use of bpf to read kernel RAM is restricted; see man kernel_lockdown.7 ``` -The second is lockdown refusing `bpf_probe_read_kernel` to PID 1. The first is what that costs: -systemd's `RestrictFileSystems=` program reads each file's filesystem type out of kernel memory -with that helper, the verifier rejects it, and systemd carries on without it. So a unit in the -guest cannot use `RestrictFileSystems=`; it is not enforced, and systemd does not say so per -unit. That is the price of confidentiality, kept on purpose: integrity would give it back by -letting root read the kernel's memory again. Logins are unaffected (`task boot:logind`). +PID 1 loads its `RestrictFileSystems=` program at every start, whatever the units ask, and the +program reads each file's filesystem type with `bpf_probe_read_kernel`, which lockdown refuses +it. The image's systemd (`image/systemd/Dockerfile`) is built without the BPF framework, so it +asks for nothing lockdown refuses and neither line appears; `task boot:lockdown` holds that for an +image built with it (#261). What that costs is the directives the framework implements: +`RestrictFileSystems=`, `SocketBind*=` and `RestrictNetworkInterfaces=` do nothing in the guest, +and nothing here uses them. Confidentiality is kept on purpose: integrity would give the helper +back by letting root read the kernel's memory again. Logins are unaffected (`task boot:logind`). Measured 2026-10-01, `task boot:initcalls` with `SPIN_KERNEL_B` set to the kernel without these options: no cost that can be told from noise. Over 20 interleaved boots each, kernel start to diff --git a/versions.yaml b/versions.yaml index 88b4ce4..496eb36 100644 --- a/versions.yaml +++ b/versions.yaml @@ -136,6 +136,20 @@ The base image's packages come from the archive, not from this image. After the bump: `task image:build`. +- name: systemd-source + kind: download + source: https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/systemd/{version}/systemd_{version}.dsc + version: 259.9-0ubuntu0.1 + pin: 17ad848e882b6b91ac0d31300181bb4ec6b221077329b0c5296c9765cce769b5 + track: page https://launchpad.net/ubuntu/resolute/+source/systemd + note: >- + image/systemd/Dockerfile: Ubuntu's systemd source package as resolute publishes it, by its + .dsc, which holds the tarballs' checksums dpkg-source checks; rebuilt with the stage1 profile + and installed in place of the archive's. The archive serves only the current version, so the + build fails once Ubuntu moves it, and task refresh:systemd proposes the new one. After the + bump: `task image:build` (its checks: the meson options taken, the packages held), then + `task boot:ssh-first` and `task boot:lockdown`. + - name: mkosi kind: download source: https://github.com/systemd/mkosi/archive/refs/tags/v{version}.tar.gz