From cb33fab4357756070c95f2172bc3a643d7f7f07e Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 10:05:04 -0300 Subject: [PATCH 1/6] experiment: systemd from Ubuntu's deb-src, stage1 profile Ubuntu's own systemd 259.9-0ubuntu0.1, from its source package, rebuilt with Debian's stage1 build profile and SELinux off: no audit, cryptsetup, coredump, AppArmor, OpenSSL, TPM2, homed, userdb and no BPF framework (socket-bind's ~25 ms in the verifier, restrict-fs's failed load, #261). Versioned +spin1. boot:ssh-first takes the packages as a row of their own beside the image as built (SPIN_SYSTEMD_DEBS), put into the overlay the way dpkg upgrades them, /etc left alone; lab.yml's systemd_run fetches them from a Base image run. A laptop, 20 boots of each interleaved, p50/p95 ms: as this branch accepts 143/148 first login 237/245 systemd stage1 accepts 119/127 first login 207/214 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 27 +++++++++++++++++ .github/workflows/lab.yml | 9 ++++-- Taskfile.yml | 8 ++++- boot/ssh_first_test.go | 57 ++++++++++++++++++++++++++++++++++++ image/Taskfile.yml | 22 ++++++++++++++ image/build.sh | 9 +++++- image/systemd/Dockerfile | 58 +++++++++++++++++++++++++++++++++++++ 7 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 image/systemd/Dockerfile diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 8f66dc8..33ba0fb 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -130,3 +130,30 @@ jobs: # Not uploaded as an artifact: it is ~860 MB and the release workflow rebuilds it # from the same inputs. What this job is for is finding out that it still builds and # still passes its own checks. + + # EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt with the stage1 profile, as an + # artifact lab.yml's systemd_run puts into boot:ssh-first as a row of its own. + systemd: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build systemd from Ubuntu's source package + run: go tool task image:systemd QEMU_JOBS="$(nproc)" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: systemd-debs-${{ github.run_id }} + path: _output/exp-systemd/debs/*.deb + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 034215a..79938eb 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -54,6 +54,10 @@ on: description: "A QEMU workflow run id whose QEMU and firmware are compared against the release's (SPIN_MACHINE_OUTPUT_B)" required: false type: string + systemd_run: + description: "EXPERIMENT: a Base image workflow run id whose systemd-debs artifact is a row of boot:ssh-first (SPIN_SYSTEMD_DEBS)" + required: false + type: string variant: description: "With a build to compare: boot it beside the release's, interleaved, or instead of it on every boot" required: false @@ -99,6 +103,7 @@ jobs: KERNEL_RUN: ${{ inputs.kernel_run }} KERNEL_RUN_C: ${{ inputs.kernel_run_c }} QEMU_RUN: ${{ inputs.qemu_run }} + SYSTEMD_RUN: ${{ inputs.systemd_run }} VARS: ${{ inputs.vars }} # GitHub reads a workflow command (::error::) only at the start of a line, where Task's # prefixed output puts the task's name. @@ -125,7 +130,7 @@ jobs: # Another build to compare: a Kernel run's vmlinux (B, and C beside it) or a QEMU run's # QEMU and firmware. qemu.yml and kernel.yml build any branch on dispatch. - name: Fetch the builds to compare - if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' + if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_run != '' run: go tool task lab:fetch # beside: the build is booted interleaved with the release's own - SPIN_KERNEL_B and @@ -135,7 +140,7 @@ jobs: # change breaks rather than what it costs: the tree the experiment boots is the release # with the build in it. - name: Put the build where the experiment boots it - if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' + if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_run != '' env: VARIANT: ${{ inputs.variant }} run: go tool task lab:place diff --git a/Taskfile.yml b/Taskfile.yml index 680d651..5473d5c 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -651,6 +651,7 @@ tasks: if [ -n "$KERNEL_RUN" ]; then fetch "$KERNEL_RUN" 'vmlinux-*' _output/lab/kernel-b; fi if [ -n "$KERNEL_RUN_C" ]; then fetch "$KERNEL_RUN_C" 'vmlinux-*' _output/lab/kernel-c; fi if [ -n "$QEMU_RUN" ]; then fetch "$QEMU_RUN" 'qemu-*-linux-amd64' _output/lab/qemu-b; fi + if [ -n "${SYSTEMD_RUN:-}" ]; then fetch "$SYSTEMD_RUN" 'systemd-debs-*' _output/lab/systemd-debs; fi lab:place: desc: >- @@ -686,6 +687,10 @@ tasks: cp -r "$(dirname "$(dirname "$q")")/qemu" "$b/qemu" [ "$VARIANT" = instead ] || echo "SPIN_MACHINE_OUTPUT_B=$b" >> "$env" fi + if [ -n "${SYSTEMD_RUN:-}" ]; then + d=$(dirname "$(one _output/lab/systemd-debs '*/systemd_*.deb')") + echo "SPIN_SYSTEMD_DEBS=$d" >> "$env" + fi if [ "$VARIANT" = instead ]; then echo "SPIN_MACHINE_OUTPUT=$b" >> "$env"; fi lab:vars: @@ -704,7 +709,7 @@ tasks: [[ "$pair" == *=* && "$key" =~ ^(REPS|TOP|GAP|ONLY|FLAGS|CPUS|TARGET|BUCKET|LINGER|MEMORY_MIB|DEBUG|FILE_MB|CACHE_TIMEOUT_S|GRAPH_[A-Z_]+|SHARE_[A-Z_]+|SPIN_[A-Z0-9_]+)$ ]] || { echo "::error::not a variable this workflow passes on: $pair"; exit 1; } case "$key" in - SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C) echo "::error::$key is set by this workflow"; exit 1 ;; + SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C|SPIN_SYSTEMD_DEBS) echo "::error::$key is set by this workflow"; exit 1 ;; esac echo "$pair" >> "${GITHUB_ENV:-/dev/stdout}" done @@ -721,6 +726,7 @@ tasks: echo "- kernel B: ${KERNEL_RUN:+run }${KERNEL_RUN:-none}" echo "- kernel C: ${KERNEL_RUN_C:+run }${KERNEL_RUN_C:-none}" echo "- QEMU B: ${QEMU_RUN:+run }${QEMU_RUN:-none}" + echo "- systemd stage1: ${SYSTEMD_RUN:+run }${SYSTEMD_RUN:-none}" if [ -n "$KERNEL_RUN$KERNEL_RUN_C$QEMU_RUN" ]; then echo "- booted: $VARIANT the release"; fi echo "- vars: ${VARS:-none}" echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs" diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 058b4c6..307a3d9 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -7,6 +7,7 @@ import ( "cmp" "crypto/ed25519" "fmt" + "maps" "math/rand/v2" "net" "os" @@ -124,6 +125,9 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} + if debs := systemdDebs(t); debs != "" { + rows = append(rows, withPackages(asBuilt, "systemd stage1", debs)) + } type samples struct{ listen, login, accepted, again, prompt []time.Duration } got := map[string]*samples{} @@ -360,3 +364,56 @@ func (vsockAddr) String() string { return "vsock" } func (vsockConn) LocalAddr() net.Addr { return vsockAddr{} } func (vsockConn) RemoteAddr() net.Addr { return vsockAddr{} } + +// systemdDebs is the directory SPIN_SYSTEMD_DEBS names, resolved, or "" when it names none: +// packages built by image/systemd (EXPERIMENT, branch systemd-stage1), put into a row of their +// own beside the image as built, interleaved with it. +func systemdDebs(t *testing.T) string { + t.Helper() + d := os.Getenv("SPIN_SYSTEMD_DEBS") + if d == "" { + return "" + } + abs, err := filepath.Abs(d) + if err != nil { + t.Fatalf("resolving SPIN_SYSTEMD_DEBS=%q: %v", d, err) + } + debs, err := filepath.Glob(filepath.Join(abs, "*.deb")) + if err != nil || len(debs) == 0 { + t.Fatalf("SPIN_SYSTEMD_DEBS=%s holds no .deb", abs) + } + return abs +} + +// withPackages is base with the packages in debs put into the overlay the way dpkg would +// upgrade them, for each one the image has installed: the files the installed version listed +// and the new one does not ship are removed, and the new files written over the old. /etc is +// left alone, because the image's own edits there are part of base and a package's defaults +// would undo them; maintainer scripts are not run, which for an upgrade between two builds of +// one source version leaves nothing out that a boot reads. +func withPackages(base variant, label, debs string) variant { + v := base + v.label = label + v.files = maps.Clone(base.files) + v.links = maps.Clone(base.links) + v.setup = base.setup + `debs=` + debs + ` +replaced=0 +for deb in "$debs"/*.deb; do + pkg=$(dpkg-deb -f "$deb" Package) + list="$MNT/var/lib/dpkg/info/$pkg.list" + [ -f "$list" ] || list="$MNT/var/lib/dpkg/info/$pkg:amd64.list" + [ -f "$list" ] || continue + new=$(mktemp) + dpkg-deb --fsys-tarfile "$deb" | tar -t | sed -e 's|^\./|/|' -e 's|/$||' | sort > "$new" + sort "$list" | comm -23 - "$new" | grep -v '^/etc/' | while read -r f; do + if [ -L "$MNT$f" ] || [ -f "$MNT$f" ]; then rm -f "$MNT$f"; fi + done + rm -f "$new" + dpkg-deb --fsys-tarfile "$deb" | tar -x -C "$MNT" --exclude=./etc + replaced=$((replaced + 1)) +done +test "$replaced" -gt 0 +grep -q 'spin1' "$MNT/usr/lib/systemd/libsystemd-shared-"*.so 2>/dev/null || strings "$MNT/usr/lib/systemd/systemd" | grep -q '+spin1' || { echo "the overlay's systemd is not the rebuilt one" >&2; exit 1; } +` + return v +} diff --git a/image/Taskfile.yml b/image/Taskfile.yml index f1c5cdd..5a652cd 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -21,6 +21,7 @@ tasks: --cap-add SYS_ADMIN \ -v {{.OUTPUT_ABS}}:/out \ -v {{.MKOSI_CACHE_DIR}}:/cache \ + {{if .SYSTEMD_DEBS}}-v {{.SYSTEMD_DEBS}}:/debs:ro{{end}} \ {{.IMAGE_BUILDER_TAG}} summary: @@ -31,6 +32,27 @@ tasks: ls -la {{.OUTPUT_ABS}}/image/rootfs.qcow2 echo "sha256: $(sha256sum {{.OUTPUT_ABS}}/image/rootfs.qcow2 | cut -d' ' -f1)" + systemd: + desc: >- + EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt from its deb-src with the stage1 + profile and SELinux off, into _output/exp-systemd/debs. boot:ssh-first takes them as a row + with SPIN_SYSTEMD_DEBS. + vars: + SYSTEMD_SOURCE_VERSION: 259.9-0ubuntu0.1 + cmds: + - rm -rf {{.OUTPUT_ABS}}/exp-systemd/debs + - mkdir -p {{.OUTPUT_ABS}}/exp-systemd/debs + - | + docker buildx build \ + --file image/systemd/Dockerfile \ + --target debs \ + --platform linux/amd64 \ + $({{.VERSIONS}} args ubuntu) \ + --build-arg SYSTEMD_SOURCE_VERSION={{.SYSTEMD_SOURCE_VERSION}} \ + --build-arg JOBS={{.QEMU_JOBS}} \ + --output type=local,dest={{.OUTPUT_ABS}}/exp-systemd/debs \ + . + builder: desc: Build the container that builds the base image (mkosi, qemu-utils). internal: true diff --git a/image/build.sh b/image/build.sh index e5d743e..5dce3ef 100755 --- a/image/build.sh +++ b/image/build.sh @@ -51,7 +51,14 @@ mkdir -p "$(dirname "$OUT")" "$SHARE" /work/out /cache cd /work echo "==> mkosi: assembling the userland" -mkosi --force build +# EXPERIMENT (branch systemd-stage1): packages built here (image/systemd), mounted at /debs by +# `task image:build SYSTEMD_DEBS=`, are a local repository apt prefers by version. +packages=() +if compgen -G '/debs/*.deb' > /dev/null; then + echo "==> mkosi: with the packages in /debs: $(cd /debs && echo *.deb)" + packages=(--package-directory=/debs) +fi +mkosi --force "${packages[@]}" build tree=/work/out/rootfs test -d "$tree" || { echo "ERROR: mkosi produced no tree at $tree" >&2; exit 1; } diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile new file mode 100644 index 0000000..f9a3313 --- /dev/null +++ b/image/systemd/Dockerfile @@ -0,0 +1,58 @@ +# EXPERIMENT (branch systemd-stage1): Ubuntu's own systemd, rebuilt from its deb-src with less in +# it, to measure what PID 1, udevd and journald stop doing at boot. Not a release input until the +# numbers say so; see docs/log.md. +# +# The source is Ubuntu's source package of the version the image would otherwise install, from +# the archive's signed Sources index: its patches, and so its security fixes, are Ubuntu's. What +# changes is the build profile: +# +# stage1 Debian's own reduced build, maintained in debian/rules and debian/control: no audit, +# cryptsetup, coredump, AppArmor, OpenSSL, TPM2, p11-kit, FIDO2, PCRE2, homed, userdb, +# importd, curl, sysupdate, boot loader or ukify, and no BPF framework. That last one +# is what PID 1 spends ~25 ms on in the verifier at every boot (socket-bind, docs/log.md) +# and what fails to load under lockdown (restrict-fs, #261). None of the packages the +# image installs is left out by it. +# SELinux off as well: the guest kernel's LSMs are lockdown, capability and bpf. +# +# The version gets a +spin1 suffix, so apt prefers it over the archive's and anything reading the +# image's package list can tell the two apart. +ARG UBUNTU_IMAGE +FROM ${UBUNTU_IMAGE} + +ARG SYSTEMD_SOURCE_VERSION +ARG JOBS=8 +ENV DEBIAN_FRONTEND=noninteractive \ + DEB_BUILD_PROFILES="stage1 nocheck noinsttest nodoc" \ + DEB_BUILD_OPTIONS="nocheck nodoc noautodbgsym" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \ + apt-get update && \ + apt-get install -y --no-install-recommends dpkg-dev devscripts && \ + apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" + +WORKDIR /build +RUN apt-get source "systemd=${SYSTEMD_SOURCE_VERSION}" && \ + ln -s "systemd-${SYSTEMD_SOURCE_VERSION%%-*}" src && test -f src/debian/rules + +WORKDIR /build/src +RUN grep -q -- '-Dselinux=enabled' debian/rules && \ + sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \ + DEBFULLNAME="spin-machine" DEBEMAIL="noreply@spin-stack.invalid" \ + dch --newversion "${SYSTEMD_SOURCE_VERSION}+spin1" --distribution resolute \ + "Rebuilt with the stage1 profile and SELinux off (spin-machine experiment)." && \ + dpkg-buildpackage -b -uc -us -j"${JOBS}" + +# What came out: the packages the image installs, at the new version, and a PID 1 without the +# BPF framework or SELinux linked in. +RUN mkdir -p /debs && cp /build/*.deb /debs/ && \ + for p in systemd systemd-sysv udev libsystemd0 libsystemd-shared libudev1 libpam-systemd; do \ + f=$(ls /debs/${p}_*+spin1_*.deb) || { echo "ERROR: no ${p} built"; exit 1; }; \ + done && \ + dpkg-deb -x /debs/libsystemd-shared_*.deb /tmp/x && \ + ! find /tmp/x -name 'libsystemd-shared-*.so' -exec ldd {} \; | grep -E 'libselinux|libapparmor|libaudit|libcrypto' && \ + ls -la /debs + +FROM scratch AS debs +COPY --from=0 /debs/ / From 71aae5c1b4ac2153cf2e2b6a3cce0863599ae9e3 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 10:16:28 -0300 Subject: [PATCH 2/6] experiment: pin systemd's source, hold what we build, and gates versions.yaml's systemd-source pins the .dsc by sha256, fetched from the archive's deb-src. The image holds every +spinN package so an apt upgrade cannot swap it for the archive's build, and build.sh checks dpkg's status says hold. The upload is declared not published. boot:waits takes SPIN_SYSTEMD_DEBS as well. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 3 ++- boot/waits_probe_test.go | 3 +++ image/Taskfile.yml | 5 +---- image/build.sh | 13 +++++++++++++ image/mkosi.postinst.chroot | 6 ++++++ image/systemd/Dockerfile | 16 ++++++++++++---- versions.yaml | 12 ++++++++++++ workflows/workflows_test.go | 1 + 8 files changed, 50 insertions(+), 9 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 33ba0fb..cee3a40 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -151,7 +151,8 @@ jobs: - name: Build systemd from Ubuntu's source package run: go tool task image:systemd QEMU_JOBS="$(nproc)" - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + - name: Upload the systemd packages + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: systemd-debs-${{ github.run_id }} path: _output/exp-systemd/debs/*.deb diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index 0186811..5745947 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -60,6 +60,9 @@ func TestBootWaits(t *testing.T) { raw := os.Getenv("RAW") v := waitsVariant(cmp.Or(os.Getenv("CPUS"), "2")) + if debs := systemdDebs(t); debs != "" { + v = withPackages(v, "waits, systemd stage1", debs) + } var boots []waitsBoot for rep := range reps { diff --git a/image/Taskfile.yml b/image/Taskfile.yml index 5a652cd..bcfb9a4 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -37,8 +37,6 @@ tasks: EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt from its deb-src with the stage1 profile and SELinux off, into _output/exp-systemd/debs. boot:ssh-first takes them as a row with SPIN_SYSTEMD_DEBS. - vars: - SYSTEMD_SOURCE_VERSION: 259.9-0ubuntu0.1 cmds: - rm -rf {{.OUTPUT_ABS}}/exp-systemd/debs - mkdir -p {{.OUTPUT_ABS}}/exp-systemd/debs @@ -47,8 +45,7 @@ tasks: --file image/systemd/Dockerfile \ --target debs \ --platform linux/amd64 \ - $({{.VERSIONS}} args ubuntu) \ - --build-arg SYSTEMD_SOURCE_VERSION={{.SYSTEMD_SOURCE_VERSION}} \ + $({{.VERSIONS}} args ubuntu systemd-source) \ --build-arg JOBS={{.QEMU_JOBS}} \ --output type=local,dest={{.OUTPUT_ABS}}/exp-systemd/debs \ . diff --git a/image/build.sh b/image/build.sh index 5dce3ef..a5678cf 100755 --- a/image/build.sh +++ b/image/build.sh @@ -113,6 +113,19 @@ dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${Package} ${Version} ${Archit 2>/dev/null | sort > "$SHARE/packages.txt" echo "==> $(wc -l < "$SHARE/packages.txt") packages recorded" +# EXPERIMENT (branch systemd-stage1): what came out of /debs. Given packages, systemd is ours, and +# every package of ours is held (mkosi.postinst.chroot): dpkg's own status, not apt-mark's word. +if [ ${#packages[@]} -gt 0 ]; then + grep -qE '^systemd [^ ]+\+spin[0-9]+ ' "$SHARE/packages.txt" || + { echo "ERROR: /debs held packages, and the image's systemd is not one of them" >&2; exit 1; } +fi +while read -r pkg version _; do + case "$version" in *+spin[0-9]*) ;; *) continue ;; esac + want=$(dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${db:Status-Want}' "$pkg") + [ "$want" = hold ] || { echo "ERROR: $pkg $version is ours and not held (it is '$want')" >&2; exit 1; } + echo "==> held: $pkg $version" +done < "$SHARE/packages.txt" + # The filesystem, sized to what the tree needs, plus a small margin, and nothing else. Free space in the base # is free space in every overlay built on it, whether or not that overlay's disk was meant to # have it: a disk larger than the image, with the filesystem grown to fill it, has the space it diff --git a/image/mkosi.postinst.chroot b/image/mkosi.postinst.chroot index 63d57ac..6952c23 100755 --- a/image/mkosi.postinst.chroot +++ b/image/mkosi.postinst.chroot @@ -34,6 +34,12 @@ lib=/usr/local/lib/spin-base done } > /var/lib/spin-machine-licenses.txt +# EXPERIMENT (branch systemd-stage1): a package this repository built (+spinN, image/systemd) is +# held, so an `apt upgrade` in a workspace cannot replace it with the archive's build of the same +# source - which would bring back what was built out, the BPF framework among it - nor half of +# it: libsystemd-shared and systemd come from one build or the boot fails. build.sh checks it. +dpkg-query -W -f='${Package} ${Version}\n' | awk '$2 ~ /\+spin[0-9]+$/ {print $1}' | xargs -r apt-mark hold + "$lib/optimize-systemd.sh" # The configuration files, copied *after* the scripts and not delivered by ExtraTrees diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile index f9a3313..3baa73d 100644 --- a/image/systemd/Dockerfile +++ b/image/systemd/Dockerfile @@ -19,7 +19,10 @@ ARG UBUNTU_IMAGE FROM ${UBUNTU_IMAGE} +# versions.yaml's systemd-source: the .dsc, by sha256. dpkg-source checks the tarballs it names +# against the checksums inside it, so the pin holds the whole source. ARG SYSTEMD_SOURCE_VERSION +ARG SYSTEMD_SOURCE_SHA256 ARG JOBS=8 ENV DEBIAN_FRONTEND=noninteractive \ DEB_BUILD_PROFILES="stage1 nocheck noinsttest nodoc" \ @@ -27,14 +30,19 @@ ENV DEBIAN_FRONTEND=noninteractive \ SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# From the archive's deb-src, which apt checks against the signed Sources index, and then the .dsc +# against the pin. The archive keeps only the version resolute publishes now: once Ubuntu moves +# systemd, this fails at the download, which is when the pin has to move with it (Launchpad keeps +# every upload, versions.yaml's source, but its librarian answered 502 to a whole build here). RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && \ - apt-get install -y --no-install-recommends dpkg-dev devscripts && \ - apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" + apt-get install -y --no-install-recommends dpkg-dev devscripts WORKDIR /build -RUN apt-get source "systemd=${SYSTEMD_SOURCE_VERSION}" && \ - ln -s "systemd-${SYSTEMD_SOURCE_VERSION%%-*}" src && test -f src/debian/rules +RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \ + echo "${SYSTEMD_SOURCE_SHA256} systemd_${SYSTEMD_SOURCE_VERSION}.dsc" | sha256sum -c - && \ + dpkg-source -x "systemd_${SYSTEMD_SOURCE_VERSION}.dsc" src && test -f src/debian/rules && \ + apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" WORKDIR /build/src RUN grep -q -- '-Dselinux=enabled' debian/rules && \ diff --git a/versions.yaml b/versions.yaml index 88b4ce4..e2fd16a 100644 --- a/versions.yaml +++ b/versions.yaml @@ -136,6 +136,18 @@ The base image's packages come from the archive, not from this image. After the bump: `task image:build`. +- name: systemd-source + kind: download + source: https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/systemd/{version}/systemd_{version}.dsc + version: 259.9-0ubuntu0.1 + pin: 17ad848e882b6b91ac0d31300181bb4ec6b221077329b0c5296c9765cce769b5 + track: page https://launchpad.net/ubuntu/resolute/+source/systemd + note: >- + EXPERIMENT (branch systemd-stage1): image/systemd/Dockerfile, Ubuntu's systemd source package as + resolute publishes it, by its .dsc, which holds the tarballs' checksums dpkg-source checks. + Launchpad keeps a superseded upload, the archive's pool does not. After the bump: + `task image:systemd`, then boot:ssh-first with SPIN_SYSTEMD_DEBS. + - name: mkosi kind: download source: https://github.com/systemd/mkosi/archive/refs/tags/v{version}.tar.gz diff --git a/workflows/workflows_test.go b/workflows/workflows_test.go index b9da264..b32689c 100644 --- a/workflows/workflows_test.go +++ b/workflows/workflows_test.go @@ -24,6 +24,7 @@ func TestEveryUploadIsAttested(t *testing.T) { NotPublished: map[string]string{ "ci.yml/gate/Upload Go coverage HTML": "a coverage page for a person, kept seven days", "kernel.yml/build/Upload the kernel": "a lane's kernel for a person to fetch; a release builds its own", + "image.yml/systemd/Upload the systemd packages": "an experiment's packages for the lab to boot (lab.yml systemd_run); no release takes them", "lab.yml/lab/Keep the log": "a measurement's log, for a person", "qemu.yml/build/Upload the extracted binaries": "a lane's QEMU for a person to fetch; a release builds its own", "qemu.yml/build/Publish the runtime image": "what ci.yml's argument check pulls (task qemu:fetch); a release compiles QEMU, it never takes this", From fb35aac5dc2a0e3ba073cc2d4007faadcd226430 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 10:24:58 -0300 Subject: [PATCH 3/6] boot: restore-first, a checkpoint restored to the first SSH login The goal's second number had no probe in the tree since #246 closed. boot:restore-first boots each of boot:ssh-first's rows cold, logs in, saves, then restores every state read from the disk and logs in again, every cold boot before every restore as #246 found the nested fleet needs. The rows are shared with boot:ssh-first (firstLoginRows), so SPIN_SYSTEMD_DEBS adds the stage1 row to both. lab.yml offers it. AGENTS.md: a socket's path fits in 108 bytes; the QMP sockets live in a short directory of their own. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/lab.yml | 1 + AGENTS.md | 5 + boot/Taskfile.yml | 11 ++ boot/restore_first_test.go | 252 +++++++++++++++++++++++++++++++++++++ boot/ssh_first_test.go | 97 +++++++------- 5 files changed, 322 insertions(+), 44 deletions(-) create mode 100644 boot/restore_first_test.go diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 79938eb..f382346 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -34,6 +34,7 @@ on: - boot:initcalls - boot:waits - boot:ssh-first + - boot:restore-first - boot:page-cache - boot:firmware - boot:firmware-stages diff --git a/AGENTS.md b/AGENTS.md index 3622b3f..a719c86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -170,6 +170,11 @@ something, ends up in `boot/` as a Task target, so the next person measures the - **Driving a guest.** `--init /bin/sh` gives a shell with nothing mounted; its console is on stdin/stdout, or on a socket with `--console unix:PATH,server=on,wait=off`. `--qmp PATH` opens the socket that `attach`, `detach` and `save` talk to. +- **A socket's path fits in 108 bytes.** `sun_path` is that long, and a QMP or console socket + under `t.TempDir()`, a scratch directory or `_output/` passes it easily. QEMU then does not + start, and what the probe reports is the guest never answering, not the path. Put sockets in a + short directory of their own (`os.MkdirTemp("/tmp", …)`) and keep overlays and states where + they belong; it has been hit more than once, again in boot:restore-first (2026-11-19). ```sh spin-machine boot --hotplug-disks 2 --qmp /tmp/q.sock --init /bin/sh & diff --git a/boot/Taskfile.yml b/boot/Taskfile.yml index 467f42b..679e382 100644 --- a/boot/Taskfile.yml +++ b/boot/Taskfile.yml @@ -174,6 +174,17 @@ tasks: cmds: - SPIN_SSH_FIRST=1 go test ./boot/ -run '^TestFirstSSH$' -count=1 -v -timeout 30m + restore-first: + desc: >- + The goal's second number: a checkpoint restored to the first SSH login over vsock, beside + the same machine booted cold, for each of boot:ssh-first's rows. Every cold boot runs and is + saved before any restore, and each state is read from the disk, not the page cache. Needs + KVM, /dev/vhost-vsock (task vsock), a built release, sudo and a TMPDIR on a disk. REPS= + repetitions. + deps: [':tools'] + cmds: + - SPIN_RESTORE_FIRST=1 go test ./boot/ -run '^TestRestoreFirstSSH$' -count=1 -v -timeout 60m + firmware-stages: desc: >- Where the time before the kernel goes on the machine a workspace boots: QEMU's start and diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go new file mode 100644 index 0000000..5f7932f --- /dev/null +++ b/boot/restore_first_test.go @@ -0,0 +1,252 @@ +// SPDX-License-Identifier: Apache-2.0 + +package boot_test + +import ( + "fmt" + "math/rand/v2" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + "unsafe" + + xssh "golang.org/x/crypto/ssh" + "golang.org/x/sys/unix" + + "github.com/spin-stack/spin-machine/boot" +) + +// The second of the two numbers in CLAUDE.md's goal: a checkpoint restored to the same first +// login. Each row of boot:ssh-first (firstLoginRows) is booted cold, logged into, saved with +// `spin-machine save` - which ends it - and later restored with `boot --incoming defer` and +// `spin-machine restore`, and logged into again, from the moment before QEMU is exec'd. +// +// Two phases, as #246's boot:score had them: every cold boot first, of every row and repetition +// interleaved, then every restore in the same order. On the bench fleet, whose hosts run KVM +// nested in a cloud VM, a cold boot run after a restored guest had run was ~1.8x slower in every +// phase alike (2026-11-13), so interleaving boot-save-restore measured that and not the machine. +// Each state is evicted from the host's page cache before its restore, and the cache is asked +// whether it was: a checkpoint kept for later is read from the disk. +// +// SPIN_RESTORE_FIRST=1 run at all +// REPS= repetitions of each row (default 10) +// CPUS= vCPUs (default 2) +// SPIN_SYSTEMD_DEBS= a row with the systemd built by image/systemd (EXPERIMENT) +func TestRestoreFirstSSH(t *testing.T) { + if os.Getenv("SPIN_RESTORE_FIRST") == "" { + t.Skip("set SPIN_RESTORE_FIRST=1: boots, saves and restores VMs with a vsock device") + } + out := releaseDir(t) + if !canSudo() { + t.Skip("this writes a key into the boot's own overlay through qemu-nbd, which needs sudo") + } + if _, err := os.Stat("/dev/vhost-vsock"); err != nil { + t.Skip("no /dev/vhost-vsock: run task vsock") + } + reps := envInt(t, "REPS", 10) + rows, signer := firstLoginRows(t) + + var runs []*checkpointRun + for i := range reps { + for _, v := range rows { + socks, err := os.MkdirTemp("/tmp", "rf") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(socks) }) + r := &checkpointRun{v: v, out: out, signer: signer, dir: t.TempDir(), socks: socks} + r.cold(t) + t.Logf("%s, repetition %d: cold login %v, saved in %v", v.label, i+1, + r.coldLogin.Round(time.Millisecond), r.save.Round(time.Millisecond)) + runs = append(runs, r) + } + } + for i, r := range runs { + r.restore(t) + t.Logf("%s, repetition %d: restored up %v, running %v, login %v", r.v.label, i/len(rows)+1, + r.up.Round(time.Millisecond), r.running.Round(time.Millisecond), r.login.Round(time.Millisecond)) + } + + ms := func(label string, f func(*checkpointRun) time.Duration) string { + var ds []time.Duration + for _, r := range runs { + if r.v.label == label { + ds = append(ds, f(r)) + } + } + p50, _ := boot.Percentile(ds, .5) + p95, _ := boot.Percentile(ds, .95) + return fmt.Sprintf("%6d/%-6d", p50.Milliseconds(), p95.Milliseconds()) + } + var b strings.Builder + fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n", + reps, rows[0].cpus) + fmt.Fprintf(&b, " %-16s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") + for _, v := range rows { + fmt.Fprintf(&b, " %-16s %s %s %s %s %s\n", v.label, + ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.save }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.up }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.running }), + ms(v.label, func(r *checkpointRun) time.Duration { return r.login })) + } + t.Log(b.String()) +} + +type checkpointRun struct { + v variant + out string + signer xssh.Signer + dir string // the overlay and the state, on the disk TMPDIR names + socks string // the QMP sockets: a Unix socket's path has to fit in 108 bytes, and dir's need not + cid uint32 + + coldLogin, save, up, running, login time.Duration +} + +func (r *checkpointRun) overlay() string { return filepath.Join(r.dir, "overlay.qcow2") } +func (r *checkpointRun) state() string { return filepath.Join(r.dir, "state") } + +// launch starts the machine with its monitor on qmp and returns a channel closed when it has +// exited, and a stop that kills its process group and waits. The test's end stops it if nothing +// has. +func (r *checkpointRun) launch(t *testing.T, qmp string, extra ...string) (exited <-chan struct{}, stop func()) { + t.Helper() + args := append([]string{"boot", "--release", r.out, "--disk", r.overlay(), "--memory", r.v.memory, + "--cpus", r.v.cpus, "--vsock-cid", fmt.Sprint(r.cid), "--qmp", qmp, "--console", "file:" + qmp + ".console", + "--append", "init=/sbin/init"}, extra...) + cmd := exec.Command(filepath.Join(r.out, "bin", "spin-machine"), args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("%s: launching the machine: %v", r.v.label, err) + } + done := make(chan struct{}) + go func() { _ = cmd.Wait(); close(done) }() + stop = func() { + select { + case <-done: + default: + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + <-done + } + } + t.Cleanup(stop) + return done, stop +} + +// loggedIn waits for sshd to accept on vsock, then logs in, and returns when the login was done. +func (r *checkpointRun) loggedIn(t *testing.T, t0 time.Time, what string) time.Duration { + t.Helper() + deadline := t0.Add(60 * time.Second) + for !vsockAccepts(r.cid, 22) { + if time.Now().After(deadline) { + t.Fatalf("%s, %s: sshd never accepted on vsock", r.v.label, what) + } + time.Sleep(2 * time.Millisecond) + } + var last error + for time.Now().Before(deadline) { + if _, last = sshLogin(r.cid, r.signer, xssh.KeyExchangeMLKEM768X25519); last == nil { + return time.Since(t0) + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("%s, %s: sshd accepted but was never logged into: %v", r.v.label, what, last) + return 0 +} + +// cold boots a fresh overlay of the release with the row's edits, logs in, saves - which ends the +// machine - and evicts the state the save has just put in the host's page cache. +func (r *checkpointRun) cold(t *testing.T) { + t.Helper() + mustRun(t, filepath.Join(r.out, "bin", "qemu-img"), "create", "-f", "qcow2", + "-F", "qcow2", "-b", filepath.Join(r.out, "image", "rootfs.qcow2"), r.overlay()) + editOverlay(t, r.overlay(), r.v) + r.cid = uint32(100000 + rand.IntN(1<<20)) + + qmp := filepath.Join(r.socks, "cold.qmp") + t0 := time.Now() + exited, _ := r.launch(t, qmp) + r.coldLogin = r.loggedIn(t, t0, "cold") + + t0 = time.Now() + mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "save", "--qmp", qmp, "--to", r.state()) + r.save = time.Since(t0) + select { // save quits QEMU, and the restore opens the same overlay + case <-exited: + case <-time.After(30 * time.Second): + t.Fatalf("%s: QEMU was still running 30s after its state was saved", r.v.label) + } + evict(t, r.state()) +} + +// restore resumes the saved state read from the disk, logs in, and kills the machine: a restored +// guest left running would be what the restores after it measured. +func (r *checkpointRun) restore(t *testing.T) { + t.Helper() + evict(t, r.state()) + qmp := filepath.Join(r.socks, "restored.qmp") + t0 := time.Now() + _, stop := r.launch(t, qmp, "--incoming", "defer") + for { + if c, err := net.Dial("unix", qmp); err == nil { + _ = c.Close() + break + } + if time.Since(t0) > 30*time.Second { + t.Fatalf("%s: the restoring QEMU never opened its QMP socket", r.v.label) + } + time.Sleep(time.Millisecond) + } + r.up = time.Since(t0) + mustRun(t, filepath.Join(r.out, "bin", "spin-machine"), "restore", "--qmp", qmp, "--from", r.state()) + r.running = time.Since(t0) + r.login = r.loggedIn(t, t0, "restored") + stop() +} + +// evict drops path from the host's page cache, and asks the cache whether it did: a restore of a +// state still in memory measures the cache and not the disk, and fadvise says nothing about what +// it dropped. +func evict(t *testing.T, path string) { + t.Helper() + f, err := os.Open(path) + if err != nil { + t.Fatal(err) + } + defer func() { _ = f.Close() }() + fd := int(f.Fd()) + if err := unix.Fsync(fd); err != nil { + t.Fatalf("syncing %s: %v", path, err) + } + if err := unix.Fadvise(fd, 0, 0, unix.FADV_DONTNEED); err != nil { + t.Fatalf("evicting %s from the page cache: %v", path, err) + } + st, err := f.Stat() + if err != nil { + t.Fatal(err) + } + m, err := unix.Mmap(fd, 0, int(st.Size()), unix.PROT_READ, unix.MAP_SHARED) + if err != nil { + t.Fatalf("mapping %s: %v", path, err) + } + defer func() { _ = unix.Munmap(m) }() + page := os.Getpagesize() + vec := make([]byte, (len(m)+page-1)/page) + // x/sys has no wrapper for mincore(2). + if _, _, errno := unix.Syscall(unix.SYS_MINCORE, uintptr(unsafe.Pointer(&m[0])), uintptr(len(m)), uintptr(unsafe.Pointer(&vec[0]))); errno != 0 { + t.Fatalf("asking which pages of %s are cached: %v", path, errno) + } + cached := 0 + for _, v := range vec { + cached += int(v & 1) + } + if cached > 0 { + t.Fatalf("%s: %d KiB still in the page cache after evicting it", path, cached*page/1024) + } +} diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 307a3d9..dc9ed7f 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -56,6 +56,58 @@ func TestFirstSSH(t *testing.T) { t.Skip("no /dev/vhost-vsock: run task vsock") } reps := envInt(t, "REPS", 10) + rows, signer := firstLoginRows(t) + + type samples struct{ listen, login, accepted, again, prompt []time.Duration } + got := map[string]*samples{} + failed, classical := map[string]int{}, map[string]int{} + for range reps { + for _, v := range rows { + r := firstSSH(t, out, v, signer) + if got[v.label] == nil { + got[v.label] = &samples{} + } + if r.listen == 0 || r.login == 0 { + failed[v.label]++ + t.Logf("%s: a boot with no login: listen %v, login %v; %s", v.label, r.listen, r.login, r.why) + continue + } + if r.classical { + classical[v.label]++ + } + g := got[v.label] + g.listen, g.login, g.again, g.prompt = append(g.listen, r.listen), append(g.login, r.login), append(g.again, r.again), append(g.prompt, r.prompt) + g.accepted = append(g.accepted, r.login-r.listen) + } + } + ms := func(ds []time.Duration, q float64) string { + d, ok := boot.Percentile(ds, q) + if !ok { + return "—" + } + return fmt.Sprintf("%.0f", float64(d)/float64(time.Millisecond)) + } + var b strings.Builder + fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) + fmt.Fprintf(&b, " %-16s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") + for _, v := range rows { + g := got[v.label] + if g == nil { + g = &samples{} + } + fmt.Fprintf(&b, " %-16s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, + ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), + ms(g.again, .5), ms(g.again, .95), + ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) + } + t.Log(b.String()) +} + +// firstLoginRows are the rows boot:ssh-first and boot:restore-first measure, and the key both log +// in with: the release as published, the release with this branch's image put into it, and that +// with the systemd SPIN_SYSTEMD_DEBS names. +func firstLoginRows(t *testing.T) ([]variant, xssh.Signer) { + t.Helper() row, signer := sshRow(t) // Rows measured against "as built" and dropped once they had answered, on the bench fleet, // 10 boots of each interleaved, lab run 43308825692 (2026-11-12), p50/p95 ms of the first @@ -128,50 +180,7 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" if debs := systemdDebs(t); debs != "" { rows = append(rows, withPackages(asBuilt, "systemd stage1", debs)) } - - type samples struct{ listen, login, accepted, again, prompt []time.Duration } - got := map[string]*samples{} - failed, classical := map[string]int{}, map[string]int{} - for range reps { - for _, v := range rows { - r := firstSSH(t, out, v, signer) - if got[v.label] == nil { - got[v.label] = &samples{} - } - if r.listen == 0 || r.login == 0 { - failed[v.label]++ - t.Logf("%s: a boot with no login: listen %v, login %v; %s", v.label, r.listen, r.login, r.why) - continue - } - if r.classical { - classical[v.label]++ - } - g := got[v.label] - g.listen, g.login, g.again, g.prompt = append(g.listen, r.listen), append(g.login, r.login), append(g.again, r.again), append(g.prompt, r.prompt) - g.accepted = append(g.accepted, r.login-r.listen) - } - } - ms := func(ds []time.Duration, q float64) string { - d, ok := boot.Percentile(ds, q) - if !ok { - return "—" - } - return fmt.Sprintf("%.0f", float64(d)/float64(time.Millisecond)) - } - var b strings.Builder - fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-16s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") - for _, v := range rows { - g := got[v.label] - if g == nil { - g = &samples{} - } - fmt.Fprintf(&b, " %-16s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, - ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), - ms(g.again, .5), ms(g.again, .95), - ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) - } - t.Log(b.String()) + return rows, signer } // sshRow makes a key to log in with and returns it with a function that makes a row of the From 13cdead697d52efbbff5783f2ccef9f2d16738dd Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 10:34:39 -0300 Subject: [PATCH 4/6] experiment: systemd variants as rows, statically linked executor image:systemd builds each variant into debs/ with meson options after the packaging's own, and checks meson took them. stage1-static links systemd-executor (exec'd for every unit started), udevd, systemctl and journalctl with libsystemd-shared instead of loading it. boot:ssh-first and boot:restore-first make a row of each variant, and one of stage1 with systemd-udev-load-credentials masked. No LTO variant: Ubuntu's build flags already have -flto=auto. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 2 +- Taskfile.yml | 3 ++- boot/restore_first_test.go | 4 ++-- boot/ssh_first_test.go | 42 ++++++++++++++++++++++++++----------- boot/waits_probe_test.go | 5 +++-- image/Taskfile.yml | 37 +++++++++++++++++++++----------- image/systemd/Dockerfile | 12 +++++++++-- image/systemd/meson-took.py | 17 +++++++++++++++ 8 files changed, 90 insertions(+), 32 deletions(-) create mode 100644 image/systemd/meson-took.py diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index cee3a40..ffdbd29 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -155,6 +155,6 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: systemd-debs-${{ github.run_id }} - path: _output/exp-systemd/debs/*.deb + path: _output/exp-systemd/debs/ if-no-files-found: error retention-days: 14 diff --git a/Taskfile.yml b/Taskfile.yml index 5473d5c..a7404ab 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -688,7 +688,8 @@ tasks: [ "$VARIANT" = instead ] || echo "SPIN_MACHINE_OUTPUT_B=$b" >> "$env" fi if [ -n "${SYSTEMD_RUN:-}" ]; then - d=$(dirname "$(one _output/lab/systemd-debs '*/systemd_*.deb')") + # //*.deb: the probes take the directory of variants. + d=$(dirname "$(dirname "$(one _output/lab/systemd-debs '*/systemd_*.deb')")") echo "SPIN_SYSTEMD_DEBS=$d" >> "$env" fi if [ "$VARIANT" = instead ]; then echo "SPIN_MACHINE_OUTPUT=$b" >> "$env"; fi diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go index 5f7932f..922f271 100644 --- a/boot/restore_first_test.go +++ b/boot/restore_first_test.go @@ -86,9 +86,9 @@ func TestRestoreFirstSSH(t *testing.T) { var b strings.Builder fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-16s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") + fmt.Fprintf(&b, " %-22s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") for _, v := range rows { - fmt.Fprintf(&b, " %-16s %s %s %s %s %s\n", v.label, + fmt.Fprintf(&b, " %-22s %s %s %s %s %s\n", v.label, ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), ms(v.label, func(r *checkpointRun) time.Duration { return r.save }), ms(v.label, func(r *checkpointRun) time.Duration { return r.up }), diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index dc9ed7f..b79ca05 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -89,13 +89,13 @@ func TestFirstSSH(t *testing.T) { } var b strings.Builder fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-16s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") + fmt.Fprintf(&b, " %-22s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") for _, v := range rows { g := got[v.label] if g == nil { g = &samples{} } - fmt.Fprintf(&b, " %-16s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, + fmt.Fprintf(&b, " %-22s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), ms(g.again, .5), ms(g.again, .95), ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) @@ -177,8 +177,17 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} - if debs := systemdDebs(t); debs != "" { - rows = append(rows, withPackages(asBuilt, "systemd stage1", debs)) + // Each systemd variant is a row, and stage1 a second time without + // systemd-udev-load-credentials: a oneshot on the critical path before udevd in 8 of 10 + // boots (boot:waits, 7.8 ms its own and 4.6 waiting), which imports credentials nobody here + // passes. A mask, not a build: it is the image's to say. + for _, b := range systemdDebs(t) { + rows = append(rows, withPackages(asBuilt, b.name, b.dir)) + if b.name == "stage1" { + r := withPackages(asBuilt, "stage1, no udev creds", b.dir) + r.mask = []string{"systemd-udev-load-credentials.service"} + rows = append(rows, r) + } } return rows, signer } @@ -374,24 +383,33 @@ func (vsockAddr) String() string { return "vsock" } func (vsockConn) LocalAddr() net.Addr { return vsockAddr{} } func (vsockConn) RemoteAddr() net.Addr { return vsockAddr{} } -// systemdDebs is the directory SPIN_SYSTEMD_DEBS names, resolved, or "" when it names none: -// packages built by image/systemd (EXPERIMENT, branch systemd-stage1), put into a row of their -// own beside the image as built, interleaved with it. -func systemdDebs(t *testing.T) string { +// systemdBuild is one variant of the systemd image/systemd builds: its name and the directory of +// its packages. +type systemdBuild struct{ name, dir string } + +// systemdDebs are the variants under the directory SPIN_SYSTEMD_DEBS names, one subdirectory of +// packages each, as `task image:systemd` writes them; none when it names nothing (EXPERIMENT, +// branch systemd-stage1). +func systemdDebs(t *testing.T) []systemdBuild { t.Helper() d := os.Getenv("SPIN_SYSTEMD_DEBS") if d == "" { - return "" + return nil } abs, err := filepath.Abs(d) if err != nil { t.Fatalf("resolving SPIN_SYSTEMD_DEBS=%q: %v", d, err) } - debs, err := filepath.Glob(filepath.Join(abs, "*.deb")) + debs, err := filepath.Glob(filepath.Join(abs, "*", "systemd_*.deb")) if err != nil || len(debs) == 0 { - t.Fatalf("SPIN_SYSTEMD_DEBS=%s holds no .deb", abs) + t.Fatalf("SPIN_SYSTEMD_DEBS=%s holds no /systemd_*.deb", abs) + } + var builds []systemdBuild + for _, deb := range debs { + dir := filepath.Dir(deb) + builds = append(builds, systemdBuild{name: filepath.Base(dir), dir: dir}) } - return abs + return builds } // withPackages is base with the packages in debs put into the overlay the way dpkg would diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index 5745947..45cd488 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -60,8 +60,9 @@ func TestBootWaits(t *testing.T) { raw := os.Getenv("RAW") v := waitsVariant(cmp.Or(os.Getenv("CPUS"), "2")) - if debs := systemdDebs(t); debs != "" { - v = withPackages(v, "waits, systemd stage1", debs) + // With SPIN_SYSTEMD_DEBS, the first variant's systemd (EXPERIMENT, branch systemd-stage1). + if builds := systemdDebs(t); len(builds) > 0 { + v = withPackages(v, "waits, systemd "+builds[0].name, builds[0].dir) } var boots []waitsBoot diff --git a/image/Taskfile.yml b/image/Taskfile.yml index bcfb9a4..0660764 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -35,20 +35,33 @@ tasks: systemd: desc: >- EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt from its deb-src with the stage1 - profile and SELinux off, into _output/exp-systemd/debs. boot:ssh-first takes them as a row - with SPIN_SYSTEMD_DEBS. + profile and SELinux off, once for each variant, into _output/exp-systemd/debs/. + boot:ssh-first and boot:restore-first take each as a row with SPIN_SYSTEMD_DEBS. cmds: - rm -rf {{.OUTPUT_ABS}}/exp-systemd/debs - - mkdir -p {{.OUTPUT_ABS}}/exp-systemd/debs - - | - docker buildx build \ - --file image/systemd/Dockerfile \ - --target debs \ - --platform linux/amd64 \ - $({{.VERSIONS}} args ubuntu systemd-source) \ - --build-arg JOBS={{.QEMU_JOBS}} \ - --output type=local,dest={{.OUTPUT_ABS}}/exp-systemd/debs \ - . + - for: + # The variants, by meson options after the packaging's own. static: the binaries the + # boot execs most - systemd-executor once for every unit started, udevd and its + # helpers - and the two command-line tools, with libsystemd-shared linked in instead + # of loaded. PID 1, journald and logind have no such option. No LTO variant: Ubuntu's + # dpkg-buildflags already compile with -flto=auto, and -Db_lto=true built the same + # packages to within a few hundred bytes (2026-11-19). + - stage1= + - stage1-static=-Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false + cmd: | + set -euo pipefail + variant='{{.ITEM}}' + name=${variant%%=*} + mkdir -p {{.OUTPUT_ABS}}/exp-systemd/debs/$name + docker buildx build \ + --file image/systemd/Dockerfile \ + --target debs \ + --platform linux/amd64 \ + $({{.VERSIONS}} args ubuntu systemd-source) \ + --build-arg MESON_EXTRA="${variant#*=}" \ + --build-arg JOBS={{.QEMU_JOBS}} \ + --output type=local,dest={{.OUTPUT_ABS}}/exp-systemd/debs/$name \ + . builder: desc: Build the container that builds the base image (mkosi, qemu-utils). diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile index 3baa73d..375d9be 100644 --- a/image/systemd/Dockerfile +++ b/image/systemd/Dockerfile @@ -44,13 +44,19 @@ RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \ dpkg-source -x "systemd_${SYSTEMD_SOURCE_VERSION}.dsc" src && test -f src/debian/rules && \ apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" +# A variant is meson options after the packaging's own, which meson takes over them: debian/rules +# hands $(CONFFLAGS_UPSTREAM) last and never sets it, so it comes from here. Each variant is its +# own image/Taskfile.yml call and its own directory of packages, a row of its own in the probes. +ARG MESON_EXTRA="" +COPY image/systemd/meson-took.py /build/meson-took.py WORKDIR /build/src RUN grep -q -- '-Dselinux=enabled' debian/rules && \ sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \ DEBFULLNAME="spin-machine" DEBEMAIL="noreply@spin-stack.invalid" \ dch --newversion "${SYSTEMD_SOURCE_VERSION}+spin1" --distribution resolute \ - "Rebuilt with the stage1 profile and SELinux off (spin-machine experiment)." && \ - dpkg-buildpackage -b -uc -us -j"${JOBS}" + "Rebuilt with the stage1 profile and SELinux off (spin-machine experiment). ${MESON_EXTRA}" && \ + CONFFLAGS_UPSTREAM="${MESON_EXTRA}" dpkg-buildpackage -b -uc -us -j"${JOBS}" && \ + meson introspect --buildoptions obj-* | python3 /build/meson-took.py ${MESON_EXTRA} # What came out: the packages the image installs, at the new version, and a PID 1 without the # BPF framework or SELinux linked in. @@ -60,6 +66,8 @@ RUN mkdir -p /debs && cp /build/*.deb /debs/ && \ done && \ dpkg-deb -x /debs/libsystemd-shared_*.deb /tmp/x && \ ! find /tmp/x -name 'libsystemd-shared-*.so' -exec ldd {} \; | grep -E 'libselinux|libapparmor|libaudit|libcrypto' && \ + dpkg-deb -x /debs/systemd_*.deb /tmp/s && \ + echo "systemd-executor: $(ldd /tmp/s/usr/lib/systemd/systemd-executor | grep -c libsystemd) libsystemd libraries" && \ ls -la /debs FROM scratch AS debs diff --git a/image/systemd/meson-took.py b/image/systemd/meson-took.py new file mode 100644 index 0000000..5417d9f --- /dev/null +++ b/image/systemd/meson-took.py @@ -0,0 +1,17 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Fail unless meson configured the build with every option a variant asked for. + +Reads `meson introspect --buildoptions` on stdin and the variant's -Dname=value arguments from +argv: an option misspelt, or one the packaging set after the variant's, would otherwise build the +stage1 packages again under the variant's name. +""" +import json +import sys + +opts = {o["name"]: o["value"] for o in json.load(sys.stdin)} +for arg in sys.argv[1:]: + name, want = arg.removeprefix("-D").split("=", 1) + got = opts.get(name) + if str(got).lower() != want.lower(): + sys.exit(f"ERROR: meson has {name}={got}, the variant asked for {want}") + print(f"meson took {name}={got}") From c01019da742a1fe11fe1a27ee3ca332bd3296835 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 10:58:54 -0300 Subject: [PATCH 5/6] boot: the udev credentials mask over stage1-static Over stage1 alone the mask moved the first login -8 ms p50 and nothing at p95 (lab 44537549305); it is asked again with the build that would ship. Co-Authored-By: Claude Opus 5.5 --- boot/restore_first_test.go | 4 ++-- boot/ssh_first_test.go | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go index 922f271..0c4dbe3 100644 --- a/boot/restore_first_test.go +++ b/boot/restore_first_test.go @@ -86,9 +86,9 @@ func TestRestoreFirstSSH(t *testing.T) { var b strings.Builder fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d of each, %s vCPUs; every cold boot before every restore\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-22s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") + fmt.Fprintf(&b, " %-30s %13s %13s %13s %13s %13s\n", "", "COLD LOGIN", "SAVE", "RESTORE: UP", "RUNNING", "LOGIN") for _, v := range rows { - fmt.Fprintf(&b, " %-22s %s %s %s %s %s\n", v.label, + fmt.Fprintf(&b, " %-30s %s %s %s %s %s\n", v.label, ms(v.label, func(r *checkpointRun) time.Duration { return r.coldLogin }), ms(v.label, func(r *checkpointRun) time.Duration { return r.save }), ms(v.label, func(r *checkpointRun) time.Duration { return r.up }), diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index b79ca05..1938d40 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -89,13 +89,13 @@ func TestFirstSSH(t *testing.T) { } var b strings.Builder fmt.Fprintf(&b, "\nms from the moment before QEMU is exec'd, p50/p95 over %d boots of each, interleaved, %s vCPUs\n\n", reps, rows[0].cpus) - fmt.Fprintf(&b, " %-22s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") + fmt.Fprintf(&b, " %-30s %15s %15s %15s %15s %15s %9s %14s\n", "", "SSHD ACCEPTS", "FIRST LOGIN", "ACCEPT TO LOGIN", "A SECOND TOOK", "CONSOLE PROMPT", "NO LOGIN", "CURVE25519 IN") for _, v := range rows { g := got[v.label] if g == nil { g = &samples{} } - fmt.Fprintf(&b, " %-22s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, + fmt.Fprintf(&b, " %-30s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %7s/%-7s %9d %14d\n", v.label, ms(g.listen, .5), ms(g.listen, .95), ms(g.login, .5), ms(g.login, .95), ms(g.accepted, .5), ms(g.accepted, .95), ms(g.again, .5), ms(g.again, .95), ms(g.prompt, .5), ms(g.prompt, .95), failed[v.label], classical[v.label]) @@ -177,14 +177,16 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} - // Each systemd variant is a row, and stage1 a second time without + // Each systemd variant is a row, and stage1-static a second time without // systemd-udev-load-credentials: a oneshot on the critical path before udevd in 8 of 10 // boots (boot:waits, 7.8 ms its own and 4.6 waiting), which imports credentials nobody here - // passes. A mask, not a build: it is the image's to say. + // passes. A mask, not a build: it is the image's to say. Over stage1 alone it moved the first + // login -8 ms p50 and nothing at p95 (lab run 44537549305), so it is asked again with the + // static build, the one that would ship. for _, b := range systemdDebs(t) { rows = append(rows, withPackages(asBuilt, b.name, b.dir)) - if b.name == "stage1" { - r := withPackages(asBuilt, "stage1, no udev creds", b.dir) + if b.name == "stage1-static" { + r := withPackages(asBuilt, "stage1-static, no udev creds", b.dir) r.mask = []string{"systemd-udev-load-credentials.service"} rows = append(rows, r) } From 0f277f069a0c11a566a7b80948baa90b40d8965c Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 11:12:42 -0300 Subject: [PATCH 6/6] image: systemd from Ubuntu's source package, stage1 and static The base image installs Ubuntu's own systemd 259.9-0ubuntu0.1 rebuilt by image/systemd/Dockerfile from its .dsc (versions.yaml systemd-source): Debian's stage1 build profile, SELinux off, and systemd-executor, udevd, systemctl and journalctl linked statically against libsystemd-shared. The packages are versioned +spin1 and held; build.sh fails an image whose systemd is not this build or whose packages are not held. On the bench fleet, 20 boots of each interleaved, the first SSH login p50 against the archive's systemd: stage1 -47, -38, -47 ms (labs 44531842773, 44537549305, 44538877783); static over stage1 -10 and -1, accept-to-login -7 in both. The restored checkpoint's guest part is unchanged (44533227886). PID 1 no longer loads socket-bind through the BPF verifier (~20 ms of its setup) nor restrict-fs, which lockdown refused at every boot: boot:lockdown holds that for an image built with it (#261). - task image:systemd, run by image:build, cached like the other builds; - mkosi takes the packages as a local repository, which needs apt-utils in the builder, and libpam-systemd is named: arriving through docker's dbus-user-session inside the chroot it was the archive's, which wants the archive's systemd exactly; - refresh:systemd and its job propose a new source the day Ubuntu publishes one, since the archive then stops serving the pinned one; - SOURCES and the pins SBOM name the source and how it was built; - the experiment's rows, lab input and CI job are removed; boot:restore-first stays. Closes #261 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 29 +----------- .github/workflows/lab.yml | 9 +--- .github/workflows/refresh.yml | 49 +++++++++++++++++++- Taskfile.yml | 16 +++---- boot/restore_first_test.go | 1 - boot/ssh_first_test.go | 80 +------------------------------- boot/testdata/lockdown-check.sh | 24 ++++++++++ boot/waits_probe_test.go | 4 -- docs/log.md | 23 ++++++++- hack/release | 22 +++++++-- image/Dockerfile | 5 +- image/README.md | 8 ++++ image/Taskfile.yml | 47 ++++++++----------- image/build.sh | 24 ++++------ image/mkosi.conf | 5 ++ image/mkosi.postinst.chroot | 3 +- image/systemd/Dockerfile | 82 +++++++++++++++++++-------------- image/systemd/meson-took.py | 10 ++-- kernel/README.md | 18 ++++---- versions.yaml | 10 ++-- workflows/workflows_test.go | 1 - 21 files changed, 240 insertions(+), 230 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index ffdbd29..1546b81 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -67,7 +67,7 @@ jobs: id: touches env: BASE: ${{ github.event.pull_request.base.sha || github.event.before }} - run: go tool task touches -- alpine e2fsprogs ubuntu mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml + run: go tool task touches -- alpine e2fsprogs ubuntu systemd-source mkosi task git-lfs buildkit -- image e2fsprogs go.mod .github/workflows/image.yml build: needs: changed @@ -131,30 +131,3 @@ jobs: # from the same inputs. What this job is for is finding out that it still builds and # still passes its own checks. - # EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt with the stage1 profile, as an - # artifact lab.yml's systemd_run puts into boot:ssh-first as a row of its own. - systemd: - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: go.mod - cache: true - - - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - - - name: Build systemd from Ubuntu's source package - run: go tool task image:systemd QEMU_JOBS="$(nproc)" - - - name: Upload the systemd packages - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: systemd-debs-${{ github.run_id }} - path: _output/exp-systemd/debs/ - if-no-files-found: error - retention-days: 14 diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index f382346..36d0a72 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -55,10 +55,6 @@ on: description: "A QEMU workflow run id whose QEMU and firmware are compared against the release's (SPIN_MACHINE_OUTPUT_B)" required: false type: string - systemd_run: - description: "EXPERIMENT: a Base image workflow run id whose systemd-debs artifact is a row of boot:ssh-first (SPIN_SYSTEMD_DEBS)" - required: false - type: string variant: description: "With a build to compare: boot it beside the release's, interleaved, or instead of it on every boot" required: false @@ -104,7 +100,6 @@ jobs: KERNEL_RUN: ${{ inputs.kernel_run }} KERNEL_RUN_C: ${{ inputs.kernel_run_c }} QEMU_RUN: ${{ inputs.qemu_run }} - SYSTEMD_RUN: ${{ inputs.systemd_run }} VARS: ${{ inputs.vars }} # GitHub reads a workflow command (::error::) only at the start of a line, where Task's # prefixed output puts the task's name. @@ -131,7 +126,7 @@ jobs: # Another build to compare: a Kernel run's vmlinux (B, and C beside it) or a QEMU run's # QEMU and firmware. qemu.yml and kernel.yml build any branch on dispatch. - name: Fetch the builds to compare - if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_run != '' + if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' run: go tool task lab:fetch # beside: the build is booted interleaved with the release's own - SPIN_KERNEL_B and @@ -141,7 +136,7 @@ jobs: # change breaks rather than what it costs: the tree the experiment boots is the release # with the build in it. - name: Put the build where the experiment boots it - if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_run != '' + if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' env: VARIANT: ${{ inputs.variant }} run: go tool task lab:place diff --git a/.github/workflows/refresh.yml b/.github/workflows/refresh.yml index 9d34537..3750f68 100644 --- a/.github/workflows/refresh.yml +++ b/.github/workflows/refresh.yml @@ -5,6 +5,9 @@ # checkpoints keep resuming, so a stable release is one versions.yaml line. kernel.yml builds it # on the pull request; a person boots it and merges it. Nothing here publishes. # +# And the systemd source package the image rebuilds (task refresh:systemd), which the archive +# stops serving the day Ubuntu replaces it. +# # And the apk locks of QEMU's and e2fsprogs' builders (task refresh:apk), because the pinned # Alpine's mirror drops the builds they name and a stale lock fails the next build - a release # found that three times. qemu.yml and image.yml build the pull request. @@ -70,7 +73,51 @@ jobs: GH_TOKEN: ${{ steps.app.outputs.token }} run: go tool task refresh:kernel - # Its own job: refresh:kernel leaves the checkout on the branch it proposed. + # Its own job, as apk's: the refresh leaves the checkout on the branch it proposed. Ubuntu's + # archive serves only the systemd source resolute publishes now, so image/systemd fails to build + # from the day it moves until the pin follows; this proposes the move that day. image.yml builds + # the pull request. + systemd: + if: github.ref == 'refs/heads/main' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + id: app + with: + client-id: Iv23licJKNgsPIM00SD0 # spin-stack-release's client ID, public: app-id is deprecated + private-key: ${{ secrets.RELEASE_APP_KEY }} + owner: spin-stack + repositories: spin-machine + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The branch is pushed as the app, below, and not with this run's token. + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + + - name: Commit as the app + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + APP_SLUG: ${{ steps.app.outputs.app-slug }} + run: go tool task git:app + + - name: Propose the systemd resolute publishes now + env: + # versions refresh opens the pull request as GITHUB_TOKEN before GH_TOKEN, and the run's + # own, set above, can neither open it nor start the checks on it; git pushes through gh, + # as GH_TOKEN. + GITHUB_TOKEN: ${{ steps.app.outputs.token }} + GH_TOKEN: ${{ steps.app.outputs.token }} + run: go tool task refresh:systemd + + # Its own job: each refresh above leaves the checkout on the branch it proposed. apk: if: github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 diff --git a/Taskfile.yml b/Taskfile.yml index a7404ab..199e18a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -107,6 +107,8 @@ vars: KERNEL_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:kernel,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/kernel,mode=max,compression=zstd{{end}}{{end}}' E2FSPROGS_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/e2fsprogs{{end}}' E2FSPROGS_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:e2fsprogs,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/e2fsprogs,mode=max,compression=zstd{{end}}{{end}}' + SYSTEMD_CACHE_FROM: '{{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd{{else}}type=local,src={{.BUILDKIT_CACHE_DIR}}/systemd{{end}}' + SYSTEMD_CACHE_TO: '{{if ne .CACHE_EXPORT "false"}}--cache-to {{if eq .CACHE_BACKEND "registry"}}type=registry,ref={{.CACHE_REGISTRY}}:systemd,mode=max,image-manifest=true,oci-mediatypes=true{{else}}type=local,dest={{.BUILDKIT_CACHE_DIR}}/systemd,mode=max,compression=zstd{{end}}{{end}}' # The container that builds the base image. Small — mkosi, qemu-utils — and cached like # the rest: uncached, every CI run reinstalls it from the archive before any image work # starts. @@ -651,7 +653,6 @@ tasks: if [ -n "$KERNEL_RUN" ]; then fetch "$KERNEL_RUN" 'vmlinux-*' _output/lab/kernel-b; fi if [ -n "$KERNEL_RUN_C" ]; then fetch "$KERNEL_RUN_C" 'vmlinux-*' _output/lab/kernel-c; fi if [ -n "$QEMU_RUN" ]; then fetch "$QEMU_RUN" 'qemu-*-linux-amd64' _output/lab/qemu-b; fi - if [ -n "${SYSTEMD_RUN:-}" ]; then fetch "$SYSTEMD_RUN" 'systemd-debs-*' _output/lab/systemd-debs; fi lab:place: desc: >- @@ -687,11 +688,6 @@ tasks: cp -r "$(dirname "$(dirname "$q")")/qemu" "$b/qemu" [ "$VARIANT" = instead ] || echo "SPIN_MACHINE_OUTPUT_B=$b" >> "$env" fi - if [ -n "${SYSTEMD_RUN:-}" ]; then - # //*.deb: the probes take the directory of variants. - d=$(dirname "$(dirname "$(one _output/lab/systemd-debs '*/systemd_*.deb')")") - echo "SPIN_SYSTEMD_DEBS=$d" >> "$env" - fi if [ "$VARIANT" = instead ]; then echo "SPIN_MACHINE_OUTPUT=$b" >> "$env"; fi lab:vars: @@ -710,7 +706,7 @@ tasks: [[ "$pair" == *=* && "$key" =~ ^(REPS|TOP|GAP|ONLY|FLAGS|CPUS|TARGET|BUCKET|LINGER|MEMORY_MIB|DEBUG|FILE_MB|CACHE_TIMEOUT_S|GRAPH_[A-Z_]+|SHARE_[A-Z_]+|SPIN_[A-Z0-9_]+)$ ]] || { echo "::error::not a variable this workflow passes on: $pair"; exit 1; } case "$key" in - SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C|SPIN_SYSTEMD_DEBS) echo "::error::$key is set by this workflow"; exit 1 ;; + SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C) echo "::error::$key is set by this workflow"; exit 1 ;; esac echo "$pair" >> "${GITHUB_ENV:-/dev/stdout}" done @@ -727,7 +723,6 @@ tasks: echo "- kernel B: ${KERNEL_RUN:+run }${KERNEL_RUN:-none}" echo "- kernel C: ${KERNEL_RUN_C:+run }${KERNEL_RUN_C:-none}" echo "- QEMU B: ${QEMU_RUN:+run }${QEMU_RUN:-none}" - echo "- systemd stage1: ${SYSTEMD_RUN:+run }${SYSTEMD_RUN:-none}" if [ -n "$KERNEL_RUN$KERNEL_RUN_C$QEMU_RUN" ]; then echo "- booted: $VARIANT the release"; fi echo "- vars: ${VARS:-none}" echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs" @@ -884,6 +879,11 @@ tasks: cmds: - '{{.VERSIONS}} refresh -propose -label kernel kernel' + refresh:systemd: + desc: Propose the systemd source package resolute now publishes as a pull request (refresh/systemd-source-); nothing when the pin is current or that version was proposed already. + cmds: + - '{{.VERSIONS}} refresh -propose -label systemd systemd-source' + # The apk locks go stale without anyone touching them: the pinned Alpine's mirror serves only the # newest build of each package, so a lock that resolved yesterday fails a build today (`unable to # select packages`), and a release found it three times (6e7ba47, 8d495c5, release run diff --git a/boot/restore_first_test.go b/boot/restore_first_test.go index 0c4dbe3..6a04ca3 100644 --- a/boot/restore_first_test.go +++ b/boot/restore_first_test.go @@ -36,7 +36,6 @@ import ( // SPIN_RESTORE_FIRST=1 run at all // REPS= repetitions of each row (default 10) // CPUS= vCPUs (default 2) -// SPIN_SYSTEMD_DEBS= a row with the systemd built by image/systemd (EXPERIMENT) func TestRestoreFirstSSH(t *testing.T) { if os.Getenv("SPIN_RESTORE_FIRST") == "" { t.Skip("set SPIN_RESTORE_FIRST=1: boots, saves and restores VMs with a vsock device") diff --git a/boot/ssh_first_test.go b/boot/ssh_first_test.go index 1938d40..e53f7d3 100644 --- a/boot/ssh_first_test.go +++ b/boot/ssh_first_test.go @@ -7,7 +7,6 @@ import ( "cmp" "crypto/ed25519" "fmt" - "maps" "math/rand/v2" "net" "os" @@ -104,8 +103,7 @@ func TestFirstSSH(t *testing.T) { } // firstLoginRows are the rows boot:ssh-first and boot:restore-first measure, and the key both log -// in with: the release as published, the release with this branch's image put into it, and that -// with the systemd SPIN_SYSTEMD_DEBS names. +// in with: the release as published, and the release with this branch's image put into it. func firstLoginRows(t *testing.T) ([]variant, xssh.Signer) { t.Helper() row, signer := sshRow(t) @@ -177,20 +175,6 @@ grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" grep -q 'pam_systemd.so.*class=user-early-light' "$MNT/etc/pam.d/common-session" ` rows := []variant{row("as released", ""), asBuilt} - // Each systemd variant is a row, and stage1-static a second time without - // systemd-udev-load-credentials: a oneshot on the critical path before udevd in 8 of 10 - // boots (boot:waits, 7.8 ms its own and 4.6 waiting), which imports credentials nobody here - // passes. A mask, not a build: it is the image's to say. Over stage1 alone it moved the first - // login -8 ms p50 and nothing at p95 (lab run 44537549305), so it is asked again with the - // static build, the one that would ship. - for _, b := range systemdDebs(t) { - rows = append(rows, withPackages(asBuilt, b.name, b.dir)) - if b.name == "stage1-static" { - r := withPackages(asBuilt, "stage1-static, no udev creds", b.dir) - r.mask = []string{"systemd-udev-load-credentials.service"} - rows = append(rows, r) - } - } return rows, signer } @@ -384,65 +368,3 @@ func (vsockAddr) String() string { return "vsock" } func (vsockConn) LocalAddr() net.Addr { return vsockAddr{} } func (vsockConn) RemoteAddr() net.Addr { return vsockAddr{} } - -// systemdBuild is one variant of the systemd image/systemd builds: its name and the directory of -// its packages. -type systemdBuild struct{ name, dir string } - -// systemdDebs are the variants under the directory SPIN_SYSTEMD_DEBS names, one subdirectory of -// packages each, as `task image:systemd` writes them; none when it names nothing (EXPERIMENT, -// branch systemd-stage1). -func systemdDebs(t *testing.T) []systemdBuild { - t.Helper() - d := os.Getenv("SPIN_SYSTEMD_DEBS") - if d == "" { - return nil - } - abs, err := filepath.Abs(d) - if err != nil { - t.Fatalf("resolving SPIN_SYSTEMD_DEBS=%q: %v", d, err) - } - debs, err := filepath.Glob(filepath.Join(abs, "*", "systemd_*.deb")) - if err != nil || len(debs) == 0 { - t.Fatalf("SPIN_SYSTEMD_DEBS=%s holds no /systemd_*.deb", abs) - } - var builds []systemdBuild - for _, deb := range debs { - dir := filepath.Dir(deb) - builds = append(builds, systemdBuild{name: filepath.Base(dir), dir: dir}) - } - return builds -} - -// withPackages is base with the packages in debs put into the overlay the way dpkg would -// upgrade them, for each one the image has installed: the files the installed version listed -// and the new one does not ship are removed, and the new files written over the old. /etc is -// left alone, because the image's own edits there are part of base and a package's defaults -// would undo them; maintainer scripts are not run, which for an upgrade between two builds of -// one source version leaves nothing out that a boot reads. -func withPackages(base variant, label, debs string) variant { - v := base - v.label = label - v.files = maps.Clone(base.files) - v.links = maps.Clone(base.links) - v.setup = base.setup + `debs=` + debs + ` -replaced=0 -for deb in "$debs"/*.deb; do - pkg=$(dpkg-deb -f "$deb" Package) - list="$MNT/var/lib/dpkg/info/$pkg.list" - [ -f "$list" ] || list="$MNT/var/lib/dpkg/info/$pkg:amd64.list" - [ -f "$list" ] || continue - new=$(mktemp) - dpkg-deb --fsys-tarfile "$deb" | tar -t | sed -e 's|^\./|/|' -e 's|/$||' | sort > "$new" - sort "$list" | comm -23 - "$new" | grep -v '^/etc/' | while read -r f; do - if [ -L "$MNT$f" ] || [ -f "$MNT$f" ]; then rm -f "$MNT$f"; fi - done - rm -f "$new" - dpkg-deb --fsys-tarfile "$deb" | tar -x -C "$MNT" --exclude=./etc - replaced=$((replaced + 1)) -done -test "$replaced" -gt 0 -grep -q 'spin1' "$MNT/usr/lib/systemd/libsystemd-shared-"*.so 2>/dev/null || strings "$MNT/usr/lib/systemd/systemd" | grep -q '+spin1' || { echo "the overlay's systemd is not the rebuilt one" >&2; exit 1; } -` - return v -} diff --git a/boot/testdata/lockdown-check.sh b/boot/testdata/lockdown-check.sh index 29c9b03..7c668a7 100644 --- a/boot/testdata/lockdown-check.sh +++ b/boot/testdata/lockdown-check.sh @@ -20,5 +20,29 @@ if echo none > /sys/kernel/security/lockdown 2>/dev/null; then fi grep -q '\[confidentiality\]' /sys/kernel/security/lockdown +# The boot asks for nothing lockdown refuses (#261). The archive's systemd loads its restrict-fs +# BPF program at every start and lockdown refuses it the bpf_probe_read_kernel it uses; the +# systemd image/systemd builds has no BPF framework. Which one PID 1 is, the installed systemd +# says: PID 1 runs that file. The journal has to hold this boot's kernel start before an absent +# line means anything - PID 1 logs at err (optimize-systemd.sh), so its own banner is not there. +# compat.yml boots the release before's image, which until a release carries image/systemd's is +# the archive's. +version=$(systemctl --version | head -1) +echo "systemd: $version" +journalctl -b -k --no-pager -o cat | grep -q '^Linux version' +case "$version" in +*+spin[0-9]*) + if journalctl -b --no-pager -o cat | grep -q 'bpf-restrict-fs'; then + echo "systemd tried restrict-fs: $(journalctl -b --no-pager -o cat | grep -m1 'bpf-restrict-fs')" + exit 1 + fi + if dmesg | grep -q 'use of bpf to read kernel RAM'; then + echo "lockdown refused a BPF read: $(dmesg | grep -m1 'use of bpf to read kernel RAM')" + exit 1 + fi + ;; +*) echo "the archive's systemd: its restrict-fs line is expected (#261)" ;; +esac + trap - EXIT echo LOCKDOWN_CHECK_OK diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index 45cd488..0186811 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -60,10 +60,6 @@ func TestBootWaits(t *testing.T) { raw := os.Getenv("RAW") v := waitsVariant(cmp.Or(os.Getenv("CPUS"), "2")) - // With SPIN_SYSTEMD_DEBS, the first variant's systemd (EXPERIMENT, branch systemd-stage1). - if builds := systemdDebs(t); len(builds) > 0 { - v = withPackages(v, "waits, systemd "+builds[0].name, builds[0].dir) - } var boots []waitsBoot for rep := range reps { diff --git a/docs/log.md b/docs/log.md index f2b11a4..5f11d58 100644 --- a/docs/log.md +++ b/docs/log.md @@ -343,6 +343,25 @@ Measured and not shipped: **11-13 · image · udev's coldplug keeps cpu (this change).** The coldplug filtered in `630fa2c` left out cpu, and a core attached before systemd starts - spin gives a workspace its cores while its own init is PID 1 - is onlined by that replay reaching `40-spin-hotadd.rules` and by nothing else: with v20261113.03 every workspace came up with one core whatever it was sized to (spin's kvm run 43527839765, four tests "still waiting for the guest to have its floor's two cores"). `task boot:hotadd` attaches a core before systemd and reads `online` after: `0` on v20261113.03's image, `0-1` with cpu in the filter. What cpu costs the first login was not measured; it is one device per core. + +### Week 11: 2026-11-14 to 11-20 + +**11-19 · image · systemd from Ubuntu's source package, stage1 (this change).** The image installs Ubuntu's own systemd 259.9-0ubuntu0.1, rebuilt from its .dsc (versions.yaml `systemd-source`) by `image/systemd/Dockerfile`. The build uses Debian's `stage1` build profile (no BPF framework, audit, AppArmor, SELinux, OpenSSL, TPM2, homed, cryptsetup…) and links systemd-executor, udevd, systemctl and journalctl statically against libsystemd-shared. The packages are versioned `+spin1` and held. *Why:* PID 1 spent ~25 ms in the BPF verifier on socket-bind at every boot (#240, "measured, nothing done"), and restrict-fs failed to load under lockdown on every boot (#261). *Measured:* +- `boot:ssh-first` on the bench fleet, 20 boots of each, interleaved, first login p50 against the archive's systemd: + - stage1: −47 (lab 44531842773), −38 (44537549305) and −47 ms (44538877783); + - static over stage1: −10 and −1 ms, within noise. Accept → login was −7 ms in both runs. +- `boot:waits` on a laptop: PID 1's setup before the generators goes from 25.8 to 5.6 ms. +- `boot:restore-first` (new, below): unchanged. The guest's part, from running to login, is ~50 ms with either systemd (44533227886). + +*Left out:* masking `systemd-udev-load-credentials` (−8, −6 ms) and `-Db_lto=true` (Ubuntu already builds with `-flto=auto`); see §4. The patch that would make restrict-fs load under lockdown, reading `s_magic` through BTF, loaded in this guest. It is for systemd upstream and is not needed here. *Cost:* a rebuild whenever Ubuntu publishes a systemd. The archive serves only the current source, so the build fails the day it moves, and `refresh:systemd` proposes the move. + +**11-19 · boot · `boot:restore-first` (this change).** The goal's second number had had no probe in the tree since #246 closed. Every row of `boot:ssh-first` is booted cold, logged into and saved, then restored with its state evicted from the page cache (checked with mincore) and logged into again. Every cold boot runs before every restore. On the bench fleet a restore reaches the first login at 841/913 ms, ~790 of it before the guest runs, reading the state from the runner's gp3 volume (44533227886). #246 read 386 ms on 11-13; what moved is not known. On a c8i.metal-48xl it is 905/942 (44535167653), so the restore is the host's disk, not the virtualization. + +**11-19 · lab · nested against metal.** `bench-metal` ran for the first time (spin-stack/ci#20 keeps it to c8i.metal-48xl, on-demand when spot has none; spin-machine#272 caps the job at 60 minutes). The same CPU (Xeon 6975P-C) on both, p50: +- first login: 378-392 ms nested (two runs) against 332 metal (44537498507), so **nesting costs ~50-60 ms**; +- `boot:firmware-stages`: QEMU start 30.5 against 28.1, the PCI walk 5.00 against 2.22 (173 exits, ~29 against 13 µs each), kernel and init 108.7 against 69.5 (44535161371, 44535156851). + +An r7i.metal-24xl (Sapphire Rapids, slower per core) read 296 ms (44534028478), faster than the c8i.metal. Unexplained; repeat before quoting it. Every number in §1 is nested. --- ## 3. Rework cycles @@ -520,6 +539,8 @@ Also the reusable builder, added and reverted on 10-13. | CPUWeight/IOWeight/Nice on the first login's lane | −2 ms over coldplug filtered; second login 63 vs 45 | lab 43518915618 | 11-13 | | One sshd for every connection (vsock Accept=no, `sshd -D` early) | First login 382 vs 381; second 40 vs 48 ms | lab 43524721490 | 11-13 | | Login without pam_systemd (diagnostic bound) | 380 vs 436, but no session, no XDG_RUNTIME_DIR | lab 43518044150 | 11-13 | +| Mask `systemd-udev-load-credentials` | First login −8 then −6 ms, p95 unchanged: not separable from noise at 20 boots | lab 44537549305, 44538877783 | 11-19 | +| systemd with `-Db_lto=true` | Ubuntu's dpkg-buildflags already pass `-flto=auto`: the same packages to within a few hundred bytes | image/systemd/Dockerfile | 11-19 | | Lower the vCPU ceiling, drop the balloon or ACPI for boot time | Not tried: hotplug and dynamic memory are an invariant | AGENTS.md | 11-13 | --- @@ -583,7 +604,7 @@ Also the reusable builder, added and reverted on 10-13. These are #242's checklist. - **Is `10-seats.conf` still needed?** The control that should fail passes on systemd 259 (#240, "Not settled here"). - **virtio-rng removal.** -13.6 ms to the prompt (lab 43321809471), a fingerprint change, not taken (#240). -- **PID 1 time.** ~25 ms in the BPF verifier for systemd's socket-bind program, "measured, nothing done" (#240). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`). +- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`); with systemd built here, it can now be instrumented. - **musl under long-running load.** Never measured (`31a57e7`). - **`page_reporting_order`.** Revisit "only if memory per host becomes the limit" (docs/memory-and-disk.md). - **Moving `MachineType` off pc-q35-11.1.** "its own change" (CLAUDE.md rule 1, `hack/machinetype`). diff --git a/hack/release b/hack/release index fd2b17a..3e367c7 100755 --- a/hack/release +++ b/hack/release @@ -16,8 +16,9 @@ VERSION="${VERSION:-$(git describe --tags --always --dirty 2>/dev/null || echo d # The versions and the upstream tarballs' SHA-256s this release was compiled from, read from # versions.yaml - the same values the builds were handed and checked the tarballs against - so # SOURCES states them without anybody reading a build file. Nothing passes these in, so there -# is no second copy to disagree: QEMU_VERSION, QEMU_SHA256, KERNEL_*, E2FSPROGS_*, QBOOT_*. -pins=$(versions env qemu kernel e2fsprogs qboot) +# is no second copy to disagree: QEMU_VERSION, QEMU_SHA256, KERNEL_*, E2FSPROGS_*, QBOOT_*, +# SYSTEMD_SOURCE_*. +pins=$(versions env qemu kernel e2fsprogs qboot systemd-source) while IFS='=' read -r key value; do declare "${key}=${value}" done <<< "${pins}" @@ -26,6 +27,7 @@ done <<< "${pins}" QEMU_URL=$(versions url qemu) KERNEL_URL=$(versions url kernel) E2FSPROGS_URL=$(versions url e2fsprogs) +SYSTEMD_SOURCE_URL=$(versions url systemd-source) RELEASE_NAME="spin-machine-${VERSION}-linux-${KERNEL_ARCH}" RELEASE_DIR="${OUTPUT_DIR}/release/${RELEASE_NAME}" @@ -209,10 +211,22 @@ Ubuntu userland image: usr/share/spin-stack/image/rootfs.qcow2 contents: usr/share/spin-stack/packages.txt — every package and its exact version licences: usr/share/spin-stack/licenses.txt — the copyright and licence text of each - source: the Ubuntu archive. For any package listed there: + source: the Ubuntu archive. For any package listed there whose version does not end + in +spinN: apt-get source = built by: image/mkosi.conf and the scripts beside it +systemd ${SYSTEMD_SOURCE_VERSION}+spin1 + packages: those in usr/share/spin-stack/packages.txt at ${SYSTEMD_SOURCE_VERSION}+spin1 + (systemd, systemd-sysv, udev, libsystemd0, libsystemd-shared, libudev1, + libpam-systemd and the rest of that source's binary packages the image installs) + source: Ubuntu's source package, unpatched: ${SYSTEMD_SOURCE_URL} + and the tarballs it names beside it + sha256: ${SYSTEMD_SOURCE_SHA256} (the .dsc, which holds the tarballs' checksums) + built by: image/systemd/Dockerfile: Debian's stage1 build profile, SELinux disabled and + systemd-executor, udevd, systemctl and journalctl linked statically against + libsystemd-shared, with a changelog entry for the +spin1 version + Written offer For three years from the date of this release, the copyright holders will provide, to anyone who asks, the complete corresponding machine-readable source for the GPL and @@ -250,7 +264,7 @@ sbom dpkg -name base-image -version "${VERSION}" -distro ubuntu -out "${PARTS}/b # pedigree, so the SBOM does not say upstream's source is what was compiled. sbom pins -name sources -version "${VERSION}" -out "${PARTS}/sources.cdx.json" \ -patches qemu=qemu/patches -patches qboot=qemu/qboot -patches kernel="${OUTPUT_DIR}/kernel/patches" \ - qemu qboot e2fsprogs kernel ubuntu task git-lfs buildkit + qemu qboot e2fsprogs kernel ubuntu systemd-source task git-lfs buildkit # The files themselves, by their hashes: the firmware is the QEMU tarball's own build of it. for f in "${SHARE}"/bin/qemu-system-x86_64 "${SHARE}"/bin/qemu-system-x86_64-tcg "${SHARE}"/bin/qemu-img \ "${SHARE}"/qemu/bios.bin "${SHARE}"/qemu/bios-256k.bin "${SHARE}"/qemu/pvh.bin "${SHARE}"/qemu/kvmvapic.bin "${SHARE}"/qemu/efi-virtio.rom; do diff --git a/image/Dockerfile b/image/Dockerfile index 7495d02..a7c6bad 100644 --- a/image/Dockerfile +++ b/image/Dockerfile @@ -27,7 +27,9 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ # What mkosi itself runs: apt and dpkg to assemble the tree, ubuntu-keyring for the archive # it comes from (gpgv, which checks the signatures, arrives with apt). curl fetches mkosi -# itself, below. qemu-utils wraps the filesystem in a qcow2. +# itself, below. qemu-utils wraps the filesystem in a qcow2. apt-utils for apt-ftparchive, with +# which mkosi makes the systemd packages image/systemd builds into a local repository +# (--package-directory, image/build.sh): without it mkosi stops at "apt-ftparchive not found". # # No e2fsprogs. The filesystem is made and checked with the static e2fsprogs this repository # builds (e2fsprogs/Dockerfile), which image/build.sh takes from the mounted _output: an ext4's @@ -45,6 +47,7 @@ RUN --mount=type=cache,sharing=locked,id=image-aptlib,target=/var/lib/apt \ ubuntu-keyring \ curl \ apt \ + apt-utils \ dpkg \ ca-certificates \ qemu-utils \ diff --git a/image/README.md b/image/README.md index 46c3acf..e7b2656 100644 --- a/image/README.md +++ b/image/README.md @@ -13,6 +13,14 @@ The contents are an Ubuntu 26.04 userland with systemd as the container's first the tools a workspace expects, and the boot optimizations in `mkosi.extra/usr/local/lib/spin-base/optimize-systemd.sh`. +Every package comes from the Ubuntu archive except systemd's. Those are Ubuntu's own source +package rebuilt here (`systemd/Dockerfile`, which says what is built out and what it measured), +versioned `+spin1` and held, so an `apt upgrade` in a workspace cannot bring back the archive's +build. `build.sh` fails an image whose systemd is not that build or whose packages are not held. +The other side of that: another package of the same source the image does not carry - +`systemd-container`, `systemd-resolved` - asks for the archive's systemd at exactly its version, +so `apt install` of it in a workspace is refused rather than mixing the two builds. + **ext4**, because the guest kernel has `EXT4_FS` and `OVERLAY_FS` and explicitly not `XFS_FS`, `BTRFS_FS`, `SQUASHFS` or `EROFS_FS`. Anything else starts with a kernel config change one directory over — which `kernel/Dockerfile` asserts. diff --git a/image/Taskfile.yml b/image/Taskfile.yml index 0660764..c145605 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -8,6 +8,7 @@ tasks: build: desc: Build rootfs.qcow2 (Ubuntu 26.04 userland, ext4, partitionless) into _output/. cmds: + - task: systemd - task: builder - mkdir -p {{.OUTPUT_ABS}} {{.MKOSI_CACHE_DIR}} # `docker run` and not `docker buildx build`, and the reason is in image/Dockerfile: @@ -21,7 +22,7 @@ tasks: --cap-add SYS_ADMIN \ -v {{.OUTPUT_ABS}}:/out \ -v {{.MKOSI_CACHE_DIR}}:/cache \ - {{if .SYSTEMD_DEBS}}-v {{.SYSTEMD_DEBS}}:/debs:ro{{end}} \ + -v {{.OUTPUT_ABS}}/systemd:/debs:ro \ {{.IMAGE_BUILDER_TAG}} summary: @@ -34,34 +35,24 @@ tasks: systemd: desc: >- - EXPERIMENT (branch systemd-stage1): Ubuntu's systemd rebuilt from its deb-src with the stage1 - profile and SELinux off, once for each variant, into _output/exp-systemd/debs/. - boot:ssh-first and boot:restore-first take each as a row with SPIN_SYSTEMD_DEBS. + Build the systemd the base image installs, Ubuntu's rebuilt from its source package + (image/systemd/Dockerfile), into _output/systemd/. cmds: - - rm -rf {{.OUTPUT_ABS}}/exp-systemd/debs - - for: - # The variants, by meson options after the packaging's own. static: the binaries the - # boot execs most - systemd-executor once for every unit started, udevd and its - # helpers - and the two command-line tools, with libsystemd-shared linked in instead - # of loaded. PID 1, journald and logind have no such option. No LTO variant: Ubuntu's - # dpkg-buildflags already compile with -flto=auto, and -Db_lto=true built the same - # packages to within a few hundred bytes (2026-11-19). - - stage1= - - stage1-static=-Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false - cmd: | - set -euo pipefail - variant='{{.ITEM}}' - name=${variant%%=*} - mkdir -p {{.OUTPUT_ABS}}/exp-systemd/debs/$name - docker buildx build \ - --file image/systemd/Dockerfile \ - --target debs \ - --platform linux/amd64 \ - $({{.VERSIONS}} args ubuntu systemd-source) \ - --build-arg MESON_EXTRA="${variant#*=}" \ - --build-arg JOBS={{.QEMU_JOBS}} \ - --output type=local,dest={{.OUTPUT_ABS}}/exp-systemd/debs/$name \ - . + # Emptied first, as qemu:build's output is: --output writes files and removes none, and a + # package this build stopped producing would still be installed by the image. + - rm -rf {{.OUTPUT_ABS}}/systemd + - mkdir -p {{.OUTPUT_ABS}}/systemd + - | + docker buildx build \ + --file image/systemd/Dockerfile \ + --target debs \ + --platform linux/amd64 \ + --cache-from {{.SYSTEMD_CACHE_FROM}} \ + {{.SYSTEMD_CACHE_TO}} \ + $({{.VERSIONS}} args ubuntu systemd-source) \ + --build-arg JOBS={{.QEMU_JOBS}} \ + --output type=local,dest={{.OUTPUT_ABS}}/systemd \ + . builder: desc: Build the container that builds the base image (mkosi, qemu-utils). diff --git a/image/build.sh b/image/build.sh index a5678cf..1b74ebc 100755 --- a/image/build.sh +++ b/image/build.sh @@ -51,14 +51,11 @@ mkdir -p "$(dirname "$OUT")" "$SHARE" /work/out /cache cd /work echo "==> mkosi: assembling the userland" -# EXPERIMENT (branch systemd-stage1): packages built here (image/systemd), mounted at /debs by -# `task image:build SYSTEMD_DEBS=`, are a local repository apt prefers by version. -packages=() -if compgen -G '/debs/*.deb' > /dev/null; then - echo "==> mkosi: with the packages in /debs: $(cd /debs && echo *.deb)" - packages=(--package-directory=/debs) -fi -mkosi --force "${packages[@]}" build +# The systemd this repository builds (image/systemd, task image:systemd), mounted at /debs: a +# local repository apt prefers by version (+spin1) for everything the tree installs from it. +compgen -G '/debs/systemd_*+spin[0-9]*_*.deb' > /dev/null || + { echo "ERROR: /debs has no systemd package of ours - task image:systemd" >&2; exit 1; } +mkosi --force --package-directory=/debs build tree=/work/out/rootfs test -d "$tree" || { echo "ERROR: mkosi produced no tree at $tree" >&2; exit 1; } @@ -113,12 +110,11 @@ dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${Package} ${Version} ${Archit 2>/dev/null | sort > "$SHARE/packages.txt" echo "==> $(wc -l < "$SHARE/packages.txt") packages recorded" -# EXPERIMENT (branch systemd-stage1): what came out of /debs. Given packages, systemd is ours, and -# every package of ours is held (mkosi.postinst.chroot): dpkg's own status, not apt-mark's word. -if [ ${#packages[@]} -gt 0 ]; then - grep -qE '^systemd [^ ]+\+spin[0-9]+ ' "$SHARE/packages.txt" || - { echo "ERROR: /debs held packages, and the image's systemd is not one of them" >&2; exit 1; } -fi +# What came out of /debs: systemd is ours, and every package of ours is held +# (mkosi.postinst.chroot), by dpkg's own status and not apt-mark's word. A tree that took the +# archive's systemd boots, and pays again for everything image/systemd/Dockerfile says it built out. +grep -qE '^systemd [^ ]+\+spin[0-9]+ ' "$SHARE/packages.txt" || + { echo "ERROR: the image's systemd is the archive's, not image/systemd's" >&2; exit 1; } while read -r pkg version _; do case "$version" in *+spin[0-9]*) ;; *) continue ;; esac want=$(dpkg-query --admindir="$tree/var/lib/dpkg" -W -f='${db:Status-Want}' "$pkg") diff --git a/image/mkosi.conf b/image/mkosi.conf index 75a29f3..6597f15 100644 --- a/image/mkosi.conf +++ b/image/mkosi.conf @@ -100,6 +100,11 @@ Packages= systemd systemd-sysv + # Named, not left to arrive: it came in with docker's dbus-user-session, asking for a + # logind, when the scripts run apt in the chroot - where image/systemd's repository is + # not, and the archive's libpam-systemd wants the archive's systemd exactly. Without it + # nothing registers a login with logind (configure-system.sh's lane checks the line). + libpam-systemd dbus-broker udev diff --git a/image/mkosi.postinst.chroot b/image/mkosi.postinst.chroot index 6952c23..9c4d614 100755 --- a/image/mkosi.postinst.chroot +++ b/image/mkosi.postinst.chroot @@ -34,8 +34,7 @@ lib=/usr/local/lib/spin-base done } > /var/lib/spin-machine-licenses.txt -# EXPERIMENT (branch systemd-stage1): a package this repository built (+spinN, image/systemd) is -# held, so an `apt upgrade` in a workspace cannot replace it with the archive's build of the same +# A package this repository built (+spinN, image/systemd) is held, so an `apt upgrade` in a workspace cannot replace it with the archive's build of the same # source - which would bring back what was built out, the BPF framework among it - nor half of # it: libsystemd-shared and systemd come from one build or the boot fails. build.sh checks it. dpkg-query -W -f='${Package} ${Version}\n' | awk '$2 ~ /\+spin[0-9]+$/ {print $1}' | xargs -r apt-mark hold diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile index 375d9be..6d4936c 100644 --- a/image/systemd/Dockerfile +++ b/image/systemd/Dockerfile @@ -1,23 +1,35 @@ -# EXPERIMENT (branch systemd-stage1): Ubuntu's own systemd, rebuilt from its deb-src with less in -# it, to measure what PID 1, udevd and journald stop doing at boot. Not a release input until the -# numbers say so; see docs/log.md. +# The systemd the base image installs: Ubuntu's own, rebuilt from its source package with less +# in it, because what PID 1, udevd and the units' executor do at every boot is the first login's +# largest share after the kernel. # -# The source is Ubuntu's source package of the version the image would otherwise install, from -# the archive's signed Sources index: its patches, and so its security fixes, are Ubuntu's. What -# changes is the build profile: +# The source is Ubuntu's source package of the version the image would otherwise install, by +# its .dsc: its patches, and so its security fixes, are Ubuntu's. What changes is the build: # -# stage1 Debian's own reduced build, maintained in debian/rules and debian/control: no audit, -# cryptsetup, coredump, AppArmor, OpenSSL, TPM2, p11-kit, FIDO2, PCRE2, homed, userdb, -# importd, curl, sysupdate, boot loader or ukify, and no BPF framework. That last one -# is what PID 1 spends ~25 ms on in the verifier at every boot (socket-bind, docs/log.md) -# and what fails to load under lockdown (restrict-fs, #261). None of the packages the -# image installs is left out by it. +# stage1 Debian's own reduced build, maintained in debian/rules and debian/control: no +# audit, cryptsetup, coredump, AppArmor, OpenSSL, TPM2, p11-kit, FIDO2, PCRE2, homed, +# userdb, importd, curl, sysupdate, boot loader or ukify, and no BPF framework. That +# last one is where the time was: PID 1's setup before the generators went from 25.8 +# to 5.6 ms (boot:waits), the BPF verifier loading socket-bind at every boot and +# restrict-fs failing to under lockdown (#261). So no unit here can use +# RestrictFileSystems=, SocketBind*= or RestrictNetworkInterfaces=; none does, and +# spin's guard loads its own programs. None of the packages the image installs is +# left out by the profile. # SELinux off as well: the guest kernel's LSMs are lockdown, capability and bpf. +# static systemd-executor - exec'd once for every unit PID 1 starts - udevd and its +# helpers, systemctl and journalctl link libsystemd-shared in instead of loading it. +# PID 1, journald and logind have no such option. # -# The version gets a +spin1 suffix, so apt prefers it over the archive's and anything reading the -# image's package list can tell the two apart. +# Measured on the bench fleet, 20 boots of each interleaved, p50 ms of the first SSH login: stage1 +# −47, −38 and −47 against the archive's systemd (lab runs 44531842773, 44537549305, 44538877783); +# static over stage1 −10 and −1, within noise, with accept-to-login −7 in both runs. A checkpoint +# restored to the first login: unchanged, its guest part ~50 ms either way (44533227886). Not +# taken: -Db_lto=true (Ubuntu's dpkg-buildflags already compile with -flto=auto, and it built the +# same packages to within a few hundred bytes). +# +# The version gets a +spin1 suffix, so apt prefers it over the archive's, a reader of +# packages.txt can tell the two apart, and image/mkosi.postinst.chroot can hold it. ARG UBUNTU_IMAGE -FROM ${UBUNTU_IMAGE} +FROM ${UBUNTU_IMAGE} AS build # versions.yaml's systemd-source: the .dsc, by sha256. dpkg-source checks the tarballs it names # against the checksums inside it, so the pin holds the whole source. @@ -32,8 +44,9 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] # From the archive's deb-src, which apt checks against the signed Sources index, and then the .dsc # against the pin. The archive keeps only the version resolute publishes now: once Ubuntu moves -# systemd, this fails at the download, which is when the pin has to move with it (Launchpad keeps -# every upload, versions.yaml's source, but its librarian answered 502 to a whole build here). +# systemd, this fails at the download, which is when the pin has to move with it (task +# refresh:systemd proposes it). Launchpad keeps every upload, versions.yaml's source, but its +# librarian answered 502 to a whole build here (2026-11-19). RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources && \ apt-get update && \ apt-get install -y --no-install-recommends dpkg-dev devscripts @@ -44,31 +57,32 @@ RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \ dpkg-source -x "systemd_${SYSTEMD_SOURCE_VERSION}.dsc" src && test -f src/debian/rules && \ apt-get build-dep -y --build-profiles="${DEB_BUILD_PROFILES// /,}" "systemd=${SYSTEMD_SOURCE_VERSION}" -# A variant is meson options after the packaging's own, which meson takes over them: debian/rules -# hands $(CONFFLAGS_UPSTREAM) last and never sets it, so it comes from here. Each variant is its -# own image/Taskfile.yml call and its own directory of packages, a row of its own in the probes. -ARG MESON_EXTRA="" +# The options after the packaging's own, which meson takes over them: debian/rules hands +# $(CONFFLAGS_UPSTREAM) last and never sets it. meson-took.py then asks the configured build +# whether it took each, so a misspelt option is a failed build and not stage1 under another name. COPY image/systemd/meson-took.py /build/meson-took.py WORKDIR /build/src -RUN grep -q -- '-Dselinux=enabled' debian/rules && \ +RUN extra="-Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false" && \ + grep -q -- '-Dselinux=enabled' debian/rules && \ sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \ DEBFULLNAME="spin-machine" DEBEMAIL="noreply@spin-stack.invalid" \ dch --newversion "${SYSTEMD_SOURCE_VERSION}+spin1" --distribution resolute \ - "Rebuilt with the stage1 profile and SELinux off (spin-machine experiment). ${MESON_EXTRA}" && \ - CONFFLAGS_UPSTREAM="${MESON_EXTRA}" dpkg-buildpackage -b -uc -us -j"${JOBS}" && \ - meson introspect --buildoptions obj-* | python3 /build/meson-took.py ${MESON_EXTRA} + "Rebuilt by spin-machine: the stage1 profile, SELinux off, ${extra}." && \ + CONFFLAGS_UPSTREAM="${extra}" dpkg-buildpackage -b -uc -us -j"${JOBS}" && \ + meson introspect --buildoptions obj-* | python3 /build/meson-took.py ${extra} -# What came out: the packages the image installs, at the new version, and a PID 1 without the -# BPF framework or SELinux linked in. +# What came out: every package the image installs, at the new version; libsystemd-shared without +# SELinux, AppArmor, audit or libcrypto; and an executor that loads no libsystemd at all. RUN mkdir -p /debs && cp /build/*.deb /debs/ && \ for p in systemd systemd-sysv udev libsystemd0 libsystemd-shared libudev1 libpam-systemd; do \ - f=$(ls /debs/${p}_*+spin1_*.deb) || { echo "ERROR: no ${p} built"; exit 1; }; \ + ls /debs/${p}_*+spin1_*.deb > /dev/null || { echo "ERROR: no ${p} built"; exit 1; }; \ done && \ - dpkg-deb -x /debs/libsystemd-shared_*.deb /tmp/x && \ - ! find /tmp/x -name 'libsystemd-shared-*.so' -exec ldd {} \; | grep -E 'libselinux|libapparmor|libaudit|libcrypto' && \ - dpkg-deb -x /debs/systemd_*.deb /tmp/s && \ - echo "systemd-executor: $(ldd /tmp/s/usr/lib/systemd/systemd-executor | grep -c libsystemd) libsystemd libraries" && \ - ls -la /debs + dpkg-deb -x /debs/libsystemd-shared_*.deb /tmp/shared && \ + so=$(find /tmp/shared -name 'libsystemd-shared-*.so') && test -n "$so" && \ + ! ldd "$so" | grep -E 'libselinux|libapparmor|libaudit|libcrypto' && \ + dpkg-deb -x /debs/systemd_*.deb /tmp/systemd && \ + ! ldd /tmp/systemd/usr/lib/systemd/systemd-executor | grep libsystemd && \ + echo "OK: $(ls /debs | wc -l) packages, libsystemd-shared without the LSMs it was built without, the executor static" FROM scratch AS debs -COPY --from=0 /debs/ / +COPY --from=build /debs/ / diff --git a/image/systemd/meson-took.py b/image/systemd/meson-took.py index 5417d9f..6f9082e 100644 --- a/image/systemd/meson-took.py +++ b/image/systemd/meson-took.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 -"""Fail unless meson configured the build with every option a variant asked for. +"""Fail unless meson configured the build with every option it was handed. -Reads `meson introspect --buildoptions` on stdin and the variant's -Dname=value arguments from -argv: an option misspelt, or one the packaging set after the variant's, would otherwise build the -stage1 packages again under the variant's name. +Reads `meson introspect --buildoptions` on stdin and the -Dname=value arguments from +argv: an option misspelt, or one the packaging set after ours, would otherwise build the +stage1 packages without it and say nothing. """ import json import sys @@ -13,5 +13,5 @@ name, want = arg.removeprefix("-D").split("=", 1) got = opts.get(name) if str(got).lower() != want.lower(): - sys.exit(f"ERROR: meson has {name}={got}, the variant asked for {want}") + sys.exit(f"ERROR: meson has {name}={got}, it was asked for {want}") print(f"meson took {name}={got}") diff --git a/kernel/README.md b/kernel/README.md index 7a4837a..6caf51f 100644 --- a/kernel/README.md +++ b/kernel/README.md @@ -69,20 +69,22 @@ release's kernel, and says so. They trace a kernel B built with of `Dockerfile`'s check, on a branch or in a scratch worktree, never in a release. The rest of the boot probes read the ring buffer and systemd, which lockdown leaves alone. -systemd notices at boot, and the console shows two lines that read like missing protection -and are lockdown working: +The archive's systemd noticed at boot, and the console showed two lines that read like missing +protection and are lockdown working: ``` bpf-restrict-fs: Failed to load BPF object: Invalid argument Lockdown: init: use of bpf to read kernel RAM is restricted; see man kernel_lockdown.7 ``` -The second is lockdown refusing `bpf_probe_read_kernel` to PID 1. The first is what that costs: -systemd's `RestrictFileSystems=` program reads each file's filesystem type out of kernel memory -with that helper, the verifier rejects it, and systemd carries on without it. So a unit in the -guest cannot use `RestrictFileSystems=`; it is not enforced, and systemd does not say so per -unit. That is the price of confidentiality, kept on purpose: integrity would give it back by -letting root read the kernel's memory again. Logins are unaffected (`task boot:logind`). +PID 1 loads its `RestrictFileSystems=` program at every start, whatever the units ask, and the +program reads each file's filesystem type with `bpf_probe_read_kernel`, which lockdown refuses +it. The image's systemd (`image/systemd/Dockerfile`) is built without the BPF framework, so it +asks for nothing lockdown refuses and neither line appears; `task boot:lockdown` holds that for an +image built with it (#261). What that costs is the directives the framework implements: +`RestrictFileSystems=`, `SocketBind*=` and `RestrictNetworkInterfaces=` do nothing in the guest, +and nothing here uses them. Confidentiality is kept on purpose: integrity would give the helper +back by letting root read the kernel's memory again. Logins are unaffected (`task boot:logind`). Measured 2026-10-01, `task boot:initcalls` with `SPIN_KERNEL_B` set to the kernel without these options: no cost that can be told from noise. Over 20 interleaved boots each, kernel start to diff --git a/versions.yaml b/versions.yaml index e2fd16a..496eb36 100644 --- a/versions.yaml +++ b/versions.yaml @@ -143,10 +143,12 @@ pin: 17ad848e882b6b91ac0d31300181bb4ec6b221077329b0c5296c9765cce769b5 track: page https://launchpad.net/ubuntu/resolute/+source/systemd note: >- - EXPERIMENT (branch systemd-stage1): image/systemd/Dockerfile, Ubuntu's systemd source package as - resolute publishes it, by its .dsc, which holds the tarballs' checksums dpkg-source checks. - Launchpad keeps a superseded upload, the archive's pool does not. After the bump: - `task image:systemd`, then boot:ssh-first with SPIN_SYSTEMD_DEBS. + image/systemd/Dockerfile: Ubuntu's systemd source package as resolute publishes it, by its + .dsc, which holds the tarballs' checksums dpkg-source checks; rebuilt with the stage1 profile + and installed in place of the archive's. The archive serves only the current version, so the + build fails once Ubuntu moves it, and task refresh:systemd proposes the new one. After the + bump: `task image:build` (its checks: the meson options taken, the packages held), then + `task boot:ssh-first` and `task boot:lockdown`. - name: mkosi kind: download diff --git a/workflows/workflows_test.go b/workflows/workflows_test.go index b32689c..b9da264 100644 --- a/workflows/workflows_test.go +++ b/workflows/workflows_test.go @@ -24,7 +24,6 @@ func TestEveryUploadIsAttested(t *testing.T) { NotPublished: map[string]string{ "ci.yml/gate/Upload Go coverage HTML": "a coverage page for a person, kept seven days", "kernel.yml/build/Upload the kernel": "a lane's kernel for a person to fetch; a release builds its own", - "image.yml/systemd/Upload the systemd packages": "an experiment's packages for the lab to boot (lab.yml systemd_run); no release takes them", "lab.yml/lab/Keep the log": "a measurement's log, for a person", "qemu.yml/build/Upload the extracted binaries": "a lane's QEMU for a person to fetch; a release builds its own", "qemu.yml/build/Publish the runtime image": "what ci.yml's argument check pulls (task qemu:fetch); a release compiles QEMU, it never takes this",