Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/lab.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,10 @@ on:
description: "A QEMU workflow run id whose QEMU and firmware are compared against the release's (SPIN_MACHINE_OUTPUT_B)"
required: false
type: string
systemd_experiment:
description: "EXPERIMENT: an image/systemd/experiments patch to build systemd with, on this runner, into each boot's overlay (SPIN_SYSTEMD_DEBS)"
required: false
type: string
variant:
description: "With a build to compare: boot it beside the release's, interleaved, or instead of it on every boot"
required: false
Expand Down Expand Up @@ -105,6 +109,7 @@ jobs:
KERNEL_RUN: ${{ inputs.kernel_run }}
KERNEL_RUN_C: ${{ inputs.kernel_run_c }}
QEMU_RUN: ${{ inputs.qemu_run }}
SYSTEMD_EXPERIMENT: ${{ inputs.systemd_experiment }}
VARS: ${{ inputs.vars }}
# GitHub reads a workflow command (::error::) only at the start of a line, where Task's
# prefixed output puts the task's name.
Expand Down Expand Up @@ -141,7 +146,7 @@ jobs:
# change breaks rather than what it costs: the tree the experiment boots is the release
# with the build in it.
- name: Put the build where the experiment boots it
if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != ''
if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_experiment != ''
env:
VARIANT: ${{ inputs.variant }}
run: go tool task lab:place
Expand Down
9 changes: 8 additions & 1 deletion Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -724,6 +724,12 @@ tasks:
cp -r "$(dirname "$(dirname "$q")")/qemu" "$b/qemu"
[ "$VARIANT" = instead ] || echo "SPIN_MACHINE_OUTPUT_B=$b" >> "$env"
fi
# EXPERIMENT: systemd built here, on this runner and before anything is measured, with
# image/systemd/experiments/$SYSTEMD_EXPERIMENT.patch; the probes put it in each overlay.
if [ -n "${SYSTEMD_EXPERIMENT:-}" ]; then
go tool task image:systemd SYSTEMD_EXPERIMENT="$SYSTEMD_EXPERIMENT" CACHE_EXPORT=false QEMU_JOBS="$(nproc)"
echo "SPIN_SYSTEMD_DEBS=$PWD/_output/systemd" >> "$env"
fi
if [ "$VARIANT" = instead ]; then echo "SPIN_MACHINE_OUTPUT=$b" >> "$env"; fi

lab:vars:
Expand All @@ -742,7 +748,7 @@ tasks:
[[ "$pair" == *=* && "$key" =~ ^(REPS|N|SETTLE|TOP|GAP|ONLY|FLAGS|CPUS|TARGET|BUCKET|LINGER|MEMORY_MIB|DEBUG|FILE_MB|CACHE_TIMEOUT_S|GRAPH_[A-Z_]+|SHARE_[A-Z_]+|SPIN_[A-Z0-9_]+)$ ]] ||
{ echo "::error::not a variable this workflow passes on: $pair"; exit 1; }
case "$key" in
SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C) echo "::error::$key is set by this workflow"; exit 1 ;;
SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C|SPIN_SYSTEMD_DEBS) echo "::error::$key is set by this workflow"; exit 1 ;;
esac
echo "$pair" >> "${GITHUB_ENV:-/dev/stdout}"
done
Expand All @@ -759,6 +765,7 @@ tasks:
echo "- kernel B: ${KERNEL_RUN:+run }${KERNEL_RUN:-none}"
echo "- kernel C: ${KERNEL_RUN_C:+run }${KERNEL_RUN_C:-none}"
echo "- QEMU B: ${QEMU_RUN:+run }${QEMU_RUN:-none}"
echo "- systemd experiment: ${SYSTEMD_EXPERIMENT:-none}"
if [ -n "$KERNEL_RUN$KERNEL_RUN_C$QEMU_RUN" ]; then echo "- booted: $VARIANT the release"; fi
echo "- vars: ${VARS:-none}"
echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs"
Expand Down
126 changes: 123 additions & 3 deletions boot/waits_probe_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,11 @@ func TestBootWaits(t *testing.T) {
raw := os.Getenv("RAW")

v := waitsVariant(cmp.Or(os.Getenv("CPUS"), "2"))
// EXPERIMENT (branch pid1-marks): a systemd image/systemd built with SYSTEMD_EXPERIMENT, put
// into each boot's overlay; with experiments/pid1-marks, PID 1 marks its own steps.
if debs := os.Getenv("SPIN_SYSTEMD_DEBS"); debs != "" {
v = withPackages(v, "waits, systemd "+filepath.Base(debs), debs)
}

var boots []waitsBoot
for rep := range reps {
Expand All @@ -73,6 +78,9 @@ func TestBootWaits(t *testing.T) {
boots = append(boots, b)
}
t.Log(waitsReport(boots, target, top, minGap))
if r := pid1Report(boots); r != "" {
t.Log(r)
}
}

// waitsVariant is the boot both probes take: the getty an echo, and spin-waits.service printing
Expand All @@ -97,7 +105,7 @@ After=multi-user.target
[Service]
Type=simple
StandardOutput=null
ExecStart=/bin/sh -c "echo SPIN-WAITS-ANCHOR > /dev/ttyS0; i=0; while [ \"$$(systemctl show -p FinishTimestampMonotonic --value)\" = 0 ] && [ $$i -lt 100 ]; do i=$$((i+1)); sleep 0.05; done; { echo SPIN-WAITS-BEGIN; systemctl show -p ` + manager + `; echo SPIN-WAITS-UNITS; systemctl show -p ` + props + ` -- $$(systemctl list-units --all --plain --no-legend --no-pager | awk '{print $$1}'); echo SPIN-WAITS-END; } > /dev/ttyS0 2>&1"
ExecStart=/bin/sh -c "echo SPIN-WAITS-ANCHOR > /dev/ttyS0; i=0; while [ \"$$(systemctl show -p FinishTimestampMonotonic --value)\" = 0 ] && [ $$i -lt 100 ]; do i=$$((i+1)); sleep 0.05; done; { echo SPIN-PID1-MARKS; cat /run/spin-pid1-marks 2>/dev/null; echo SPIN-WAITS-BEGIN; systemctl show -p ` + manager + `; echo SPIN-WAITS-UNITS; systemctl show -p ` + props + ` -- $$(systemctl list-units --all --plain --no-legend --no-pager | awk '{print $$1}'); echo SPIN-WAITS-END; } > /dev/ttyS0 2>&1"
[Install]
WantedBy=multi-user.target
`
Expand All @@ -117,6 +125,7 @@ type waitsBoot struct {
w boot.Waits
dump string
console string
pid1 []pid1Mark // PID 1's own marks, when its systemd was built with experiments/pid1-marks
kernel, usable time.Duration
login string // the unit whose start is the login
start time.Time // the moment before the machine was launched
Expand Down Expand Up @@ -251,7 +260,11 @@ func bootWaits(t *testing.T, out string, v variant, target, end string) waitsBoo
t.Fatalf("%s was not started in this boot: TARGET names a unit this image does not start", target)
}
kernel := stamps["SPIN-WAITS-ANCHOR"] - probe.Start
return waitsBoot{w: wt, dump: dump, console: s, start: t0, ram: ram, kernel: kernel, usable: stamps["login:"] - kernel, login: target}
var pid1 []pid1Mark
if i := strings.Index(s, "SPIN-PID1-MARKS"); i >= 0 && i < begin {
pid1 = parsePID1Marks(s[i+len("SPIN-PID1-MARKS") : begin])
}
return waitsBoot{w: wt, dump: dump, console: s, pid1: pid1, start: t0, ram: ram, kernel: kernel, usable: stamps["login:"] - kernel, login: target}
}

// waitsReport prints the boots as p50s, then the median boot verbatim: a critical path or a
Expand Down Expand Up @@ -388,7 +401,10 @@ func waitsReport(boots []waitsBoot, target string, top int, minGap time.Duration
}
}
// The first gap, from the end of the unit load to the first job (32 ms p50 on the bench
// fleet, lab run 43361486792), is PID 1 and nothing else. systemd stamps UnitsLoadFinish
// fleet, lab run 43361486792), is PID 1 and nothing else. Measured inside PID 1 (11-19, lab run
// 44561762624, experiments/pid1-marks): 28.4 ms loading the 192 units default.target reaches,
// 20.9 of it PID 1's own CPU, and 0.3 ms building the transaction - so what follows, from one
// debug-logged boot, named the wrong step. systemd stamps UnitsLoadFinish
// after enumerating mounts, devices and slices, and only then loads the units the boot
// starts: coldplug, then "Activating default unit: default.target" parses every unit file
// default.target reaches, and a stub for each name an After= or RequiresMountsFor= gives,
Expand Down Expand Up @@ -440,3 +456,107 @@ func clipUnit(s string) string {
}
return s
}

// pid1Mark is one line of /run/spin-pid1-marks: PID 1's monotonic clock and its own CPU time,
// the units it had loaded and the time spent in unit_load so far, and the step it had done.
type pid1Mark struct {
at, cpu, inLoad time.Duration
units int
step string
}

func parsePID1Marks(text string) []pid1Mark {
var marks []pid1Mark
for _, line := range strings.Split(text, "\n") {
f := strings.Fields(line)
if len(f) != 5 {
continue
}
at, err1 := strconv.ParseInt(f[0], 10, 64)
cpu, err2 := strconv.ParseInt(f[1], 10, 64)
units, err3 := strconv.Atoi(f[2])
inLoad, err4 := strconv.ParseInt(f[3], 10, 64)
if err1 != nil || err2 != nil || err3 != nil || err4 != nil {
continue
}
marks = append(marks, pid1Mark{time.Duration(at) * time.Microsecond, time.Duration(cpu) * time.Microsecond,
time.Duration(inLoad) * time.Microsecond, units, f[4]})
}
return marks
}

// pid1Report is, for each of PID 1's steps from the one before it, the p50 of the wall time, of
// PID 1's own CPU time, of the time in unit_load and of the units loaded, over the boots that carry
// marks; "" when none does. Wall time well above CPU time is waiting, not parsing.
func pid1Report(boots []waitsBoot) string {
type step struct{ wall, cpu, load, units []time.Duration }
var order []string
steps := map[string]*step{}
for _, b := range boots {
for i, m := range b.pid1 {
if i == 0 {
continue
}
p := b.pid1[i-1]
st := steps[m.step]
if st == nil {
st = &step{}
steps[m.step] = st
order = append(order, m.step)
}
st.wall = append(st.wall, m.at-p.at)
st.cpu = append(st.cpu, m.cpu-p.cpu)
st.load = append(st.load, m.inLoad-p.inLoad)
st.units = append(st.units, time.Duration(m.units-p.units))
}
}
if len(order) == 0 {
return ""
}
ms := func(ds []time.Duration) string {
p50, _ := boot.Percentile(ds, 0.5)
return fmt.Sprintf("%.2f", float64(p50)/float64(time.Millisecond))
}
var b strings.Builder
fmt.Fprintf(&b, "\nPID 1's steps from the unit load to the first job (experiments/pid1-marks), from the step before, p50 over %d boots\n\n", len(steps[order[0]].wall))
fmt.Fprintf(&b, " %-22s %9s %9s %12s %7s\n", "", "WALL ms", "CPU ms", "UNIT_LOAD ms", "UNITS")
for _, name := range order {
st := steps[name]
units, _ := boot.Percentile(st.units, 0.5)
fmt.Fprintf(&b, " %-22s %9s %9s %12s %7d\n", name, ms(st.wall), ms(st.cpu), ms(st.load), int(units))
}
return b.String()
}

// withPackages is base with the packages in debs put into the overlay the way dpkg would
// upgrade them, for each one the image has installed: the files the installed version listed
// and the new one does not ship are removed, and the new files written over the old. /etc is
// left alone, because the image's own edits there are part of base and a package's defaults
// would undo them; maintainer scripts are not run, which for an upgrade between two builds of
// one source version leaves nothing out that a boot reads.
func withPackages(base variant, label, debs string) variant {
v := base
v.label = label
v.files = maps.Clone(base.files)
v.links = maps.Clone(base.links)
v.setup = base.setup + `debs=` + debs + `
replaced=0
for deb in "$debs"/*.deb; do
pkg=$(dpkg-deb -f "$deb" Package)
list="$MNT/var/lib/dpkg/info/$pkg.list"
[ -f "$list" ] || list="$MNT/var/lib/dpkg/info/$pkg:amd64.list"
[ -f "$list" ] || continue
new=$(mktemp)
dpkg-deb --fsys-tarfile "$deb" | tar -t | sed -e 's|^\./|/|' -e 's|/$||' | sort > "$new"
sort "$list" | comm -23 - "$new" | grep -v '^/etc/' | while read -r f; do
if [ -L "$MNT$f" ] || [ -f "$MNT$f" ]; then rm -f "$MNT$f"; fi
done
rm -f "$new"
dpkg-deb --fsys-tarfile "$deb" | tar -x -C "$MNT" --exclude=./etc
replaced=$((replaced + 1))
done
test "$replaced" -gt 0
grep -q 'spin1' "$MNT/usr/lib/systemd/libsystemd-shared-"*.so 2>/dev/null || strings "$MNT/usr/lib/systemd/systemd" | grep -q '+spin1' || { echo "the overlay's systemd is not the rebuilt one" >&2; exit 1; }
`
return v
}
10 changes: 9 additions & 1 deletion docs/log.md
Original file line number Diff line number Diff line change
Expand Up @@ -374,6 +374,14 @@ The socket without default dependencies (`systemd-logind-varlink.socket.d/20-lan
- the first login: 332/355 → 328/337. The p50 moves little because logind starting sooner takes the CPU sshd was starting on.

Masking `modprobe@drm`, which logind is ordered after, added nothing (327/345). What is left of the login is CPU contention on 2 vCPUs, `user@1000` (38 ms) and the host key generation (21 ms) starting in the same window, rather than waits.

**11-19 · systemd/boot · where PID 1's gap between the unit load and the first job goes (experiment, tooling kept).** `image/systemd/experiments/pid1-marks.patch` makes PID 1 append its monotonic clock, its own CPU time, the units loaded and the time in `unit_load` after each step of `manager_startup` and `do_queue_default_job` to `/run/spin-pid1-marks`. `boot:waits` reads the file. `lab.yml`'s `systemd_experiment` builds the patched systemd on the runner and puts it into each overlay. Bench fleet, 10 boots (lab 44561762624), p50 ms:
- bus, varlink, coldplug, vacuum, ready: < 0.2 together;
- **loading what `default.target` reaches: 28.4 wall, 20.9 of it PID 1's CPU, 192 units** (~0.11 ms of CPU each; the rest is likely cold reads);
- building the transaction: 0.30;
- the first dispatch: 0.87.

The gap is unit loading, not the transaction that 11-13's single debug-logged boot named. 96 of the units a boot lists are inactive, loaded because another unit names them in `After=`/`Wants=`: `initrd-*`, `plymouth-*`, `modprobe@*`, hibernate, cryptsetup's targets, `firewalld` and `auditd` named by docker. Masking does not spare a load (a masked unit is still looked up and loaded as masked), which is why masking 11 units moved nothing (§4).
---

## 3. Rework cycles
Expand Down Expand Up @@ -617,7 +625,7 @@ Also the reusable builder, added and reverted on 10-13.
These are #242's checklist.
- **Is `10-seats.conf` still needed?** The control that should fail passes on systemd 259 (#240, "Not settled here").
- **virtio-rng removal.** -13.6 ms to the prompt (lab 43321809471), a fingerprint change, not taken (#240).
- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`); with systemd built here, it can now be instrumented.
- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms gap after the unit load is loading 192 units, ~21 ms of CPU (11-19, experiments/pid1-marks), not building the transaction (0.3 ms). It shrinks only with fewer units reached at boot.
- **musl under long-running load.** Never measured (`31a57e7`).
- **`page_reporting_order`.** Revisit "only if memory per host becomes the limit" (docs/memory-and-disk.md).
- **Moving `MachineType` off pc-q35-11.1.** "its own change" (CLAUDE.md rule 1, `hack/machinetype`).
Expand Down
1 change: 1 addition & 0 deletions image/Taskfile.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ tasks:
{{.SYSTEMD_CACHE_TO}} \
$({{.VERSIONS}} args ubuntu systemd-source) \
--build-arg JOBS={{.QEMU_JOBS}} \
--build-arg SYSTEMD_EXPERIMENT={{.SYSTEMD_EXPERIMENT}} \
--output type=local,dest={{.OUTPUT_ABS}}/systemd \
.

Expand Down
7 changes: 7 additions & 0 deletions image/systemd/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,14 @@ RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \
# $(CONFFLAGS_UPSTREAM) last and never sets it. meson-took.py then asks the configured build
# whether it took each, so a misspelt option is a failed build and not stage1 under another name.
COPY image/systemd/meson-took.py /build/meson-took.py
# An experiment's patch, applied only when SYSTEMD_EXPERIMENT names one: a build for the lab
# (task lab:place), never the image's. Each says what it measures and is removed with its answer.
ARG SYSTEMD_EXPERIMENT=""
COPY image/systemd/experiments/ /build/experiments/
WORKDIR /build/src
RUN if [ -n "${SYSTEMD_EXPERIMENT}" ]; then \
patch -p1 --fuzz=0 < "/build/experiments/${SYSTEMD_EXPERIMENT}.patch"; \
fi
RUN extra="-Dpcre2=enabled -Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false" && \
grep -q -- '-Dselinux=enabled' debian/rules && \
sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \
Expand Down
Loading