From 2f8d9d7c96f15259644c1789a3996b3d80ac8c17 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 12:56:39 -0300 Subject: [PATCH 1/3] experiment: PID 1 marks its steps between the unit load and the first job image/systemd builds an experiment's patch when SYSTEMD_EXPERIMENT names one; experiments/pid1-marks makes PID 1 append its monotonic clock to /run/spin-pid1-marks after each step of manager_startup and do_queue_default_job. boot:waits puts SPIN_SYSTEMD_DEBS into each overlay and gives each step's p50; lab.yml's systemd_experiment builds the packages on the runner before anything is measured. The ~32 ms gap had been "explained, not removable" from one debug-logged boot. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/lab.yml | 7 +- Taskfile.yml | 9 +- boot/waits_probe_test.go | 98 +++++++++++++++- image/Taskfile.yml | 1 + image/systemd/Dockerfile | 7 ++ image/systemd/experiments/pid1-marks.patch | 123 +++++++++++++++++++++ 6 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 image/systemd/experiments/pid1-marks.patch diff --git a/.github/workflows/lab.yml b/.github/workflows/lab.yml index 9c53a10..3915a63 100644 --- a/.github/workflows/lab.yml +++ b/.github/workflows/lab.yml @@ -57,6 +57,10 @@ on: description: "A QEMU workflow run id whose QEMU and firmware are compared against the release's (SPIN_MACHINE_OUTPUT_B)" required: false type: string + systemd_experiment: + description: "EXPERIMENT: an image/systemd/experiments patch to build systemd with, on this runner, into each boot's overlay (SPIN_SYSTEMD_DEBS)" + required: false + type: string variant: description: "With a build to compare: boot it beside the release's, interleaved, or instead of it on every boot" required: false @@ -105,6 +109,7 @@ jobs: KERNEL_RUN: ${{ inputs.kernel_run }} KERNEL_RUN_C: ${{ inputs.kernel_run_c }} QEMU_RUN: ${{ inputs.qemu_run }} + SYSTEMD_EXPERIMENT: ${{ inputs.systemd_experiment }} VARS: ${{ inputs.vars }} # GitHub reads a workflow command (::error::) only at the start of a line, where Task's # prefixed output puts the task's name. @@ -141,7 +146,7 @@ jobs: # change breaks rather than what it costs: the tree the experiment boots is the release # with the build in it. - name: Put the build where the experiment boots it - if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' + if: inputs.kernel_run != '' || inputs.kernel_run_c != '' || inputs.qemu_run != '' || inputs.systemd_experiment != '' env: VARIANT: ${{ inputs.variant }} run: go tool task lab:place diff --git a/Taskfile.yml b/Taskfile.yml index 0a2f208..42c040a 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -724,6 +724,12 @@ tasks: cp -r "$(dirname "$(dirname "$q")")/qemu" "$b/qemu" [ "$VARIANT" = instead ] || echo "SPIN_MACHINE_OUTPUT_B=$b" >> "$env" fi + # EXPERIMENT: systemd built here, on this runner and before anything is measured, with + # image/systemd/experiments/$SYSTEMD_EXPERIMENT.patch; the probes put it in each overlay. + if [ -n "${SYSTEMD_EXPERIMENT:-}" ]; then + go tool task image:systemd SYSTEMD_EXPERIMENT="$SYSTEMD_EXPERIMENT" CACHE_EXPORT=false QEMU_JOBS="$(nproc)" + echo "SPIN_SYSTEMD_DEBS=$PWD/_output/systemd" >> "$env" + fi if [ "$VARIANT" = instead ]; then echo "SPIN_MACHINE_OUTPUT=$b" >> "$env"; fi lab:vars: @@ -742,7 +748,7 @@ tasks: [[ "$pair" == *=* && "$key" =~ ^(REPS|N|SETTLE|TOP|GAP|ONLY|FLAGS|CPUS|TARGET|BUCKET|LINGER|MEMORY_MIB|DEBUG|FILE_MB|CACHE_TIMEOUT_S|GRAPH_[A-Z_]+|SHARE_[A-Z_]+|SPIN_[A-Z0-9_]+)$ ]] || { echo "::error::not a variable this workflow passes on: $pair"; exit 1; } case "$key" in - SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C) echo "::error::$key is set by this workflow"; exit 1 ;; + SPIN_MACHINE_OUTPUT|SPIN_MACHINE_OUTPUT_B|SPIN_KERNEL_B|SPIN_KERNEL_C|SPIN_SYSTEMD_DEBS) echo "::error::$key is set by this workflow"; exit 1 ;; esac echo "$pair" >> "${GITHUB_ENV:-/dev/stdout}" done @@ -759,6 +765,7 @@ tasks: echo "- kernel B: ${KERNEL_RUN:+run }${KERNEL_RUN:-none}" echo "- kernel C: ${KERNEL_RUN_C:+run }${KERNEL_RUN_C:-none}" echo "- QEMU B: ${QEMU_RUN:+run }${QEMU_RUN:-none}" + echo "- systemd experiment: ${SYSTEMD_EXPERIMENT:-none}" if [ -n "$KERNEL_RUN$KERNEL_RUN_C$QEMU_RUN" ]; then echo "- booted: $VARIANT the release"; fi echo "- vars: ${VARS:-none}" echo "- CPU: $(sed -n 's/^model name\s*: //p' /proc/cpuinfo | head -1), $(nproc) CPUs" diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index 0186811..6c262d6 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -60,6 +60,11 @@ func TestBootWaits(t *testing.T) { raw := os.Getenv("RAW") v := waitsVariant(cmp.Or(os.Getenv("CPUS"), "2")) + // EXPERIMENT (branch pid1-marks): a systemd image/systemd built with SYSTEMD_EXPERIMENT, put + // into each boot's overlay; with experiments/pid1-marks, PID 1 marks its own steps. + if debs := os.Getenv("SPIN_SYSTEMD_DEBS"); debs != "" { + v = withPackages(v, "waits, systemd "+filepath.Base(debs), debs) + } var boots []waitsBoot for rep := range reps { @@ -73,6 +78,9 @@ func TestBootWaits(t *testing.T) { boots = append(boots, b) } t.Log(waitsReport(boots, target, top, minGap)) + if r := pid1Report(boots); r != "" { + t.Log(r) + } } // waitsVariant is the boot both probes take: the getty an echo, and spin-waits.service printing @@ -97,7 +105,7 @@ After=multi-user.target [Service] Type=simple StandardOutput=null -ExecStart=/bin/sh -c "echo SPIN-WAITS-ANCHOR > /dev/ttyS0; i=0; while [ \"$$(systemctl show -p FinishTimestampMonotonic --value)\" = 0 ] && [ $$i -lt 100 ]; do i=$$((i+1)); sleep 0.05; done; { echo SPIN-WAITS-BEGIN; systemctl show -p ` + manager + `; echo SPIN-WAITS-UNITS; systemctl show -p ` + props + ` -- $$(systemctl list-units --all --plain --no-legend --no-pager | awk '{print $$1}'); echo SPIN-WAITS-END; } > /dev/ttyS0 2>&1" +ExecStart=/bin/sh -c "echo SPIN-WAITS-ANCHOR > /dev/ttyS0; i=0; while [ \"$$(systemctl show -p FinishTimestampMonotonic --value)\" = 0 ] && [ $$i -lt 100 ]; do i=$$((i+1)); sleep 0.05; done; { echo SPIN-PID1-MARKS; cat /run/spin-pid1-marks 2>/dev/null; echo SPIN-WAITS-BEGIN; systemctl show -p ` + manager + `; echo SPIN-WAITS-UNITS; systemctl show -p ` + props + ` -- $$(systemctl list-units --all --plain --no-legend --no-pager | awk '{print $$1}'); echo SPIN-WAITS-END; } > /dev/ttyS0 2>&1" [Install] WantedBy=multi-user.target ` @@ -117,6 +125,7 @@ type waitsBoot struct { w boot.Waits dump string console string + pid1 []pid1Mark // PID 1's own marks, when its systemd was built with experiments/pid1-marks kernel, usable time.Duration login string // the unit whose start is the login start time.Time // the moment before the machine was launched @@ -251,7 +260,11 @@ func bootWaits(t *testing.T, out string, v variant, target, end string) waitsBoo t.Fatalf("%s was not started in this boot: TARGET names a unit this image does not start", target) } kernel := stamps["SPIN-WAITS-ANCHOR"] - probe.Start - return waitsBoot{w: wt, dump: dump, console: s, start: t0, ram: ram, kernel: kernel, usable: stamps["login:"] - kernel, login: target} + var pid1 []pid1Mark + if i := strings.Index(s, "SPIN-PID1-MARKS"); i >= 0 && i < begin { + pid1 = parsePID1Marks(s[i+len("SPIN-PID1-MARKS") : begin]) + } + return waitsBoot{w: wt, dump: dump, console: s, pid1: pid1, start: t0, ram: ram, kernel: kernel, usable: stamps["login:"] - kernel, login: target} } // waitsReport prints the boots as p50s, then the median boot verbatim: a critical path or a @@ -440,3 +453,84 @@ func clipUnit(s string) string { } return s } + +// pid1Mark is one line of /run/spin-pid1-marks: PID 1's monotonic clock and the step it had done. +type pid1Mark struct { + at time.Duration + step string +} + +func parsePID1Marks(text string) []pid1Mark { + var marks []pid1Mark + for _, line := range strings.Split(text, "\n") { + us, step, ok := strings.Cut(strings.TrimSpace(line), " ") + n, err := strconv.ParseInt(us, 10, 64) + if !ok || err != nil { + continue + } + marks = append(marks, pid1Mark{time.Duration(n) * time.Microsecond, step}) + } + return marks +} + +// pid1Report is the p50 of each of PID 1's steps, from the one before it, over the boots that +// carry marks; "" when none does. +func pid1Report(boots []waitsBoot) string { + var order []string + took := map[string][]time.Duration{} + for _, b := range boots { + for i, m := range b.pid1 { + if i == 0 { + continue + } + if _, seen := took[m.step]; !seen { + order = append(order, m.step) + } + took[m.step] = append(took[m.step], m.at-b.pid1[i-1].at) + } + } + if len(order) == 0 { + return "" + } + var b strings.Builder + fmt.Fprintf(&b, "\nPID 1's steps from the unit load to the first job (experiments/pid1-marks), ms from the step before, p50/p95 over %d boots\n\n", len(took[order[0]])) + for _, step := range order { + p50, _ := boot.Percentile(took[step], 0.5) + p95, _ := boot.Percentile(took[step], 0.95) + fmt.Fprintf(&b, " %-22s %7.2f/%-7.2f\n", step, float64(p50)/float64(time.Millisecond), float64(p95)/float64(time.Millisecond)) + } + return b.String() +} + +// withPackages is base with the packages in debs put into the overlay the way dpkg would +// upgrade them, for each one the image has installed: the files the installed version listed +// and the new one does not ship are removed, and the new files written over the old. /etc is +// left alone, because the image's own edits there are part of base and a package's defaults +// would undo them; maintainer scripts are not run, which for an upgrade between two builds of +// one source version leaves nothing out that a boot reads. +func withPackages(base variant, label, debs string) variant { + v := base + v.label = label + v.files = maps.Clone(base.files) + v.links = maps.Clone(base.links) + v.setup = base.setup + `debs=` + debs + ` +replaced=0 +for deb in "$debs"/*.deb; do + pkg=$(dpkg-deb -f "$deb" Package) + list="$MNT/var/lib/dpkg/info/$pkg.list" + [ -f "$list" ] || list="$MNT/var/lib/dpkg/info/$pkg:amd64.list" + [ -f "$list" ] || continue + new=$(mktemp) + dpkg-deb --fsys-tarfile "$deb" | tar -t | sed -e 's|^\./|/|' -e 's|/$||' | sort > "$new" + sort "$list" | comm -23 - "$new" | grep -v '^/etc/' | while read -r f; do + if [ -L "$MNT$f" ] || [ -f "$MNT$f" ]; then rm -f "$MNT$f"; fi + done + rm -f "$new" + dpkg-deb --fsys-tarfile "$deb" | tar -x -C "$MNT" --exclude=./etc + replaced=$((replaced + 1)) +done +test "$replaced" -gt 0 +grep -q 'spin1' "$MNT/usr/lib/systemd/libsystemd-shared-"*.so 2>/dev/null || strings "$MNT/usr/lib/systemd/systemd" | grep -q '+spin1' || { echo "the overlay's systemd is not the rebuilt one" >&2; exit 1; } +` + return v +} diff --git a/image/Taskfile.yml b/image/Taskfile.yml index c145605..7d732e9 100644 --- a/image/Taskfile.yml +++ b/image/Taskfile.yml @@ -51,6 +51,7 @@ tasks: {{.SYSTEMD_CACHE_TO}} \ $({{.VERSIONS}} args ubuntu systemd-source) \ --build-arg JOBS={{.QEMU_JOBS}} \ + --build-arg SYSTEMD_EXPERIMENT={{.SYSTEMD_EXPERIMENT}} \ --output type=local,dest={{.OUTPUT_ABS}}/systemd \ . diff --git a/image/systemd/Dockerfile b/image/systemd/Dockerfile index af7066f..ca0c076 100644 --- a/image/systemd/Dockerfile +++ b/image/systemd/Dockerfile @@ -67,7 +67,14 @@ RUN apt-get source --download-only "systemd=${SYSTEMD_SOURCE_VERSION}" && \ # $(CONFFLAGS_UPSTREAM) last and never sets it. meson-took.py then asks the configured build # whether it took each, so a misspelt option is a failed build and not stage1 under another name. COPY image/systemd/meson-took.py /build/meson-took.py +# An experiment's patch, applied only when SYSTEMD_EXPERIMENT names one: a build for the lab +# (task lab:place), never the image's. Each says what it measures and is removed with its answer. +ARG SYSTEMD_EXPERIMENT="" +COPY image/systemd/experiments/ /build/experiments/ WORKDIR /build/src +RUN if [ -n "${SYSTEMD_EXPERIMENT}" ]; then \ + patch -p1 --fuzz=0 < "/build/experiments/${SYSTEMD_EXPERIMENT}.patch"; \ + fi RUN extra="-Dpcre2=enabled -Dlink-executor-shared=false -Dlink-udev-shared=false -Dlink-systemctl-shared=false -Dlink-journalctl-shared=false" && \ grep -q -- '-Dselinux=enabled' debian/rules && \ sed -i 's/-Dselinux=enabled/-Dselinux=disabled/' debian/rules && \ diff --git a/image/systemd/experiments/pid1-marks.patch b/image/systemd/experiments/pid1-marks.patch new file mode 100644 index 0000000..2b48750 --- /dev/null +++ b/image/systemd/experiments/pid1-marks.patch @@ -0,0 +1,123 @@ +diff -ruN a/src/core/main.c b/src/core/main.c +--- a/src/core/main.c 2026-11-19 12:53:06.697308531 -0300 ++++ b/src/core/main.c 2026-11-19 12:53:27.097674529 -0300 +@@ -105,6 +105,7 @@ + #include "version.h" + #include "virt.h" + #include "watchdog.h" ++#include "spin-mark.h" + + #if HAS_FEATURE_ADDRESS_SANITIZER + #include +@@ -2609,7 +2610,9 @@ + + log_debug("Activating default unit: %s", unit); + ++ spin_mark("default-unit-start"); + r = manager_load_startable_unit_or_warn(m, unit, NULL, &target); ++ spin_mark("default-unit-loaded"); + if (r < 0 && in_initrd() && !arg_default_unit) { + /* Fall back to default.target, which we used to always use by default. Only do this if no + * explicit configuration was given. */ +@@ -2632,6 +2635,7 @@ + assert(target->load_state == UNIT_LOADED); + + r = manager_add_job(m, JOB_START, target, JOB_ISOLATE, &error, &job); ++ spin_mark("transaction-built"); + if (r == -EPERM) { + log_debug_errno(r, "Default target could not be isolated, starting instead: %s", bus_error_message(&error, r)); + +diff -ruN a/src/core/manager.c b/src/core/manager.c +--- a/src/core/manager.c 2026-11-19 12:53:06.697237252 -0300 ++++ b/src/core/manager.c 2026-11-19 12:53:27.097245995 -0300 +@@ -96,6 +96,7 @@ + #include "varlink.h" + #include "virt.h" + #include "watchdog.h" ++#include "spin-mark.h" + + /* Make sure clients notifying us don't block */ + #define MANAGER_SOCKET_RCVBUF_SIZE (8*U64_MB) +@@ -2049,6 +2050,7 @@ + manager_enumerate_perpetual(m); + manager_enumerate(m); + dual_timestamp_now(m->timestamps + manager_timestamp_initrd_mangle(MANAGER_TIMESTAMP_UNITS_LOAD_FINISH)); ++ spin_mark("units-loaded"); + + /* Second, deserialize if there is something to deserialize */ + if (serialization) { +@@ -2097,18 +2099,23 @@ + /* This shouldn't fail, except if things are really broken. */ + return r; + ++ spin_mark("fds-and-notify"); + /* Connect to the bus if we are good for it */ + manager_setup_bus(m); ++ spin_mark("bus"); + + r = manager_varlink_init(m); + if (r < 0) + log_warning_errno(r, "Failed to set up Varlink, ignoring: %m"); + ++ spin_mark("varlink"); + /* Third, fire things up! */ + manager_coldplug(m); ++ spin_mark("coldplug"); + + /* Clean up runtime objects */ + manager_vacuum(m); ++ spin_mark("vacuum"); + + if (serialization) + /* Let's wait for the UnitNew/JobNew messages being sent, before we notify that the +@@ -2117,6 +2124,7 @@ + } + + manager_ready(m); ++ spin_mark("ready"); + + manager_set_switching_root(m, false); + +@@ -2543,6 +2551,12 @@ + + assert(source); + ++ static bool spin_first = true; ++ if (spin_first && prioq_peek(m->run_queue)) { ++ spin_first = false; ++ spin_mark("first-dispatch"); ++ } ++ + while ((j = prioq_peek(m->run_queue))) { + assert(j->installed); + assert(j->in_run_queue); +diff -ruN a/src/core/spin-mark.h b/src/core/spin-mark.h +--- a/src/core/spin-mark.h 1969-12-31 21:00:00.000000000 -0300 ++++ b/src/core/spin-mark.h 2026-11-19 12:53:27.084515904 -0300 +@@ -0,0 +1,26 @@ ++/* SPDX-License-Identifier: LGPL-2.1-or-later */ ++#pragma once ++ ++/* spin-machine experiment: PID 1's steps between the unit load and the first job, appended to ++ * /run/spin-pid1-marks as " ", which boot:waits reads after the boot. ++ * A file and not /dev/kmsg, whose user writes the kernel rate-limits. Not for a release. */ ++ ++#include ++#include ++#include ++#include ++ ++static inline void spin_mark(const char *what) { ++ struct timespec ts; ++ int fd; ++ ++ if (getpid() != 1) ++ return; ++ fd = open("/run/spin-pid1-marks", O_WRONLY|O_CREAT|O_APPEND|O_CLOEXEC|O_NOCTTY, 0644); ++ if (fd < 0) ++ return; ++ (void) clock_gettime(CLOCK_MONOTONIC, &ts); ++ (void) dprintf(fd, "%llu %s\n", ++ (unsigned long long) ts.tv_sec * 1000000ULL + (unsigned long long) ts.tv_nsec / 1000ULL, what); ++ (void) close(fd); ++} From b04730ebac4b070f8e911d0114ec92d203c45bf1 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 13:03:25 -0300 Subject: [PATCH 2/3] experiment: PID 1's marks carry its CPU time, units loaded and time in unit_load Wall time well above PID 1's own CPU time in a step is waiting - on the disk or for a CPU - rather than parsing. Co-Authored-By: Claude Opus 5.5 --- boot/waits_probe_test.go | 57 +++++++++++++++------- image/systemd/experiments/pid1-marks.patch | 50 ++++++++++++++++--- 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index 6c262d6..e665d9c 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -454,50 +454,73 @@ func clipUnit(s string) string { return s } -// pid1Mark is one line of /run/spin-pid1-marks: PID 1's monotonic clock and the step it had done. +// pid1Mark is one line of /run/spin-pid1-marks: PID 1's monotonic clock and its own CPU time, +// the units it had loaded and the time spent in unit_load so far, and the step it had done. type pid1Mark struct { - at time.Duration - step string + at, cpu, inLoad time.Duration + units int + step string } func parsePID1Marks(text string) []pid1Mark { var marks []pid1Mark for _, line := range strings.Split(text, "\n") { - us, step, ok := strings.Cut(strings.TrimSpace(line), " ") - n, err := strconv.ParseInt(us, 10, 64) - if !ok || err != nil { + f := strings.Fields(line) + if len(f) != 5 { continue } - marks = append(marks, pid1Mark{time.Duration(n) * time.Microsecond, step}) + at, err1 := strconv.ParseInt(f[0], 10, 64) + cpu, err2 := strconv.ParseInt(f[1], 10, 64) + units, err3 := strconv.Atoi(f[2]) + inLoad, err4 := strconv.ParseInt(f[3], 10, 64) + if err1 != nil || err2 != nil || err3 != nil || err4 != nil { + continue + } + marks = append(marks, pid1Mark{time.Duration(at) * time.Microsecond, time.Duration(cpu) * time.Microsecond, + time.Duration(inLoad) * time.Microsecond, units, f[4]}) } return marks } -// pid1Report is the p50 of each of PID 1's steps, from the one before it, over the boots that -// carry marks; "" when none does. +// pid1Report is, for each of PID 1's steps from the one before it, the p50 of the wall time, of +// PID 1's own CPU time, of the time in unit_load and of the units loaded, over the boots that carry +// marks; "" when none does. Wall time well above CPU time is waiting, not parsing. func pid1Report(boots []waitsBoot) string { + type step struct{ wall, cpu, load, units []time.Duration } var order []string - took := map[string][]time.Duration{} + steps := map[string]*step{} for _, b := range boots { for i, m := range b.pid1 { if i == 0 { continue } - if _, seen := took[m.step]; !seen { + p := b.pid1[i-1] + st := steps[m.step] + if st == nil { + st = &step{} + steps[m.step] = st order = append(order, m.step) } - took[m.step] = append(took[m.step], m.at-b.pid1[i-1].at) + st.wall = append(st.wall, m.at-p.at) + st.cpu = append(st.cpu, m.cpu-p.cpu) + st.load = append(st.load, m.inLoad-p.inLoad) + st.units = append(st.units, time.Duration(m.units-p.units)) } } if len(order) == 0 { return "" } + ms := func(ds []time.Duration) string { + p50, _ := boot.Percentile(ds, 0.5) + return fmt.Sprintf("%.2f", float64(p50)/float64(time.Millisecond)) + } var b strings.Builder - fmt.Fprintf(&b, "\nPID 1's steps from the unit load to the first job (experiments/pid1-marks), ms from the step before, p50/p95 over %d boots\n\n", len(took[order[0]])) - for _, step := range order { - p50, _ := boot.Percentile(took[step], 0.5) - p95, _ := boot.Percentile(took[step], 0.95) - fmt.Fprintf(&b, " %-22s %7.2f/%-7.2f\n", step, float64(p50)/float64(time.Millisecond), float64(p95)/float64(time.Millisecond)) + fmt.Fprintf(&b, "\nPID 1's steps from the unit load to the first job (experiments/pid1-marks), from the step before, p50 over %d boots\n\n", len(steps[order[0]].wall)) + fmt.Fprintf(&b, " %-22s %9s %9s %12s %7s\n", "", "WALL ms", "CPU ms", "UNIT_LOAD ms", "UNITS") + for _, name := range order { + st := steps[name] + units, _ := boot.Percentile(st.units, 0.5) + fmt.Fprintf(&b, " %-22s %9s %9s %12s %7d\n", name, ms(st.wall), ms(st.cpu), ms(st.load), int(units)) } return b.String() } diff --git a/image/systemd/experiments/pid1-marks.patch b/image/systemd/experiments/pid1-marks.patch index 2b48750..9e22069 100644 --- a/image/systemd/experiments/pid1-marks.patch +++ b/image/systemd/experiments/pid1-marks.patch @@ -93,22 +93,32 @@ diff -ruN a/src/core/manager.c b/src/core/manager.c assert(j->in_run_queue); diff -ruN a/src/core/spin-mark.h b/src/core/spin-mark.h --- a/src/core/spin-mark.h 1969-12-31 21:00:00.000000000 -0300 -+++ b/src/core/spin-mark.h 2026-11-19 12:53:27.084515904 -0300 -@@ -0,0 +1,26 @@ ++++ b/src/core/spin-mark.h 2026-11-19 13:01:04.896028361 -0300 +@@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later */ +#pragma once + +/* spin-machine experiment: PID 1's steps between the unit load and the first job, appended to -+ * /run/spin-pid1-marks as " ", which boot:waits reads after the boot. -+ * A file and not /dev/kmsg, whose user writes the kernel rate-limits. Not for a release. */ ++ * /run/spin-pid1-marks as " ", which boot:waits reads after the boot. Wall time far above CPU time is PID 1 ++ * waiting - on the disk or for a CPU - rather than parsing. A file and not /dev/kmsg, whose user ++ * writes the kernel rate-limits. Not for a release. */ + +#include +#include +#include +#include + -+static inline void spin_mark(const char *what) { ++extern unsigned spin_units_loaded; ++extern unsigned long long spin_unit_load_usec; ++ ++static inline unsigned long long spin_usec(clockid_t clock) { + struct timespec ts; ++ (void) clock_gettime(clock, &ts); ++ return (unsigned long long) ts.tv_sec * 1000000ULL + (unsigned long long) ts.tv_nsec / 1000ULL; ++} ++ ++static inline void spin_mark(const char *what) { + int fd; + + if (getpid() != 1) @@ -116,8 +126,32 @@ diff -ruN a/src/core/spin-mark.h b/src/core/spin-mark.h + fd = open("/run/spin-pid1-marks", O_WRONLY|O_CREAT|O_APPEND|O_CLOEXEC|O_NOCTTY, 0644); + if (fd < 0) + return; -+ (void) clock_gettime(CLOCK_MONOTONIC, &ts); -+ (void) dprintf(fd, "%llu %s\n", -+ (unsigned long long) ts.tv_sec * 1000000ULL + (unsigned long long) ts.tv_nsec / 1000ULL, what); ++ (void) dprintf(fd, "%llu %llu %u %llu %s\n", spin_usec(CLOCK_MONOTONIC), spin_usec(CLOCK_THREAD_CPUTIME_ID), ++ spin_units_loaded, spin_unit_load_usec, what); + (void) close(fd); +} +diff -ruN a/src/core/unit.c b/src/core/unit.c +--- a/src/core/unit.c 2026-11-19 13:01:04.885596937 -0300 ++++ b/src/core/unit.c 2026-11-19 13:01:04.897713974 -0300 +@@ -61,6 +61,10 @@ + #include "tmpfile-util.h" + #include "umask-util.h" + #include "unit.h" ++#include "spin-mark.h" ++ ++unsigned spin_units_loaded = 0; ++unsigned long long spin_unit_load_usec = 0; + #include "unit-name.h" + #include "user-util.h" + #include "varlink.h" +@@ -1696,7 +1700,10 @@ + u->fragment_mtime = now(CLOCK_REALTIME); + } + ++ unsigned long long spin_t0 = spin_usec(CLOCK_MONOTONIC); + r = UNIT_VTABLE(u)->load(u); ++ spin_unit_load_usec += spin_usec(CLOCK_MONOTONIC) - spin_t0; ++ spin_units_loaded++; + if (r < 0) + goto fail; + From 2a0f132daedc341c1fdbd0b4f36c7037be38bb32 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Thu, 19 Nov 2026 13:11:21 -0300 Subject: [PATCH 3/3] docs/log: PID 1's gap is loading 192 units, not the transaction Co-Authored-By: Claude Opus 5.5 --- boot/waits_probe_test.go | 5 ++++- docs/log.md | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/boot/waits_probe_test.go b/boot/waits_probe_test.go index e665d9c..e9a8406 100644 --- a/boot/waits_probe_test.go +++ b/boot/waits_probe_test.go @@ -401,7 +401,10 @@ func waitsReport(boots []waitsBoot, target string, top int, minGap time.Duration } } // The first gap, from the end of the unit load to the first job (32 ms p50 on the bench - // fleet, lab run 43361486792), is PID 1 and nothing else. systemd stamps UnitsLoadFinish + // fleet, lab run 43361486792), is PID 1 and nothing else. Measured inside PID 1 (11-19, lab run + // 44561762624, experiments/pid1-marks): 28.4 ms loading the 192 units default.target reaches, + // 20.9 of it PID 1's own CPU, and 0.3 ms building the transaction - so what follows, from one + // debug-logged boot, named the wrong step. systemd stamps UnitsLoadFinish // after enumerating mounts, devices and slices, and only then loads the units the boot // starts: coldplug, then "Activating default unit: default.target" parses every unit file // default.target reaches, and a stub for each name an After= or RequiresMountsFor= gives, diff --git a/docs/log.md b/docs/log.md index e30b122..a3ae515 100644 --- a/docs/log.md +++ b/docs/log.md @@ -374,6 +374,14 @@ The socket without default dependencies (`systemd-logind-varlink.socket.d/20-lan - the first login: 332/355 → 328/337. The p50 moves little because logind starting sooner takes the CPU sshd was starting on. Masking `modprobe@drm`, which logind is ordered after, added nothing (327/345). What is left of the login is CPU contention on 2 vCPUs, `user@1000` (38 ms) and the host key generation (21 ms) starting in the same window, rather than waits. + +**11-19 · systemd/boot · where PID 1's gap between the unit load and the first job goes (experiment, tooling kept).** `image/systemd/experiments/pid1-marks.patch` makes PID 1 append its monotonic clock, its own CPU time, the units loaded and the time in `unit_load` after each step of `manager_startup` and `do_queue_default_job` to `/run/spin-pid1-marks`. `boot:waits` reads the file. `lab.yml`'s `systemd_experiment` builds the patched systemd on the runner and puts it into each overlay. Bench fleet, 10 boots (lab 44561762624), p50 ms: +- bus, varlink, coldplug, vacuum, ready: < 0.2 together; +- **loading what `default.target` reaches: 28.4 wall, 20.9 of it PID 1's CPU, 192 units** (~0.11 ms of CPU each; the rest is likely cold reads); +- building the transaction: 0.30; +- the first dispatch: 0.87. + +The gap is unit loading, not the transaction that 11-13's single debug-logged boot named. 96 of the units a boot lists are inactive, loaded because another unit names them in `After=`/`Wants=`: `initrd-*`, `plymouth-*`, `modprobe@*`, hibernate, cryptsetup's targets, `firewalld` and `auditd` named by docker. Masking does not spare a load (a masked unit is still looked up and loaded as masked), which is why masking 11 units moved nothing (§4). --- ## 3. Rework cycles @@ -617,7 +625,7 @@ Also the reusable builder, added and reverted on 10-13. These are #242's checklist. - **Is `10-seats.conf` still needed?** The control that should fail passes on systemd 259 (#240, "Not settled here"). - **virtio-rng removal.** -13.6 ms to the prompt (lab 43321809471), a fingerprint change, not taken (#240). -- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms transaction-build gap is "explained, not removable" (`boot/waits_probe_test.go:391-399`); with systemd built here, it can now be instrumented. +- **PID 1 time.** The ~25 ms in the BPF verifier for systemd's socket-bind program (#240) is gone with the systemd image/systemd builds (11-19). The ~32 ms gap after the unit load is loading 192 units, ~21 ms of CPU (11-19, experiments/pid1-marks), not building the transaction (0.3 ms). It shrinks only with fewer units reached at boot. - **musl under long-running load.** Never measured (`31a57e7`). - **`page_reporting_order`.** Revisit "only if memory per host becomes the limit" (docs/memory-and-disk.md). - **Moving `MachineType` off pc-q35-11.1.** "its own change" (CLAUDE.md rule 1, `hack/machinetype`).