rk selftests, and kselftest units in rk test

.github/workflows/nightly.yml

@@ -1,5 +1,6 @@
 # The nightly: every pin in the current set on X64, built with gcc-14 and with rucc from tinyconfig
-# and the test fragment, and graded by rk test on boot, smoke and KUnit.
+# and the test fragment, and graded by rk test on boot, smoke, KUnit and the selftests, which are
+# built with gcc-14 and with rucc.
 #
 # Both builds of a version run in one job, because rk test and rk mixed need both build directories
 # and the objects are too big to move between jobs. When the rucc kernel fails a unit the reference
@@ -77,7 +78,7 @@ jobs:
         run: |
           sudo apt-get update -q
           sudo apt-get install -qy --no-install-recommends gcc-14 flex bison bc libelf-dev libssl-dev \
-            cpio qemu-system-x86 busybox-static
+            cpio qemu-system-x86 busybox-static libcap-dev libnuma-dev rsync
       - name: let the runner user open /dev/kvm
         run: |
           echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
@@ -120,11 +121,25 @@ jobs:
         run: |
           target/release/rk build ${{ matrix.version }} --row X64 --config "$CONFIG" --fragment test \
             --cc rucc --targets "bzImage modules" --out "$RUNNER_TEMP/rucc" --keep-going
+      - name: rk selftests with gcc-14
+        run: |
+          target/release/rk selftests --build "$RUNNER_TEMP/gcc" --cc gcc-14 \
+            --out "$RUNNER_TEMP/selftests-gcc"
+      - name: rk selftests with rucc
+        continue-on-error: true
+        run: |
+          target/release/rk selftests --build "$RUNNER_TEMP/gcc" --cc rucc \
+            --out "$RUNNER_TEMP/selftests-rucc"
       - name: rk test
         id: test
         run: |
+          rucc_selftests=()
+          if [ -f "$RUNNER_TEMP/selftests-rucc/selftests.json" ]; then
+            rucc_selftests=(--rucc-selftests "$RUNNER_TEMP/selftests-rucc")
+          fi
           target/release/rk test --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
-            --kinds boot,smoke,kunit --out "$RUNNER_TEMP/test"
+            --kinds boot,smoke,kunit,kselftest --selftests "$RUNNER_TEMP/selftests-gcc" \
+            "${rucc_selftests[@]}" --out "$RUNNER_TEMP/test"
       - name: rk cross-modules
         if: always() && steps.test.outcome != 'skipped'
         continue-on-error: true
@@ -138,16 +153,20 @@ jobs:
           unit=$(python3 -c 'import json, sys; t = json.load(open(sys.argv[1])); print(next((u["name"] for u in t["units"] if u["graded"] and not u["passed"] and u.get("attribution", "kernel") == "kernel"), ""))' "$RUNNER_TEMP/test/test.json")
           if [ -z "$unit" ]; then echo "no kernel failure to bisect"; exit 0; fi
           target/release/rk mixed --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \
-            --unit "$unit" --out "$RUNNER_TEMP/mixed"
+            --unit "$unit" --selftests "$RUNNER_TEMP/selftests-gcc" --out "$RUNNER_TEMP/mixed"
       - name: keep the record
         if: always()
         run: |
-          mkdir -p record/gcc record/rucc record/test record/mixed record/cross
+          mkdir -p record/gcc record/rucc record/test record/mixed record/cross record/selftests
           for side in gcc rucc; do
             cp "$RUNNER_TEMP/$side"/{build.json,summary.md,build.log,.config} "record/$side/" 2>/dev/null || true
           done
           cp -r "$RUNNER_TEMP/test/." record/test/ 2>/dev/null || true
           cp -r "$RUNNER_TEMP/cross/." record/cross/ 2>/dev/null || true
+          for side in gcc rucc; do
+            mkdir -p "record/selftests/$side"
+            cp -r "$RUNNER_TEMP/selftests-$side"/{selftests.json,logs} "record/selftests/$side/" 2>/dev/null || true
+          done
           cp "$RUNNER_TEMP"/mixed/{mixed.json,summary.md,relink.jsonl} "$RUNNER_TEMP"/mixed/*.log record/mixed/ 2>/dev/null || true
           cp -r "$RUNNER_TEMP/mixed/fuel" record/mixed/ 2>/dev/null || true
           find record/mixed/fuel -name '*.o' -delete 2>/dev/null || true

README.md

@@ -50,10 +50,12 @@ The reference compilers live in era containers built from `provision/eras` and p
 
 `rk boot --build DIR` boots a build under QEMU with `rk-init/init.sh` as PID 1 and a static busybox, and reads the smoke checks off the serial console. `rk baseline` builds and boots with the reference three times and writes the result under `results/baseline`.
 
-`rk test --reference DIR --other DIR --kinds boot,smoke,kunit` is the graded run. It boots the reference kernel and the rucc kernel once per suite, with `rk.suite=` on the command line, and compares them unit by unit: `boot`, each smoke check, each KUnit result read from the TAP on the console, and each test module loading. The kunit boot carries the build's modules, so build with `--targets "bzImage modules"` when the test fragment makes KUnit tests modules. The units the reference passes in every run (`--runs`) are graded and the rucc kernel must pass all of them. The splats in both consoles are normalized, and one that only the rucc kernel prints fails the run too. Given a rucc-built busybox with `--rucc-busybox`, every failure is attributed to the kernel, the userland or the two together by running the other two cells of the 2x2. kselftest and LTP are refused until K5. The run directory gets `test.json`, `summary.md` and the console of every boot.
+`rk test --reference DIR --other DIR --kinds boot,smoke,kunit` is the graded run. It boots the reference kernel and the rucc kernel once per suite, with `rk.suite=` on the command line, and compares them unit by unit: `boot`, each smoke check, each KUnit result read from the TAP on the console, and each test module loading. The kunit boot carries the build's modules, so build with `--targets "bzImage modules"` when the test fragment makes KUnit tests modules. The units the reference passes in every run (`--runs`) are graded and the rucc kernel must pass all of them. The splats in both consoles are normalized, and one that only the rucc kernel prints fails the run too. Given a rucc-built busybox with `--rucc-busybox`, every failure is attributed to the kernel, the userland or the two together by running the other two cells of the 2x2. With `--kinds kselftest` and `--selftests DIR` from `rk selftests`, every pinned collection is a boot of its own, and each program `run_kselftest.sh` reports is a unit, `kselftest:<collection>:<program>`. A rucc-built set passed with `--rucc-selftests` counts as part of the rucc userland for attribution. LTP is not run yet. The run directory gets `test.json`, `summary.md` and the console of every boot.
 
 `rk mixed --reference DIR --other DIR --unit UNIT` finds what makes one unit that `rk test` failed go wrong. Every object the two builds have with different bytes is a candidate. A trial copies the reference build, puts some rucc objects in place of the reference's, relinks with the reference's own make command so that nothing is compiled again, and boots the suite that reports the unit. Delta debugging finds a smallest set of rucc objects that still fails. When one object is left and the other build is rucc's, the object is compiled again with `-fpass-fuel-global=N` and the fuel is bisected to the first transformation that breaks the unit, and the `-fdump-ir=all` dumps on each side of it name the pass and the functions it changed. A trial that does not link is a finding of its own and ends the search. The run directory gets `mixed.json`, `summary.md`, the console of every boot and the dumps.
 
+`rk selftests --build DIR --cc COMPILER` builds the kselftest collections that `rows.toml` pins for the build's row, from the build's own kernel tree and against its exported headers, and installs them the way `run_kselftest.sh` expects. Each collection is built on its own with `make -k`, so one program that needs a library the machine lacks leaves the rest standing, and the programs that did not build are listed in `selftests.json`. The programs are linked dynamically, because static linking breaks real collections, so the initramfs that `rk test` makes for a collection carries the interpreter and the shared libraries its programs need. `--collections` picks others, and the output goes to `selftests` in the build directory unless `--out` says otherwise.
+
 `rk cross-modules --reference DIR --other DIR` loads the rucc modules into the reference kernel and the reference modules into the rucc kernel. It first checks every module against the other kernel without booting: the relocation types, the whole `vermagic`, symbols the kernel does not export and CRCs that differ from its `Module.symvers`. Then it runs the kunit suite three times per run: the reference kernel with its own modules, which finds the units that pass in QEMU without hardware, and then each kernel with the other build's modules. Both crossed boots must pass every unit the first one passed, and a splat only a crossed boot prints fails the run. Both builds need `--targets "bzImage modules"`. The run directory gets `cross-modules.json`, `summary.md` and the console of every boot.
 
 `rk config-diff`, `rk probes`, `rk flags-diff`, `rk demands` and `rk asm-inventory` read build directories and write markdown tables: the `.config` differences, the compiler probes the two compilers answered differently, the flags each unit was compiled with on one side only (read from the `.cmd` files kbuild writes), the failed units by error, and the instructions the kernel writes itself. `rk config-diff --why` also names the Kconfig expressions behind each difference and the probe that decided them, and needs the kernel tree, which it finds through `build.json` or `--source`. `rk syntax --build DIR --cc rucc` replays every unit of a reference build through rucc's front end with the reference's own command lines, `-fsyntax-only` for C and `-E` for assembly, and writes the failing units grouped by their first error. Units listed with an issue in `syntax-known.toml` are known failures, so the command fails only on new ones.

crates/rk/src/boot.rs

@@ -72,7 +72,8 @@ pub fn initramfs(busybox: &[u8]) -> Vec<u8> {
 }
 
 /// The initramfs for a busybox binary with more files, each given by its path inside and its
-/// bytes. The directories they need are made first.
+/// bytes. The directories they need are made first. An ELF file or a script can be run, and
+/// anything else is only read.
 #[must_use]
 pub fn initramfs_with(busybox: &[u8], files: &[(String, Vec<u8>)]) -> Vec<u8> {
     let base = ["bin", "dev", "proc", "sys", "tmp"];
@@ -143,7 +144,11 @@ pub fn initramfs_with(busybox: &[u8], files: &[(String, Vec<u8>)]) -> Vec<u8> {
     }));
     entries.extend(files.iter().map(|(path, data)| Entry {
         name: path,
-        mode: 0o100_644,
+        mode: if data.starts_with(b"\x7fELF") || data.starts_with(b"#!") {
+            0o100_755
+        } else {
+            0o100_644
+        },
         data,
         rdev: (0, 0),
     }));

crates/rk/src/build.rs

@@ -406,7 +406,7 @@ pub fn make_in(
 }
 
 /// Whether the machine we run on is the row's architecture.
-fn host_is(arch: &str) -> bool {
+pub fn host_is(arch: &str) -> bool {
     matches!(
         (std::env::consts::ARCH, arch),
         ("x86_64", "x86_64" | "i386" | "x86") | ("aarch64", "arm64") | ("riscv64", "riscv")

crates/rk/src/cli.rs

@@ -79,11 +79,14 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> {
                 "kinds",
                 "busybox",
                 "rucc-busybox",
+                "selftests",
+                "rucc-selftests",
                 "runs",
                 "timeout",
                 "out",
             ],
         ),
+        "selftests" => (false, &["build", "cc", "collections", "jobs", "out"]),
         "mixed" => (
             false,
             &[
@@ -92,6 +95,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> {
                 "row",
                 "unit",
                 "busybox",
+                "selftests",
                 "timeout",
                 "jobs",
                 "out",
@@ -153,11 +157,14 @@ usage:
   rk syntax --build DIR [--cc rucc] [--allow FILE] [--jobs N]
   rk boot --build DIR [--row X64] [--initramfs FILE | --busybox PATH] [--timeout 300]
           [--append WORDS]
-  rk test --reference DIR --other DIR [--kinds boot,smoke,kunit] [--row X64] [--runs 1]
+  rk test --reference DIR --other DIR [--kinds boot,smoke,kunit,kselftest] [--row X64] [--runs 1]
+          [--selftests DIR] [--rucc-selftests DIR]
   rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--timeout 600] [--no-fuel]
+          [--selftests DIR]
           [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR]
   rk cross-modules --reference DIR --other DIR [--row X64] [--runs 1] [--timeout 600]
           [--busybox PATH] [--out DIR]
+  rk selftests --build DIR [--cc COMPILER] [--collections timers,size] [--jobs N] [--out DIR]
   rk initramfs --out FILE [--busybox PATH]
   rk baseline [VERSION] --cc COMPILER [--row X64] [--config defconfig] [--fragment test]
               [--runs 3] [--timeout 600] [--stack-usage]

crates/rk/src/main.rs

@@ -22,6 +22,7 @@ mod pins;
 mod probes;
 mod repo;
 mod sections;
+mod selftests;
 mod sets;
 mod symvers;
 mod syntax;
@@ -70,6 +71,7 @@ fn main() -> ExitCode {
             "test" => test_command(&repo, &args),
             "mixed" => mixed_command(&repo, &args),
             "cross-modules" => cross_modules(&repo, &args),
+            "selftests" => selftests_command(&repo, &args),
             "asm-inventory" => asm_inventory(&repo, &args),
             "baseline" => baseline_command(&repo, &args),
             "initramfs" => initramfs_command(&args).map(|_| ExitCode::SUCCESS),
@@ -757,6 +759,19 @@ fn test_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
         std::fs::canonicalize(path).map_err(|e| format!("resolving {path}: {e}"))
     };
     let kinds = testrun::parse_kinds(args.get("kinds").unwrap_or("boot,smoke,kunit"))?;
+    let selftests = args
+        .get("selftests")
+        .map(|_| dir("selftests"))
+        .transpose()?;
+    let rucc_selftests = args
+        .get("rucc-selftests")
+        .map(|_| dir("rucc-selftests"))
+        .transpose()?;
+    if kinds.contains(&testrun::Kind::Kselftest) && selftests.is_none() {
+        return Err(
+            "--kinds kselftest needs --selftests, a directory rk selftests wrote".to_string(),
+        );
+    }
     let reference = dir("reference")?;
     let other = dir("other")?;
     let built = outcome_of(&other).ok();
@@ -799,6 +814,8 @@ fn test_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
             .get("rucc-busybox")
             .map(|p| read_busybox(Some(p)))
             .transpose()?,
+        selftests,
+        rucc_selftests,
         kinds,
         runs,
         timeout: number("timeout", 600)?,
@@ -812,6 +829,67 @@ fn test_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
     Ok(verdict(outcome.passed()))
 }
 
+/// Build the selftests of a build's tree with a compiler, for the initramfs.
+fn selftests_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
+    let path = args
+        .get("build")
+        .ok_or("rk selftests needs --build, a build directory")?;
+    let dir = std::fs::canonicalize(path).map_err(|e| format!("resolving {path}: {e}"))?;
+    let made = outcome_of(&dir)?;
+    let rows = personas::Rows::load(&repo.file("rows.toml"))?;
+    let row = rows.get(&made.row)?.clone();
+    let collections: Vec<String> = match args.get("collections") {
+        Some(list) => list
+            .split(',')
+            .map(str::trim)
+            .filter(|c| !c.is_empty())
+            .map(str::to_string)
+            .collect(),
+        None => row.selftests.clone(),
+    };
+    if collections.is_empty() {
+        return Err(format!(
+            "row {} pins no selftests in rows.toml; pass --collections",
+            row.name
+        ));
+    }
+    let cc = args
+        .get("cc")
+        .map_or_else(|| made.compiler.path.display().to_string(), str::to_string);
+    let jobs = match args.get("jobs") {
+        Some(n) => n
+            .parse()
+            .map_err(|_| format!("--jobs {n} is not a number"))?,
+        None => std::thread::available_parallelism().map_or(4, std::num::NonZero::get),
+    };
+    let out = args
+        .get("out")
+        .map_or_else(|| dir.join("selftests"), std::path::PathBuf::from);
+    std::fs::create_dir_all(&out).map_err(|e| format!("creating {}: {e}", out.display()))?;
+    let out =
+        std::fs::canonicalize(&out).map_err(|e| format!("resolving {}: {e}", out.display()))?;
+    let plan = selftests::Plan {
+        source: made.source.clone(),
+        build: dir,
+        arch: row.arch.clone(),
+        cross: if build::host_is(&row.arch) {
+            String::new()
+        } else {
+            row.cross.clone()
+        },
+        cc,
+        collections,
+        jobs,
+        out,
+    };
+    let outcome = selftests::run(&plan)?;
+    let summary = selftests::summary(&outcome);
+    print!("{summary}");
+    step_summary(&summary);
+    println!("wrote {}", plan.out.join("selftests.json").display());
+    Ok(verdict(!outcome.runnable().is_empty()))
+}
+
 /// Load each build's modules into the other build's kernel and run their KUnit suites.
 fn cross_modules(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
     let dir = |name: &str| -> Result<std::path::PathBuf, String> {
@@ -915,6 +993,10 @@ fn mixed_command(repo: &Repo, args: &Args) -> Result<ExitCode, String> {
         row,
         unit,
         busybox: read_busybox(args.get("busybox"))?,
+        selftests: args
+            .get("selftests")
+            .map(|p| std::fs::canonicalize(p).map_err(|e| format!("resolving {p}: {e}")))
+            .transpose()?,
         timeout: number("timeout", 600)?,
         jobs,
         fuel: !args.has("no-fuel"),

crates/rk/src/mixed.rs

@@ -286,10 +286,17 @@ pub fn recompile_args(record: &CompileRecord, object: &Path, extra: &[String]) -
 
 /// The suite a unit is reported by.
 #[must_use]
-pub fn suite_of(unit: &str) -> &'static str {
+pub fn suite_of(unit: &str) -> String {
     match testrun::kind_of(unit) {
-        Kind::Kunit => "kunit",
-        Kind::Boot | Kind::Smoke => "smoke",
+        Kind::Kunit => "kunit".to_string(),
+        Kind::Boot | Kind::Smoke => "smoke".to_string(),
+        Kind::Kselftest => {
+            let collection = unit
+                .strip_prefix("kselftest:")
+                .and_then(|rest| rest.split(':').next())
+                .unwrap_or_default();
+            format!("kselftest:{collection}")
+        }
     }
 }
 
@@ -306,6 +313,8 @@ pub struct Plan {
     pub unit: String,
     /// The userland, a static busybox.
     pub busybox: Vec<u8>,
+    /// The selftests, an `rk selftests` directory, for a `kselftest:` unit.
+    pub selftests: Option<PathBuf>,
     /// Seconds each boot may take.
     pub timeout: u64,
     /// Parallel jobs for the relink.
@@ -522,6 +531,13 @@ impl Trials<'_> {
         let suite = suite_of(&self.plan.unit);
         let files = if suite == "kunit" {
             testrun::modules(&self.tree)?
+        } else if let Some(collection) = suite.strip_prefix("kselftest:") {
+            let dir = self
+                .plan
+                .selftests
+                .as_deref()
+                .ok_or("a kselftest unit needs --selftests, an rk selftests directory")?;
+            crate::selftests::files(dir, collection)?
         } else {
             Vec::new()
         };
@@ -540,7 +556,7 @@ impl Trials<'_> {
         let outcome = outcome?;
         let console = std::fs::read_to_string(stem.with_extension("log")).unwrap_or_default();
         let kind = testrun::kind_of(&self.plan.unit);
-        let units = testrun::units_of(suite, &[kind], &outcome, &console);
+        let units = testrun::units_of(&suite, &[kind], &outcome, &console);
         Ok(units.get(&self.plan.unit) != Some(&Status::Pass))
     }
 }
@@ -1010,6 +1026,10 @@ mod tests {
         assert_eq!(suite_of("smoke:proc"), "smoke");
         assert_eq!(suite_of("kunit:list.list_add"), "kunit");
         assert_eq!(suite_of("kunit-module:lib/test_list.ko"), "kunit");
+        assert_eq!(
+            suite_of("kselftest:timers:posix_timers"),
+            "kselftest:timers"
+        );
     }
 
     #[test]