diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index b365474..9fa08a8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,6 +1,6 @@ # The nightly: every pin in the current set on X64, built with gcc-14 and with rucc from tinyconfig -# and the test fragment, and graded by rk test on boot, smoke, KUnit and the selftests, which are -# built with gcc-14 and with rucc. +# and the test fragment, and graded by rk test on boot, smoke, KUnit, the selftests and LTP, which +# are built with gcc-14 and with rucc. # # Both builds of a version run in one job, because rk test and rk mixed need both build directories # and the objects are too big to move between jobs. When the rucc kernel fails a unit the reference @@ -78,7 +78,8 @@ jobs: run: | sudo apt-get update -q sudo apt-get install -qy --no-install-recommends gcc-14 flex bison bc libelf-dev libssl-dev \ - cpio qemu-system-x86 busybox-static libcap-dev libnuma-dev rsync + cpio qemu-system-x86 busybox-static libcap-dev libnuma-dev rsync libaio-dev libacl1-dev \ + libkeyutils-dev - name: let the runner user open /dev/kvm run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ @@ -130,16 +131,30 @@ jobs: run: | target/release/rk selftests --build "$RUNNER_TEMP/gcc" --cc rucc \ --out "$RUNNER_TEMP/selftests-rucc" + - uses: actions/cache@v6.1.0 + with: + path: ${{ env.RK_CACHE }}/ltp + key: ltp-${{ hashFiles('ltp.toml') }} + - name: rk ltp with gcc-14 + run: | + target/release/rk ltp --build "$RUNNER_TEMP/gcc" --cc gcc-14 --out "$RUNNER_TEMP/ltp-gcc" + - name: rk ltp with rucc + continue-on-error: true + run: | + target/release/rk ltp --build "$RUNNER_TEMP/gcc" --cc rucc --out "$RUNNER_TEMP/ltp-rucc" - name: rk test id: test run: | - rucc_selftests=() + rucc_userland=() if [ -f "$RUNNER_TEMP/selftests-rucc/selftests.json" ]; then - rucc_selftests=(--rucc-selftests "$RUNNER_TEMP/selftests-rucc") + rucc_userland+=(--rucc-selftests "$RUNNER_TEMP/selftests-rucc") + fi + if [ -f "$RUNNER_TEMP/ltp-rucc/ltp.json" ]; then + rucc_userland+=(--rucc-ltp "$RUNNER_TEMP/ltp-rucc") fi target/release/rk test --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ - --kinds boot,smoke,kunit,kselftest --selftests "$RUNNER_TEMP/selftests-gcc" \ - "${rucc_selftests[@]}" --out "$RUNNER_TEMP/test" + --kinds boot,smoke,kunit,kselftest,ltp --selftests "$RUNNER_TEMP/selftests-gcc" \ + --ltp "$RUNNER_TEMP/ltp-gcc" "${rucc_userland[@]}" --out "$RUNNER_TEMP/test" - name: rk cross-modules if: always() && steps.test.outcome != 'skipped' continue-on-error: true @@ -153,11 +168,12 @@ jobs: unit=$(python3 -c 'import json, sys; t = json.load(open(sys.argv[1])); print(next((u["name"] for u in t["units"] if u["graded"] and not u["passed"] and u.get("attribution", "kernel") == "kernel"), ""))' "$RUNNER_TEMP/test/test.json") if [ -z "$unit" ]; then echo "no kernel failure to bisect"; exit 0; fi target/release/rk mixed --reference "$RUNNER_TEMP/gcc" --other "$RUNNER_TEMP/rucc" \ - --unit "$unit" --selftests "$RUNNER_TEMP/selftests-gcc" --out "$RUNNER_TEMP/mixed" + --unit "$unit" --selftests "$RUNNER_TEMP/selftests-gcc" --ltp "$RUNNER_TEMP/ltp-gcc" \ + --out "$RUNNER_TEMP/mixed" - name: keep the record if: always() run: | - mkdir -p record/gcc record/rucc record/test record/mixed record/cross record/selftests + mkdir -p record/gcc record/rucc record/test record/mixed record/cross record/selftests record/ltp for side in gcc rucc; do cp "$RUNNER_TEMP/$side"/{build.json,summary.md,build.log,.config} "record/$side/" 2>/dev/null || true done @@ -166,6 +182,8 @@ jobs: for side in gcc rucc; do mkdir -p "record/selftests/$side" cp -r "$RUNNER_TEMP/selftests-$side"/{selftests.json,logs} "record/selftests/$side/" 2>/dev/null || true + mkdir -p "record/ltp/$side" + cp -r "$RUNNER_TEMP/ltp-$side"/{ltp.json,logs} "record/ltp/$side/" 2>/dev/null || true done cp "$RUNNER_TEMP"/mixed/{mixed.json,summary.md,relink.jsonl} "$RUNNER_TEMP"/mixed/*.log record/mixed/ 2>/dev/null || true cp -r "$RUNNER_TEMP/mixed/fuel" record/mixed/ 2>/dev/null || true diff --git a/README.md b/README.md index c0b80d6..3ce6fd3 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,14 @@ The reference compilers live in era containers built from `provision/eras` and p `rk boot --build DIR` boots a build under QEMU with `rk-init/init.sh` as PID 1 and a static busybox, and reads the smoke checks off the serial console. `rk baseline` builds and boots with the reference three times and writes the result under `results/baseline`. -`rk test --reference DIR --other DIR --kinds boot,smoke,kunit` is the graded run. It boots the reference kernel and the rucc kernel once per suite, with `rk.suite=` on the command line, and compares them unit by unit: `boot`, each smoke check, each KUnit result read from the TAP on the console, and each test module loading. The kunit boot carries the build's modules, so build with `--targets "bzImage modules"` when the test fragment makes KUnit tests modules. The units the reference passes in every run (`--runs`) are graded and the rucc kernel must pass all of them. The splats in both consoles are normalized, and one that only the rucc kernel prints fails the run too. Given a rucc-built busybox with `--rucc-busybox`, every failure is attributed to the kernel, the userland or the two together by running the other two cells of the 2x2. With `--kinds kselftest` and `--selftests DIR` from `rk selftests`, every pinned collection is a boot of its own, and each program `run_kselftest.sh` reports is a unit, `kselftest::`. A rucc-built set passed with `--rucc-selftests` counts as part of the rucc userland for attribution. LTP is not run yet. The run directory gets `test.json`, `summary.md` and the console of every boot. +`rk test --reference DIR --other DIR --kinds boot,smoke,kunit` is the graded run. It boots the reference kernel and the rucc kernel once per suite, with `rk.suite=` on the command line, and compares them unit by unit: `boot`, each smoke check, each KUnit result read from the TAP on the console, and each test module loading. The kunit boot carries the build's modules, so build with `--targets "bzImage modules"` when the test fragment makes KUnit tests modules. The units the reference passes in every run (`--runs`) are graded and the rucc kernel must pass all of them. The splats in both consoles are normalized, and one that only the rucc kernel prints fails the run too. Given a rucc-built busybox with `--rucc-busybox`, every failure is attributed to the kernel, the userland or the two together by running the other two cells of the 2x2. With `--kinds kselftest` and `--selftests DIR` from `rk selftests`, every pinned collection is a boot of its own, and each program `run_kselftest.sh` reports is a unit, `kselftest::`. A rucc-built set passed with `--rucc-selftests` counts as part of the rucc userland for attribution. `--kinds ltp` with `--ltp DIR` from `rk ltp` does the same for LTP: every pinned runtest file is a boot, every test in it is a unit, `ltp::`, and `--rucc-ltp` is the rucc-built LTP. The run directory gets `test.json`, `summary.md` and the console of every boot. `rk mixed --reference DIR --other DIR --unit UNIT` finds what makes one unit that `rk test` failed go wrong. Every object the two builds have with different bytes is a candidate. A trial copies the reference build, puts some rucc objects in place of the reference's, relinks with the reference's own make command so that nothing is compiled again, and boots the suite that reports the unit. Delta debugging finds a smallest set of rucc objects that still fails. When one object is left and the other build is rucc's, the object is compiled again with `-fpass-fuel-global=N` and the fuel is bisected to the first transformation that breaks the unit, and the `-fdump-ir=all` dumps on each side of it name the pass and the functions it changed. A trial that does not link is a finding of its own and ends the search. The run directory gets `mixed.json`, `summary.md`, the console of every boot and the dumps. `rk selftests --build DIR --cc COMPILER` builds the kselftest collections that `rows.toml` pins for the build's row, from the build's own kernel tree and against its exported headers, and installs them the way `run_kselftest.sh` expects. Each collection is built on its own with `make -k`, so one program that needs a library the machine lacks leaves the rest standing, and the programs that did not build are listed in `selftests.json`. The programs are linked dynamically, because static linking breaks real collections, so the initramfs that `rk test` makes for a collection carries the interpreter and the shared libraries its programs need. `--collections` picks others, and the output goes to `selftests` in the build directory unless `--out` says otherwise. +`rk ltp --build DIR --cc COMPILER` builds the LTP release pinned in `ltp.toml` with that compiler. The tarball is checked against its hash and unpacked into the cache once, and each build configures and builds a copy of it with `make -k`, then installs it under `/ltp`. `ltp.json` lists the tests of each runtest file `rows.toml` pins for the row, and which of them were not built. The initramfs for a runtest file carries only the programs its commands name, the shared scripts and `tst_` helpers, the tests' data and the libraries they need, since the whole install does not fit in the guest. Each test runs with a ten minute limit, its output goes to the console only when it fails, and LTP's exit code 32, nothing to test here, is a skip. `--runtests` picks other files, and the output goes to `ltp` in the build directory unless `--out` says otherwise. + `rk cross-modules --reference DIR --other DIR` loads the rucc modules into the reference kernel and the reference modules into the rucc kernel. It first checks every module against the other kernel without booting: the relocation types, the whole `vermagic`, symbols the kernel does not export and CRCs that differ from its `Module.symvers`. Then it runs the kunit suite three times per run: the reference kernel with its own modules, which finds the units that pass in QEMU without hardware, and then each kernel with the other build's modules. Both crossed boots must pass every unit the first one passed, and a splat only a crossed boot prints fails the run. Both builds need `--targets "bzImage modules"`. The run directory gets `cross-modules.json`, `summary.md` and the console of every boot. `rk config-diff`, `rk probes`, `rk flags-diff`, `rk demands` and `rk asm-inventory` read build directories and write markdown tables: the `.config` differences, the compiler probes the two compilers answered differently, the flags each unit was compiled with on one side only (read from the `.cmd` files kbuild writes), the failed units by error, and the instructions the kernel writes itself. `rk config-diff --why` also names the Kconfig expressions behind each difference and the probe that decided them, and needs the kernel tree, which it finds through `build.json` or `--source`. `rk syntax --build DIR --cc rucc` replays every unit of a reference build through rucc's front end with the reference's own command lines, `-fsyntax-only` for C and `-E` for assembly, and writes the failing units grouped by their first error. Units listed with an issue in `syntax-known.toml` are known failures, so the command fails only on new ones. diff --git a/crates/rk/src/boot.rs b/crates/rk/src/boot.rs index fd63a6e..142a38e 100644 --- a/crates/rk/src/boot.rs +++ b/crates/rk/src/boot.rs @@ -303,6 +303,15 @@ fn kvm_usable(row: &Row) -> bool { .is_ok() } +/// The guest's memory in MiB: 1 GiB, and more when the initramfs is big, since the kernel holds +/// the archive and its unpacked files at the same time while it boots. An LTP runtest file can +/// bring a few hundred megabytes of programs. +#[must_use] +pub fn memory_mib(initramfs: u64) -> u64 { + let archive = initramfs.div_ceil(1 << 20); + (512 + 2 * archive).max(1024).next_multiple_of(256) +} + /// The QEMU command line. #[must_use] pub fn qemu_command(plan: &Plan, image: &Path, kvm: bool) -> Vec { @@ -311,12 +320,14 @@ pub fn qemu_command(plan: &Plan, image: &Path, kvm: bool) -> Vec { } else { plan.row.machine.clone() }; + let initramfs = std::fs::metadata(&plan.initramfs).map_or(0, |m| m.len()); + let memory = format!("{}M", memory_mib(initramfs)); let mut words: Vec = [ plan.row.qemu.as_str(), "-M", machine.as_str(), "-m", - "1G", + memory.as_str(), "-smp", "2", "-nographic", @@ -517,6 +528,14 @@ mod tests { assert!(!o.passed()); } + #[test] + fn a_big_initramfs_gets_more_memory() { + assert_eq!(memory_mib(0), 1024); + assert_eq!(memory_mib(200 << 20), 1024); + assert_eq!(memory_mib(400 << 20), 1536); + assert_eq!(memory_mib((400 << 20) + 1), 1536); + } + #[test] fn x86_images_live_under_arch_x86() { assert_eq!(srcarch("x86_64"), "x86"); diff --git a/crates/rk/src/cli.rs b/crates/rk/src/cli.rs index 0ec0e9f..e5be285 100644 --- a/crates/rk/src/cli.rs +++ b/crates/rk/src/cli.rs @@ -33,30 +33,30 @@ const SWITCHES: &[&str] = &[ "no-fuel", ]; +/// The flags of `rk build`. +const BUILD: &[&str] = &[ + "row", + "config", + "cc", + "out", + "jobs", + "targets", + "bringup", + "bringup-cc", + "fragment", + "keep-going", + "twice", + "stack-usage", + "config-only", + "no-upstream-check", +]; + /// Options each command accepts, and whether it takes a positional word. fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { Some(match command { "fetch" => (true, &["set", "no-upstream-check", "all"]), "sets" => (false, &["releases", "write"]), - "build" => ( - true, - &[ - "row", - "config", - "cc", - "out", - "jobs", - "targets", - "bringup", - "bringup-cc", - "fragment", - "keep-going", - "twice", - "stack-usage", - "config-only", - "no-upstream-check", - ], - ), + "build" => (true, BUILD), "config-diff" => (false, &["reference", "other", "why", "source"]), "probes" | "flags-diff" | "symvers-diff" | "objtool-report" => { (false, &["reference", "other"]) @@ -81,12 +81,15 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "rucc-busybox", "selftests", "rucc-selftests", + "ltp", + "rucc-ltp", "runs", "timeout", "out", ], ), "selftests" => (false, &["build", "cc", "collections", "jobs", "out"]), + "ltp" => (false, &["build", "cc", "runtests", "jobs", "out"]), "mixed" => ( false, &[ @@ -96,6 +99,7 @@ fn accepted(command: &str) -> Option<(bool, &'static [&'static str])> { "unit", "busybox", "selftests", + "ltp", "timeout", "jobs", "out", @@ -157,14 +161,15 @@ usage: rk syntax --build DIR [--cc rucc] [--allow FILE] [--jobs N] rk boot --build DIR [--row X64] [--initramfs FILE | --busybox PATH] [--timeout 300] [--append WORDS] - rk test --reference DIR --other DIR [--kinds boot,smoke,kunit,kselftest] [--row X64] [--runs 1] - [--selftests DIR] [--rucc-selftests DIR] - rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--timeout 600] [--no-fuel] - [--selftests DIR] + rk test --reference DIR --other DIR [--kinds boot,smoke,kunit,kselftest,ltp] [--row X64] + [--runs 1] [--selftests DIR] [--rucc-selftests DIR] [--ltp DIR] [--rucc-ltp DIR] [--busybox PATH] [--rucc-busybox PATH] [--timeout 600] [--out DIR] + rk mixed --reference DIR --other DIR --unit UNIT [--row X64] [--timeout 600] [--no-fuel] + [--selftests DIR] [--ltp DIR] rk cross-modules --reference DIR --other DIR [--row X64] [--runs 1] [--timeout 600] [--busybox PATH] [--out DIR] rk selftests --build DIR [--cc COMPILER] [--collections timers,size] [--jobs N] [--out DIR] + rk ltp --build DIR [--cc COMPILER] [--runtests syscalls,mm] [--jobs N] [--out DIR] rk initramfs --out FILE [--busybox PATH] rk baseline [VERSION] --cc COMPILER [--row X64] [--config defconfig] [--fragment test] [--runs 3] [--timeout 600] [--stack-usage] diff --git a/crates/rk/src/ltp.rs b/crates/rk/src/ltp.rs new file mode 100644 index 0000000..b27cba1 --- /dev/null +++ b/crates/rk/src/ltp.rs @@ -0,0 +1,557 @@ +//! `rk ltp`: the Linux Test Project built as user programs, for the initramfs (plan 11.5). +//! +//! The release pinned in `ltp.toml` is downloaded into the cache, checked against its hash and +//! unpacked once. `rk ltp` copies the tree into its output directory and configures it with the +//! compiler asked for. LTP's kernel modules are left out, since they would be built for this +//! machine's kernel rather than the one under test. The build runs with `make -k`, so that a test +//! that does not build leaves the rest standing, and the install with `make -i`, under `/ltp`, +//! the way it sits in the initramfs. +//! +//! A runtest file is a list of tests, a tag and a command per line. The runtest files pinned for +//! the row in `rows.toml` are each one boot of `rk test --kinds ltp`, and every tag in them is a +//! unit, `ltp::`. A tag whose program was not installed stays in the file, so a boot +//! reports it as failed rather than leaving it out, and `ltp.json` lists it as not built. +//! +//! The initramfs carries the runtest file, the programs its commands name, the shell scripts and +//! `tst_` helpers the tests share, the data of the tests it runs and the shared libraries all of +//! them need. The whole install does not fit next to a kernel in the guest's memory. +//! +//! What comes out is the install, `ltp.json` and the configure, build and install logs. + +use crate::selftests::libraries; +use rk_shim::digest::sha256_file; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Where the install goes inside the initramfs, and the prefix it is configured for. +pub const ROOT: &str = "ltp"; + +/// The file in an unpacked tree that holds the hash of the tarball it came from. +const MARKER: &str = ".rk-sha256"; + +/// The pinned release, `ltp.toml`. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +pub struct Pin { + /// The release, as in `20260529`. + pub version: String, + /// The tarball. + pub url: String, + /// The tarball's SHA-256. + pub sha256: String, +} + +impl Pin { + /// Read `ltp.toml`. + pub fn load(path: &Path) -> Result { + let text = std::fs::read_to_string(path) + .map_err(|e| format!("reading {}: {e}", path.display()))?; + let pin: Self = toml::from_str(&text).map_err(|e| format!("{}: {e}", path.display()))?; + if pin.sha256.len() != 64 { + return Err(format!("{}: sha256 is not a SHA-256", path.display())); + } + Ok(pin) + } + + /// The unpacked tree in the cache, downloading and unpacking the tarball when the tree is not + /// there yet. + pub fn fetch(&self, cache: &Path) -> Result { + let dir = cache.join("ltp"); + let tree = dir.join(format!("ltp-full-{}", self.version)); + if std::fs::read_to_string(tree.join(MARKER)).is_ok_and(|h| h.trim() == self.sha256) { + return Ok(tree); + } + let archive = dir.join(format!("ltp-full-{}.tar.xz", self.version)); + if !archive.is_file() { + crate::kernelorg::download(&self.url, &archive)?; + } + let got = + sha256_file(&archive).map_err(|e| format!("hashing {}: {e}", archive.display()))?; + if got != self.sha256 { + let _ = std::fs::remove_file(&archive); + return Err(format!( + "{} has SHA-256 {got}, and ltp.toml says {}; the file was removed", + archive.display(), + self.sha256 + )); + } + if tree.exists() { + std::fs::remove_dir_all(&tree) + .map_err(|e| format!("removing {}: {e}", tree.display()))?; + } + let status = Command::new("tar") + .arg("-xJf") + .arg(&archive) + .arg("-C") + .arg(&dir) + .status() + .map_err(|e| format!("running tar: {e}"))?; + if !status.success() || !tree.join("configure").is_file() { + return Err(format!( + "unpacking {} did not give {}", + archive.display(), + tree.display() + )); + } + std::fs::write(tree.join(MARKER), &self.sha256) + .map_err(|e| format!("writing the marker in {}: {e}", tree.display()))?; + Ok(tree) + } +} + +/// One test of a runtest file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Test { + /// The tag, which names the unit. + pub tag: String, + /// The command, run with `sh -c`. + pub command: String, +} + +/// Read a runtest file. Blank lines and comments are not tests. +#[must_use] +pub fn parse_runtest(text: &str) -> Vec { + text.lines() + .map(str::trim) + .filter(|line| !line.is_empty() && !line.starts_with('#')) + .filter_map(|line| { + let (tag, command) = line.split_once(char::is_whitespace)?; + Some(Test { + tag: tag.to_string(), + command: command.trim().to_string(), + }) + }) + .collect() +} + +/// The words of a command that may name a program, without the quotes and operators of the +/// shell around them. +fn words(command: &str) -> impl Iterator { + command + .split(|c: char| c.is_whitespace() || ";|&()<>\"'`".contains(c)) + .filter(|w| !w.is_empty() && !w.starts_with('-') && !w.contains('/') && !w.contains('$')) +} + +/// One runtest file's build. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Runtest { + /// Its name, the file under `runtest/`. + pub name: String, + /// The tags of its tests. + pub tests: Vec, + /// The tags whose program was not installed. + pub missing: Vec, +} + +/// What `rk ltp` did, written to `ltp.json` in the output directory. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Outcome { + /// The release. + pub version: String, + /// The compiler. + pub cc: String, + /// Every runtest file asked for, in the order asked. + pub runtests: Vec, +} + +impl Outcome { + /// Read `ltp.json` from an output directory. + pub fn load(dir: &Path) -> Result { + let path = dir.join("ltp.json"); + let text = std::fs::read_to_string(&path) + .map_err(|e| format!("reading {}: {e}; run rk ltp first", path.display()))?; + serde_json::from_str(&text).map_err(|e| format!("reading {}: {e}", path.display())) + } + + /// The names of the runtest files that have at least one test built. + #[must_use] + pub fn runnable(&self) -> Vec { + self.runtests + .iter() + .filter(|r| r.missing.len() < r.tests.len()) + .map(|r| r.name.clone()) + .collect() + } +} + +/// Everything `rk ltp` was asked to do. +#[derive(Debug, Clone)] +pub struct Plan { + /// The unpacked release. + pub source: PathBuf, + /// The release. + pub version: String, + /// The GNU triplet to configure for, empty for a native build. + pub host: String, + /// The `strip` for the programs, which keep a static copy of LTP's library and its debug + /// information each. + pub strip: String, + /// The compiler, as `CC`. + pub cc: String, + /// The runtest files. + pub runtests: Vec, + /// Parallel jobs. + pub jobs: usize, + /// The output directory. + pub out: PathBuf, +} + +/// Run a command in a directory with its output going to a log file, and say whether it +/// succeeded. +fn logged(mut command: Command, dir: &Path, log: &Path) -> Result { + let file = + std::fs::File::create(log).map_err(|e| format!("creating {}: {e}", log.display()))?; + let err = file + .try_clone() + .map_err(|e| format!("opening {}: {e}", log.display()))?; + let status = command + .current_dir(dir) + .stdout(file) + .stderr(err) + .status() + .map_err(|e| format!("running {}: {e}", command.get_program().display()))?; + Ok(status.success()) +} + +/// Whether a test's program is in the install. A command that names no installed program at all +/// is a test that was not built. +fn built(test: &Test, bin: &Path) -> bool { + words(&test.command).any(|w| bin.join(w).is_file()) +} + +/// Whether a file starts like an ELF file. +fn is_elf(path: &Path) -> bool { + use std::io::Read as _; + let mut magic = [0u8; 4]; + std::fs::File::open(path).is_ok_and(|mut f| f.read_exact(&mut magic).is_ok()) + && magic == *b"\x7fELF" +} + +/// Take the debug information out of every program in a directory, which leaves about a third of +/// each. A program `strip` cannot read is left as it is. +fn strip_debug(strip: &str, bin: &Path) -> Result<(), String> { + let entries = std::fs::read_dir(bin).map_err(|e| format!("reading {}: {e}", bin.display()))?; + let programs: Vec = entries + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| is_elf(p)) + .collect(); + for chunk in programs.chunks(200) { + Command::new(strip) + .arg("--strip-debug") + .args(chunk) + .status() + .map_err(|e| format!("running {strip}: {e}"))?; + } + Ok(()) +} + +/// Seconds a boot running one runtest file may take: `base`, and three more for each test in it, +/// since a file holds anything from one test to fifteen hundred. +#[must_use] +pub fn timeout(base: u64, files: &[(String, Vec)]) -> u64 { + let tests = files + .iter() + .find(|(name, _)| name.starts_with(&format!("{ROOT}/runtest/"))) + .map_or(0, |(_, data)| { + parse_runtest(&String::from_utf8_lossy(data)).len() + }); + base + 3 * u64::try_from(tests).unwrap_or(u64::MAX / 4) +} + +/// Copy the tree, configure, build and install it, and write `ltp.json`. +pub fn run(plan: &Plan) -> Result { + let logs = plan.out.join("logs"); + std::fs::create_dir_all(&logs).map_err(|e| format!("creating {}: {e}", logs.display()))?; + let build = plan.out.join("build"); + let _ = std::fs::remove_dir_all(&build); + let status = Command::new("cp") + .arg("-R") + .arg(&plan.source) + .arg(&build) + .status() + .map_err(|e| format!("running cp: {e}"))?; + if !status.success() { + return Err(format!("copying {} failed", plan.source.display())); + } + let mut configure = Command::new("./configure"); + configure + .arg(format!("--prefix=/{ROOT}")) + .arg("--without-modules") + .arg(format!("CC={}", plan.cc)); + if !plan.host.is_empty() { + configure.arg(format!("--host={}", plan.host)); + } + eprintln!("rk: ltp configure"); + if !logged(configure, &build, &logs.join("configure.log"))? { + return Err(format!( + "configuring LTP with {} failed; see {}", + plan.cc, + logs.join("configure.log").display() + )); + } + eprintln!("rk: ltp build"); + let mut make = Command::new("make"); + make.arg("-k").arg(format!("-j{}", plan.jobs)); + logged(make, &build, &logs.join("build.log"))?; + let install = plan.out.join("install"); + let _ = std::fs::remove_dir_all(&install); + let mut make = Command::new("make"); + make.arg("-i") + .arg(format!("DESTDIR={}", install.display())) + .arg("install"); + logged(make, &build, &logs.join("install.log"))?; + let root = install.join(ROOT); + let bin = root.join("testcases/bin"); + strip_debug(&plan.strip, &bin)?; + let mut runtests = Vec::new(); + for name in &plan.runtests { + let text = std::fs::read_to_string(root.join("runtest").join(name)).unwrap_or_default(); + let tests = parse_runtest(&text); + runtests.push(Runtest { + name: name.clone(), + tests: tests.iter().map(|t| t.tag.clone()).collect(), + missing: tests + .iter() + .filter(|t| !built(t, &bin)) + .map(|t| t.tag.clone()) + .collect(), + }); + } + let outcome = Outcome { + version: plan.version.clone(), + cc: plan.cc.clone(), + runtests, + }; + let json = serde_json::to_string_pretty(&outcome).unwrap_or_default(); + std::fs::write(plan.out.join("ltp.json"), json + "\n") + .map_err(|e| format!("writing ltp.json: {e}"))?; + Ok(outcome) +} + +/// The outcome as markdown. +#[must_use] +pub fn summary(o: &Outcome) -> String { + let mut s = format!( + "### rk ltp {}\n\n| runtest | tests | not built |\n|---|---|---|\n", + o.version + ); + for r in &o.runtests { + let _ = writeln!( + s, + "| {} | {} | {} |", + r.name, + r.tests.len(), + r.missing.len() + ); + } + let empty: Vec<&str> = o + .runtests + .iter() + .filter(|r| r.tests.is_empty()) + .map(|r| r.name.as_str()) + .collect(); + if !empty.is_empty() { + let _ = writeln!( + s, + "\n{} list no tests, so this release may not have them.", + empty.join(", ") + ); + } + s +} + +/// Whether a file in `testcases/bin` goes in every LTP initramfs: the shell libraries and +/// scripts, which are small, and the `tst_` helpers they call. +fn shared(name: &str, data: &[u8]) -> bool { + name.starts_with("tst_") || !data.starts_with(b"\x7fELF") +} + +/// The initramfs files for one runtest file: the file, the programs its commands name, the shared +/// scripts and helpers, the data of its tests and the libraries the programs need. +pub fn files(out: &Path, runtest: &str) -> Result)>, String> { + let root = out.join("install").join(ROOT); + let path = root.join("runtest").join(runtest); + let text = + std::fs::read_to_string(&path).map_err(|e| format!("reading {}: {e}", path.display()))?; + let named: BTreeSet = parse_runtest(&text) + .iter() + .flat_map(|t| words(&t.command).map(str::to_string).collect::>()) + .collect(); + let mut files = vec![( + format!("{ROOT}/runtest/{runtest}"), + text.clone().into_bytes(), + )]; + let bin = root.join("testcases/bin"); + let entries = std::fs::read_dir(&bin).map_err(|e| format!("reading {}: {e}", bin.display()))?; + let mut programs: BTreeMap> = BTreeMap::new(); + for entry in entries.filter_map(Result::ok) { + let name = entry.file_name().to_string_lossy().into_owned(); + if !entry.path().is_file() { + continue; + } + let data = std::fs::read(entry.path()) + .map_err(|e| format!("reading {}: {e}", entry.path().display()))?; + if named.contains(&name) || shared(&name, &data) { + programs.insert(name, data); + } + } + let data = root.join("testcases/data"); + for name in programs.keys() { + let dir = data.join(name); + if dir.is_dir() { + let mut inside = Vec::new(); + crate::selftests::walk(&dir, &root, &mut inside)?; + files.extend( + inside + .into_iter() + .map(|(name, data)| (format!("{ROOT}/{name}"), data)), + ); + } + } + let libs = libraries(programs.values().map(Vec::as_slice)); + files.extend( + programs + .into_iter() + .map(|(name, data)| (format!("{ROOT}/testcases/bin/{name}"), data)), + ); + files.extend(libs); + Ok(files) +} + +/// The results `rk-init` printed for one runtest file, as `ltp::`. A test that ran +/// into nothing to test on this machine, LTP's exit code 32, is a skip. +#[must_use] +pub fn units(console: &str) -> BTreeMap { + use crate::tap::Status; + let mut out = BTreeMap::new(); + for line in console.lines() { + let text = crate::boot::strip_timestamp(line.trim_end_matches('\r')).trim(); + let Some(rest) = text.strip_prefix("RK-LTP ") else { + continue; + }; + let mut words = rest.split_whitespace(); + let (Some(runtest), Some(tag), Some(result)) = (words.next(), words.next(), words.next()) + else { + continue; + }; + let status = match result { + "pass" => Status::Pass, + "skip" => Status::Skip, + _ => Status::Fail, + }; + out.insert(format!("ltp:{runtest}:{tag}"), status); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tap::Status; + + #[test] + fn a_runtest_file_reads_as_tags_and_commands() { + let tests = parse_runtest( + "#DESCRIPTION:Kernel system calls\nabort01 abort01\n\naccess01 access01 -i 2\n # a comment\nbad\n", + ); + assert_eq!( + tests, + [ + Test { + tag: "abort01".to_string(), + command: "abort01".to_string() + }, + Test { + tag: "access01".to_string(), + command: "access01 -i 2".to_string() + }, + ] + ); + } + + #[test] + fn a_command_names_its_programs() { + let named: Vec<&str> = + words("export TCbin=$LTPROOT/testcases/bin; fsx-linux -N 10000 -o 8192 \"a b\"") + .collect(); + assert!(named.contains(&"fsx-linux"), "{named:?}"); + assert!(named.contains(&"export"), "{named:?}"); + assert!(!named.iter().any(|w| w.contains('/') || w.starts_with('-'))); + } + + #[test] + fn a_test_is_built_when_its_program_was_installed() { + let dir = std::env::temp_dir().join(format!("rk-ltp-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("abort01"), b"\x7fELF").unwrap(); + let test = |command: &str| Test { + tag: "t".to_string(), + command: command.to_string(), + }; + assert!(built(&test("abort01"), &dir)); + assert!(built(&test("sh -c \"abort01 -i 2\""), &dir)); + assert!(!built(&test("access01"), &dir)); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn results_are_read_per_tag() { + let console = "[ 2.000000] RK-LTP syscalls abort01 pass 0\n# abort01.c:52: TFAIL: oops\nRK-LTP syscalls access01 fail 1\nRK-LTP syscalls acct02 skip 32\nRK-LTP syscalls\n"; + let units = units(console); + assert_eq!(units.len(), 3, "{units:?}"); + assert_eq!(units["ltp:syscalls:abort01"], Status::Pass); + assert_eq!(units["ltp:syscalls:access01"], Status::Fail); + assert_eq!(units["ltp:syscalls:acct02"], Status::Skip); + } + + #[test] + fn a_runtest_file_runs_when_one_of_its_tests_was_built() { + let o = Outcome { + version: "20260529".to_string(), + cc: "gcc".to_string(), + runtests: vec![ + Runtest { + name: "syscalls".to_string(), + tests: vec!["a".to_string(), "b".to_string()], + missing: vec!["b".to_string()], + }, + Runtest { + name: "gone".to_string(), + tests: Vec::new(), + missing: Vec::new(), + }, + ], + }; + assert_eq!(o.runnable(), ["syscalls"]); + let text = summary(&o); + assert!(text.contains("| syscalls | 2 | 1 |"), "{text}"); + assert!(text.contains("gone list no tests"), "{text}"); + } + + #[test] + fn a_boot_gets_time_for_every_test() { + let files = vec![ + ("bin/x".to_string(), Vec::new()), + ( + "ltp/runtest/syscalls".to_string(), + b"# a\nabort01 abort01\naccess01 access01\n".to_vec(), + ), + ]; + assert_eq!(timeout(600, &files), 606); + assert_eq!(timeout(600, &[]), 600); + } + + #[test] + fn the_shipped_pin_reads() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../ltp.toml"); + let pin = Pin::load(&path).unwrap(); + assert!( + pin.url + .ends_with(&format!("ltp-full-{}.tar.xz", pin.version)) + ); + } +} diff --git a/crates/rk/src/main.rs b/crates/rk/src/main.rs index e3f29f4..d212aa1 100644 --- a/crates/rk/src/main.rs +++ b/crates/rk/src/main.rs @@ -14,6 +14,7 @@ mod flags; mod frames; mod kconfig; mod kernelorg; +mod ltp; mod mixed; mod modules; mod objects; @@ -73,6 +74,7 @@ fn main() -> ExitCode { "mixed" => mixed_command(&repo, &args), "cross-modules" => cross_modules(&repo, &args), "selftests" => selftests_command(&repo, &args), + "ltp" => ltp_command(&repo, &args), "asm-inventory" => asm_inventory(&repo, &args), "baseline" => baseline_command(&repo, &args), "initramfs" => initramfs_command(&args).map(|_| ExitCode::SUCCESS), @@ -781,6 +783,11 @@ fn test_command(repo: &Repo, args: &Args) -> Result { "--kinds kselftest needs --selftests, a directory rk selftests wrote".to_string(), ); } + let ltp = args.get("ltp").map(|_| dir("ltp")).transpose()?; + let rucc_ltp = args.get("rucc-ltp").map(|_| dir("rucc-ltp")).transpose()?; + if kinds.contains(&testrun::Kind::Ltp) && ltp.is_none() { + return Err("--kinds ltp needs --ltp, a directory rk ltp wrote".to_string()); + } let reference = dir("reference")?; let other = dir("other")?; let built = outcome_of(&other).ok(); @@ -825,6 +832,8 @@ fn test_command(repo: &Repo, args: &Args) -> Result { .transpose()?, selftests, rucc_selftests, + ltp, + rucc_ltp, kinds, runs, timeout: number("timeout", 600)?, @@ -899,6 +908,74 @@ fn selftests_command(repo: &Repo, args: &Args) -> Result { Ok(verdict(!outcome.runnable().is_empty())) } +/// Build the pinned LTP release with a compiler, for the initramfs. +fn ltp_command(repo: &Repo, args: &Args) -> Result { + let path = args + .get("build") + .ok_or("rk ltp needs --build, a build directory")?; + let dir = std::fs::canonicalize(path).map_err(|e| format!("resolving {path}: {e}"))?; + let made = outcome_of(&dir)?; + let rows = personas::Rows::load(&repo.file("rows.toml"))?; + let row = rows.get(&made.row)?.clone(); + let runtests: Vec = match args.get("runtests") { + Some(list) => list + .split(',') + .map(str::trim) + .filter(|r| !r.is_empty()) + .map(str::to_string) + .collect(), + None => row.ltp.clone(), + }; + if runtests.is_empty() { + return Err(format!( + "row {} pins no LTP runtest files in rows.toml; pass --runtests", + row.name + )); + } + let pin = ltp::Pin::load(&repo.file("ltp.toml"))?; + let source = pin.fetch(&repo::cache_dir())?; + let cc = args + .get("cc") + .map_or_else(|| made.compiler.path.display().to_string(), str::to_string); + let jobs = match args.get("jobs") { + Some(n) => n + .parse() + .map_err(|_| format!("--jobs {n} is not a number"))?, + None => std::thread::available_parallelism().map_or(4, std::num::NonZero::get), + }; + let out = args + .get("out") + .map_or_else(|| dir.join("ltp"), std::path::PathBuf::from); + std::fs::create_dir_all(&out).map_err(|e| format!("creating {}: {e}", out.display()))?; + let out = + std::fs::canonicalize(&out).map_err(|e| format!("resolving {}: {e}", out.display()))?; + let native = build::host_is(&row.arch); + let plan = ltp::Plan { + source, + version: pin.version, + host: if native { + String::new() + } else { + row.cross.trim_end_matches('-').to_string() + }, + strip: if native { + "strip".to_string() + } else { + format!("{}strip", row.cross) + }, + cc, + runtests, + jobs, + out, + }; + let outcome = ltp::run(&plan)?; + let summary = ltp::summary(&outcome); + print!("{summary}"); + step_summary(&summary); + println!("wrote {}", plan.out.join("ltp.json").display()); + Ok(verdict(!outcome.runnable().is_empty())) +} + /// Load each build's modules into the other build's kernel and run their KUnit suites. fn cross_modules(repo: &Repo, args: &Args) -> Result { let dir = |name: &str| -> Result { @@ -1006,6 +1083,10 @@ fn mixed_command(repo: &Repo, args: &Args) -> Result { .get("selftests") .map(|p| std::fs::canonicalize(p).map_err(|e| format!("resolving {p}: {e}"))) .transpose()?, + ltp: args + .get("ltp") + .map(|p| std::fs::canonicalize(p).map_err(|e| format!("resolving {p}: {e}"))) + .transpose()?, timeout: number("timeout", 600)?, jobs, fuel: !args.has("no-fuel"), diff --git a/crates/rk/src/mixed.rs b/crates/rk/src/mixed.rs index 5be2760..5fe67d9 100644 --- a/crates/rk/src/mixed.rs +++ b/crates/rk/src/mixed.rs @@ -297,6 +297,13 @@ pub fn suite_of(unit: &str) -> String { .unwrap_or_default(); format!("kselftest:{collection}") } + Kind::Ltp => { + let runtest = unit + .strip_prefix("ltp:") + .and_then(|rest| rest.split(':').next()) + .unwrap_or_default(); + format!("ltp:{runtest}") + } } } @@ -315,6 +322,8 @@ pub struct Plan { pub busybox: Vec, /// The selftests, an `rk selftests` directory, for a `kselftest:` unit. pub selftests: Option, + /// LTP, an `rk ltp` directory, for an `ltp:` unit. + pub ltp: Option, /// Seconds each boot may take. pub timeout: u64, /// Parallel jobs for the relink. @@ -538,6 +547,13 @@ impl Trials<'_> { .as_deref() .ok_or("a kselftest unit needs --selftests, an rk selftests directory")?; crate::selftests::files(dir, collection)? + } else if let Some(runtest) = suite.strip_prefix("ltp:") { + let dir = self + .plan + .ltp + .as_deref() + .ok_or("an ltp unit needs --ltp, an rk ltp directory")?; + crate::ltp::files(dir, runtest)? } else { Vec::new() }; @@ -548,7 +564,11 @@ impl Trials<'_> { build: self.tree.clone(), row: self.plan.row.clone(), initramfs: initramfs.clone(), - timeout: self.plan.timeout, + timeout: if suite.starts_with("ltp:") { + crate::ltp::timeout(self.plan.timeout, &files) + } else { + self.plan.timeout + }, append: format!("rk.suite={suite}"), stem: Some(stem.to_path_buf()), }); @@ -1030,6 +1050,7 @@ mod tests { suite_of("kselftest:timers:posix_timers"), "kselftest:timers" ); + assert_eq!(suite_of("ltp:syscalls:abort01"), "ltp:syscalls"); } #[test] diff --git a/crates/rk/src/personas.rs b/crates/rk/src/personas.rs index 2a7858f..d577dde 100644 --- a/crates/rk/src/personas.rs +++ b/crates/rk/src/personas.rs @@ -167,6 +167,9 @@ pub struct Row { /// The kselftest collections pinned for the row. #[serde(default)] pub selftests: Vec, + /// The LTP runtest files pinned for the row. + #[serde(default)] + pub ltp: Vec, } impl Rows { diff --git a/crates/rk/src/selftests.rs b/crates/rk/src/selftests.rs index f826ad6..57cb60d 100644 --- a/crates/rk/src/selftests.rs +++ b/crates/rk/src/selftests.rs @@ -338,7 +338,7 @@ pub fn libraries<'a>(programs: impl IntoIterator) -> Vec<(Strin } /// Every file under a directory, by its path relative to `base`. -fn walk(dir: &Path, base: &Path, out: &mut Vec<(String, Vec)>) -> Result<(), String> { +pub fn walk(dir: &Path, base: &Path, out: &mut Vec<(String, Vec)>) -> Result<(), String> { let entries = std::fs::read_dir(dir).map_err(|e| format!("reading {}: {e}", dir.display()))?; let mut paths: Vec = entries.filter_map(|e| e.ok().map(|e| e.path())).collect(); paths.sort(); diff --git a/crates/rk/src/testrun.rs b/crates/rk/src/testrun.rs index a310818..44b63fd 100644 --- a/crates/rk/src/testrun.rs +++ b/crates/rk/src/testrun.rs @@ -23,7 +23,11 @@ //! The selftests are units too, `kselftest::`, one boot per collection, with //! the programs `rk selftests` built put under `/kselftest` in the initramfs with the libraries //! they need. They are user programs, so a rucc-built set passed with `--rucc-selftests` is part -//! of the rucc userland for attribution, the way `--rucc-busybox` is. LTP is refused for now. +//! of the rucc userland for attribution, the way `--rucc-busybox` is. +//! +//! LTP is the same, one boot per runtest file pinned for the row, with what `rk ltp` installed for +//! that file put under `/ltp`. Every tag is a unit, `ltp::`, and a rucc-built LTP +//! passed with `--rucc-ltp` is part of the rucc userland too. //! //! What comes out is `test.json` and `summary.md` in the run directory, next to the console of //! every boot. @@ -48,6 +52,8 @@ pub enum Kind { Kunit, /// The kernel's selftests, one boot per collection. Kselftest, + /// LTP, one boot per runtest file. + Ltp, } impl Kind { @@ -59,6 +65,7 @@ impl Kind { Self::Smoke => "smoke", Self::Kunit => "kunit", Self::Kselftest => "kselftest", + Self::Ltp => "ltp", } } } @@ -72,14 +79,10 @@ pub fn parse_kinds(text: &str) -> Result, String> { "smoke" => Kind::Smoke, "kunit" => Kind::Kunit, "kselftest" => Kind::Kselftest, - "ltp" => { - return Err( - "ltp is not run yet; rk test runs boot, smoke, kunit and kselftest".to_string(), - ); - } + "ltp" => Kind::Ltp, other => { return Err(format!( - "unknown test kind {other}; the kinds are boot, smoke, kunit and kselftest" + "unknown test kind {other}; the kinds are boot, smoke, kunit, kselftest and ltp" )); } }; @@ -95,9 +98,9 @@ pub fn parse_kinds(text: &str) -> Result, String> { } /// The suites, each one boot, that the kinds need, with one `kselftest:` suite for -/// each selftest collection. +/// each selftest collection and one `ltp:` suite for each runtest file. #[must_use] -pub fn suites(kinds: &[Kind], collections: &[String]) -> Vec { +pub fn suites(kinds: &[Kind], collections: &[String], runtests: &[String]) -> Vec { let mut out = Vec::new(); if kinds.contains(&Kind::Boot) || kinds.contains(&Kind::Smoke) { out.push("smoke".to_string()); @@ -108,6 +111,9 @@ pub fn suites(kinds: &[Kind], collections: &[String]) -> Vec { if kinds.contains(&Kind::Kselftest) { out.extend(collections.iter().map(|c| format!("kselftest:{c}"))); } + if kinds.contains(&Kind::Ltp) { + out.extend(runtests.iter().map(|r| format!("ltp:{r}"))); + } out } @@ -118,6 +124,8 @@ pub fn kind_of(unit: &str) -> Kind { Kind::Smoke } else if unit.starts_with("kselftest:") { Kind::Kselftest + } else if unit.starts_with("ltp:") { + Kind::Ltp } else if unit.starts_with("kunit") { Kind::Kunit } else { @@ -176,6 +184,9 @@ pub fn units_of(suite: &str, kinds: &[Kind], outcome: &boot::Outcome, console: & if suite.starts_with("kselftest:") { units.extend(crate::selftests::units(console)); } + if suite.starts_with("ltp:") { + units.extend(crate::ltp::units(console)); + } units.retain(|unit, _| kinds.contains(&kind_of(unit))); units } @@ -269,6 +280,10 @@ pub struct Plan { pub selftests: Option, /// The rucc-built selftests. Without them both kernels run `selftests`. pub rucc_selftests: Option, + /// The GCC-built LTP, an `rk ltp` output directory. + pub ltp: Option, + /// The rucc-built LTP. Without it both kernels run `ltp`. + pub rucc_ltp: Option, /// The kinds of test. pub kinds: Vec, /// How many times each kernel runs every suite. @@ -310,15 +325,35 @@ pub fn modules(build: &Path) -> Result)>, String> { } impl Plan { + /// The suites to boot, with the selftest collections and runtest files that have something + /// built. + fn suites(&self) -> Result, String> { + let collections = match &self.selftests { + Some(dir) if self.kinds.contains(&Kind::Kselftest) => { + crate::selftests::Outcome::load(dir)?.runnable() + } + _ => Vec::new(), + }; + let runtests = match &self.ltp { + Some(dir) if self.kinds.contains(&Kind::Ltp) => { + crate::ltp::Outcome::load(dir)?.runnable() + } + _ => Vec::new(), + }; + Ok(suites(&self.kinds, &collections, &runtests)) + } + /// The GCC userland and the rucc one, which is the GCC one where nothing else was given. fn userlands(&self) -> (Userland<'_>, Userland<'_>) { let gcc = Userland { busybox: &self.busybox, selftests: self.selftests.as_deref(), + ltp: self.ltp.as_deref(), }; let rucc = Userland { busybox: self.rucc_busybox.as_deref().unwrap_or(&self.busybox), selftests: self.rucc_selftests.as_deref().or(gcc.selftests), + ltp: self.rucc_ltp.as_deref().or(gcc.ltp), }; (gcc, rucc) } @@ -331,15 +366,18 @@ pub fn suite_kind(suite: &str) -> Kind { "smoke" => Kind::Smoke, "kunit" => Kind::Kunit, _ if suite.starts_with("kselftest:") => Kind::Kselftest, + _ if suite.starts_with("ltp:") => Kind::Ltp, _ => Kind::Boot, } } -/// A userland: the busybox and, for the selftests, an `rk selftests` output directory. +/// A userland: the busybox and, for the selftests and LTP, the output directories of +/// `rk selftests` and `rk ltp`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct Userland<'a> { busybox: &'a [u8], selftests: Option<&'a Path>, + ltp: Option<&'a Path>, } /// One boot of one kernel with one userland running one suite, named by `stem` in the run @@ -358,9 +396,19 @@ fn boot_suite( .selftests .ok_or("the kselftest kind needs --selftests, an rk selftests directory")?; crate::selftests::files(dir, collection)? + } else if let Some(runtest) = suite.strip_prefix("ltp:") { + let dir = userland + .ltp + .ok_or("the ltp kind needs --ltp, an rk ltp directory")?; + crate::ltp::files(dir, runtest)? } else { Vec::new() }; + let timeout = if suite.starts_with("ltp:") { + crate::ltp::timeout(plan.timeout, &files) + } else { + plan.timeout + }; let busybox = userland.busybox; let initramfs = plan.out.join(format!("{stem}.cpio")); std::fs::write(&initramfs, boot::initramfs_with(busybox, &files)) @@ -370,7 +418,7 @@ fn boot_suite( build: build.to_path_buf(), row: plan.row.clone(), initramfs: initramfs.clone(), - timeout: plan.timeout, + timeout, append: format!("rk.suite={suite}"), stem: Some(plan.out.join(stem)), }); @@ -440,13 +488,7 @@ pub fn run(plan: &Plan) -> Result { .map_err(|e| format!("creating {}: {e}", plan.out.display()))?; let (gcc, rucc) = plan.userlands(); let two_userlands = gcc != rucc; - let collections = match &plan.selftests { - Some(dir) if plan.kinds.contains(&Kind::Kselftest) => { - crate::selftests::Outcome::load(dir)?.runnable() - } - _ => Vec::new(), - }; - let suites = suites(&plan.kinds, &collections); + let suites = plan.suites()?; let mut reference = Results::new(); let mut other = Results::new(); let mut splats = Splats::default(); @@ -617,15 +659,19 @@ mod tests { parse_kinds("boot,kselftest").unwrap(), [Kind::Boot, Kind::Kselftest] ); - assert!(parse_kinds("ltp").unwrap_err().contains("not run yet")); + assert_eq!(parse_kinds("ltp,boot").unwrap(), [Kind::Boot, Kind::Ltp]); assert!(parse_kinds("fuzz").unwrap_err().contains("unknown")); let timers = ["timers".to_string(), "size".to_string()]; - assert_eq!(suites(&[Kind::Boot], &timers), ["smoke"]); - assert_eq!(suites(&[Kind::Kunit], &timers), ["kunit"]); + let syscalls = ["syscalls".to_string()]; + assert_eq!(suites(&[Kind::Boot], &timers, &syscalls), ["smoke"]); + assert_eq!(suites(&[Kind::Kunit], &timers, &syscalls), ["kunit"]); assert_eq!( - suites(&[Kind::Kselftest], &timers), + suites(&[Kind::Kselftest], &timers, &syscalls), ["kselftest:timers", "kselftest:size"] ); + assert_eq!(suites(&[Kind::Ltp], &timers, &syscalls), ["ltp:syscalls"]); + assert_eq!(kind_of("ltp:syscalls:abort01"), Kind::Ltp); + assert_eq!(suite_kind("ltp:syscalls"), Kind::Ltp); assert_eq!(kind_of("kselftest:timers:posix_timers"), Kind::Kselftest); assert_eq!(suite_kind("kselftest:timers"), Kind::Kselftest); assert_eq!(suite_kind("kunit"), Kind::Kunit); diff --git a/ltp.toml b/ltp.toml new file mode 100644 index 0000000..8e25307 --- /dev/null +++ b/ltp.toml @@ -0,0 +1,9 @@ +# The LTP release `rk ltp` builds (plan 11.5). The tarball is the full one from the release page, +# checked against `sha256` before it is unpacked into the cache. The runtest files each row runs +# are pinned in rows.toml. +# +# To move the pin, take the hash from the `.sha256` file published next to the tarball. + +version = "20260529" +url = "https://github.com/linux-test-project/ltp/releases/download/20260529/ltp-full-20260529.tar.xz" +sha256 = "685d83c6e370ac09201fb79593412f868fe031ee2890e204b5727fedcf51fb47" diff --git a/rk-init/init.sh b/rk-init/init.sh index 7129283..b2afd0f 100644 --- a/rk-init/init.sh +++ b/rk-init/init.sh @@ -7,7 +7,8 @@ # in the order kbuild built them, which runs the KUnit tests they hold, and tries again with the # ones that failed for as long as that loads more, since a module can need one later in the # order. Every line rk boot and rk test read starts with RK-, apart from the TAP of KUnit and of the -# selftests, which the kselftest: suite runs from /kselftest. The +# selftests, which the kselftest: suite runs from /kselftest. The ltp: suite +# runs one LTP runtest file from /ltp and prints an RK-LTP line per test. The # C version that docs/plan/11-boot-and-tests.md describes replaces this once rucc can build it, # and until then both kernels run the same busybox, so the userland is never the difference. @@ -81,10 +82,39 @@ kselftest() { cd / } +# One runtest file of LTP from /ltp. Each test runs in an empty directory under /tmp with its +# output kept aside, which goes to the console behind "# " only when the test fails. LTP exits 32 +# when there was nothing to test here, which is a skip. +ltp() { + file=/ltp/runtest/$1 + [ -f "$file" ] || return 0 + mkdir -p /dev/shm + mount -t tmpfs tmpfs /dev/shm 2>/dev/null + export LTPROOT=/ltp TMPDIR=/tmp PATH=/ltp/testcases/bin:/bin + grep -v '^[[:space:]]*#' "$file" | while read -r tag command; do + [ -n "$tag" ] && [ -n "$command" ] || continue + mkdir -p /tmp/rk-ltp + cd /tmp/rk-ltp + timeout 600 sh -c "$command" /tmp/rk-ltp.out 2>&1 + code=$? + cd / + case $code in + 0) result=pass ;; + 32) result=skip ;; + *) result=fail ;; + esac + [ $result = fail ] && sed 's/^/# /' /tmp/rk-ltp.out + echo "RK-LTP $1 $tag $result $code" + rm -rf /tmp/rk-ltp /tmp/rk-ltp.out + done + export PATH=/bin +} + case $suite in smoke) smoke ;; kunit) kunit ;; kselftest:*) kselftest "${suite#kselftest:}" ;; + ltp:*) ltp "${suite#ltp:}" ;; *) echo "RK-SUITE $suite unknown" ;; esac diff --git a/rows.toml b/rows.toml index 6848846..3a5abab 100644 --- a/rows.toml +++ b/rows.toml @@ -4,7 +4,8 @@ # leaves it out when the host is the row's own architecture, so a row can name it and still build # natively. `image` is the make target for the boot image. The QEMU fields are read by `rk boot`. # `selftests` are the kselftest collections pinned for the row (plan 11.5), which `rk selftests` -# builds and `rk test --kinds kselftest` runs. +# builds and `rk test --kinds kselftest` runs. `ltp` are the LTP runtest files, which `rk ltp` +# builds and `rk test --kinds ltp` runs, one boot each. [[row]] name = "X64" @@ -20,6 +21,7 @@ selftests = [ "mqueue", "pidfd", "proc", "rseq", "seccomp", "sigaltstack", "size", "timens", "timers", "vDSO", "ptrace", "syscall_user_dispatch", "landlock", "splice", "sync", "user_events", "x86", ] +ltp = ["syscalls", "syscalls-ipc", "mm", "fs", "pty", "sched", "math", "cve", "containers"] [[row]] name = "A64" @@ -35,6 +37,7 @@ selftests = [ "mqueue", "pidfd", "proc", "rseq", "seccomp", "sigaltstack", "size", "timens", "timers", "vDSO", "ptrace", "syscall_user_dispatch", "landlock", "splice", "sync", "user_events", "arm64", ] +ltp = ["syscalls", "syscalls-ipc", "mm", "fs", "pty", "sched", "math", "cve", "containers"] [[row]] name = "X32"