From 5dae66276a16d59cd4978c02aee1133d29d16a98 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 5 Nov 2026 16:34:51 -0800 Subject: [PATCH] test-microvm: cap a busy loop with cpu.max The guest kernel builds the CPU controller's bandwidth limit, so that a guest agent can cap an execution's CPU time with its cgroup's cpu.max (#1193), but no scenario showed that the limit holds. The cpu-bandwidth scenario enables the controller in the root cgroup, limits a cgroup to a quarter of a CPU, 25 ms of every 100 ms period, and runs a busy loop in it for a second. The guest reports the limit that cpu.max reads back, the wall time, and the cgroup's cpu.stat, and the harness requires the limit, a throttled period, and at most half a CPU, twice the limit's share, which a loop without the limit exceeds. The scenario runs in the default suite, in the shorter shard of each backend, which plan-microvm-shards chose with an estimate of 2 s; it took 1.5 to 1.8 s. doc/ci.md describes it, and the kernel feature inventory names it among the CPU controller's validation. A unit test checks the limit that the scenario's script writes, its marker, and the check's bound and failures. On nvx-kvm-test (Linux/KVM), nvx-mshv-test (Linux/MSHV), and nvx-whp-test (Windows/WHP), all virtual machines, with the release artifacts of 4cfe8330e, the scenario failed with the release kernel, which has no CPU controller, and passed with the kernel of the CPU controller's change. In each of 11 runs, among them one each of the Ubuntu and the Azure Linux guest on nvx-mshv-test, its limit held the loop to 26.8% to 28.1% of the wall time and throttled it in 9 or 10 of 10 periods. The default suite passed with that kernel on all three. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/ci.md | 10 +++ scripts/nvx_tools/kernel_features.py | 2 +- scripts/nvx_tools/microvm_shards.json | 12 ++- .../microvm_test_scripts/cpu-bandwidth.sh.in | 35 ++++++++ scripts/nvx_tools/microvm_tests.py | 81 +++++++++++++++++++ scripts/test_microvm_tests.py | 48 +++++++++++ 6 files changed, 184 insertions(+), 4 deletions(-) create mode 100644 scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in diff --git a/doc/ci.md b/doc/ci.md index 298d9f2e8..e71277160 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -468,6 +468,16 @@ the guest must have installed the IPv6 default route that the last token, `virtnet_gw6=`, names. OpenVMM must refuse a command line one byte longer before boot, with an error that names its length and the limit. +The `cpu-bandwidth` scenario covers the guest kernel's +[CPU controller](design/4.3-isolation-and-resource-model.md#433-cgroups-and-resources), +through which a guest agent caps an execution's CPU time with its cgroup's +`cpu.max` (#1193). The guest enables the controller in the root cgroup, +limits a cgroup to a quarter of a CPU, 25 ms of every 100 ms period, and runs +a busy loop in it for a second. The cgroup's `cpu.max` must hold the limit, +its `cpu.stat` must show that the kernel throttled the loop, and the loop must +have run for at most half of the wall time, twice the limit's share, which a +loop without the limit exceeds. + The `virtio-features` scenario covers the guest-visible virtio features of [Table 2.8](design/2.5-device-topology-transport-and-discovery.md). It boots a guest with every fixed virtio device: the NIC, the dormant host filesystem diff --git a/scripts/nvx_tools/kernel_features.py b/scripts/nvx_tools/kernel_features.py index c4f4b2918..22563481c 100644 --- a/scripts/nvx_tools/kernel_features.py +++ b/scripts/nvx_tools/kernel_features.py @@ -485,7 +485,7 @@ class KernelFeature: "task group unless a guest agent sets a limit." ), consumers=("nvx::guest::cgroup", "guest agents of runtime adapters"), - validation=("build-kernel's sandbox group",), + validation=("build-kernel's sandbox group", "the cpu-bandwidth scenario"), settings=( "CONFIG_CGROUP_SCHED=y", "CONFIG_FAIR_GROUP_SCHED=y", diff --git a/scripts/nvx_tools/microvm_shards.json b/scripts/nvx_tools/microvm_shards.json index dd15bf727..097ee649f 100644 --- a/scripts/nvx_tools/microvm_shards.json +++ b/scripts/nvx_tools/microvm_shards.json @@ -27,6 +27,7 @@ "command-line-limit": 1.5, "console-exit": 25.2, "console-snapshot": 49.1, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 1.5, "denied-filesystem-paths": 0.8, "directional-network-policy": 5.8, @@ -74,10 +75,11 @@ }, "shards": [ { - "seconds": 269.1, + "seconds": 271.1, "scenarios": [ "command-line-limit", "console-snapshot", + "cpu-bandwidth", "directional-network-policy", "endpoint-policy-snapshot", "filesystem-comma-paths", @@ -137,6 +139,7 @@ "command-line-limit": 1.2, "console-exit": 27.2, "console-snapshot": 1.0, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 1.7, "denied-filesystem-paths": 0.6, "directional-network-policy": 5.7, @@ -209,9 +212,10 @@ ] }, { - "seconds": 249.8, + "seconds": 251.8, "scenarios": [ "console-exit", + "cpu-bandwidth", "cpu-profile-fallback", "denied-filesystem-paths", "filesystem-comma-paths", @@ -247,6 +251,7 @@ "command-line-limit": 2.3, "console-exit": 87.5, "console-snapshot": 3.9, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 4.3, "denied-filesystem-paths": 1.3, "directional-network-policy": 7.0, @@ -294,8 +299,9 @@ }, "shards": [ { - "seconds": 356.1, + "seconds": 358.1, "scenarios": [ + "cpu-bandwidth", "denied-filesystem-paths", "directional-network-policy", "endpoint-policy-snapshot", diff --git a/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in b/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in new file mode 100644 index 000000000..a562e9604 --- /dev/null +++ b/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in @@ -0,0 +1,35 @@ +set -eu +fail() { + echo "NVX-CPU-BANDWIDTH-FAIL $1" + nvx-exit 97 + exit 97 +} +cgroups=/sys/fs/cgroup +grep -q " $cgroups cgroup2 " /proc/mounts || + mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 "$cgroups" +echo +cpu >"$cgroups/cgroup.subtree_control" || + fail "the kernel has no CPU controller" +limited=$cgroups/nvx-cpu-bandwidth +mkdir "$limited" +echo "@QUOTA_US@ @PERIOD_US@" >"$limited/cpu.max" || + fail "cpu.max refused the limit" +read -r start _ "$limited/cgroup.procs" + while :; do :; done +) & +loop=$! +sleep 1 +read -r end _ str: ) +def run_cpu_bandwidth( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + memory_mib: int, + timeout: float, + log_path: Path, +) -> None: + """Cap a busy loop at a quarter of a CPU with its cgroup's cpu.max (#1193). + + The guest runs the loop for a second in a cgroup whose cpu.max allows + CPU_BANDWIDTH_QUOTA_US of every CPU_BANDWIDTH_PERIOD_US, and reports the + limit, the wall time, and the cgroup's cpu.stat to check_cpu_bandwidth. + """ + result = run_guest_script( + workload_boot_command( + executable, backend, kernel, initrd, memory_mib, "quiet loglevel=0" + ), + _render_script( + "cpu-bandwidth.sh.in", + QUOTA_US=str(CPU_BANDWIDTH_QUOTA_US), + PERIOD_US=str(CPU_BANDWIDTH_PERIOD_US), + ), + CPU_BANDWIDTH_COMPLETION_MARKER, + timeout=timeout, + log_path=log_path, + ) + check_cpu_bandwidth(result["output"]) + + +def check_cpu_bandwidth(output: str) -> None: + """Require the one report of the cpu-bandwidth guest's ``output`` to hold + the limit that it wrote, and its loop to have been throttled to at most + twice the limit's share of the wall time, which a loop without it exceeds. + """ + report = _single_marker_value( + output.encode("utf-8"), CPU_BANDWIDTH_REPORT_PREFIX + ).decode("utf-8") + try: + fields = parse_fields(report) + limit = (int(fields["quota_us"]), int(fields["period_us"])) + wall_us = int(fields["wall_cs"]) * 10_000 + usage_us = int(fields["usage_usec"]) + throttled = int(fields["nr_throttled"]) + except (KeyError, ValueError) as error: + raise RuntimeError(f"malformed CPU bandwidth report {report!r}") from error + if limit != (CPU_BANDWIDTH_QUOTA_US, CPU_BANDWIDTH_PERIOD_US): + raise RuntimeError(f"cpu.max holds {limit}, not the limit that the guest wrote") + if throttled < 1: + raise RuntimeError("the kernel never throttled the busy loop") + if usage_us * CPU_BANDWIDTH_PERIOD_US > 2 * wall_us * CPU_BANDWIDTH_QUOTA_US: + raise RuntimeError( + f"the busy loop ran for {usage_us} us in {wall_us} us, more than twice " + "its limit's share" + ) + + def _directional_network_command( executable: Path, kernel: Path, @@ -11635,3 +11702,17 @@ def _run_scenarios( timeout=args.timeout, output_dir=output_dir, ) + if "cpu-bandwidth" in scenarios: + report.start( + "cpu-bandwidth", + f"Running the microVM CPU bandwidth limit on OpenVMM/{args.backend}", + ) + run_cpu_bandwidth( + executable, + kernel, + initrd, + args.backend, + memory_mib=args.memory_mib, + timeout=args.timeout, + log_path=output_dir / "cpu-bandwidth.log", + ) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 5526b590f..6a3dd801c 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -8692,6 +8692,54 @@ def require(path: Path, _description: str) -> Path: self.assertEqual(run.call_args.args[3], "mshv") self.assertEqual(run.call_args.kwargs["output_dir"], Path(temporary)) + def test_cpu_bandwidth_requires_cpu_max_to_throttle_a_busy_loop(self): + self.assertIn("cpu-bandwidth", microvm_tests.MICROVM_TEST_SCENARIOS) + # A second at a quarter of a CPU: 25 ms of each 100 ms period, and the + # overruns of the 10 ms ticks. + report = ( + "NVX-CPU-BANDWIDTH quota_us=25000 period_us=100000 wall_cs=101 " + "usage_usec=290000 user_usec=290000 system_usec=0 nr_periods=10 " + "nr_throttled=9 throttled_usec=690000 nr_bursts=0 burst_usec=0\r\n" + ) + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + microvm_tests, + "run_guest_script", + return_value=_guest_script_result(f"{report}NVX-CPU-BANDWIDTH-OK\r\n"), + ) as run_guest_script, + ): + microvm_tests.run_cpu_bandwidth( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + memory_mib=128, + timeout=60, + log_path=Path(temporary) / "cpu-bandwidth.log", + ) + _, script, marker = run_guest_script.call_args.args + self.assertIn('echo "25000 100000" >"$limited/cpu.max"', script) + self.assertEqual(marker, microvm_tests.CPU_BANDWIDTH_COMPLETION_MARKER) + + # Half a CPU, twice the limit's share, is the most that the loop may use. + microvm_tests.check_cpu_bandwidth( + report.replace("usage_usec=290000", "usage_usec=505000") + ) + for (old, new), message in ( + (("nr_throttled=9", "nr_throttled=0"), "never throttled"), + (("usage_usec=290000", "usage_usec=505001"), "more than twice"), + (("period_us=100000", "period_us=50000"), "not the limit"), + (("quota_us=25000", "quota_us=max"), "malformed"), + ((report, ""), "found 0"), + ((report, report * 2), "found 2"), + ): + with ( + self.subTest(message=message), + self.assertRaisesRegex(RuntimeError, message), + ): + microvm_tests.check_cpu_bandwidth(report.replace(old, new)) + def test_directional_network_commands_map_generic_default_actions(self): with patch.object( microvm_tests,