diff --git a/doc/ci.md b/doc/ci.md index 298d9f2e8..e71277160 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -468,6 +468,16 @@ the guest must have installed the IPv6 default route that the last token, `virtnet_gw6=`, names. OpenVMM must refuse a command line one byte longer before boot, with an error that names its length and the limit. +The `cpu-bandwidth` scenario covers the guest kernel's +[CPU controller](design/4.3-isolation-and-resource-model.md#433-cgroups-and-resources), +through which a guest agent caps an execution's CPU time with its cgroup's +`cpu.max` (#1193). The guest enables the controller in the root cgroup, +limits a cgroup to a quarter of a CPU, 25 ms of every 100 ms period, and runs +a busy loop in it for a second. The cgroup's `cpu.max` must hold the limit, +its `cpu.stat` must show that the kernel throttled the loop, and the loop must +have run for at most half of the wall time, twice the limit's share, which a +loop without the limit exceeds. + The `virtio-features` scenario covers the guest-visible virtio features of [Table 2.8](design/2.5-device-topology-transport-and-discovery.md). It boots a guest with every fixed virtio device: the NIC, the dormant host filesystem diff --git a/scripts/nvx_tools/kernel_features.py b/scripts/nvx_tools/kernel_features.py index c4f4b2918..22563481c 100644 --- a/scripts/nvx_tools/kernel_features.py +++ b/scripts/nvx_tools/kernel_features.py @@ -485,7 +485,7 @@ class KernelFeature: "task group unless a guest agent sets a limit." ), consumers=("nvx::guest::cgroup", "guest agents of runtime adapters"), - validation=("build-kernel's sandbox group",), + validation=("build-kernel's sandbox group", "the cpu-bandwidth scenario"), settings=( "CONFIG_CGROUP_SCHED=y", "CONFIG_FAIR_GROUP_SCHED=y", diff --git a/scripts/nvx_tools/microvm_shards.json b/scripts/nvx_tools/microvm_shards.json index dd15bf727..097ee649f 100644 --- a/scripts/nvx_tools/microvm_shards.json +++ b/scripts/nvx_tools/microvm_shards.json @@ -27,6 +27,7 @@ "command-line-limit": 1.5, "console-exit": 25.2, "console-snapshot": 49.1, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 1.5, "denied-filesystem-paths": 0.8, "directional-network-policy": 5.8, @@ -74,10 +75,11 @@ }, "shards": [ { - "seconds": 269.1, + "seconds": 271.1, "scenarios": [ "command-line-limit", "console-snapshot", + "cpu-bandwidth", "directional-network-policy", "endpoint-policy-snapshot", "filesystem-comma-paths", @@ -137,6 +139,7 @@ "command-line-limit": 1.2, "console-exit": 27.2, "console-snapshot": 1.0, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 1.7, "denied-filesystem-paths": 0.6, "directional-network-policy": 5.7, @@ -209,9 +212,10 @@ ] }, { - "seconds": 249.8, + "seconds": 251.8, "scenarios": [ "console-exit", + "cpu-bandwidth", "cpu-profile-fallback", "denied-filesystem-paths", "filesystem-comma-paths", @@ -247,6 +251,7 @@ "command-line-limit": 2.3, "console-exit": 87.5, "console-snapshot": 3.9, + "cpu-bandwidth": 2.0, "cpu-profile-fallback": 4.3, "denied-filesystem-paths": 1.3, "directional-network-policy": 7.0, @@ -294,8 +299,9 @@ }, "shards": [ { - "seconds": 356.1, + "seconds": 358.1, "scenarios": [ + "cpu-bandwidth", "denied-filesystem-paths", "directional-network-policy", "endpoint-policy-snapshot", diff --git a/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in b/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in new file mode 100644 index 000000000..a562e9604 --- /dev/null +++ b/scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in @@ -0,0 +1,35 @@ +set -eu +fail() { + echo "NVX-CPU-BANDWIDTH-FAIL $1" + nvx-exit 97 + exit 97 +} +cgroups=/sys/fs/cgroup +grep -q " $cgroups cgroup2 " /proc/mounts || + mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 "$cgroups" +echo +cpu >"$cgroups/cgroup.subtree_control" || + fail "the kernel has no CPU controller" +limited=$cgroups/nvx-cpu-bandwidth +mkdir "$limited" +echo "@QUOTA_US@ @PERIOD_US@" >"$limited/cpu.max" || + fail "cpu.max refused the limit" +read -r start _ "$limited/cgroup.procs" + while :; do :; done +) & +loop=$! +sleep 1 +read -r end _ str: ) +def run_cpu_bandwidth( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + memory_mib: int, + timeout: float, + log_path: Path, +) -> None: + """Cap a busy loop at a quarter of a CPU with its cgroup's cpu.max (#1193). + + The guest runs the loop for a second in a cgroup whose cpu.max allows + CPU_BANDWIDTH_QUOTA_US of every CPU_BANDWIDTH_PERIOD_US, and reports the + limit, the wall time, and the cgroup's cpu.stat to check_cpu_bandwidth. + """ + result = run_guest_script( + workload_boot_command( + executable, backend, kernel, initrd, memory_mib, "quiet loglevel=0" + ), + _render_script( + "cpu-bandwidth.sh.in", + QUOTA_US=str(CPU_BANDWIDTH_QUOTA_US), + PERIOD_US=str(CPU_BANDWIDTH_PERIOD_US), + ), + CPU_BANDWIDTH_COMPLETION_MARKER, + timeout=timeout, + log_path=log_path, + ) + check_cpu_bandwidth(result["output"]) + + +def check_cpu_bandwidth(output: str) -> None: + """Require the one report of the cpu-bandwidth guest's ``output`` to hold + the limit that it wrote, and its loop to have been throttled to at most + twice the limit's share of the wall time, which a loop without it exceeds. + """ + report = _single_marker_value( + output.encode("utf-8"), CPU_BANDWIDTH_REPORT_PREFIX + ).decode("utf-8") + try: + fields = parse_fields(report) + limit = (int(fields["quota_us"]), int(fields["period_us"])) + wall_us = int(fields["wall_cs"]) * 10_000 + usage_us = int(fields["usage_usec"]) + throttled = int(fields["nr_throttled"]) + except (KeyError, ValueError) as error: + raise RuntimeError(f"malformed CPU bandwidth report {report!r}") from error + if limit != (CPU_BANDWIDTH_QUOTA_US, CPU_BANDWIDTH_PERIOD_US): + raise RuntimeError(f"cpu.max holds {limit}, not the limit that the guest wrote") + if throttled < 1: + raise RuntimeError("the kernel never throttled the busy loop") + if usage_us * CPU_BANDWIDTH_PERIOD_US > 2 * wall_us * CPU_BANDWIDTH_QUOTA_US: + raise RuntimeError( + f"the busy loop ran for {usage_us} us in {wall_us} us, more than twice " + "its limit's share" + ) + + def _directional_network_command( executable: Path, kernel: Path, @@ -11635,3 +11702,17 @@ def _run_scenarios( timeout=args.timeout, output_dir=output_dir, ) + if "cpu-bandwidth" in scenarios: + report.start( + "cpu-bandwidth", + f"Running the microVM CPU bandwidth limit on OpenVMM/{args.backend}", + ) + run_cpu_bandwidth( + executable, + kernel, + initrd, + args.backend, + memory_mib=args.memory_mib, + timeout=args.timeout, + log_path=output_dir / "cpu-bandwidth.log", + ) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 5526b590f..6a3dd801c 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -8692,6 +8692,54 @@ def require(path: Path, _description: str) -> Path: self.assertEqual(run.call_args.args[3], "mshv") self.assertEqual(run.call_args.kwargs["output_dir"], Path(temporary)) + def test_cpu_bandwidth_requires_cpu_max_to_throttle_a_busy_loop(self): + self.assertIn("cpu-bandwidth", microvm_tests.MICROVM_TEST_SCENARIOS) + # A second at a quarter of a CPU: 25 ms of each 100 ms period, and the + # overruns of the 10 ms ticks. + report = ( + "NVX-CPU-BANDWIDTH quota_us=25000 period_us=100000 wall_cs=101 " + "usage_usec=290000 user_usec=290000 system_usec=0 nr_periods=10 " + "nr_throttled=9 throttled_usec=690000 nr_bursts=0 burst_usec=0\r\n" + ) + with ( + tempfile.TemporaryDirectory() as temporary, + patch.object( + microvm_tests, + "run_guest_script", + return_value=_guest_script_result(f"{report}NVX-CPU-BANDWIDTH-OK\r\n"), + ) as run_guest_script, + ): + microvm_tests.run_cpu_bandwidth( + Path("openvmm"), + Path("vmlinux"), + Path("initrd"), + "kvm", + memory_mib=128, + timeout=60, + log_path=Path(temporary) / "cpu-bandwidth.log", + ) + _, script, marker = run_guest_script.call_args.args + self.assertIn('echo "25000 100000" >"$limited/cpu.max"', script) + self.assertEqual(marker, microvm_tests.CPU_BANDWIDTH_COMPLETION_MARKER) + + # Half a CPU, twice the limit's share, is the most that the loop may use. + microvm_tests.check_cpu_bandwidth( + report.replace("usage_usec=290000", "usage_usec=505000") + ) + for (old, new), message in ( + (("nr_throttled=9", "nr_throttled=0"), "never throttled"), + (("usage_usec=290000", "usage_usec=505001"), "more than twice"), + (("period_us=100000", "period_us=50000"), "not the limit"), + (("quota_us=25000", "quota_us=max"), "malformed"), + ((report, ""), "found 0"), + ((report, report * 2), "found 2"), + ): + with ( + self.subTest(message=message), + self.assertRaisesRegex(RuntimeError, message), + ): + microvm_tests.check_cpu_bandwidth(report.replace(old, new)) + def test_directional_network_commands_map_generic_default_actions(self): with patch.object( microvm_tests,