Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions doc/ci.md
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,16 @@ the guest must have installed the IPv6 default route that the last token,
`virtnet_gw6=`, names. OpenVMM must refuse a command line one byte longer
before boot, with an error that names its length and the limit.

The `cpu-bandwidth` scenario covers the guest kernel's
[CPU controller](design/4.3-isolation-and-resource-model.md#433-cgroups-and-resources),
through which a guest agent caps an execution's CPU time with its cgroup's
`cpu.max` (#1193). The guest enables the controller in the root cgroup,
limits a cgroup to a quarter of a CPU, 25 ms of every 100 ms period, and runs
a busy loop in it for a second. The cgroup's `cpu.max` must hold the limit,
its `cpu.stat` must show that the kernel throttled the loop, and the loop must
have run for at most half of the wall time, twice the limit's share, which a
loop without the limit exceeds.

The `virtio-features` scenario covers the guest-visible virtio features of
[Table 2.8](design/2.5-device-topology-transport-and-discovery.md). It boots a
guest with every fixed virtio device: the NIC, the dormant host filesystem
Expand Down
2 changes: 1 addition & 1 deletion scripts/nvx_tools/kernel_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -485,7 +485,7 @@ class KernelFeature:
"task group unless a guest agent sets a limit."
),
consumers=("nvx::guest::cgroup", "guest agents of runtime adapters"),
validation=("build-kernel's sandbox group",),
validation=("build-kernel's sandbox group", "the cpu-bandwidth scenario"),
settings=(
"CONFIG_CGROUP_SCHED=y",
"CONFIG_FAIR_GROUP_SCHED=y",
Expand Down
12 changes: 9 additions & 3 deletions scripts/nvx_tools/microvm_shards.json
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
"command-line-limit": 1.5,
"console-exit": 25.2,
"console-snapshot": 49.1,
"cpu-bandwidth": 2.0,
"cpu-profile-fallback": 1.5,
"denied-filesystem-paths": 0.8,
"directional-network-policy": 5.8,
Expand Down Expand Up @@ -74,10 +75,11 @@
},
"shards": [
{
"seconds": 269.1,
"seconds": 271.1,
"scenarios": [
"command-line-limit",
"console-snapshot",
"cpu-bandwidth",
"directional-network-policy",
"endpoint-policy-snapshot",
"filesystem-comma-paths",
Expand Down Expand Up @@ -137,6 +139,7 @@
"command-line-limit": 1.2,
"console-exit": 27.2,
"console-snapshot": 1.0,
"cpu-bandwidth": 2.0,
"cpu-profile-fallback": 1.7,
"denied-filesystem-paths": 0.6,
"directional-network-policy": 5.7,
Expand Down Expand Up @@ -209,9 +212,10 @@
]
},
{
"seconds": 249.8,
"seconds": 251.8,
"scenarios": [
"console-exit",
"cpu-bandwidth",
"cpu-profile-fallback",
"denied-filesystem-paths",
"filesystem-comma-paths",
Expand Down Expand Up @@ -247,6 +251,7 @@
"command-line-limit": 2.3,
"console-exit": 87.5,
"console-snapshot": 3.9,
"cpu-bandwidth": 2.0,
"cpu-profile-fallback": 4.3,
"denied-filesystem-paths": 1.3,
"directional-network-policy": 7.0,
Expand Down Expand Up @@ -294,8 +299,9 @@
},
"shards": [
{
"seconds": 356.1,
"seconds": 358.1,
"scenarios": [
"cpu-bandwidth",
"denied-filesystem-paths",
"directional-network-policy",
"endpoint-policy-snapshot",
Expand Down
35 changes: 35 additions & 0 deletions scripts/nvx_tools/microvm_test_scripts/cpu-bandwidth.sh.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
set -eu
fail() {
echo "NVX-CPU-BANDWIDTH-FAIL $1"
nvx-exit 97
exit 97
}
cgroups=/sys/fs/cgroup
grep -q " $cgroups cgroup2 " /proc/mounts ||
mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 "$cgroups"
echo +cpu >"$cgroups/cgroup.subtree_control" ||
fail "the kernel has no CPU controller"
limited=$cgroups/nvx-cpu-bandwidth
mkdir "$limited"
echo "@QUOTA_US@ @PERIOD_US@" >"$limited/cpu.max" ||
fail "cpu.max refused the limit"
read -r start _ </proc/uptime
# The loop moves itself into the limited cgroup, writing 0, before it spins.
(
echo 0 >"$limited/cgroup.procs"
while :; do :; done
) &
loop=$!
sleep 1
read -r end _ </proc/uptime
kill "$loop" || fail "the busy loop ended early"
read -r quota period <"$limited/cpu.max"
# /proc/uptime has two decimals, which a leading 1 keeps from reading as octal.
wall=$(((${end%.*} - ${start%.*}) * 100 + 1${end#*.} - 1${start#*.}))
report="quota_us=$quota period_us=$period wall_cs=$wall"
while read -r key value; do
report="$report $key=$value"
done <"$limited/cpu.stat"
echo "NVX-CPU-BANDWIDTH $report"
echo NVX-CPU-BANDWIDTH-OK
nvx-exit 0
81 changes: 81 additions & 0 deletions scripts/nvx_tools/microvm_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@
"command-line-limit",
"console-exit",
"console-snapshot",
"cpu-bandwidth",
"cpu-profile-fallback",
"directional-network-policy",
"denied-filesystem-paths",
Expand Down Expand Up @@ -277,6 +278,13 @@ class MicrovmVirtioDevice(NamedTuple):
b"microVM kernel command line is 2048 bytes; "
b"the limit is 2047 bytes plus the terminating NUL"
)
CPU_BANDWIDTH_COMPLETION_MARKER = b"NVX-CPU-BANDWIDTH-OK"
# The cpu.max limit of the cpu-bandwidth scenario's busy loop, a quarter of a
# CPU, and the prefix of the line on which its guest reports the limit that it
# reads back, the wall time in centiseconds, and the cgroup's cpu.stat.
CPU_BANDWIDTH_QUOTA_US = 25_000
CPU_BANDWIDTH_PERIOD_US = 100_000
CPU_BANDWIDTH_REPORT_PREFIX = b"NVX-CPU-BANDWIDTH "
DIRECTIONAL_NETWORK_ALLOW_MARKER = b"NVX-DIRECTIONAL-NETWORK-ALLOW-OK"
DIRECTIONAL_NETWORK_DENY_MARKER = b"NVX-DIRECTIONAL-NETWORK-DENY-OK"
DIRECTIONAL_NETWORK_INGRESS_READY_MARKER = b"NVX-DIRECTIONAL-INGRESS-READY"
Expand Down Expand Up @@ -2149,6 +2157,65 @@ def boot(padding: int, log_name: str) -> str:
)


def run_cpu_bandwidth(
executable: Path,
kernel: Path,
initrd: Path,
backend: str,
*,
memory_mib: int,
timeout: float,
log_path: Path,
) -> None:
"""Cap a busy loop at a quarter of a CPU with its cgroup's cpu.max (#1193).

The guest runs the loop for a second in a cgroup whose cpu.max allows
CPU_BANDWIDTH_QUOTA_US of every CPU_BANDWIDTH_PERIOD_US, and reports the
limit, the wall time, and the cgroup's cpu.stat to check_cpu_bandwidth.
"""
result = run_guest_script(
workload_boot_command(
executable, backend, kernel, initrd, memory_mib, "quiet loglevel=0"
),
_render_script(
"cpu-bandwidth.sh.in",
QUOTA_US=str(CPU_BANDWIDTH_QUOTA_US),
PERIOD_US=str(CPU_BANDWIDTH_PERIOD_US),
),
CPU_BANDWIDTH_COMPLETION_MARKER,
timeout=timeout,
log_path=log_path,
)
check_cpu_bandwidth(result["output"])


def check_cpu_bandwidth(output: str) -> None:
"""Require the one report of the cpu-bandwidth guest's ``output`` to hold
the limit that it wrote, and its loop to have been throttled to at most
twice the limit's share of the wall time, which a loop without it exceeds.
"""
report = _single_marker_value(
output.encode("utf-8"), CPU_BANDWIDTH_REPORT_PREFIX
).decode("utf-8")
try:
fields = parse_fields(report)
limit = (int(fields["quota_us"]), int(fields["period_us"]))
wall_us = int(fields["wall_cs"]) * 10_000
usage_us = int(fields["usage_usec"])
throttled = int(fields["nr_throttled"])
except (KeyError, ValueError) as error:
raise RuntimeError(f"malformed CPU bandwidth report {report!r}") from error
if limit != (CPU_BANDWIDTH_QUOTA_US, CPU_BANDWIDTH_PERIOD_US):
raise RuntimeError(f"cpu.max holds {limit}, not the limit that the guest wrote")
if throttled < 1:
raise RuntimeError("the kernel never throttled the busy loop")
if usage_us * CPU_BANDWIDTH_PERIOD_US > 2 * wall_us * CPU_BANDWIDTH_QUOTA_US:
raise RuntimeError(
f"the busy loop ran for {usage_us} us in {wall_us} us, more than twice "
"its limit's share"
)


def _directional_network_command(
executable: Path,
kernel: Path,
Expand Down Expand Up @@ -11635,3 +11702,17 @@ def _run_scenarios(
timeout=args.timeout,
output_dir=output_dir,
)
if "cpu-bandwidth" in scenarios:
report.start(
"cpu-bandwidth",
f"Running the microVM CPU bandwidth limit on OpenVMM/{args.backend}",
)
run_cpu_bandwidth(
executable,
kernel,
initrd,
args.backend,
memory_mib=args.memory_mib,
timeout=args.timeout,
log_path=output_dir / "cpu-bandwidth.log",
)
48 changes: 48 additions & 0 deletions scripts/test_microvm_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -8692,6 +8692,54 @@ def require(path: Path, _description: str) -> Path:
self.assertEqual(run.call_args.args[3], "mshv")
self.assertEqual(run.call_args.kwargs["output_dir"], Path(temporary))

def test_cpu_bandwidth_requires_cpu_max_to_throttle_a_busy_loop(self):
self.assertIn("cpu-bandwidth", microvm_tests.MICROVM_TEST_SCENARIOS)
# A second at a quarter of a CPU: 25 ms of each 100 ms period, and the
# overruns of the 10 ms ticks.
report = (
"NVX-CPU-BANDWIDTH quota_us=25000 period_us=100000 wall_cs=101 "
"usage_usec=290000 user_usec=290000 system_usec=0 nr_periods=10 "
"nr_throttled=9 throttled_usec=690000 nr_bursts=0 burst_usec=0\r\n"
)
with (
tempfile.TemporaryDirectory() as temporary,
patch.object(
microvm_tests,
"run_guest_script",
return_value=_guest_script_result(f"{report}NVX-CPU-BANDWIDTH-OK\r\n"),
) as run_guest_script,
):
microvm_tests.run_cpu_bandwidth(
Path("openvmm"),
Path("vmlinux"),
Path("initrd"),
"kvm",
memory_mib=128,
timeout=60,
log_path=Path(temporary) / "cpu-bandwidth.log",
)
_, script, marker = run_guest_script.call_args.args
self.assertIn('echo "25000 100000" >"$limited/cpu.max"', script)
self.assertEqual(marker, microvm_tests.CPU_BANDWIDTH_COMPLETION_MARKER)

# Half a CPU, twice the limit's share, is the most that the loop may use.
microvm_tests.check_cpu_bandwidth(
report.replace("usage_usec=290000", "usage_usec=505000")
)
for (old, new), message in (
(("nr_throttled=9", "nr_throttled=0"), "never throttled"),
(("usage_usec=290000", "usage_usec=505001"), "more than twice"),
(("period_us=100000", "period_us=50000"), "not the limit"),
(("quota_us=25000", "quota_us=max"), "malformed"),
((report, ""), "found 0"),
((report, report * 2), "found 2"),
):
with (
self.subTest(message=message),
self.assertRaisesRegex(RuntimeError, message),
):
microvm_tests.check_cpu_bandwidth(report.replace(old, new))

def test_directional_network_commands_map_generic_default_actions(self):
with patch.object(
microvm_tests,
Expand Down
Loading