From d665c29367ff351a6e1aba329ad18b32e48c9804 Mon Sep 17 00:00:00 2001 From: Manu Nicholas Jacob Date: Tue, 10 Nov 2026 10:19:00 -0600 Subject: [PATCH] system, check: warn when a benchmark ran under a cgroup CPU quota pyperf system show registers ten operations and pyperf check warns on four statistical conditions, but nothing looks at whether the run was throttled. A cgroup CPU quota (docker run --cpus, a Kubernetes CPU limit, systemd CPUQuota=) throttles the benchmark: it moves the mean and leaves the relative spread alone, so a capped run comes back looking statistically stable while being wrong, and the advice offered on an unstable result (more runs, system tune) does not help because the machine was not jittery, it was capped. Add a "CPU quota" operation next to "Power supply" that reports a binding quota and, from cgroup v2 cpu.stat, whether the benchmark cgroup was throttled. Reuse get_cgroup_cpu_quota() from the cpu_quota metadata (gh-265) and add get_cgroup_cpu_throttling() beside it. pyperf check emits the same warning from the cpu_quota metadata of a saved result, so an already stored result can be judged after the fact. Signed-off-by: Manu Nicholas Jacob --- doc/system.rst | 6 +++ pyperf/_cli.py | 15 +++++++ pyperf/_collect_metadata.py | 73 +++++++++++++++++++++++++++++++++++ pyperf/_system.py | 39 +++++++++++++++++++ pyperf/tests/test_cli.py | 20 ++++++++++ pyperf/tests/test_metadata.py | 53 +++++++++++++++++++++++++ pyperf/tests/test_system.py | 39 +++++++++++++++++++ 7 files changed, 245 insertions(+) diff --git a/doc/system.rst b/doc/system.rst index b5642970..04b136f2 100644 --- a/doc/system.rst +++ b/doc/system.rst @@ -191,6 +191,12 @@ The :ref:`pyperf system command ` implements the following operation * "Power supply": check that the power cable is plugged. If the power cable is unplugged (a laptop running only on a battery), the CPU speed can change when the battery level becomes too low. +* "CPU quota": report a cgroup CPU bandwidth limit (``docker run --cpus``, a + Kubernetes CPU limit, systemd ``CPUQuota=``) and whether the benchmark cgroup + has been throttled. A CPU quota throttles the benchmark, which moves the mean + without widening the relative spread, so a capped run can look statistically + stable while being wrong. ``pyperf check`` emits the same warning from the + ``cpu_quota`` metadata of a saved result. * "Turbo Boost (MSR)": use ``/dev/cpu/N/msr`` to read/write the Turbo Boost mode of Intel CPUs * "Turbo Boost (intel_pstate driver)": read from/write into diff --git a/pyperf/_cli.py b/pyperf/_cli.py index b9ccaa91..24f77be8 100644 --- a/pyperf/_cli.py +++ b/pyperf/_cli.py @@ -493,6 +493,21 @@ def format_checks(bench, lines=None, check_too_many_processes=False): warn("See https://bugzilla.redhat.com/show_bug.cgi?id=1378529") break + # Warn if the benchmark ran under a cgroup CPU quota (cpu_quota metadata). + # A quota throttles the run, which moves the mean but leaves the relative + # spread alone, so the statistical checks above cannot see it: a capped run + # comes back looking stable. + for run in bench._runs: + cpu_quota = run._metadata.get('cpu_quota') + if cpu_quota: + empty_line(lines) + lines.append("WARNING: the benchmark ran under a CPU quota (%s)" + % cpu_quota) + lines.append("A CPU quota throttles the benchmark: the result can " + "look stable while being capped. Run without a CPU " + "quota for reliable results.") + break + return lines diff --git a/pyperf/_collect_metadata.py b/pyperf/_collect_metadata.py index 8516cd13..e17b8d96 100644 --- a/pyperf/_collect_metadata.py +++ b/pyperf/_collect_metadata.py @@ -422,6 +422,79 @@ def collect_cpu_quota(metadata): metadata['cpu_quota'] = '%s CPUs' % format(quota, 'g') +def _read_cgroup_cpu_stat(path): + # Parse a cgroup "cpu.stat" file ("key value" lines) into a dict of ints. + # Return None if the file cannot be read. + stat = {} + found = False + try: + with open_text(path) as fp: + for line in fp: + fields = line.split() + if len(fields) == 2: + try: + stat[fields[0]] = int(fields[1]) + found = True + except ValueError: + continue + except OSError: + return None + return stat if found else None + + +def get_cgroup_cpu_throttling(): + """Get the Linux cgroup CPU throttling counters of the current process. + + Return a dict with 'nr_throttled' (number of enforcement periods in which + the cgroup was throttled) and 'throttled_usec' (total time spent throttled, + in microseconds), or None if the counters cannot be read (ex: not Linux, no + cgroup, or no cpu controller). The counters are cumulative since the cgroup + was created, so None means "could not be checked", which is distinct from + zero ("checked, never throttled"). + + With cgroup v2 a limit set on a parent cgroup also throttles, and the + throttling is recorded in the cgroup where the limit binds while the root + counters stay at zero, so the largest counters in the hierarchy are used. + """ + nr_throttled = None + throttled_usec = None + for line in read_proc("self/cgroup"): + parts = line.split(":", 2) + if len(parts) != 3: + continue + hierarchy, controllers, path = parts + path = path.strip("/") + names = path.split("/") if path else [] + if hierarchy == "0" and not controllers: + # cgroup v2: check the cgroup and each of its parents + bases = [os.path.join("/sys/fs/cgroup", *names[:index]) + for index in range(len(names), -1, -1)] + elif "cpu" in controllers.split(","): + # cgroup v1: the cgroup path is not visible inside a container + bases = [os.path.join("/sys/fs/cgroup/cpu", *names), + "/sys/fs/cgroup/cpu"] + else: + continue + for base in bases: + stat = _read_cgroup_cpu_stat(os.path.join(base, "cpu.stat")) + if stat is None: + continue + if "nr_throttled" in stat: + nr_throttled = max(nr_throttled or 0, stat["nr_throttled"]) + if "throttled_usec" in stat: + # cgroup v2 + throttled_usec = max(throttled_usec or 0, + stat["throttled_usec"]) + elif "throttled_time" in stat: + # cgroup v1 reports nanoseconds + throttled_usec = max(throttled_usec or 0, + stat["throttled_time"] // 1000) + if nr_throttled is None and throttled_usec is None: + return None + return {"nr_throttled": nr_throttled or 0, + "throttled_usec": throttled_usec or 0} + + def collect_cpu_model(metadata): for line in read_proc("cpuinfo"): if line.startswith('model name'): diff --git a/pyperf/_system.py b/pyperf/_system.py index d0f8e526..8af5c4fb 100644 --- a/pyperf/_system.py +++ b/pyperf/_system.py @@ -983,6 +983,44 @@ def write(self, tune): self.log_action("Max sample rate set to %s per second" % new_rate) +class CPUQuota(Operation): + """ + Detect a cgroup CPU bandwidth limit: docker run --cpus, a Kubernetes CPU + limit, systemd CPUQuota=. A quota throttles the benchmark, which moves the + mean without widening the relative spread, so a capped run can look + statistically stable while being wrong. Nothing pyperf can tune fixes it, + so this only reports the state, like the power supply check. + """ + + @staticmethod + def available(): + return OS_LINUX and os.path.exists('/sys/fs/cgroup') + + def __init__(self, system): + super().__init__('CPU quota', system) + + def show(self): + from pyperf._collect_metadata import (get_cgroup_cpu_quota, + get_cgroup_cpu_throttling) + + quota = get_cgroup_cpu_quota() + if quota is None: + return + + self.log_state('a CPU quota of %s CPUs is set' % format(quota, 'g')) + + throttling = get_cgroup_cpu_throttling() + if throttling and throttling['nr_throttled']: + self.log_state('the benchmark cgroup was CPU-throttled %s times ' + '(%.1f ms total)' + % (throttling['nr_throttled'], + throttling['throttled_usec'] / 1000.0)) + + self.advice('A CPU quota throttles the benchmark and can make a ' + 'capped run look stable; run without a CPU quota to get ' + 'reliable results') + + OPERATIONS = [ # Generic operations PerfEvent, @@ -991,6 +1029,7 @@ def write(self, tune): CPUFrequency, IRQAffinity, PowerSupply, + CPUQuota, # Setting the CPU scaling governor resets no_turbo # and so must be set before Turbo Boost diff --git a/pyperf/tests/test_cli.py b/pyperf/tests/test_cli.py index 4452bca1..89b18381 100644 --- a/pyperf/tests/test_cli.py +++ b/pyperf/tests/test_cli.py @@ -27,6 +27,26 @@ def test_format_result_calibration(self): 'Calibration: 100 loops') self.assertRaises(ValueError, bench.median) + def _quota_bench(self, metadata): + md = {'name': 'bench'} + md.update(metadata) + runs = [pyperf.Run([1.0, 1.0, 1.0], metadata=md, + collect_metadata=False) + for _ in range(3)] + return pyperf.Benchmark(runs) + + def test_checks_warns_on_cpu_quota(self): + # a run under a CPU quota must warn even though the values look stable + bench = self._quota_bench({'cpu_quota': '2 CPUs'}) + text = '\n'.join(cli.format_checks(bench)) + self.assertIn('CPU quota', text) + self.assertIn('2 CPUs', text) + + def test_checks_no_cpu_quota(self): + bench = self._quota_bench({}) + text = '\n'.join(cli.format_checks(bench)) + self.assertNotIn('CPU quota', text) + if __name__ == "__main__": unittest.main() diff --git a/pyperf/tests/test_metadata.py b/pyperf/tests/test_metadata.py index ae1d7bc9..3ac7f0e5 100644 --- a/pyperf/tests/test_metadata.py +++ b/pyperf/tests/test_metadata.py @@ -102,6 +102,59 @@ def test_cpu_quota_no_limit(self): # no cgroup information at all (ex: not Linux) self.assertIsNone(self.check_cpu_quota({})) + def check_cpu_throttling(self, files): + def mock_open(filename, *args, **kw): + filename = filename.replace('\\', '/') + if filename not in files: + raise FileNotFoundError(filename) + return io.StringIO(files[filename]) + + with mock.patch('pyperf._utils.open', create=True, + side_effect=mock_open): + return perf_metadata.get_cgroup_cpu_throttling() + + def test_cpu_throttling_cgroup_v2(self): + # the quota binds on the process cgroup and throttling is recorded + # there, while the root counters stay at zero + files = { + '/proc/self/cgroup': '0::/user.slice/run-1.scope\n', + '/sys/fs/cgroup/user.slice/run-1.scope/cpu.stat': + ('usage_usec 999\nnr_periods 2470\n' + 'nr_throttled 3306\nthrottled_usec 247631465\n'), + '/sys/fs/cgroup/user.slice/cpu.stat': + 'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + '/sys/fs/cgroup/cpu.stat': + 'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 3306, 'throttled_usec': 247631465}) + + def test_cpu_throttling_zero_is_not_none(self): + # checked and never throttled: zero, distinct from "could not check" + files = { + '/proc/self/cgroup': '0::/init.scope\n', + '/sys/fs/cgroup/init.scope/cpu.stat': + 'usage_usec 5\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 0, 'throttled_usec': 0}) + + def test_cpu_throttling_cgroup_v1(self): + # cgroup v1 reports throttled_time in nanoseconds + files = { + '/proc/self/cgroup': '5:cpu,cpuacct:/docker/abc\n', + '/sys/fs/cgroup/cpu/docker/abc/cpu.stat': + 'nr_periods 100\nnr_throttled 7\nthrottled_time 456789000\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 7, 'throttled_usec': 456789}) + + def test_cpu_throttling_no_cgroup(self): + # no cpu.stat to read (ex: not Linux): could not be checked + self.assertIsNone(self.check_cpu_throttling({})) + files = {'/proc/self/cgroup': '0::/init.scope\n'} + self.assertIsNone(self.check_cpu_throttling(files)) + class CpuFunctionsTests(unittest.TestCase): INTEL_CPU_INFO = textwrap.dedent(""" diff --git a/pyperf/tests/test_system.py b/pyperf/tests/test_system.py index e8d519a3..ef8a83ad 100644 --- a/pyperf/tests/test_system.py +++ b/pyperf/tests/test_system.py @@ -1,10 +1,49 @@ import os.path import sys import unittest +from unittest import mock from pyperf.tests import get_output +class CPUQuotaTests(unittest.TestCase): + def _show(self, quota, throttling): + from pyperf._system import CPUQuota + + system = mock.Mock() + op = CPUQuota(system) + with mock.patch('pyperf._collect_metadata.get_cgroup_cpu_quota', + return_value=quota), \ + mock.patch('pyperf._collect_metadata.get_cgroup_cpu_throttling', + return_value=throttling): + op.show() + states = ' '.join(call.args[0] + for call in system.log_state.call_args_list) + advices = ' '.join(call.args[0] + for call in system.advice.call_args_list) + return states, advices + + def test_no_quota_is_silent(self): + states, advices = self._show(None, None) + self.assertEqual(states, '') + self.assertEqual(advices, '') + + def test_quota_set_but_not_throttled(self): + states, advices = self._show(2.0, {'nr_throttled': 0, + 'throttled_usec': 0}) + self.assertIn('2 CPUs', states) + self.assertNotIn('throttled', states) + self.assertIn('CPU quota', advices) + + def test_quota_and_throttled(self): + states, advices = self._show(2.0, {'nr_throttled': 3306, + 'throttled_usec': 247631465}) + self.assertIn('2 CPUs', states) + self.assertIn('3306', states) + self.assertIn('throttled', states.lower()) + self.assertIn('CPU quota', advices) + + class SystemTests(unittest.TestCase): def test_show(self): args = [sys.executable, '-m', 'pyperf', 'system', 'show']