diff --git a/doc/system.rst b/doc/system.rst index b5642970..04b136f2 100644 --- a/doc/system.rst +++ b/doc/system.rst @@ -191,6 +191,12 @@ The :ref:`pyperf system command ` implements the following operation * "Power supply": check that the power cable is plugged. If the power cable is unplugged (a laptop running only on a battery), the CPU speed can change when the battery level becomes too low. +* "CPU quota": report a cgroup CPU bandwidth limit (``docker run --cpus``, a + Kubernetes CPU limit, systemd ``CPUQuota=``) and whether the benchmark cgroup + has been throttled. A CPU quota throttles the benchmark, which moves the mean + without widening the relative spread, so a capped run can look statistically + stable while being wrong. ``pyperf check`` emits the same warning from the + ``cpu_quota`` metadata of a saved result. * "Turbo Boost (MSR)": use ``/dev/cpu/N/msr`` to read/write the Turbo Boost mode of Intel CPUs * "Turbo Boost (intel_pstate driver)": read from/write into diff --git a/pyperf/_cli.py b/pyperf/_cli.py index b9ccaa91..24f77be8 100644 --- a/pyperf/_cli.py +++ b/pyperf/_cli.py @@ -493,6 +493,21 @@ def format_checks(bench, lines=None, check_too_many_processes=False): warn("See https://bugzilla.redhat.com/show_bug.cgi?id=1378529") break + # Warn if the benchmark ran under a cgroup CPU quota (cpu_quota metadata). + # A quota throttles the run, which moves the mean but leaves the relative + # spread alone, so the statistical checks above cannot see it: a capped run + # comes back looking stable. + for run in bench._runs: + cpu_quota = run._metadata.get('cpu_quota') + if cpu_quota: + empty_line(lines) + lines.append("WARNING: the benchmark ran under a CPU quota (%s)" + % cpu_quota) + lines.append("A CPU quota throttles the benchmark: the result can " + "look stable while being capped. Run without a CPU " + "quota for reliable results.") + break + return lines diff --git a/pyperf/_collect_metadata.py b/pyperf/_collect_metadata.py index 8516cd13..e17b8d96 100644 --- a/pyperf/_collect_metadata.py +++ b/pyperf/_collect_metadata.py @@ -422,6 +422,79 @@ def collect_cpu_quota(metadata): metadata['cpu_quota'] = '%s CPUs' % format(quota, 'g') +def _read_cgroup_cpu_stat(path): + # Parse a cgroup "cpu.stat" file ("key value" lines) into a dict of ints. + # Return None if the file cannot be read. + stat = {} + found = False + try: + with open_text(path) as fp: + for line in fp: + fields = line.split() + if len(fields) == 2: + try: + stat[fields[0]] = int(fields[1]) + found = True + except ValueError: + continue + except OSError: + return None + return stat if found else None + + +def get_cgroup_cpu_throttling(): + """Get the Linux cgroup CPU throttling counters of the current process. + + Return a dict with 'nr_throttled' (number of enforcement periods in which + the cgroup was throttled) and 'throttled_usec' (total time spent throttled, + in microseconds), or None if the counters cannot be read (ex: not Linux, no + cgroup, or no cpu controller). The counters are cumulative since the cgroup + was created, so None means "could not be checked", which is distinct from + zero ("checked, never throttled"). + + With cgroup v2 a limit set on a parent cgroup also throttles, and the + throttling is recorded in the cgroup where the limit binds while the root + counters stay at zero, so the largest counters in the hierarchy are used. + """ + nr_throttled = None + throttled_usec = None + for line in read_proc("self/cgroup"): + parts = line.split(":", 2) + if len(parts) != 3: + continue + hierarchy, controllers, path = parts + path = path.strip("/") + names = path.split("/") if path else [] + if hierarchy == "0" and not controllers: + # cgroup v2: check the cgroup and each of its parents + bases = [os.path.join("/sys/fs/cgroup", *names[:index]) + for index in range(len(names), -1, -1)] + elif "cpu" in controllers.split(","): + # cgroup v1: the cgroup path is not visible inside a container + bases = [os.path.join("/sys/fs/cgroup/cpu", *names), + "/sys/fs/cgroup/cpu"] + else: + continue + for base in bases: + stat = _read_cgroup_cpu_stat(os.path.join(base, "cpu.stat")) + if stat is None: + continue + if "nr_throttled" in stat: + nr_throttled = max(nr_throttled or 0, stat["nr_throttled"]) + if "throttled_usec" in stat: + # cgroup v2 + throttled_usec = max(throttled_usec or 0, + stat["throttled_usec"]) + elif "throttled_time" in stat: + # cgroup v1 reports nanoseconds + throttled_usec = max(throttled_usec or 0, + stat["throttled_time"] // 1000) + if nr_throttled is None and throttled_usec is None: + return None + return {"nr_throttled": nr_throttled or 0, + "throttled_usec": throttled_usec or 0} + + def collect_cpu_model(metadata): for line in read_proc("cpuinfo"): if line.startswith('model name'): diff --git a/pyperf/_system.py b/pyperf/_system.py index d0f8e526..8af5c4fb 100644 --- a/pyperf/_system.py +++ b/pyperf/_system.py @@ -983,6 +983,44 @@ def write(self, tune): self.log_action("Max sample rate set to %s per second" % new_rate) +class CPUQuota(Operation): + """ + Detect a cgroup CPU bandwidth limit: docker run --cpus, a Kubernetes CPU + limit, systemd CPUQuota=. A quota throttles the benchmark, which moves the + mean without widening the relative spread, so a capped run can look + statistically stable while being wrong. Nothing pyperf can tune fixes it, + so this only reports the state, like the power supply check. + """ + + @staticmethod + def available(): + return OS_LINUX and os.path.exists('/sys/fs/cgroup') + + def __init__(self, system): + super().__init__('CPU quota', system) + + def show(self): + from pyperf._collect_metadata import (get_cgroup_cpu_quota, + get_cgroup_cpu_throttling) + + quota = get_cgroup_cpu_quota() + if quota is None: + return + + self.log_state('a CPU quota of %s CPUs is set' % format(quota, 'g')) + + throttling = get_cgroup_cpu_throttling() + if throttling and throttling['nr_throttled']: + self.log_state('the benchmark cgroup was CPU-throttled %s times ' + '(%.1f ms total)' + % (throttling['nr_throttled'], + throttling['throttled_usec'] / 1000.0)) + + self.advice('A CPU quota throttles the benchmark and can make a ' + 'capped run look stable; run without a CPU quota to get ' + 'reliable results') + + OPERATIONS = [ # Generic operations PerfEvent, @@ -991,6 +1029,7 @@ def write(self, tune): CPUFrequency, IRQAffinity, PowerSupply, + CPUQuota, # Setting the CPU scaling governor resets no_turbo # and so must be set before Turbo Boost diff --git a/pyperf/tests/test_cli.py b/pyperf/tests/test_cli.py index 4452bca1..89b18381 100644 --- a/pyperf/tests/test_cli.py +++ b/pyperf/tests/test_cli.py @@ -27,6 +27,26 @@ def test_format_result_calibration(self): 'Calibration: 100 loops') self.assertRaises(ValueError, bench.median) + def _quota_bench(self, metadata): + md = {'name': 'bench'} + md.update(metadata) + runs = [pyperf.Run([1.0, 1.0, 1.0], metadata=md, + collect_metadata=False) + for _ in range(3)] + return pyperf.Benchmark(runs) + + def test_checks_warns_on_cpu_quota(self): + # a run under a CPU quota must warn even though the values look stable + bench = self._quota_bench({'cpu_quota': '2 CPUs'}) + text = '\n'.join(cli.format_checks(bench)) + self.assertIn('CPU quota', text) + self.assertIn('2 CPUs', text) + + def test_checks_no_cpu_quota(self): + bench = self._quota_bench({}) + text = '\n'.join(cli.format_checks(bench)) + self.assertNotIn('CPU quota', text) + if __name__ == "__main__": unittest.main() diff --git a/pyperf/tests/test_metadata.py b/pyperf/tests/test_metadata.py index ae1d7bc9..3ac7f0e5 100644 --- a/pyperf/tests/test_metadata.py +++ b/pyperf/tests/test_metadata.py @@ -102,6 +102,59 @@ def test_cpu_quota_no_limit(self): # no cgroup information at all (ex: not Linux) self.assertIsNone(self.check_cpu_quota({})) + def check_cpu_throttling(self, files): + def mock_open(filename, *args, **kw): + filename = filename.replace('\\', '/') + if filename not in files: + raise FileNotFoundError(filename) + return io.StringIO(files[filename]) + + with mock.patch('pyperf._utils.open', create=True, + side_effect=mock_open): + return perf_metadata.get_cgroup_cpu_throttling() + + def test_cpu_throttling_cgroup_v2(self): + # the quota binds on the process cgroup and throttling is recorded + # there, while the root counters stay at zero + files = { + '/proc/self/cgroup': '0::/user.slice/run-1.scope\n', + '/sys/fs/cgroup/user.slice/run-1.scope/cpu.stat': + ('usage_usec 999\nnr_periods 2470\n' + 'nr_throttled 3306\nthrottled_usec 247631465\n'), + '/sys/fs/cgroup/user.slice/cpu.stat': + 'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + '/sys/fs/cgroup/cpu.stat': + 'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 3306, 'throttled_usec': 247631465}) + + def test_cpu_throttling_zero_is_not_none(self): + # checked and never throttled: zero, distinct from "could not check" + files = { + '/proc/self/cgroup': '0::/init.scope\n', + '/sys/fs/cgroup/init.scope/cpu.stat': + 'usage_usec 5\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 0, 'throttled_usec': 0}) + + def test_cpu_throttling_cgroup_v1(self): + # cgroup v1 reports throttled_time in nanoseconds + files = { + '/proc/self/cgroup': '5:cpu,cpuacct:/docker/abc\n', + '/sys/fs/cgroup/cpu/docker/abc/cpu.stat': + 'nr_periods 100\nnr_throttled 7\nthrottled_time 456789000\n', + } + self.assertEqual(self.check_cpu_throttling(files), + {'nr_throttled': 7, 'throttled_usec': 456789}) + + def test_cpu_throttling_no_cgroup(self): + # no cpu.stat to read (ex: not Linux): could not be checked + self.assertIsNone(self.check_cpu_throttling({})) + files = {'/proc/self/cgroup': '0::/init.scope\n'} + self.assertIsNone(self.check_cpu_throttling(files)) + class CpuFunctionsTests(unittest.TestCase): INTEL_CPU_INFO = textwrap.dedent(""" diff --git a/pyperf/tests/test_system.py b/pyperf/tests/test_system.py index e8d519a3..ef8a83ad 100644 --- a/pyperf/tests/test_system.py +++ b/pyperf/tests/test_system.py @@ -1,10 +1,49 @@ import os.path import sys import unittest +from unittest import mock from pyperf.tests import get_output +class CPUQuotaTests(unittest.TestCase): + def _show(self, quota, throttling): + from pyperf._system import CPUQuota + + system = mock.Mock() + op = CPUQuota(system) + with mock.patch('pyperf._collect_metadata.get_cgroup_cpu_quota', + return_value=quota), \ + mock.patch('pyperf._collect_metadata.get_cgroup_cpu_throttling', + return_value=throttling): + op.show() + states = ' '.join(call.args[0] + for call in system.log_state.call_args_list) + advices = ' '.join(call.args[0] + for call in system.advice.call_args_list) + return states, advices + + def test_no_quota_is_silent(self): + states, advices = self._show(None, None) + self.assertEqual(states, '') + self.assertEqual(advices, '') + + def test_quota_set_but_not_throttled(self): + states, advices = self._show(2.0, {'nr_throttled': 0, + 'throttled_usec': 0}) + self.assertIn('2 CPUs', states) + self.assertNotIn('throttled', states) + self.assertIn('CPU quota', advices) + + def test_quota_and_throttled(self): + states, advices = self._show(2.0, {'nr_throttled': 3306, + 'throttled_usec': 247631465}) + self.assertIn('2 CPUs', states) + self.assertIn('3306', states) + self.assertIn('throttled', states.lower()) + self.assertIn('CPU quota', advices) + + class SystemTests(unittest.TestCase): def test_show(self): args = [sys.executable, '-m', 'pyperf', 'system', 'show']