From 6a6d1fd49709788259776b572cca6eca48e3d2e2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 25 Mar 2026 21:13:04 +0100 Subject: [PATCH 001/111] Moved kernprof pre-import code to another module - `line_profiler/curated_profiling.py` New module for setting up profiling in a curated environment - `ClassifiedPreimportTargets.from_targets()` Method for creating a `ClassifiedPreimportTargets` instance, facilitating writing pre-import modules in a replicable and portable manner - `ClassifiedPreimportTargets.write_preimport_module()` Method for writing a pre-import module based on an instance; also fixed bug where the body of the written module was intercepted without appearing in the debug output - `kernprof.py` - `_gather_preimport_targets()` Migrated to `line_profiler.curated_profiling` - `_write_preimports()` Now using the new `ClassifiedPreimportTargets` class, moving esp. the logic to the `write_preimport_module()` method --- kernprof.py | 84 ++--------------- line_profiler/curated_profiling.py | 140 +++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 76 deletions(-) create mode 100644 line_profiler/curated_profiling.py diff --git a/kernprof.py b/kernprof.py index 8a7c4d6a..0a198acb 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1043,72 +1043,16 @@ def _write_tempfile(source, content, options): ) -def _gather_preimport_targets(options, exclude): - """ - Used in _write_preimports - """ - from line_profiler.autoprofile.util_static import modpath_to_modname - from line_profiler.autoprofile.eager_preimports import is_dotted_path - - filtered_targets = [] - recurse_targets = [] - invalid_targets = [] - for target in options.prof_mod: - if is_dotted_path(target): - modname = target - else: - # Paths already normalized by - # `_normalize_profiling_targets()` - if not os.path.exists(target): - invalid_targets.append(target) - continue - if any(os.path.samefile(target, excluded) for excluded in exclude): - # Ignore the script to be run in eager importing - # (`line_profiler.autoprofile.autoprofile.run()` will - # handle it) - continue - modname = modpath_to_modname(target, hide_init=False) - if modname is None: # Not import-able - invalid_targets.append(target) - continue - if modname.endswith('.__init__'): - modname = modname.rpartition('.')[0] - filtered_targets.append(modname) - else: - recurse_targets.append(modname) - if invalid_targets: - invalid_targets = sorted(set(invalid_targets)) - msg = ( - '{} profile-on-import target{} cannot be converted to ' - 'dotted-path form: {!r}'.format( - len(invalid_targets), - '' if len(invalid_targets) == 1 else 's', - invalid_targets, - ) - ) - warnings.warn(msg) - diagnostics.log.warning(msg) - - return filtered_targets, recurse_targets - - def _write_preimports(prof, options, exclude): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. """ - from line_profiler.autoprofile.eager_preimports import ( - write_eager_import_module, - ) from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) + from line_profiler.curated_profiling import ClassifiedPreimportTargets - filtered_targets, recurse_targets = _gather_preimport_targets( - options, exclude - ) - if not (filtered_targets or recurse_targets): - return # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; # so we write to a tempfile and `execfile()` it @@ -1116,26 +1060,14 @@ def _write_preimports(prof, options, exclude): temp_mod_path = _touch_tempfile( dir=options.tmpdir, prefix='kernprof-eager-preimports-', suffix='.py' ) - write_module_kwargs = { - 'dotted_paths': filtered_targets, - 'recurse': recurse_targets, - 'static': options.static, - } - temp_file = open(temp_mod_path, mode='w') - if options.debug: - with StringIO() as sio: - write_eager_import_module(stream=sio, **write_module_kwargs) - code = sio.getvalue() - with temp_file as fobj: - print(code, file=fobj) - diagnostics.log.debug( - 'Wrote temporary module for pre-imports to ' - f'{short_string_path(temp_mod_path)!r}' + with open(temp_mod_path, mode='w') as fobj: + preimports = ClassifiedPreimportTargets.from_targets( + options.prof_mod, exclude, ) - else: - with temp_file as fobj: - write_eager_import_module(stream=fobj, **write_module_kwargs) - if not options.dryrun: + preimports.write_preimport_module( + fobj, debug=options.debug, static=options.static, + ) + if preimports and not options.dryrun: ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py new file mode 100644 index 00000000..d5bf5b3c --- /dev/null +++ b/line_profiler/curated_profiling.py @@ -0,0 +1,140 @@ +""" +Tools for setting up profiling in a curated environment (e.g. with +the use of :py:mod:`kernprof`). +""" +from __future__ import annotations + +import dataclasses +import os +import warnings +from collections.abc import Collection +from io import StringIO +from textwrap import indent +from typing import TextIO +from typing_extensions import Self + +from . import _diagnostics as diagnostics +from .autoprofile.util_static import modpath_to_modname +from .autoprofile.eager_preimports import ( + is_dotted_path, write_eager_import_module, +) +from .cli_utils import short_string_path + + +__all__ = ('ClassifiedPreimportTargets',) + + +@dataclasses.dataclass +class ClassifiedPreimportTargets: + """ + Pre-import targets classified into three bins: ``regular`` targets, + targets to ``recurse`` into, and ``invalid`` targets + """ + regular: list[str] = dataclasses.field(default_factory=list) + recurse: list[str] = dataclasses.field(default_factory=list) + invalid: list[str] = dataclasses.field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.regular or self.recurse) + + def write_preimport_module( + self, fobj: TextIO, *, debug: bool | None = None, **kwargs + ) -> None: + """ + Convenience interface with + :py:func:`~.write_eager_import_module`, writing a module which + when imported sets up profiling of the targets. + + Args: + fobj (TextIO): + File object to write said module to. + debug (Optional[bool]): + Whether to generate debugging outputs. + kwargs: + Passed to :py:func:`~.write_eager_import_module`. + """ + if self.invalid: + invalid_targets = sorted(set(self.invalid)) + msg = ( + '{} profile-on-import target{} cannot be converted to ' + 'dotted-path form: {!r}'.format( + len(invalid_targets), + '' if len(invalid_targets) == 1 else 's', + invalid_targets, + ) + ) + warnings.warn(msg) + diagnostics.log.warning(msg) + + if not self: + return None + write_module_kwargs = { + 'dotted_paths': self.regular, + 'recurse': self.recurse, + **kwargs, + } + if diagnostics.DEBUG if debug is None else debug: + with StringIO() as sio: + write_eager_import_module(stream=sio, **write_module_kwargs) + code = sio.getvalue() + print(code, file=fobj) + if hasattr(fobj, 'name'): + fobj_repr = repr(short_string_path(str(fobj.name))) + else: + fobj_repr = repr(fobj) # Fall back + diagnostics.log.debug( + f'Wrote temporary module for pre-imports to {fobj_repr}:\n' + + indent(code, ' ') + ) + else: + write_eager_import_module(stream=fobj, **write_module_kwargs) + + @classmethod + def from_targets( + cls, + targets: Collection[str], + exclude: Collection[os.PathLike[str] | str] = (), + ) -> Self: + """ + Create an instance based on a collection of targets + (like what is supplied to :cmd:`kernprof --prof-mod=...`). + + Args: + targets (Collection[str]) + Collection of dotted paths and filenames to profile. + exclude (Collection[str]) + Collections of filenames which are explicitly excluded + from being profiled. + + Return: + New instance. + """ + filtered_targets = [] + recurse_targets = [] + invalid_targets = [] + for target in targets: + if is_dotted_path(target): + modname = target + else: + # Paths already normalized by + # `_normalize_profiling_targets()` + if not os.path.exists(target): + invalid_targets.append(target) + continue + if any( + os.path.samefile(target, excluded) for excluded in exclude + ): + # Ignore the script to be run in eager importing + # (`line_profiler.autoprofile.autoprofile.run()` + # will handle it) + continue + modname = modpath_to_modname(target, hide_init=False) + if modname is None: # Not import-able + invalid_targets.append(target) + continue + if modname.endswith('.__init__'): + modname = modname.rpartition('.')[0] + filtered_targets.append(modname) + else: + recurse_targets.append(modname) + return cls(filtered_targets, recurse_targets, invalid_targets) From f2c8565e7cda360778a76a16703a80623882283f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 26 Mar 2026 00:11:02 +0100 Subject: [PATCH 002/111] Removed redundant check in kernprof --- kernprof.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 0a198acb..bd4ef5b1 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1363,8 +1363,7 @@ def _post_profile(options, prof): for _ in range(prof.enable_count): prof.disable_by_count() # Restore the state of the global `@line_profiler.profile` - if options.global_profiler: - options.install_profiler(None) + options.install_profiler(None) if __name__ == '__main__': From fa817a711af0cfddb04c48c0504f88889e43232d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 26 Mar 2026 03:34:04 +0100 Subject: [PATCH 003/111] Move code between kernprof & `~.curated_profiling` - `kernprof.py::_manage_profiler` `line_profiler/curated_profiling.py::CuratedProfilerContext` New context-manager classes for handling profiler setup and teardown - `kernprof.py::_pre_profile()` Refactored into the above context managers and other private functions (`_prepare_profiler()`, `_prepare_exec_script()`) --- kernprof.py | 65 ++++++++++++++--------- line_profiler/curated_profiling.py | 83 ++++++++++++++++++++++++++++-- 2 files changed, 119 insertions(+), 29 deletions(-) diff --git a/kernprof.py b/kernprof.py index bd4ef5b1..5a873571 100755 --- a/kernprof.py +++ b/kernprof.py @@ -187,7 +187,6 @@ def main(): """ # noqa: E501 import atexit -import builtins import functools import os import sys @@ -198,9 +197,7 @@ def main(): import shutil import tempfile import time -import warnings from argparse import ArgumentParser -from io import StringIO from operator import methodcaller from runpy import run_module from pathlib import Path @@ -1163,13 +1160,42 @@ def _call_with_diagnostics(options, func, *args, **kwargs): return func(*args, **kwargs) -def _pre_profile(options, module, exit_on_error): +class _manage_profiler: """ Prepare the environment to execute profiling with requested options. Note: modifies ``options`` with extra attributes. """ + def __init__(self, options, module, exit_on_error): + self.options = options + self.module = module + self.exit_on_error = exit_on_error + + def __enter__(self): + from line_profiler.curated_profiling import CuratedProfilerContext + + self.prof = _prepare_profiler( + self.options, self.module, self.exit_on_error, + ) + self._ctx = CuratedProfilerContext( + self.prof, insert_builtin=self.options.builtin, + ) + self._ctx.install() + script_file = _prepare_exec_script( + self.options, self.module, self.prof, self.exit_on_error, + ) + return self.prof, script_file + + def __exit__(self, *_, **__): + _post_profile(self.options, self.prof) + self._ctx.uninstall() + + +def _prepare_profiler(options, module, exit_on_error): + """ + Set up the appropriate profiler instance. + """ if not options.outfile: extension = 'lprof' if options.line_by_line else 'prof' options.outfile = f'{os.path.basename(options.script)}.{extension}' @@ -1199,24 +1225,21 @@ def _pre_profile(options, module, exit_on_error): execfile(setup_file, ns, ns) if options.line_by_line: - prof = line_profiler.LineProfiler() options.builtin = True + return line_profiler.LineProfiler() elif Profile.__module__ == 'profile': raise RuntimeError( 'non-line-by-line profiling depends on cProfile, ' 'which is not available on this platform' ) else: - prof = ContextualProfile() + return ContextualProfile() - # Overwrite the explicit decorator - global_profiler = line_profiler.profile - install_profiler = global_profiler._kernprof_overwrite - install_profiler(prof) - - if options.builtin: - builtins.__dict__['profile'] = prof +def _prepare_exec_script(options, module, prof, exit_on_error): + """ + Set up the script to be executed among other things. + """ if module: script_file = find_module_script( options.script, static=options.static, exit_on_error=exit_on_error @@ -1245,8 +1268,6 @@ def _pre_profile(options, module, exit_on_error): exclude = set() if module else {script_file} _write_preimports(prof, options, exclude) - options.global_profiler = global_profiler - options.install_profiler = install_profiler if options.output_interval and not options.dryrun: options.rt = RepeatedTimer( max(options.output_interval, 1), prof.dump_stats, options.outfile @@ -1254,7 +1275,7 @@ def _pre_profile(options, module, exit_on_error): else: options.rt = None options.original_stdout = sys.stdout - return script_file, prof + return script_file def _main_profile(options, module=False, exit_on_error=True): @@ -1262,9 +1283,10 @@ def _main_profile(options, module=False, exit_on_error=True): Called by :py:func:`main()` for the actual execution and profiling of code after initial parsing of options; not to be invoked on its own. """ - script_file, prof = _pre_profile(options, module, exit_on_error) call = functools.partial(_call_with_diagnostics, options) - try: + with _manage_profiler( + options, module, exit_on_error, + ) as (prof, script_file): rmod = functools.partial( run_module, run_name='__main__', alter_sys=True ) @@ -1315,8 +1337,6 @@ def _main_profile(options, module=False, exit_on_error=True): module_ns, module_ns, ) - finally: - _post_profile(options, prof) def _post_profile(options, prof): @@ -1359,11 +1379,6 @@ def _post_profile(options, prof): f'{quote(py_exe)} -m {show_mod} ' f'{quote(short_outfile)}' ) - # Fully disable the profiler - for _ in range(prof.enable_count): - prof.disable_by_count() - # Restore the state of the global `@line_profiler.profile` - options.install_profiler(None) if __name__ == '__main__': diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index d5bf5b3c..ecad97a9 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -4,24 +4,27 @@ """ from __future__ import annotations +import builtins import dataclasses +import functools import os import warnings -from collections.abc import Collection +from collections.abc import Callable, Collection from io import StringIO from textwrap import indent -from typing import TextIO +from typing import Any, TextIO from typing_extensions import Self -from . import _diagnostics as diagnostics +from . import _diagnostics as diagnostics, profile as _global_profiler from .autoprofile.util_static import modpath_to_modname from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, ) from .cli_utils import short_string_path +from .profiler_mixin import ByCountProfilerMixin -__all__ = ('ClassifiedPreimportTargets',) +__all__ = ('ClassifiedPreimportTargets', 'CuratedProfilerContext') @dataclasses.dataclass @@ -138,3 +141,75 @@ def from_targets( else: recurse_targets.append(modname) return cls(filtered_targets, recurse_targets, invalid_targets) + + +class CuratedProfilerContext: + """ + Context manager for handling various bookkeeping tasks when setting + up and tearing down profiling: + + - Slipping ``prof`` into the builtin namespace (if + ``insert_builtin`` is true) and :py::deco:`~.profile` + - At exit, clearing the ``enable_count`` of ``prof``, properly + disabling it + + Note: + The attributes on this object are to be considered + implementation details, but not its methods and their + signatures. + """ + def __init__( + self, + prof: ByCountProfilerMixin, + insert_builtin: bool = False, + builtin_loc: str = 'profile', + ) -> None: + self.prof = prof + self.insert_builtin = insert_builtin + self.builtin_loc = builtin_loc + self._installed = False + self._global_install = _global_profiler._kernprof_overwrite + + def install(self) -> None: + def del_builtin_profile() -> None: + delattr(builtins, self.builtin_loc) + + def set_builtin_profile(old: Any) -> None: + setattr(builtins, self.builtin_loc, old) + + if self._installed: + return + # Overwrite the explicit profiler (`@line_profiler.profile`) + self._global_install(self.prof) # type: ignore[arg-type] + # Set up hooks to deal with inserting `.prof` as a builtin name + if self.insert_builtin: + try: + old = getattr(builtins, self.builtin_loc) + except AttributeError: + self._restore: Callable[[], None] = del_builtin_profile + else: + self._restore = functools.partial(set_builtin_profile, old) + set_builtin_profile(self.prof) + self._installed = True + + def uninstall(self) -> None: + if not self._installed: + return + # Restore the `builtins` namespace + if ( + self.insert_builtin + and getattr(builtins, self.builtin_loc, None) is self.prof + ): + self._restore() + # Fully disable the profiler + for _i in range(getattr(self.prof, 'enable_count', 0)): + self.prof.disable_by_count() + # Restore the state of the global `@line_profiler.profile` + self._global_install(None) # type: ignore[arg-type] + self._installed = False + + def __enter__(self) -> None: + self.install() + + def __exit__(self, *_, **__) -> None: + self.uninstall() From 1af7db56f5a2c728c27b0b4451de5031a199aaab Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 02:55:21 +0100 Subject: [PATCH 004/111] New subpackage for profiling child processes line_profiler/_child_process_profiling/cache.py::LineProfilingCache New class for passing info onto child processes so that profiling can resume there line_profiler/pth_hook.py New submodule for the .pth-file-based solution to propagating profiling into child processes: write_pth_hook() In the main process, write the temporary .pth file to be loaded in child processes load_pth_hook() Called by the .pth in child process, loading the cache and setting up profiling based thereon --- .../_child_process_profiling/__init__.py | 8 + .../_child_process_profiling/cache.py | 142 ++++++++++++ .../_child_process_profiling/pth_hook.py | 206 ++++++++++++++++++ 3 files changed, 356 insertions(+) create mode 100644 line_profiler/_child_process_profiling/__init__.py create mode 100644 line_profiler/_child_process_profiling/cache.py create mode 100644 line_profiler/_child_process_profiling/pth_hook.py diff --git a/line_profiler/_child_process_profiling/__init__.py b/line_profiler/_child_process_profiling/__init__.py new file mode 100644 index 00000000..4ff89c0a --- /dev/null +++ b/line_profiler/_child_process_profiling/__init__.py @@ -0,0 +1,8 @@ +""" +Tooling for profiling child Python processes and gathering their +profiling results. + +Note: + All contents of this subpackage is to be considered implementation + details. +""" diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py new file mode 100644 index 00000000..74a1cf4a --- /dev/null +++ b/line_profiler/_child_process_profiling/cache.py @@ -0,0 +1,142 @@ +""" +A cache object to be used by for propagating profiling down to child +processes. +""" +from __future__ import annotations + +import dataclasses +import os +try: + import _pickle as pickle +except ImportError: + import pickle # type: ignore[assignment,no-redef] +from collections.abc import Collection, Callable +from functools import partial +from pickle import HIGHEST_PROTOCOL +from typing import Any +from typing_extensions import Self, ParamSpec + +# Note: this should have been defined here in this file, but we moved it +# over to `~._child_process_hook` because that module contains the .pth +# hook, which must run with minimal overhead when a Python process isn't +# associated with a profiled process +from .pth_hook import INHERITED_PID_ENV_VARNAME + + +__all__ = ('LineProfilingCache',) + + +PS = ParamSpec('PS') + +INHERITED_CACHE_ENV_VARNAME_PREFIX = ( + 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' +) +CACHE_FILENAME = 'line_profiler_cache.pkl' + + +@dataclasses.dataclass +class LineProfilingCache: + cache_dir: os.PathLike[str] | str + profiling_targets: Collection[str] = dataclasses.field( + default_factory=list, + ) + preimports_module: os.PathLike[str] | str | None = None + main_pid: int = dataclasses.field(default_factory=os.getpid) + insert_builtin: bool = True + _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( + default_factory=list, init=False, + ) + + def cleanup(self) -> None: + """ + Pop all the cleanup callbacks from the internal stack added via + :py:meth:`~.add_cleanup` and call them in order. + """ + callbacks = self._cleanup_stack + while callbacks: + callback = callbacks.pop() + try: + callback() + except Exception: + pass + + def add_cleanup( + self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Add a cleanup callback to the internal stack; they can be later + called by :py:meth:`~.cleanup`. + """ + if args or kwargs: + callback = partial(callback, *args, **kwargs) + self._cleanup_stack.append(callback) + + def copy( + self, *, inherit_cleanups: bool = False, **replacements + ) -> Self: + """ + Make a copy with optionally replaced fields; + if ``inherit_cleanups`` is set to true, the copy also makes a + (shallow) copy of the clean-callback stack. + """ + init_args: dict[str, Any] = {} + for field, value in self._get_init_args().items(): + init_args[field] = replacements.get(field, value) + copy = type(self)(**init_args) + if inherit_cleanups: + copy._cleanup_stack[:] = self._cleanup_stack + return copy + + @classmethod + def load(cls) -> Self: + """ + Reconstruct the instance from the environment variables + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID` and + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR_`. + These should have been set from an ancestral Python process. + """ + pid = os.environ[INHERITED_PID_ENV_VARNAME] + cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] + with open(cls._get_filename(cache_dir), mode='rb') as fobj: + return cls(**pickle.load(fobj)) + + def dump(self) -> None: + """ + Serialize the cache instance and dump into the default location + as indicated by :py:attr:`~.cache_dir`, so that they can be + :py:meth:`~.load`-ed by child processes. + + Note: + Cleanup callbacks are not serialized. + """ + with open(self.filename, mode='wb') as fobj: + pickle.dump( + self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, + ) + + def _get_init_args(self) -> dict[str, Any]: + init_fields = [ + field_obj.name for field_obj in dataclasses.fields(self) + if field_obj.init + ] + return {name: getattr(self, name) for name in init_fields} + + @staticmethod + def _get_filename(cache_dir: os.PathLike[str] | str) -> str: + return os.path.join(cache_dir, CACHE_FILENAME) + + @property + def environ(self) -> dict[str, str]: + """ + Environment variables to be injected into and inherited by child + processes. + """ + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{self.main_pid}' + return { + INHERITED_PID_ENV_VARNAME: str(self.main_pid), + cache_varname: str(self.cache_dir), + } + + @property + def filename(self) -> str: + return self._get_filename(self.cache_dir) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py new file mode 100644 index 00000000..82261512 --- /dev/null +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -0,0 +1,206 @@ +""" +Hooks to set up shop in a child Python process and extend profiling +to therein. + +Note: + - The current implementation writes temporary .pth files to the + site-packages directory, which are executed for all Python + processes referring to the same `lib/`. However, only processes + originating from a parent which set the requisite environment + variables will execute to the profiling code. + - Said .pth file always import this module; hence, this file is kept + intentionally lean to reduce overhead: + - Imports in this file are deferred to being as late as possible. + - Type annotations are replaced with type comments. + - Non-essential functionalities are split into small separate + submodules (e.g. :py:mod:`~.cache`). + - Inspired by similar code in `coverage.control` and + `pytest_autoprofile.startup_hook`. +""" +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .cache import LineProfilingCache # noqa: F401 + from pathlib import Path # noqa: F401 + + +__all__ = ( + 'write_pth_hook', 'load_pth_hook', '_setup_in_child_process' +) + +INHERITED_PID_ENV_VARNAME = ( + 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' +) + + +def write_pth_hook(cache): # type: (LineProfilingCache) -> Path + """ + Write a .pth file which allows for setting up profiling in child + Python processes. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + + Returns: + fpath (Path): + Path to the written .pth file + + Note: + - To be called in the main process. + - The ``cache`` is responsible for deleting the written .pth + file via the registered cleanup callback. + """ + import os + from pathlib import Path # noqa: F811 + from sysconfig import get_path + from tempfile import mkstemp + + if not os.path.exists(cache.filename): + cache.dump() + assert os.path.exists(cache.filename) + + handle, fname = mkstemp( + prefix='_line_profiler_profiling_hook_', + suffix='.pth', + dir=get_path('purelib'), + ) + fpath = Path(fname) + try: + pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( + (lambda: None).__module__, cache.main_pid, + ) + fpath.write_text(pth_content) + cache.add_cleanup(fpath.unlink, missing_ok=True) + except Exception: + os.remove(fpath) + raise + finally: # Not closing the handle causes issues on Windows + os.close(handle) + return fpath + + +def load_pth_hook(ppid): # type: (int) -> None + """ + Function imported and called by the written .pth file; to reduce + overhead, we immediately return if ``ppid`` doesn't match + :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID`. + """ + from os import environ + + try: + env_ppid = int(environ[INHERITED_PID_ENV_VARNAME]) + except (KeyError, ValueError): + return + if env_ppid != ppid: + return + + # If we're here, we're most probably in a descendent process of a + # profiled Python process, so we can be more liberal with the + # imports without worrying about overhead + import warnings + from .._diagnostics import DEBUG, log + from .cache import LineProfilingCache # noqa: F811 + + # Note: .pth files may be double-loaded in a virtual environment + # (see https://stackoverflow.com/questions/58807569), so work around + # that; + # also see similar check in `coverage.control.process_startup()` + if getattr(load_pth_hook, 'called', False): + return + try: + _setup_in_child_process(LineProfilingCache.load()) + except Exception as e: + if DEBUG: + msg = f'{type(e)}: {e}' + warnings.warn(msg) + log.warning(msg) + finally: + load_pth_hook.called = True # type: ignore[attr-defined] + + +def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None + """ + Create a wrapper around :py:func:`os.fork` which handles profiling. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + + Side effects: + - :py:func:`os.fork` (if available) replaced with the wrapper + - Cleanup callback registered at ``cache`` undoing that + """ + import os + from functools import wraps + + try: + fork = os.fork + except AttributeError: # Can't fork on this platform + return + + @wraps(fork) + def wrapper(): + result = fork() + if not result: # In the fork + _setup_in_child_process(cache.copy(), False) + return result + + os.fork = wrapper + cache.add_cleanup(setattr, os, 'fork', fork) + + +def _setup_in_child_process(cache, wrap_os_fork=True): + # type: (LineProfilingCache, bool) -> None + """ + Set up shop in a forked/spawned child process so that + (line-)profiling can extend therein. + + Args: + cache (:py:class:`~.LineProfilingCache`): + Cache object + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles profiling; + already-forked child processes should set this to false + """ + import os + from atexit import register + from tempfile import mkstemp + from ..curated_profiling import CuratedProfilerContext + from ..line_profiler import LineProfiler + + # Create a profiler instance and manage it with + # `CuratedProfilerContext` + prof = LineProfiler() + ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) + ctx.install() + cache.add_cleanup(ctx.uninstall) + + # Do the preimports at `cache.preimports_module` where appropriate + if cache.preimports_module: + with open(cache.preimports_module, mode='rb') as fobj: + code = compile(fobj.read(), cache.preimports_module, 'exec') + exec(code, {}) # Use a fresh, empty namespace + + # Set up the importer for rewriting `__main__` (TODO) + ... + + # Occupy a tempfile slot in `cache.cache_dir` and set the profiler + # up to write thereto when the process terminates + handle, prof_outfile = mkstemp( + prefix='line-profiler-child-prof-output-', + suffix='.lprof', + dir=cache.cache_dir, + ) + try: + cache.add_cleanup(prof.dump_stats, prof_outfile) + finally: + os.close(handle) + + # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) + if wrap_os_fork: + _wrap_os_fork(cache) + + # Set `cache.cleanup()` as an atexit hook to handle everything when + # the child process is about to terminate + register(cache.cleanup) From 4bf52a5d85beac8c448a3098e14788030af25a34 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 05:00:32 +0100 Subject: [PATCH 005/111] WIP: set up child-proc profiling via import sys line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added new `.profile_imports` attribute to correspond to `kernprof`'s `--prof-imports` flag line_profiler/_child_process_profiling/meta_path_finder.py New submodule defining the `RewritingFinder` class, a meta path finder which rewrites a single module on import line_profiler/_child_process_profiling/pth_hook.py write_pth_hook() Now also handling the `os.fork()` patching/wrapping _setup_in_child_process() Now creating a `RewritingFinder` to mirror what `~.autoprofile.autoprofile.run()` does in the main process . --- .../_child_process_profiling/cache.py | 3 + .../meta_path_finder.py | 221 ++++++++++++++++++ .../_child_process_profiling/pth_hook.py | 30 ++- 3 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 line_profiler/_child_process_profiling/meta_path_finder.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 74a1cf4a..bd55e65f 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -40,8 +40,11 @@ class LineProfilingCache: profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) + profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) + # Note: if we're using the line profiler, `kernprof` always set + # `builtin` to true insert_builtin: bool = True _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( default_factory=list, init=False, diff --git a/line_profiler/_child_process_profiling/meta_path_finder.py b/line_profiler/_child_process_profiling/meta_path_finder.py new file mode 100644 index 00000000..1436fc06 --- /dev/null +++ b/line_profiler/_child_process_profiling/meta_path_finder.py @@ -0,0 +1,221 @@ +""" +A meta path finder object which rewrites a specific module. + +Note: + Based on the implementation in + :py:mod:`pytest_autoprofile.importers`. +""" +from __future__ import annotations + +import ast +import os +import sys +from collections.abc import Callable +from importlib.abc import MetaPathFinder, SourceLoader +from importlib.machinery import ModuleSpec +from pathlib import Path +from types import CodeType, ModuleType +from typing import TYPE_CHECKING + +from .. import _diagnostics as diagnostics +from ..autoprofile.run_module import AstTreeModuleProfiler +from ..line_profiler import LineProfiler +from .cache import LineProfilingCache + + +__all__ = ('RewritingFinder',) + + +class RewritingFinder(MetaPathFinder, SourceLoader): + """ + Meta path finder to be set up in child processes, so that the + ``module_to_rewrite`` is rewritten for profiling as + :py:func:`line_profiler.autoprofile.autoprofile.run` does. + """ + _cached_code_obj: CodeType + + def __init__( + self, + prof: LineProfiler, + lp_cache: LineProfilingCache, + module_to_rewrite: str = '__main__', + ) -> None: + self.prof = prof + self.lp_cache = lp_cache + self.module_to_rewrite = module_to_rewrite + + def install(self, *, index: int = 0) -> None: + """ + Install the importer into :py:data:`sys.meta_path` at the + specified ``index``; + if it's already there, it's first removed and re-inserted at + the requested position. + """ + self.uninstall(invalidate_caches=False) + sys.meta_path.insert(index, self) + + def uninstall(self, *, invalidate_caches: bool = True) -> None: + """ + Uninstall the importer from :py:data:`sys.meta_path`, and + optionally also invalidate the caches. + """ + try: + sys.meta_path.remove(self) + except ValueError: # Not in the list + return + if invalidate_caches: + self.invalidate_caches() + + @classmethod + def find_spec_by_path(cls, *args, **kwargs) -> ModuleSpec | None: + """ + Implementation of + :py:meth:`MetaPathFinder.find_spec` which looks + for module specs with the other meta-path finders. + + Returns: + maybe_spec (ModuleSpec | None) + Module spec if found, :py:const:`None` otherwise + """ + Implementation = Callable[..., ModuleSpec | None] + impls: list[Implementation] = [ + finder.find_spec for finder in sys.meta_path + if callable(getattr(finder, 'find_spec', None)) + if not isinstance(finder, cls) + ] + for impl in impls: + try: + spec = impl(*args, **kwargs) + except Exception: + continue + if spec is not None: + return spec + return None + + def write_code(self, spec: ModuleSpec) -> CodeType: + """ + Rewrite the module code that ``spec`` points to with + :py:class:`ast.autoprofile.run_module.AstTreeModuleProfiler`. + + Args: + spec (ModuleSpec) + Module spec + + Returns: + code (CodeType) + Code object + """ + assert spec.origin + fname = str(Path(spec.origin)) + module = AstTreeModuleProfiler( + spec.origin, + list(self.lp_cache.profiling_targets), + self.lp_cache.profile_imports, + ).profile() + # Slip in a helper node so as to ensure the availability of + # `@profile` + # Note: `@profile` is slipped into the local namespace by + # `RewritingFinder.exec_module()`, but that may not be + # enough for applications directly using the code objects (e.g. + # `runpy`. Hence, we provide an out by falling back to + # `@line_profiler.profile`. + ensure_profile_node, = ast.parse( + 'if profile not in globals():\n' + ' from line_profiler import profile' + ).body + module.body.insert(0, ensure_profile_node) + return compile(module, fname, 'exec') + + # Methods as dictated by the interface + + def invalidate_caches(self) -> None: + try: + del self._cached_code_obj + except AttributeError: + pass + super().invalidate_caches() + + def exec_module(self, module: ModuleType) -> None: + namespace = module.__dict__ + namespace['profile'] = prof = self.prof + spec: ModuleSpec | None = module.__spec__ + if spec is None: + raise RuntimeError(f'module = {module!r}: empty `.__spec__`') + if TYPE_CHECKING: # Appease `mypy` + assert hasattr(prof, 'object_count') + count = prof.object_count + exec(self.get_code(spec), namespace, namespace) + if prof.object_count > count: + msg = ( + 'main PID = {} / child PID = {}: ' + 'profiled {} code object{} in the `{}` module' + ).format( + self.lp_cache.main_pid, + os.getpid(), + prof.object_count, + '' if prof.object_count == 1 else 's', + self.module_to_rewrite, + ) + diagnostics.log.debug(msg) + + def find_spec(self, *args, **kwargs) -> ModuleSpec | None: + spec = self.find_spec_by_path(*args, **kwargs) + if (spec is not None and spec.name == self.module_to_rewrite): + return spec + return None + + @staticmethod + def get_data(path: os.PathLike[str] | str) -> bytes: + return Path(path).read_bytes() + + @classmethod + def get_filename(cls, name: str) -> str: + spec = cls.find_spec_by_path(name) + if spec is None: + raise ImportError(name) + origin = spec.origin + if origin is None: + raise ImportError(name) + if origin == 'frozen' or not os.path.exists(origin): + raise ImportError(name) + return origin + + def get_code(self, name_or_spec: str | ModuleSpec) -> CodeType: + if isinstance(name_or_spec, str): + spec = self.find_spec_by_path(name_or_spec) + if spec is None: + raise ImportError(name_or_spec) + else: + spec = name_or_spec + try: + try: + return self._cached_code_obj + except AttributeError: + self._cached_code_obj = code = self.write_code(spec) + return code + except Exception as e: + raise ImportError(name_or_spec) from e + + if TYPE_CHECKING: + def source_to_code( # type: ignore[override] + self, *args, **kwargs + ) -> CodeType: + """ + Notes + ----- + :py:mod:`mypy` reports that + :py:meth:`SourceLoader.source_to_code`, an instance method, + clashes with + :py:meth:`importlib.abc.InspectLoader.source_to_code`, a + static method. + + Since: + - The method is functionally only used in the superclasses + as an instance method, and + - :py:class:`importlib.abc.InspectLoader`` is merely + included as a base class because :py:class:`SourceLoader` + inherits from it, + just explicitly override the method here so that we can + catch and suppress the :py:mod:`mypy` error. + """ + return super().source_to_code(*args, **kwargs) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 82261512..f19a8c33 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -5,17 +5,17 @@ Note: - The current implementation writes temporary .pth files to the site-packages directory, which are executed for all Python - processes referring to the same `lib/`. However, only processes - originating from a parent which set the requisite environment - variables will execute to the profiling code. + processes referring to the same :path:`lib/`. However, only + processes originating from a parent which set the requisite + environment variables will execute to the profiling code. - Said .pth file always import this module; hence, this file is kept intentionally lean to reduce overhead: - Imports in this file are deferred to being as late as possible. - Type annotations are replaced with type comments. - Non-essential functionalities are split into small separate submodules (e.g. :py:mod:`~.cache`). - - Inspired by similar code in `coverage.control` and - `pytest_autoprofile.startup_hook`. + - Inspired by similar code in :py:mod:`coverage.control` and + :py:mod:`pytest_autoprofile.startup_hook`. """ from typing import TYPE_CHECKING @@ -50,6 +50,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - To be called in the main process. - The ``cache`` is responsible for deleting the written .pth file via the registered cleanup callback. + - For convenience, we also wrap :py:func:`os.fork` when this + function is called. """ import os from pathlib import Path # noqa: F811 @@ -77,6 +79,9 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path raise finally: # Not closing the handle causes issues on Windows os.close(handle) + + _wrap_os_fork(cache) + return fpath @@ -166,12 +171,21 @@ def _setup_in_child_process(cache, wrap_os_fork=True): import os from atexit import register from tempfile import mkstemp + from ..autoprofile.autoprofile import ( + # Note: we need this to equip the profiler with the + # `.add_imported_function_or_module()` pseudo-method + # (see `kernprof.py::_write_preimports()`), which is required + # for the preimports to work + _extend_line_profiler_for_profiling_imports as upgrade_profiler, + ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler + from .meta_path_finder import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` prof = LineProfiler() + upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() cache.add_cleanup(ctx.uninstall) @@ -182,8 +196,10 @@ def _setup_in_child_process(cache, wrap_os_fork=True): code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Set up the importer for rewriting `__main__` (TODO) - ... + # Set up the importer for rewriting `__main__` + finder = RewritingFinder(prof, cache) + finder.install() + cache.add_cleanup(finder.uninstall) # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates From 0869ab4fb35831cd82800f273d08094ea4309c57 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 19:07:08 +0100 Subject: [PATCH 006/111] WIP: `multiprocessing` patches line_profiler/_child_process_profiling/cache::LineProfilingCache Refactored `.load()` line_profiler/_child_process_profiling/multiprocessing_patches.py New submodule for applying patches to the `multiprocessing` package, so that profiling is automatically set up in child processes created by it --- .../_child_process_profiling/cache.py | 8 +- .../multiprocessing_patches.py | 188 ++++++++++++++++++ 2 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 line_profiler/_child_process_profiling/multiprocessing_patches.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index bd55e65f..3a4f22eb 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -100,8 +100,7 @@ def load(cls) -> Self: """ pid = os.environ[INHERITED_PID_ENV_VARNAME] cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] - with open(cls._get_filename(cache_dir), mode='rb') as fobj: - return cls(**pickle.load(fobj)) + return cls._from_path(cls._get_filename(cache_dir)) def dump(self) -> None: """ @@ -117,6 +116,11 @@ def dump(self) -> None: self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, ) + @classmethod + def _from_path(cls, fname: os.PathLike[str] | str) -> Self: + with open(cls._get_filename(fname), mode='rb') as fobj: + return cls(**pickle.load(fobj)) + def _get_init_args(self) -> dict[str, Any]: init_fields = [ field_obj.name for field_obj in dataclasses.fields(self) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py new file mode 100644 index 00000000..39cabe2a --- /dev/null +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -0,0 +1,188 @@ +""" +Patch :py:mod:`multiprocessing` so that profiling extends into processes +it creates. + +Notes +----- +- Based on the implementations in :py:mod:`coverage.multiproc` and + :py:mod:`pytest_autoprofile._multiprocessing`. +- Results may vary if the process pool is not properly + :py:meth:`multiprocessing.pool.Pool.close`-d and + :py:meth:`multiprocessing.pool.Pool.join`-ed; + see `this caveat `__. +""" +from __future__ import annotations + +import multiprocessing.process +from collections.abc import Callable +from functools import partial, partialmethod +from typing import Any, TypedDict, TypeVar +from typing_extensions import Concatenate, ParamSpec + +from .cache import LineProfilingCache +from .pth_hook import _setup_in_child_process + + +__all__ = ('apply',) + + +T = TypeVar('T') +PS = ParamSpec('PS') + + +class _HookState(TypedDict): + cache_path: str # Cache to be loaded from here + + +class PickleHook: + """ + Object which, when unpickled, sets up profiling in the + :py:mod:`multiprocessing`-created process. + + See also: + :py:class:`coverage.multiproc.Stowaway` + """ + def __init__(self, cache_path: str) -> None: + self.cache_path = cache_path + + def __getstate__(self) -> _HookState: + return {'cache_path': self.cache_path} + + def __setstate__(self, state: _HookState) -> None: + self.cache_path = path = state['cache_path'] + apply(path) + + +def bootstrap( + self: multiprocessing.process.BaseProcess, + vanilla_impl: Callable[ + Concatenate[multiprocessing.process.BaseProcess, PS], T + ], + lp_cache: LineProfilingCache, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + """ + Wrap around + :py:meth:`multiprocessing.process.BaseProcess._bootstrap`, + writing the profiling results after it is run. + + Args: + self (multiprocessing.process.BaseProcess) + :py:class:`~.BaseProcess` + vanilla_impl (Callable) + Vanilla :py:meth:`~.BaseProcess._bootstrap` + lp_cache (LineProfilingCache) + Cache recovered by :py:meth:`~.LineProfilingCache.load` + *args + **kwargs + Passed to :py:meth:`~.BaseProcess._bootstrap` + + Returns: + Return value of ``vanilla_impl(*args, **kwargs)`` + + Side effects: + Profiling results are written + """ + try: + return vanilla_impl(self, *args, **kwargs) + finally: # Write profiling results + lp_cache.cleanup() + + +def get_preparation_data( + vanilla_impl: Callable[PS, dict[str, Any]], + cache_path: str, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> dict[str, Any]: + """ + Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, + slipping a :py:class:`PickleHook` into the returned dictionary so + that profiling is + triggered upon unpickling. + + Args: + vanilla_impl + Vanilla + :py:func:`multiprocessing.spawn.get_preparation_data` + cache_path + File from which the :py:class:`LineProfilingCache` should be + loaded + *args + **kwargs + Passed to + :py:func:`multiprocessing.spawn.get_preparation_data` + + Returns + Dictionary returned by + ``get_preparation_data(*args, **kwargs)`` with an extra key + """ + key = 'line_profiler_pickle_hook' # Doesn't matter + data = vanilla_impl(*args, **kwargs) + assert key not in data + data[key] = PickleHook(cache_path) + return data + + +def apply( + cache_path: str, *, lp_cache: LineProfilingCache | None = None, +) -> None: + """ + Set up profiling in :py:mod:`multiprocessing` child processes by + applying patches to the module. + + Args: + cache_path + Path to the file whence a :py:class:`LineProfilingCache` + object can be loaded + lp_cache + Optional :py:class:`LineProfilingCache` instance; + if not provided, it is loaded from `cache_path`, and + profiling is set up therefrom in the (sub-)process + + Side effects: + - :py:mod:`multiprocessing` marked as having been set up + - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + patched + - :py:func:`multiprocessing.spawn.get_preparation_data` patched + - Cleanup callbacks registered via `lp_cache.add_cleanup()` + """ + patched_marker = '_line_profiler_patched_multiprocessing' + if getattr(multiprocessing, patched_marker, False): + return + if lp_cache is None: + lp_cache = LineProfilingCache._from_path(cache_path) + _setup_in_child_process(lp_cache) + + vanilla: Callable[..., Any] | None + + # Patch `multiprocessing.process.BaseProcess._bootstrap()` + Proc = multiprocessing.process.BaseProcess + vanilla = Proc._bootstrap # type: ignore[attr-defined] + Proc._bootstrap = ( # type: ignore[attr-defined] + partialmethod(bootstrap, vanilla, lp_cache) + ) + lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) + + # Patch `multiprocessing.spawn.get_preparation_data()` + try: + from multiprocessing import spawn + except ImportError: # Incompatible platforms + pass + else: + vanilla = getattr(spawn, 'get_preparation_data', None) + if vanilla: + spawn.get_preparation_data = partial( + get_preparation_data, vanilla, cache_path, + ) + lp_cache.add_cleanup( + setattr, spawn, 'get_preparation_data', vanilla, + ) + + # Mark `multiprocessing` as having been patched + setattr(multiprocessing, patched_marker, True) + lp_cache.add_cleanup(vars(multiprocessing).pop, patched_marker, None) From e56d23bf733c8e67deffc2827dfddee5ba26ff4c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 28 Mar 2026 20:54:35 +0100 Subject: [PATCH 007/111] Child-process stat gathering; cache refactoring line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added debug logging to various methods gather_stats() New method for gathering profiling stats from child processes inject_env_vars() New method for injecting `.environ` into `os.environ` line_profiler/line_profiler.py::LineStats get_empty_instance() New convenience method for creating an empty instance from_files() Added new argument `on_defective` to allow for processing a group of files that cannot all be correctly read --- .../_child_process_profiling/cache.py | 84 +++++++++++++++++-- line_profiler/line_profiler.py | 54 +++++++++++- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 3a4f22eb..1c611fc5 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -11,11 +11,15 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable -from functools import partial +from functools import partial, cached_property +from operator import setitem +from pathlib import Path from pickle import HIGHEST_PROTOCOL -from typing import Any +from typing import Any, cast from typing_extensions import Self, ParamSpec +from .. import _diagnostics as diagnostics +from ..line_profiler import LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -60,8 +64,11 @@ def cleanup(self) -> None: callback = callbacks.pop() try: callback() - except Exception: - pass + except Exception as e: + msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' + else: + msg = f'Cleanup succeeded: {callback}' + self._debug(msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -73,6 +80,7 @@ def add_cleanup( if args or kwargs: callback = partial(callback, *args, **kwargs) self._cleanup_stack.append(callback) + self._debug(f'Cleanup callback added: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -99,7 +107,13 @@ def load(cls) -> Self: These should have been set from an ancestral Python process. """ pid = os.environ[INHERITED_PID_ENV_VARNAME] - cache_dir = os.environ[f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}'] + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' + cache_dir = os.environ[cache_varname] + msg = ( + f'PID {os.getpid()} (from {pid}): ' + f'Loading instance from ${{{cache_varname}}} = {cache_dir}' + ) + diagnostics.log.debug(msg) return cls._from_path(cls._get_filename(cache_dir)) def dump(self) -> None: @@ -111,14 +125,59 @@ def dump(self) -> None: Note: Cleanup callbacks are not serialized. """ + content = self._get_init_args() + msg = f'Dumping instance data to {self.filename}: {content!r}' + self._debug(msg) with open(self.filename, mode='wb') as fobj: - pickle.dump( - self._get_init_args(), fobj, protocol=HIGHEST_PROTOCOL, - ) + pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) + + def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: + """ + Gather the profiling output files matching ``glob_pattern`` from + :py:attr:`~.cache_dir`, consolidating them into a single + :py:class:`LineStats` object. + """ + fnames = list(Path(self.cache_dir).glob(glob_pattern)) + self._debug( + 'Loading results from {} child profiling file(s): {!r}' + .format(len(fnames), fnames) + ) + if not fnames: + return LineStats.get_empty_instance() + return LineStats.from_files(*fnames, on_defective='ignore') + + def inject_env_vars( + self, env: dict[str, str] | None = None, + ) -> None: + """ + Inject the :py:attr:`~.environ` variables into ``env`` and add + cleanup callbacks to reverse them. + + Args: + env (dict[str, str] | None): + Dictionary in the format of :py:data:`os.environ`; + default is to use that + """ + if env is None: + env = cast(dict[str, str], os.environ) + for name, value in self.environ.items(): + try: + old = env[name] + except KeyError: + self.add_cleanup(env.pop, name, None) + change = f'{value!r} (new)' + else: + self.add_cleanup(setitem, env, name, old) + change = f'{old!r} -> {value!r}' + self._debug(f'Injecting env var ${{{name}}}: {change}') + env[name] = value + + def _debug(self, msg: str) -> None: + diagnostics.log.debug(f'{self._debug_message_header}: {msg}') @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: - with open(cls._get_filename(fname), mode='rb') as fobj: + with open(fname, mode='rb') as fobj: return cls(**pickle.load(fobj)) def _get_init_args(self) -> dict[str, Any]: @@ -147,3 +206,10 @@ def environ(self) -> dict[str, str]: @property def filename(self) -> str: return self._get_filename(self.cache_dir) + + @cached_property + def _debug_message_header(self) -> str: + pid = os.getpid() + if self.main_pid == pid: + return f'PID {pid} (main process)' + return f'PID {pid} (from {self.main_pid})' diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index bede6f6c..03899261 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -18,6 +18,7 @@ import tempfile import types import tokenize +import warnings from argparse import ArgumentParser from datetime import datetime from os import PathLike @@ -367,17 +368,64 @@ def to_file(self, filename: PathLike[str] | str) -> None: with open(filename, 'wb') as f: pickle.dump(self, f, pickle.HIGHEST_PROTOCOL) + @classmethod + def get_empty_instance(cls) -> Self: + """ + Returns: + instance (LineStats): + New instance without any profiling data. + """ + unit = LineProfiler().timer_unit # type: ignore[attr-defined] + return cls({}, unit) + @classmethod def from_files( - cls, file: PathLike[str] | str, /, *files: PathLike[str] | str + cls, + file: PathLike[str] | str, + /, + *files: PathLike[str] | str, + on_defective: Literal['ignore', 'warn', 'error'] = 'error', ) -> Self: """ Utility function to load an instance from the given filenames. + + Args: + file (PathLike[str] | str): + File to load profiling data from + *files (PathLike[str] | str): + Ditto above + on_defective (Literal['ignore', 'warn', 'error']): + What to do if some files fail to load: ``'ignore'`` + those files, skip them but with a ``'warn'``-ing, or + raise the ``'error'`` as soon as one is encountered + + Returns: + instance (LineStats): + New instance """ stats_objs = [] - for file in [file, *files]: + failures: dict[str, str] = {} + all_files = [file, *files] + for file in all_files: with open(file, 'rb') as f: - stats_objs.append(pickle.load(f)) + try: + stats_objs.append(pickle.load(f)) + except Exception as e: + if on_defective == 'error': + raise + failures[str(file)] = f'{type(e).__name__}: {e}' + if failures: + msg = ( + '{} file(s) out of {} failed to load and are skipped: {!r}' + .format(len(failures), len(all_files), failures) + ) + if on_defective == 'warn': + warnings.warn(msg) + diagnostics.log.warning(msg) + else: # 'ignore' + diagnostics.log.debug(msg) + if not stats_objs: + return cls.get_empty_instance() return cls.from_stats_objects(*stats_objs) @classmethod From 6f43316845d1825543676e201cb12692e7380484 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 29 Mar 2026 03:32:42 +0200 Subject: [PATCH 008/111] WIP: kernprof now sets up child-proc profiling line_profiler/rc/line_profiler.toml::[tool.line_profiler.kernprof] Added new key-value pair `prof-child-procs` for the default value of `kernprof --prof-child-procs` kernprof.py - New boolean flags `[--prof-child-procs[=...] | --no-prof-child-procs]` for controlling whether to set up profiling in child processes - Fixed bug in `_manage_profiler.__exit__()` where `CuratedProfilerContext.uninstall()` can be skipped if the preceding code raises an error --- kernprof.py | 158 +++++++++++++++++++++++----- line_profiler/rc/line_profiler.toml | 3 + 2 files changed, 136 insertions(+), 25 deletions(-) diff --git a/kernprof.py b/kernprof.py index 5a873571..37cc14a0 100755 --- a/kernprof.py +++ b/kernprof.py @@ -79,6 +79,7 @@ def main(): [-s SETUP] [-p {path/to/script | object.dotted.path}[,...]] [--preimports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [--prof-imports [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] + [--prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [-o OUTFILE] [-v] [-q] [--rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0]] [-u UNIT] @@ -151,7 +152,7 @@ def main(): -q, --quiet Decrease verbosity level (default: 0). At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, silence the stderr. - --rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] +--rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Use rich formatting if viewing output. (Default: False; short form: -r) -u, --unit UNIT Output unit (in seconds) in which the timing info is @@ -226,6 +227,7 @@ def main(): short_string_path, ) from line_profiler.profiler_mixin import ByCountProfilerMixin +from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._logger import Logger from line_profiler import _diagnostics as diagnostics @@ -327,6 +329,7 @@ def resolve_module_path(mod_name): # type: (str) -> str | None fname = mod_spec.origin # type: str | None if fname and os.path.exists(fname): return fname + return None get_module_path = modname_to_modpath if static else resolve_module_path @@ -678,6 +681,14 @@ def _add_core_parser_arguments(parser): 'Only works with line profiling (`-l`/`--line-by-line`). ' f'(Default: {default.conf_dict["prof_imports"]})', ) + add_argument( + prof_opts, + '--prof-child-procs', + action='store_true', + help='Extend profiling into child Python processes. ' + 'Only works with line profiling (`-l`/`--line-by-line`). ' + f'(Default: {default.conf_dict["prof_child_procs"]})', + ) out_opts = parser.add_argument_group('output options') if default.conf_dict['outfile']: def_outfile = repr(default.conf_dict['outfile']) @@ -800,8 +811,8 @@ def _build_parsers(args=None): # We've already consumed the `-m `, so we need a dummy # parser for generating the help text; # but the real parser should not consume the `options.script` - # positional arg, and it it got the `--help` option, it should - # hand off the the dummy parser + # positional arg, and if it got the `--help` option, it should + # hand off to the dummy parser real_parser = ArgumentParser(add_help=False, **parser_kwargs) real_parser.add_argument('-h', '--help', action='store_true') help_parser = ArgumentParser(**parser_kwargs) @@ -1040,7 +1051,7 @@ def _write_tempfile(source, content, options): ) -def _write_preimports(prof, options, exclude): +def _write_preimports(prof, options, exclude, keep=False): """ Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. @@ -1068,8 +1079,9 @@ def _write_preimports(prof, options, exclude): ns = {} # Use a fresh namespace execfile(temp_mod_path, ns, ns) # Delete the tempfile ASAP if its execution succeeded - if not diagnostics.KEEP_TEMPDIRS: + if not (keep or diagnostics.KEEP_TEMPDIRS): _remove(temp_mod_path) + return temp_mod_path def _remove(path, *, recursive=False, missing_ok=False): @@ -1083,9 +1095,8 @@ def _remove(path, *, recursive=False, missing_ok=False): path.unlink(missing_ok=missing_ok) -def _dump_filtered_stats(tmpdir, prof, filename): +def _dump_filtered_stats(tmpdir, prof, filename, extra_line_stats=None): import os - import pickle # Build list of known temp file paths tempfile_paths = [ @@ -1103,21 +1114,25 @@ def _dump_filtered_stats(tmpdir, prof, filename): prof.dump_stats(filename) return + line_stats = prof.get_stats() + if extra_line_stats is not None: + line_stats += extra_line_stats + _dump_filtered_line_stats(line_stats, tempfile_paths, filename) + + +def _dump_filtered_line_stats(stats, exclude, filename): # Filter the filenames to remove data from tempfiles, which will # have been deleted by the time the results are viewed in a # separate process - stats = prof.get_stats() timings = stats.timings for key in set(timings): fname = key[0] try: - if any(os.path.samefile(fname, tmp) for tmp in tempfile_paths): + if any(os.path.samefile(fname, tmp) for tmp in exclude): del timings[key] except OSError: del timings[key] - - with open(filename, 'wb') as f: - pickle.dump(stats, f, protocol=pickle.HIGHEST_PROTOCOL) + stats.to_file(filename) def _format_call_message(func, *args, **kwargs): @@ -1167,6 +1182,8 @@ class _manage_profiler: Note: modifies ``options`` with extra attributes. """ + cache: LineProfilingCache + def __init__(self, options, module, exit_on_error): self.options = options self.module = module @@ -1182,14 +1199,36 @@ def __enter__(self): self.prof, insert_builtin=self.options.builtin, ) self._ctx.install() - script_file = _prepare_exec_script( - self.options, self.module, self.prof, self.exit_on_error, + # Keep the generated pre-imports file to be reused in child + # processes + script_file, preimports_file = _prepare_exec_script( + self.options, self.module, self.prof, + exit_on_error=self.exit_on_error, + keep_preimports_file=self.set_up_child_profiling, ) + if self.set_up_child_profiling: + self.cache = _prepare_child_profiling_cache( + self.options, self.prof, preimports_file, + ) return self.prof, script_file def __exit__(self, *_, **__): - _post_profile(self.options, self.prof) - self._ctx.uninstall() + try: + extra_stats = None + if self.set_up_child_profiling: + try: + extra_stats = self.cache.gather_stats() + finally: + self.cache.cleanup() + _post_profile(self.options, self.prof, extra_stats) + finally: + self._ctx.uninstall() + + @property + def set_up_child_profiling(self): + return bool( + self.options.line_by_line and self.options.prof_child_procs + ) def _prepare_profiler(options, module, exit_on_error): @@ -1236,7 +1275,12 @@ def _prepare_profiler(options, module, exit_on_error): return ContextualProfile() -def _prepare_exec_script(options, module, prof, exit_on_error): +def _prepare_exec_script( + options, module, prof, + *, + exit_on_error=False, + keep_preimports_file=False, +): """ Set up the script to be executed among other things. """ @@ -1259,6 +1303,8 @@ def _prepare_exec_script(options, module, prof, exit_on_error): options.prof_mod = _normalize_profiling_targets(options.prof_mod) if not options.prof_mod: options.preimports = False + + preimports_file = None if options.line_by_line and options.preimports: # We assume most items in `.prof_mod` to be import-able without # significant side effects, but the same cannot be said if it @@ -1266,7 +1312,9 @@ def _prepare_exec_script(options, module, prof, exit_on_error): # even have a `if __name__ == '__main__': ...` guard. So don't # eager-import it. exclude = set() if module else {script_file} - _write_preimports(prof, options, exclude) + preimports_file = _write_preimports( + prof, options, exclude, keep=keep_preimports_file, + ) if options.output_interval and not options.dryrun: options.rt = RepeatedTimer( @@ -1275,7 +1323,60 @@ def _prepare_exec_script(options, module, prof, exit_on_error): else: options.rt = None options.original_stdout = sys.stdout - return script_file + return script_file, preimports_file + + +def _prepare_child_profiling_cache(options, prof, preimports_file): + """ + Handle the (line-)profiling of spawned/forked child Python + processes. + """ + from line_profiler._child_process_profiling import ( + pth_hook, multiprocessing_patches, + ) + + # Create the cache dir and cache file here; the cache instance will + # be responsible for managing their lifetimes, while derivative + # instances in child processes will merely inherit and use them + cache = LineProfilingCache( + tempfile.mkdtemp(), + options.prof_mod, + options.prof_imports, + preimports_file, + insert_builtin=options.builtin, + ) + cache.add_cleanup( + _remove, cache.cache_dir, recursive=True, missing_ok=True, + ) + cache.dump() + cache.add_cleanup(_remove, cache.filename, missing_ok=True) + + # This file is handed to us at the end of + # `_manage_profiler.__enter__()`; + # normally it is deleted before `.__enter__()` returns, but when + # child-process profiling is used, it is to persist for the lifetime + # of the cache (so that child processes can do the same preimports) + if preimports_file is not None: + cache.add_cleanup(_remove, preimports_file, missing_ok=True) + + # Note: the following functions/methods all clean up after + # themselves, so there is no need to explicitly call + # `cache.add_cleanup()` + + # Inject environment variables so that child Python processes can + # inherit them and resume profiling + cache.inject_env_vars() + + # Create the .pth file which allows for setting up profiling in all + # the child Python processes (e.g. those created by `os.system()` or + # `subprocess.run()` + pth_hook.write_pth_hook(cache) + + # Patch `multiprocessing` so that child processes are properly + # handled across all "start methods" + multiprocessing_patches.apply(cache.filename, lp_cache=cache) + + return cache def _main_profile(options, module=False, exit_on_error=True): @@ -1339,14 +1440,16 @@ def _main_profile(options, module=False, exit_on_error=True): ) -def _post_profile(options, prof): +def _post_profile(options, prof, extra_line_stats=None): """ Cleanup setup after executing a main profile """ if options.rt is not None: options.rt.stop() if not options.dryrun: - _dump_filtered_stats(options.tmpdir, prof, options.outfile) + _dump_filtered_stats( + options.tmpdir, prof, options.outfile, extra_line_stats, + ) short_outfile = short_string_path(options.outfile) diagnostics.log.info( ( @@ -1357,9 +1460,15 @@ def _post_profile(options, prof): + f'to {short_outfile!r}' ) if options.verbose > 0 and not options.dryrun: - kwargs = {} - if not isinstance(prof, ContextualProfile): - kwargs.update( + if isinstance(prof, ContextualProfile): + _call_with_diagnostics(options, prof.print_stats) + else: + stats = prof.get_stats() + if extra_line_stats is not None: + stats += extra_line_stats + _call_with_diagnostics( + options, + stats.print, output_unit=options.unit, stripzeros=options.skip_zero, summarize=options.summarize, @@ -1367,7 +1476,6 @@ def _post_profile(options, prof): stream=options.original_stdout, config=options.config, ) - _call_with_diagnostics(options, prof.print_stats, **kwargs) else: py_exe = _python_command() if isinstance(prof, ContextualProfile): diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 6680c06a..a5b7740c 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -94,6 +94,9 @@ preimports = true # - `prof-imports` (bool): # `--prof-imports` (true) or `--no-prof-imports` (false) prof-imports = false +# - `prof-child-procs` (bool): +# `--prof-child-procs` (true) or `--no-prof-child-procs` (false) +prof-child-procs = false # - Misc flags # - `verbose` (count): From 4e8ea92015508f06182ec5d6d26ad4a53c64576b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 1 Apr 2026 22:31:13 +0200 Subject: [PATCH 009/111] Debugging support in `kernprof` kernprof.py::_prepare_child_profiling_cache() - Now respecting ${LINE_PROFILER_KEEP_TEMPDIRS} - Now setting `LineProfilingCache.debug` line_profiler/_child_process_profiling/cache.py::LineProfilingCache - Added new attributes `.debug` and `._debug_log` - Now diverting debug messages to log files in `.cache_dir` --- kernprof.py | 13 ++++---- .../_child_process_profiling/cache.py | 31 ++++++++++++++----- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/kernprof.py b/kernprof.py index 37cc14a0..4f323ecb 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1344,20 +1344,21 @@ def _prepare_child_profiling_cache(options, prof, preimports_file): options.prof_imports, preimports_file, insert_builtin=options.builtin, + debug=options.debug, ) - cache.add_cleanup( - _remove, cache.cache_dir, recursive=True, missing_ok=True, - ) + clean_up = functools.partial(cache.add_cleanup, _remove) + if not diagnostics.KEEP_TEMPDIRS: + clean_up(cache.cache_dir, recursive=True, missing_ok=True) cache.dump() - cache.add_cleanup(_remove, cache.filename, missing_ok=True) + clean_up(cache.filename, missing_ok=True) # This file is handed to us at the end of # `_manage_profiler.__enter__()`; # normally it is deleted before `.__enter__()` returns, but when # child-process profiling is used, it is to persist for the lifetime # of the cache (so that child processes can do the same preimports) - if preimports_file is not None: - cache.add_cleanup(_remove, preimports_file, missing_ok=True) + if not (preimports_file is None or diagnostics.KEEP_TEMPDIRS): + clean_up(preimports_file, missing_ok=True) # Note: the following functions/methods all clean up after # themselves, so there is no need to explicitly call diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 1c611fc5..b8fb1818 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -50,6 +50,7 @@ class LineProfilingCache: # Note: if we're using the line profiler, `kernprof` always set # `builtin` to true insert_builtin: bool = True + debug: bool = diagnostics.DEBUG _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( default_factory=list, init=False, ) @@ -68,7 +69,7 @@ def cleanup(self) -> None: msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' else: msg = f'Cleanup succeeded: {callback}' - self._debug(msg) + self._debug_output(msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -80,7 +81,7 @@ def add_cleanup( if args or kwargs: callback = partial(callback, *args, **kwargs) self._cleanup_stack.append(callback) - self._debug(f'Cleanup callback added: {callback}') + self._debug_output(f'Cleanup callback added: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -127,7 +128,7 @@ def dump(self) -> None: """ content = self._get_init_args() msg = f'Dumping instance data to {self.filename}: {content!r}' - self._debug(msg) + self._debug_output(msg) with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) @@ -138,7 +139,7 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: :py:class:`LineStats` object. """ fnames = list(Path(self.cache_dir).glob(glob_pattern)) - self._debug( + self._debug_output( 'Loading results from {} child profiling file(s): {!r}' .format(len(fnames), fnames) ) @@ -169,11 +170,20 @@ def inject_env_vars( else: self.add_cleanup(setitem, env, name, old) change = f'{old!r} -> {value!r}' - self._debug(f'Injecting env var ${{{name}}}: {change}') + self._debug_output(f'Injecting env var ${{{name}}}: {change}') env[name] = value - def _debug(self, msg: str) -> None: - diagnostics.log.debug(f'{self._debug_message_header}: {msg}') + def _debug_output(self, msg: str) -> None: + msg = f'{self._debug_message_header}: {msg}' + diagnostics.log.debug(msg) + try: + with self._debug_log.open(mode='a') as fobj: + print(msg, file=fobj) + except ( + AttributeError, # `._debug_log` is None in non-debug mode + OSError, # Cache dir may have been removed during cleanup + ): + pass @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: @@ -207,6 +217,13 @@ def environ(self) -> dict[str, str]: def filename(self) -> str: return self._get_filename(self.cache_dir) + @property + def _debug_log(self) -> Path | None: + if not self.debug: + return None + fname = f'debug_log_{self.main_pid}_{os.getpid()}.log' + return Path(self.cache_dir) / fname + @cached_property def _debug_message_header(self) -> str: pid = os.getpid() From 5d89868d4ba878a673fbd1d518f071523a036fed Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 6 Apr 2026 22:43:24 +0200 Subject: [PATCH 010/111] Cleanup prioritization and more debug messages line_profiler/_child_process_profiling/cache.py::LineProfilingCache add_cleanup() Now deferring to a `._add_cleanup()` method which allows for cleanup-function prioritization _debug_output() Fixed type-checking line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply() Added debug output before `_setup_in_child_process()` is called to help with tracing line_profiler/_child_process_profiling/pth_hook.py load_pth_hook() _wrap_os_fork() Added debug output before `_setup_in_child_process()` is called to help with tracing _setup_in_child_process() - `wrap_os_fork` now defaults to false - `prof.dump_stats()` now has increased priority over other callbacks (doesn't seem to help with the malformed prof files though...) - Child-process profiling output now written to a less randomized filename to facilitate debugging --- .../_child_process_profiling/cache.py | 50 ++++++++++++------- .../multiprocessing_patches.py | 1 + .../_child_process_profiling/pth_hook.py | 17 ++++--- 3 files changed, 43 insertions(+), 25 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index b8fb1818..76eb62aa 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -51,8 +51,8 @@ class LineProfilingCache: # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG - _cleanup_stack: list[Callable[[], Any]] = dataclasses.field( - default_factory=list, init=False, + _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( + default_factory=dict, init=False, ) def cleanup(self) -> None: @@ -60,16 +60,17 @@ def cleanup(self) -> None: Pop all the cleanup callbacks from the internal stack added via :py:meth:`~.add_cleanup` and call them in order. """ - callbacks = self._cleanup_stack - while callbacks: - callback = callbacks.pop() - try: - callback() - except Exception as e: - msg = f'Cleanup failed: {callback}: {type(e).__name__}: {e}' - else: - msg = f'Cleanup succeeded: {callback}' - self._debug_output(msg) + for priority in sorted(self._cleanup_stacks): + callbacks = self._cleanup_stacks.pop(priority) + while callbacks: + callback = callbacks.pop() + try: + callback() + except Exception as e: + msg = f'failed: {callback}: {type(e).__name__}: {e}' + else: + msg = f'succeeded: {callback}' + self._debug_output('Cleanup ' + msg) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -78,10 +79,19 @@ def add_cleanup( Add a cleanup callback to the internal stack; they can be later called by :py:meth:`~.cleanup`. """ + self._add_cleanup(callback, 0, *args, **kwargs) + + def _add_cleanup( + self, callback: Callable[PS, Any], priority: float, + *args: PS.args, **kwargs: PS.kwargs, + ) -> None: if args or kwargs: callback = partial(callback, *args, **kwargs) - self._cleanup_stack.append(callback) - self._debug_output(f'Cleanup callback added: {callback}') + self._cleanup_stacks.setdefault(priority, []).append(callback) + header = 'Cleanup callback added' + if priority: + header = f'{header} (priority: {priority})' + self._debug_output(f'{header}: {callback}') def copy( self, *, inherit_cleanups: bool = False, **replacements @@ -96,7 +106,10 @@ def copy( init_args[field] = replacements.get(field, value) copy = type(self)(**init_args) if inherit_cleanups: - copy._cleanup_stack[:] = self._cleanup_stack + copy._cleanup_stacks = { + priority: list(callbacks) + for priority, callbacks in self._cleanup_stacks.items() + } return copy @classmethod @@ -176,13 +189,12 @@ def inject_env_vars( def _debug_output(self, msg: str) -> None: msg = f'{self._debug_message_header}: {msg}' diagnostics.log.debug(msg) + if not self._debug_log: + return try: with self._debug_log.open(mode='a') as fobj: print(msg, file=fobj) - except ( - AttributeError, # `._debug_log` is None in non-debug mode - OSError, # Cache dir may have been removed during cleanup - ): + except OSError: # Cache dir may have been rm-ed during cleanup pass @classmethod diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 39cabe2a..a617cb3e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -156,6 +156,7 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) + lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') _setup_in_child_process(lp_cache) vanilla: Callable[..., Any] | None diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index f19a8c33..e073d820 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -114,7 +114,10 @@ def load_pth_hook(ppid): # type: (int) -> None if getattr(load_pth_hook, 'called', False): return try: - _setup_in_child_process(LineProfilingCache.load()) + cache = LineProfilingCache.load() + cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') + _setup_in_child_process(cache, True) + # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: msg = f'{type(e)}: {e}' @@ -148,14 +151,15 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None def wrapper(): result = fork() if not result: # In the fork - _setup_in_child_process(cache.copy(), False) + cache._debug_output(f'cache {id(cache):#x} setting up (fork)...') + _setup_in_child_process(cache.copy()) return result os.fork = wrapper cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=True): +def _setup_in_child_process(cache, wrap_os_fork=False): # type: (LineProfilingCache, bool) -> None """ Set up shop in a forked/spawned child process so that @@ -204,12 +208,13 @@ def _setup_in_child_process(cache, wrap_os_fork=True): # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates handle, prof_outfile = mkstemp( - prefix='line-profiler-child-prof-output-', + prefix='child-prof-output-{}-{}-{:#x}-' + .format(cache.main_pid, os.getpid(), id(prof)), suffix='.lprof', dir=cache.cache_dir, ) - try: - cache.add_cleanup(prof.dump_stats, prof_outfile) + try: # Whatever else we do, write the profiling stats first + cache._add_cleanup(prof.dump_stats, -1, prof_outfile) finally: os.close(handle) From e6faa2a0c64c5bcc8e525266cec98866d465a292 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 7 Apr 2026 02:20:54 +0200 Subject: [PATCH 011/111] Try to keep one cache per process line_profiler/_child_process_profiling/cache.py::LineProfilingCache profiler New attribute for the profiler instance copy(..., inherit_profiler=...) New argument for inheriting the `.profiler` load() Now keeping track of the loaded instance and returning it in subsequent calls line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply(..., lp_cache=None) - If the `LineProfilingCache.load()`-ed instance is consistent with that loaded from `cache_path`, the former is used - Added more debugging output line_profiler/_child_process_profiling/pth_hook.py load_pth_hook() Added more debugging output _wrap_os_fork() Updated debugging output _setup_in_child_process() - Now returning a boolean (whether setup has been newly done) - Now setting `.profiler` of the cache instance - Added moew debugging output --- .../_child_process_profiling/cache.py | 58 ++++++++++++++----- .../multiprocessing_patches.py | 12 +++- .../_child_process_profiling/pth_hook.py | 32 ++++++++-- 3 files changed, 80 insertions(+), 22 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 76eb62aa..589e54ab 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,11 +15,11 @@ from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL -from typing import Any, cast +from typing import Any, ClassVar, cast from typing_extensions import Self, ParamSpec from .. import _diagnostics as diagnostics -from ..line_profiler import LineStats +from ..line_profiler import LineProfiler, LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -51,9 +51,12 @@ class LineProfilingCache: # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG + + profiler: LineProfiler | None = dataclasses.field(default=None, init=False) _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( default_factory=dict, init=False, ) + _loaded_instance: ClassVar[Self | None] = None def cleanup(self) -> None: """ @@ -94,12 +97,27 @@ def _add_cleanup( self._debug_output(f'{header}: {callback}') def copy( - self, *, inherit_cleanups: bool = False, **replacements + self, *, + inherit_cleanups: bool = False, + inherit_profiler: bool = False, + **replacements ) -> Self: """ - Make a copy with optionally replaced fields; - if ``inherit_cleanups`` is set to true, the copy also makes a - (shallow) copy of the clean-callback stack. + Make a copy with optionally replaced fields. + + Args: + inherit_cleanups (bool): + If true, the copy also makes a (shallow) copy of the + cleanup-callback stack. + inherit_profiler (bool): + If true, the copy also gets a reference to + :py:attr:`~.profiler` + **replacements (Any): + Optional fields to replace + + Return: + inst (LineProfilingCache): + New instance """ init_args: dict[str, Any] = {} for field, value in self._get_init_args().items(): @@ -110,6 +128,8 @@ def copy( priority: list(callbacks) for priority, callbacks in self._cleanup_stacks.items() } + if inherit_profiler: + copy.profiler = self.profiler return copy @classmethod @@ -119,16 +139,24 @@ def load(cls) -> Self: :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID` and :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR_`. These should have been set from an ancestral Python process. + + Note: + If a previously :py:meth:`.~.load`-ed instance exists, it is + returned instead of a new instance. """ - pid = os.environ[INHERITED_PID_ENV_VARNAME] - cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' - cache_dir = os.environ[cache_varname] - msg = ( - f'PID {os.getpid()} (from {pid}): ' - f'Loading instance from ${{{cache_varname}}} = {cache_dir}' - ) - diagnostics.log.debug(msg) - return cls._from_path(cls._get_filename(cache_dir)) + instance = cls._loaded_instance + if instance is None: + pid = os.environ[INHERITED_PID_ENV_VARNAME] + cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' + cache_dir = os.environ[cache_varname] + msg = ( + f'PID {os.getpid()} (from {pid}): ' + f'Loading instance from ${{{cache_varname}}} = {cache_dir}' + ) + diagnostics.log.debug(msg) + instance = cls._from_path(cls._get_filename(cache_dir)) + cls._loaded_instance = instance + return instance def dump(self) -> None: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index a617cb3e..8fa88db3 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -156,8 +156,18 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) + # Hack to retrieve the `.load()`-ed instance if one exists + loaded_cache = LineProfilingCache._loaded_instance + if ( + loaded_cache is not None + and loaded_cache._get_init_args() == lp_cache._get_init_args() + ): + lp_cache = loaded_cache lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - _setup_in_child_process(lp_cache) + has_set_up = _setup_in_child_process(lp_cache) + lp_cache._debug_output('cache {:#x} setup {}'.format( + id(lp_cache), 'done' if has_set_up else 'aborted', + )) vanilla: Callable[..., Any] | None diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index e073d820..75a74ed9 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -116,7 +116,10 @@ def load_pth_hook(ppid): # type: (int) -> None try: cache = LineProfilingCache.load() cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') - _setup_in_child_process(cache, True) + has_set_up = _setup_in_child_process(cache, True) + cache._debug_output('cache {:#x} setup {}'.format( + id(cache), 'done' if has_set_up else 'aborted', + )) # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: @@ -150,9 +153,15 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None @wraps(fork) def wrapper(): result = fork() - if not result: # In the fork - cache._debug_output(f'cache {id(cache):#x} setting up (fork)...') - _setup_in_child_process(cache.copy()) + if result: + return result + # If we're here, we are in the fork + forked = cache.copy() + forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') + has_set_up = _setup_in_child_process(forked) + forked._debug_output('cache {:#x} setup {}'.format( + id(forked), 'done' if has_set_up else 'aborted', + )) return result os.fork = wrapper @@ -160,7 +169,7 @@ def wrapper(): def _setup_in_child_process(cache, wrap_os_fork=False): - # type: (LineProfilingCache, bool) -> None + # type: (LineProfilingCache, bool) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. @@ -171,7 +180,15 @@ def _setup_in_child_process(cache, wrap_os_fork=False): wrap_os_fork (bool): Whether to wrap :py:func:`os.fork` which handles profiling; already-forked child processes should set this to false + + Returns: + has_set_up (bool): + False is ``cache`` has already been set up prior to calling + this function, true otherwise """ + if cache.profiler is not None: # Already set up + return False + import os from atexit import register from tempfile import mkstemp @@ -188,7 +205,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Create a profiler instance and manage it with # `CuratedProfilerContext` - prof = LineProfiler() + cache.profiler = prof = LineProfiler() upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() @@ -196,6 +213,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: + cache._debug_output(f'cache {id(cache):#x} loading preimports...') with open(cache.preimports_module, mode='rb') as fobj: code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace @@ -225,3 +243,5 @@ def _setup_in_child_process(cache, wrap_os_fork=False): # Set `cache.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate register(cache.cleanup) + + return True From 4d6900ae54759f4a00f9ad08d813db3aad991e88 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 7 Apr 2026 04:17:54 +0200 Subject: [PATCH 012/111] Renamed submodule --- .../{meta_path_finder.py => import_machinery.py} | 0 line_profiler/_child_process_profiling/pth_hook.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename line_profiler/_child_process_profiling/{meta_path_finder.py => import_machinery.py} (100%) diff --git a/line_profiler/_child_process_profiling/meta_path_finder.py b/line_profiler/_child_process_profiling/import_machinery.py similarity index 100% rename from line_profiler/_child_process_profiling/meta_path_finder.py rename to line_profiler/_child_process_profiling/import_machinery.py diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 75a74ed9..0c6a8b19 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -201,7 +201,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False): ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler - from .meta_path_finder import RewritingFinder + from .import_machinery import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` From 1bd36edfbc7b8cdd25d6d079550472ccd36a29dd Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 00:10:02 +0200 Subject: [PATCH 013/111] WIP: pass along more data kernprof.py::_manage_profiler.__enter__() Updated so that the created `LineProfilingCache` instance carries a `.rewrite_module` line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added an optional `.rewrite_module` attribute line_profiler/_child_process_profiling/import_machinery.py ::RewritingFinder.find_spec() Now looking at `.lp_cache.rewrite_module` (where available) to check for specs to return --- kernprof.py | 13 +++--- .../_child_process_profiling/cache.py | 1 + .../import_machinery.py | 45 ++++++++++++++----- .../multiprocessing_patches.py | 3 +- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/kernprof.py b/kernprof.py index 4f323ecb..cc7ebe5c 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1208,7 +1208,7 @@ def __enter__(self): ) if self.set_up_child_profiling: self.cache = _prepare_child_profiling_cache( - self.options, self.prof, preimports_file, + self.options, self.prof, preimports_file, script_file, ) return self.prof, script_file @@ -1326,7 +1326,7 @@ def _prepare_exec_script( return script_file, preimports_file -def _prepare_child_profiling_cache(options, prof, preimports_file): +def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): """ Handle the (line-)profiling of spawned/forked child Python processes. @@ -1339,10 +1339,11 @@ def _prepare_child_profiling_cache(options, prof, preimports_file): # be responsible for managing their lifetimes, while derivative # instances in child processes will merely inherit and use them cache = LineProfilingCache( - tempfile.mkdtemp(), - options.prof_mod, - options.prof_imports, - preimports_file, + cache_dir=tempfile.mkdtemp(), + profiling_targets=options.prof_mod, + rewrite_module=script_file, + profile_imports=options.prof_imports, + preimports_module=preimports_file, insert_builtin=options.builtin, debug=options.debug, ) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 589e54ab..784cdf5c 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -44,6 +44,7 @@ class LineProfilingCache: profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) + rewrite_module: os.PathLike[str] | str | None = None profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) diff --git a/line_profiler/_child_process_profiling/import_machinery.py b/line_profiler/_child_process_profiling/import_machinery.py index 1436fc06..d4ddc1a0 100644 --- a/line_profiler/_child_process_profiling/import_machinery.py +++ b/line_profiler/_child_process_profiling/import_machinery.py @@ -11,13 +11,13 @@ import os import sys from collections.abc import Callable +from functools import partial from importlib.abc import MetaPathFinder, SourceLoader from importlib.machinery import ModuleSpec from pathlib import Path from types import CodeType, ModuleType from typing import TYPE_CHECKING -from .. import _diagnostics as diagnostics from ..autoprofile.run_module import AstTreeModuleProfiler from ..line_profiler import LineProfiler from .cache import LineProfilingCache @@ -26,6 +26,18 @@ __all__ = ('RewritingFinder',) +def _check_module_name(name: str, spec: ModuleSpec) -> bool: + return spec.name == name + + +def _check_module_origin( + path: os.PathLike[str] | str, spec: ModuleSpec, +) -> bool: + if spec.origin is None: + return False + return os.path.samefile(path, spec.origin) + + class RewritingFinder(MetaPathFinder, SourceLoader): """ Meta path finder to be set up in child processes, so that the @@ -33,6 +45,7 @@ class RewritingFinder(MetaPathFinder, SourceLoader): :py:func:`line_profiler.autoprofile.autoprofile.run` does. """ _cached_code_obj: CodeType + _checks: list[Callable[[ModuleSpec], bool]] def __init__( self, @@ -42,7 +55,12 @@ def __init__( ) -> None: self.prof = prof self.lp_cache = lp_cache - self.module_to_rewrite = module_to_rewrite + self._checks = [] + self._checks.append(partial(_check_module_name, module_to_rewrite)) + if lp_cache.rewrite_module: + self._checks.append( + partial(_check_module_origin, lp_cache.rewrite_module) + ) def install(self, *, index: int = 0) -> None: """ @@ -146,22 +164,25 @@ def exec_module(self, module: ModuleType) -> None: count = prof.object_count exec(self.get_code(spec), namespace, namespace) if prof.object_count > count: - msg = ( - 'main PID = {} / child PID = {}: ' - 'profiled {} code object{} in the `{}` module' - ).format( - self.lp_cache.main_pid, - os.getpid(), + msg = '{}: profiled {} code object{} in the `{}` module'.format( + type(self).__name__, prof.object_count, '' if prof.object_count == 1 else 's', - self.module_to_rewrite, + spec.name, ) - diagnostics.log.debug(msg) + self.lp_cache._debug_output(msg) def find_spec(self, *args, **kwargs) -> ModuleSpec | None: spec = self.find_spec_by_path(*args, **kwargs) - if (spec is not None and spec.name == self.module_to_rewrite): - return spec + if spec is None: + return None + for check in self._checks: + try: + if check(spec): + spec.loader = self + return spec + except Exception: + pass return None @staticmethod diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 8fa88db3..3abe0a82 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -102,8 +102,7 @@ def get_preparation_data( """ Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, slipping a :py:class:`PickleHook` into the returned dictionary so - that profiling is - triggered upon unpickling. + that profiling is triggered upon unpickling. Args: vanilla_impl From c26f9efa7258a4ee83082d9c2dbc2ef9e50bbf05 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 02:07:47 +0200 Subject: [PATCH 014/111] WIP: fix profiling for forked processes line_profiler/_child_process_profiling/ cache.py::LineProfilingCache _replace_loaded_instance() New convenience method for an instance in a fork to replace the instance to be `.load()`-ed _consistent_with_loaded_instance New attribute for checking whether the instance is consistent with what would have been `.load()`-ed multiprocessing_patches.py bootstrap(..., lp_cache=...) Can now be `None`, which defers the `.load()`-ing of the cache instance apply() - Streamlined logic for retrieving the loaded instance - Now using the above deferred loading whenever appropriate, so that cleanup and profiling is preserved in forked processes pth_hook.py::_wrap_os_fork() Now using `._replace_loaded_instance()`, so that future calls to `.load()` in the forked process retrieves the newly-created instance --- .../_child_process_profiling/cache.py | 10 +++++++ .../multiprocessing_patches.py | 26 ++++++++++--------- .../_child_process_profiling/pth_hook.py | 5 ++++ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 784cdf5c..58801457 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -226,6 +226,12 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def _replace_loaded_instance(self) -> bool: + if self._consistent_with_loaded_instance: + type(self)._loaded_instance = self + return True + return False + @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: with open(fname, mode='rb') as fobj: @@ -271,3 +277,7 @@ def _debug_message_header(self) -> str: if self.main_pid == pid: return f'PID {pid} (main process)' return f'PID {pid} (from {self.main_pid})' + + @cached_property + def _consistent_with_loaded_instance(self) -> bool: + return type(self).load()._get_init_args() == self._get_init_args() diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 3abe0a82..ca55c6e7 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -59,7 +59,7 @@ def bootstrap( vanilla_impl: Callable[ Concatenate[multiprocessing.process.BaseProcess, PS], T ], - lp_cache: LineProfilingCache, + lp_cache: LineProfilingCache | None, /, *args: PS.args, **kwargs: PS.kwargs @@ -74,8 +74,10 @@ def bootstrap( :py:class:`~.BaseProcess` vanilla_impl (Callable) Vanilla :py:meth:`~.BaseProcess._bootstrap` - lp_cache (LineProfilingCache) - Cache recovered by :py:meth:`~.LineProfilingCache.load` + lp_cache (LineProfilingCache | None) + Cache recovered by :py:meth:`~.LineProfilingCache.load`; + if :py:const:`None`, the ``.load()`` is deferred to after + ``vanilla_impl()`` is run *args **kwargs Passed to :py:meth:`~.BaseProcess._bootstrap` @@ -89,6 +91,8 @@ def bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: # Write profiling results + if lp_cache is None: + lp_cache = LineProfilingCache.load() lp_cache.cleanup() @@ -155,13 +159,8 @@ def apply( return if lp_cache is None: lp_cache = LineProfilingCache._from_path(cache_path) - # Hack to retrieve the `.load()`-ed instance if one exists - loaded_cache = LineProfilingCache._loaded_instance - if ( - loaded_cache is not None - and loaded_cache._get_init_args() == lp_cache._get_init_args() - ): - lp_cache = loaded_cache + if lp_cache._consistent_with_loaded_instance: + lp_cache = LineProfilingCache.load() lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') has_set_up = _setup_in_child_process(lp_cache) lp_cache._debug_output('cache {:#x} setup {}'.format( @@ -173,8 +172,11 @@ def apply( # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess vanilla = Proc._bootstrap # type: ignore[attr-defined] - Proc._bootstrap = ( # type: ignore[attr-defined] - partialmethod(bootstrap, vanilla, lp_cache) + Proc._bootstrap = partialmethod( # type: ignore[attr-defined] + bootstrap, vanilla, + # Always defer to the `.load()`-ed instance where possible + # (helps with keeping forked processes consistent) + None if lp_cache._consistent_with_loaded_instance else lp_cache, ) lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 0c6a8b19..29ac7822 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -157,6 +157,11 @@ def wrapper(): return result # If we're here, we are in the fork forked = cache.copy() + if forked._replace_loaded_instance(): + forked._debug_output( + f'cache {id(forked):#x} in fork ' + 'superseded cached `.load()`-ed instance' + ) forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') has_set_up = _setup_in_child_process(forked) forked._debug_output('cache {:#x} setup {}'.format( From 74d310f987d01f8e5b991af1eb0b6536fc0967e8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 02:59:59 +0200 Subject: [PATCH 015/111] Cleanup: `[...].multiprocessing_patches` kernprof.py::_prepare_child_profiling_cache() - Updated call to `[...].multiprocessing_patches.apply()` - Now always setting up the created instance as the one returned by further calls to `.load()` line_profiler/_child_process_profiling/multiprocessing_patches.py PickleHook - Refactored to contain no instance variables - Now always using `LineProfilingCache.load()` to retrieve the appropriate cache instance bootstrap() Removed argument `lp_cache` get_preparation_data() Removed arguemnt `cache_path` apply() - Removed argument `cache_path` - Argument `lp_cache` now required - Simplified implementation --- kernprof.py | 5 +- .../multiprocessing_patches.py | 100 +++++++----------- 2 files changed, 44 insertions(+), 61 deletions(-) diff --git a/kernprof.py b/kernprof.py index cc7ebe5c..55c0d8d9 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1376,7 +1376,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # Patch `multiprocessing` so that child processes are properly # handled across all "start methods" - multiprocessing_patches.apply(cache.filename, lp_cache=cache) + multiprocessing_patches.apply(cache) + + # Set up the cache instance to be the default one `.load()`-ed + cache._replace_loaded_instance() return cache diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ca55c6e7..3210ef39 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,7 +17,7 @@ import multiprocessing.process from collections.abc import Callable from functools import partial, partialmethod -from typing import Any, TypedDict, TypeVar +from typing import Any, TypeVar from typing_extensions import Concatenate, ParamSpec from .cache import LineProfilingCache @@ -30,9 +30,7 @@ T = TypeVar('T') PS = ParamSpec('PS') - -class _HookState(TypedDict): - cache_path: str # Cache to be loaded from here +_PATCHED_MARKER = '_line_profiler_patched_multiprocessing' class PickleHook: @@ -43,15 +41,24 @@ class PickleHook: See also: :py:class:`coverage.multiproc.Stowaway` """ - def __init__(self, cache_path: str) -> None: - self.cache_path = cache_path - - def __getstate__(self) -> _HookState: - return {'cache_path': self.cache_path} - - def __setstate__(self, state: _HookState) -> None: - self.cache_path = path = state['cache_path'] - apply(path) + def __getstate__(_) -> int: + # Cannot return `None`, or nothing will be pickled and + # `.__getstate__()` will not be invoked in the child + return 1 + + def __setstate__(*_) -> None: + # We're in a child process created by `multiprocessing`, so set + # up shop here... + lp_cache = LineProfilingCache.load() + lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') + has_set_up = _setup_in_child_process(lp_cache) + lp_cache._debug_output('cache {:#x} setup {}'.format( + id(lp_cache), 'done' if has_set_up else 'aborted', + )) + # ... and we don't care about polluting the `multiprocessing` + # namespace either, so don't bother with cleanup + if not getattr(multiprocessing, _PATCHED_MARKER, False): + _apply_mp_patches(lp_cache, _no_op) def bootstrap( @@ -59,7 +66,6 @@ def bootstrap( vanilla_impl: Callable[ Concatenate[multiprocessing.process.BaseProcess, PS], T ], - lp_cache: LineProfilingCache | None, /, *args: PS.args, **kwargs: PS.kwargs @@ -74,10 +80,6 @@ def bootstrap( :py:class:`~.BaseProcess` vanilla_impl (Callable) Vanilla :py:meth:`~.BaseProcess._bootstrap` - lp_cache (LineProfilingCache | None) - Cache recovered by :py:meth:`~.LineProfilingCache.load`; - if :py:const:`None`, the ``.load()`` is deferred to after - ``vanilla_impl()`` is run *args **kwargs Passed to :py:meth:`~.BaseProcess._bootstrap` @@ -91,14 +93,11 @@ def bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: # Write profiling results - if lp_cache is None: - lp_cache = LineProfilingCache.load() - lp_cache.cleanup() + LineProfilingCache.load().cleanup() def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], - cache_path: str, /, *args: PS.args, **kwargs: PS.kwargs @@ -112,9 +111,6 @@ def get_preparation_data( vanilla_impl Vanilla :py:func:`multiprocessing.spawn.get_preparation_data` - cache_path - File from which the :py:class:`LineProfilingCache` should be - loaded *args **kwargs Passed to @@ -127,25 +123,18 @@ def get_preparation_data( key = 'line_profiler_pickle_hook' # Doesn't matter data = vanilla_impl(*args, **kwargs) assert key not in data - data[key] = PickleHook(cache_path) + data[key] = PickleHook() return data -def apply( - cache_path: str, *, lp_cache: LineProfilingCache | None = None, -) -> None: +def apply(lp_cache: LineProfilingCache) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - cache_path - Path to the file whence a :py:class:`LineProfilingCache` - object can be loaded - lp_cache - Optional :py:class:`LineProfilingCache` instance; - if not provided, it is loaded from `cache_path`, and - profiling is set up therefrom in the (sub-)process + lp_cache (LineProfilingCache) + Cache instance governing the profiling run Side effects: - :py:mod:`multiprocessing` marked as having been set up @@ -154,19 +143,13 @@ def apply( - :py:func:`multiprocessing.spawn.get_preparation_data` patched - Cleanup callbacks registered via `lp_cache.add_cleanup()` """ - patched_marker = '_line_profiler_patched_multiprocessing' - if getattr(multiprocessing, patched_marker, False): - return - if lp_cache is None: - lp_cache = LineProfilingCache._from_path(cache_path) - if lp_cache._consistent_with_loaded_instance: - lp_cache = LineProfilingCache.load() - lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - has_set_up = _setup_in_child_process(lp_cache) - lp_cache._debug_output('cache {:#x} setup {}'.format( - id(lp_cache), 'done' if has_set_up else 'aborted', - )) + if not getattr(multiprocessing, _PATCHED_MARKER, False): + _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + +def _apply_mp_patches( + lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], +) -> None: vanilla: Callable[..., Any] | None # Patch `multiprocessing.process.BaseProcess._bootstrap()` @@ -174,11 +157,8 @@ def apply( vanilla = Proc._bootstrap # type: ignore[attr-defined] Proc._bootstrap = partialmethod( # type: ignore[attr-defined] bootstrap, vanilla, - # Always defer to the `.load()`-ed instance where possible - # (helps with keeping forked processes consistent) - None if lp_cache._consistent_with_loaded_instance else lp_cache, ) - lp_cache.add_cleanup(setattr, Proc, '_bootstrap', vanilla) + add_cleanup(setattr, Proc, '_bootstrap', vanilla) # Patch `multiprocessing.spawn.get_preparation_data()` try: @@ -188,13 +168,13 @@ def apply( else: vanilla = getattr(spawn, 'get_preparation_data', None) if vanilla: - spawn.get_preparation_data = partial( - get_preparation_data, vanilla, cache_path, - ) - lp_cache.add_cleanup( - setattr, spawn, 'get_preparation_data', vanilla, - ) + spawn.get_preparation_data = partial(get_preparation_data, vanilla) + add_cleanup(setattr, spawn, 'get_preparation_data', vanilla) # Mark `multiprocessing` as having been patched - setattr(multiprocessing, patched_marker, True) - lp_cache.add_cleanup(vars(multiprocessing).pop, patched_marker, None) + setattr(multiprocessing, _PATCHED_MARKER, True) + add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) + + +def _no_op(*_, **__) -> None: + pass From 21ec2182625a46bfe9ee5302f28538dab362ae21 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 05:00:15 +0200 Subject: [PATCH 016/111] Removed `import_machinery` line_profiler/_child_process_profiling/import_machinery.py Removed line_profiler/_child_process_profiling/pth_hook.py ::_setup_in_child_process() No longer set up the `RewritingFinder` because messing with the import system doesn't help with propagating autoprofiling rewrites to child processes... --- .../import_machinery.py | 242 ------------------ .../_child_process_profiling/pth_hook.py | 6 - 2 files changed, 248 deletions(-) delete mode 100644 line_profiler/_child_process_profiling/import_machinery.py diff --git a/line_profiler/_child_process_profiling/import_machinery.py b/line_profiler/_child_process_profiling/import_machinery.py deleted file mode 100644 index d4ddc1a0..00000000 --- a/line_profiler/_child_process_profiling/import_machinery.py +++ /dev/null @@ -1,242 +0,0 @@ -""" -A meta path finder object which rewrites a specific module. - -Note: - Based on the implementation in - :py:mod:`pytest_autoprofile.importers`. -""" -from __future__ import annotations - -import ast -import os -import sys -from collections.abc import Callable -from functools import partial -from importlib.abc import MetaPathFinder, SourceLoader -from importlib.machinery import ModuleSpec -from pathlib import Path -from types import CodeType, ModuleType -from typing import TYPE_CHECKING - -from ..autoprofile.run_module import AstTreeModuleProfiler -from ..line_profiler import LineProfiler -from .cache import LineProfilingCache - - -__all__ = ('RewritingFinder',) - - -def _check_module_name(name: str, spec: ModuleSpec) -> bool: - return spec.name == name - - -def _check_module_origin( - path: os.PathLike[str] | str, spec: ModuleSpec, -) -> bool: - if spec.origin is None: - return False - return os.path.samefile(path, spec.origin) - - -class RewritingFinder(MetaPathFinder, SourceLoader): - """ - Meta path finder to be set up in child processes, so that the - ``module_to_rewrite`` is rewritten for profiling as - :py:func:`line_profiler.autoprofile.autoprofile.run` does. - """ - _cached_code_obj: CodeType - _checks: list[Callable[[ModuleSpec], bool]] - - def __init__( - self, - prof: LineProfiler, - lp_cache: LineProfilingCache, - module_to_rewrite: str = '__main__', - ) -> None: - self.prof = prof - self.lp_cache = lp_cache - self._checks = [] - self._checks.append(partial(_check_module_name, module_to_rewrite)) - if lp_cache.rewrite_module: - self._checks.append( - partial(_check_module_origin, lp_cache.rewrite_module) - ) - - def install(self, *, index: int = 0) -> None: - """ - Install the importer into :py:data:`sys.meta_path` at the - specified ``index``; - if it's already there, it's first removed and re-inserted at - the requested position. - """ - self.uninstall(invalidate_caches=False) - sys.meta_path.insert(index, self) - - def uninstall(self, *, invalidate_caches: bool = True) -> None: - """ - Uninstall the importer from :py:data:`sys.meta_path`, and - optionally also invalidate the caches. - """ - try: - sys.meta_path.remove(self) - except ValueError: # Not in the list - return - if invalidate_caches: - self.invalidate_caches() - - @classmethod - def find_spec_by_path(cls, *args, **kwargs) -> ModuleSpec | None: - """ - Implementation of - :py:meth:`MetaPathFinder.find_spec` which looks - for module specs with the other meta-path finders. - - Returns: - maybe_spec (ModuleSpec | None) - Module spec if found, :py:const:`None` otherwise - """ - Implementation = Callable[..., ModuleSpec | None] - impls: list[Implementation] = [ - finder.find_spec for finder in sys.meta_path - if callable(getattr(finder, 'find_spec', None)) - if not isinstance(finder, cls) - ] - for impl in impls: - try: - spec = impl(*args, **kwargs) - except Exception: - continue - if spec is not None: - return spec - return None - - def write_code(self, spec: ModuleSpec) -> CodeType: - """ - Rewrite the module code that ``spec`` points to with - :py:class:`ast.autoprofile.run_module.AstTreeModuleProfiler`. - - Args: - spec (ModuleSpec) - Module spec - - Returns: - code (CodeType) - Code object - """ - assert spec.origin - fname = str(Path(spec.origin)) - module = AstTreeModuleProfiler( - spec.origin, - list(self.lp_cache.profiling_targets), - self.lp_cache.profile_imports, - ).profile() - # Slip in a helper node so as to ensure the availability of - # `@profile` - # Note: `@profile` is slipped into the local namespace by - # `RewritingFinder.exec_module()`, but that may not be - # enough for applications directly using the code objects (e.g. - # `runpy`. Hence, we provide an out by falling back to - # `@line_profiler.profile`. - ensure_profile_node, = ast.parse( - 'if profile not in globals():\n' - ' from line_profiler import profile' - ).body - module.body.insert(0, ensure_profile_node) - return compile(module, fname, 'exec') - - # Methods as dictated by the interface - - def invalidate_caches(self) -> None: - try: - del self._cached_code_obj - except AttributeError: - pass - super().invalidate_caches() - - def exec_module(self, module: ModuleType) -> None: - namespace = module.__dict__ - namespace['profile'] = prof = self.prof - spec: ModuleSpec | None = module.__spec__ - if spec is None: - raise RuntimeError(f'module = {module!r}: empty `.__spec__`') - if TYPE_CHECKING: # Appease `mypy` - assert hasattr(prof, 'object_count') - count = prof.object_count - exec(self.get_code(spec), namespace, namespace) - if prof.object_count > count: - msg = '{}: profiled {} code object{} in the `{}` module'.format( - type(self).__name__, - prof.object_count, - '' if prof.object_count == 1 else 's', - spec.name, - ) - self.lp_cache._debug_output(msg) - - def find_spec(self, *args, **kwargs) -> ModuleSpec | None: - spec = self.find_spec_by_path(*args, **kwargs) - if spec is None: - return None - for check in self._checks: - try: - if check(spec): - spec.loader = self - return spec - except Exception: - pass - return None - - @staticmethod - def get_data(path: os.PathLike[str] | str) -> bytes: - return Path(path).read_bytes() - - @classmethod - def get_filename(cls, name: str) -> str: - spec = cls.find_spec_by_path(name) - if spec is None: - raise ImportError(name) - origin = spec.origin - if origin is None: - raise ImportError(name) - if origin == 'frozen' or not os.path.exists(origin): - raise ImportError(name) - return origin - - def get_code(self, name_or_spec: str | ModuleSpec) -> CodeType: - if isinstance(name_or_spec, str): - spec = self.find_spec_by_path(name_or_spec) - if spec is None: - raise ImportError(name_or_spec) - else: - spec = name_or_spec - try: - try: - return self._cached_code_obj - except AttributeError: - self._cached_code_obj = code = self.write_code(spec) - return code - except Exception as e: - raise ImportError(name_or_spec) from e - - if TYPE_CHECKING: - def source_to_code( # type: ignore[override] - self, *args, **kwargs - ) -> CodeType: - """ - Notes - ----- - :py:mod:`mypy` reports that - :py:meth:`SourceLoader.source_to_code`, an instance method, - clashes with - :py:meth:`importlib.abc.InspectLoader.source_to_code`, a - static method. - - Since: - - The method is functionally only used in the superclasses - as an instance method, and - - :py:class:`importlib.abc.InspectLoader`` is merely - included as a base class because :py:class:`SourceLoader` - inherits from it, - just explicitly override the method here so that we can - catch and suppress the :py:mod:`mypy` error. - """ - return super().source_to_code(*args, **kwargs) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 29ac7822..a46b4dac 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -206,7 +206,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False): ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler - from .import_machinery import RewritingFinder # Create a profiler instance and manage it with # `CuratedProfilerContext` @@ -223,11 +222,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False): code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Set up the importer for rewriting `__main__` - finder = RewritingFinder(prof, cache) - finder.install() - cache.add_cleanup(finder.uninstall) - # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates handle, prof_outfile = mkstemp( From a51411fec69f4468d529aca3e67674b84f61bcca Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 07:54:14 +0200 Subject: [PATCH 017/111] Fixed stat-aggregation bug kernprof.py _dump_filtered_stats() Fixed bug where if no tempfile remains, the `extra_line_stats` are not merged into the dumped stats _prepare_child_profiling_cache() Now setting the `.profiler` of the returned cache object --- kernprof.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/kernprof.py b/kernprof.py index 55c0d8d9..50097fde 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1098,25 +1098,30 @@ def _remove(path, *, recursive=False, missing_ok=False): def _dump_filtered_stats(tmpdir, prof, filename, extra_line_stats=None): import os + if isinstance(prof, ContextualProfile): + # - Not using `line_profiler` + # -> doesn't matter if the source lines can't be retrieved + # -> no need to filter anything + prof.dump_stats(filename) + return + + # Remember to incorporate extra stats where available + line_stats = prof.get_stats() + if extra_line_stats is not None: + line_stats += extra_line_stats + # Build list of known temp file paths tempfile_paths = [ os.path.join(dirpath, fname) for dirpath, _, fnames in os.walk(tmpdir) for fname in fnames ] - - if not tempfile_paths or isinstance(prof, ContextualProfile): + if not tempfile_paths: # - No tempfiles written -> no function lives in tempfiles # -> no need to filter anything - # - Not using `line_profiler` - # -> doesn't matter if the source lines can't be retrieved - # -> no need to filter anything - prof.dump_stats(filename) + line_stats.to_file(filename) return - line_stats = prof.get_stats() - if extra_line_stats is not None: - line_stats += extra_line_stats _dump_filtered_line_stats(line_stats, tempfile_paths, filename) @@ -1378,8 +1383,11 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # handled across all "start methods" multiprocessing_patches.apply(cache) - # Set up the cache instance to be the default one `.load()`-ed + # Misc. setup: + # - Set up the cache instance to be the default one `.load()`-ed cache._replace_loaded_instance() + # - Set `cache.profiler` + cache.profiler = prof return cache From c06eea6e7dcd2e8086c02681292af973805b960d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 08:20:07 +0200 Subject: [PATCH 018/111] More forked-process fixes line_profiler/_child_process_profiling/multiprocessing_patches.py ::_apply_mp_patches() - Added debugging output for the patches - Now patching the copy of `runpy` imported by `multiprocessing.runpy` line_profiler/_child_process_profiling/pth_hook.py _wrap_os_fork() No longer creating a new `LineProfiler` instance (helps with handling forked processes) _setup_in_child_process(..., prof=...) New argument for avoiding instantiating a new profiler when not necessary (e.g. in a forked process) line_profiler/_child_process_profiling/runpy_patches.py New submodule for the aforementioned patching of `runpy` --- .../multiprocessing_patches.py | 45 ++++-- .../_child_process_profiling/pth_hook.py | 22 ++- .../_child_process_profiling/runpy_patches.py | 149 ++++++++++++++++++ 3 files changed, 199 insertions(+), 17 deletions(-) create mode 100644 line_profiler/_child_process_profiling/runpy_patches.py diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 3210ef39..28ce9acc 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -22,6 +22,7 @@ from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process +from .runpy_patches import create_runpy_wrapper __all__ = ('apply',) @@ -150,26 +151,50 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], ) -> None: - vanilla: Callable[..., Any] | None + def replace( + obj: Any, attr: str, value: Any, obj_name: str | None = None, + ) -> None: + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(setattr, obj, attr, old) + else: + add_cleanup(delattr, obj, attr) + setattr(obj, attr, value) + if obj_name is None: + obj_name = repr(obj) + lp_cache._debug_output('cache {:#x}: patched `{}.{}` -> `{}`'.format( + id(lp_cache), obj_name, attr, value, + )) # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - vanilla = Proc._bootstrap # type: ignore[attr-defined] - Proc._bootstrap = partialmethod( # type: ignore[attr-defined] - bootstrap, vanilla, + bootstrap_wrapper = partialmethod( + bootstrap, + Proc._bootstrap, # type: ignore[attr-defined] + ) + replace( + Proc, '_bootstrap', bootstrap_wrapper, + f'{Proc.__module__}.{Proc.__qualname__}', ) - add_cleanup(setattr, Proc, '_bootstrap', vanilla) - # Patch `multiprocessing.spawn.get_preparation_data()` + # Patch `multiprocessing.spawn` try: from multiprocessing import spawn except ImportError: # Incompatible platforms pass else: - vanilla = getattr(spawn, 'get_preparation_data', None) - if vanilla: - spawn.get_preparation_data = partial(get_preparation_data, vanilla) - add_cleanup(setattr, spawn, 'get_preparation_data', vanilla) + # Patch `get_preparation_data()` + gpd_wrapper = partial( # type: ignore[call-arg] + get_preparation_data, spawn.get_preparation_data, + ) + replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) + # Patch `runpy` (do it locally instead of tempering with the + # global `runpy` mmodule) + if hasattr(spawn, 'runpy'): + replace( + spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, + ) # Mark `multiprocessing` as having been patched setattr(multiprocessing, _PATCHED_MARKER, True) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index a46b4dac..e8dafaae 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -20,8 +20,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from .cache import LineProfilingCache # noqa: F401 + from collections.abc import Callable # noqa: F401 from pathlib import Path # noqa: F401 + from typing import Any # noqa: F401 + from ..line_profiler import LineProfiler # noqa: F401 + from .cache import LineProfilingCache # noqa: F401 __all__ = ( @@ -151,7 +154,7 @@ def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None return @wraps(fork) - def wrapper(): + def wrapper(): # type: () -> int result = fork() if result: return result @@ -162,8 +165,11 @@ def wrapper(): f'cache {id(forked):#x} in fork ' 'superseded cached `.load()`-ed instance' ) + # Note: we can reuse the profiler instance in the fork, but it + # needs to go through setup so that the separate profiling + # results are dumped into another output file forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') - has_set_up = _setup_in_child_process(forked) + has_set_up = _setup_in_child_process(forked, prof=cache.profiler) forked._debug_output('cache {:#x} setup {}'.format( id(forked), 'done' if has_set_up else 'aborted', )) @@ -173,8 +179,8 @@ def wrapper(): cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=False): - # type: (LineProfilingCache, bool) -> bool +def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): + # type: (LineProfilingCache, bool, LineProfiler | None) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. @@ -205,11 +211,13 @@ def _setup_in_child_process(cache, wrap_os_fork=False): _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) from ..curated_profiling import CuratedProfilerContext - from ..line_profiler import LineProfiler + from ..line_profiler import LineProfiler # noqa: F811 # Create a profiler instance and manage it with # `CuratedProfilerContext` - cache.profiler = prof = LineProfiler() + if prof is None: + prof = LineProfiler() + cache.profiler = prof upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py new file mode 100644 index 00000000..69d38da4 --- /dev/null +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -0,0 +1,149 @@ +""" +Patches for :py:mod:`runpy` to be patched into the namespace of +:py:mod:`multiprocessing.spawn`, so that the rewriting of ``__main__`` +can be continued into child processes. +""" +from __future__ import annotations + +import os +from collections.abc import Callable +from functools import partial +from importlib.util import find_spec +from types import ModuleType +from typing import cast, TypeVar +from typing_extensions import Concatenate, ParamSpec + +from ..autoprofile.ast_tree_profiler import AstTreeProfiler +from ..autoprofile.run_module import AstTreeModuleProfiler +from ..autoprofile.util_static import modname_to_modpath +from .cache import LineProfilingCache + + +__all__ = ('create_runpy_wrapper',) + + +PS = ParamSpec('PS') +T = TypeVar('T') + + +THIS_MODULE = (lambda: None).__module__ + + +def _copy_module(name: str) -> ModuleType: + """ + Returns: + module (ModuleType): + Module object, which is a fresh copy of the module named + ``name`` + """ + spec = find_spec(name) + if spec is None: + raise ModuleNotFoundError(name) + assert spec.loader + assert callable(getattr(spec.loader, 'exec_module', None)) + module = ModuleType(spec.name) + for attr, value in { + '__spec__': spec, + '__name__': spec.name, + '__file__': spec.origin, + '__path__': spec.submodule_search_locations, + }.items(): + if value is not None: + setattr(module, attr, value) + spec.loader.exec_module(module) + return module + + +def _exec( + cache: LineProfilingCache, + CodeWriter: type[AstTreeProfiler], + _code, # This represents the first pos arg to `exec()` (ignored) + /, + *args, **kwargs, +) -> None: + """ + To be monkey-patched into :py:mod:`runpy`'s namespace as `exec()` + so that rewritten and autoprofiled code at ``cache.rewrite_module`` + is always executed. + """ + assert cache.rewrite_module + cache._debug_output('Calling via {}: `exec({})`'.format( + THIS_MODULE, + ', '.join( + [repr(a) for a in (_code, *args)] + + [f'{k}={v!r}' for k, v in kwargs.items()] + ), + )) + fname = str(cache.rewrite_module) + code_writer = CodeWriter( + fname, + list(cache.profiling_targets), + cache.profile_imports, + ) + code = compile(code_writer.profile(), fname, 'exec') + exec(code, *args, **kwargs) + + +def _run( + cache: LineProfilingCache, + runpy: ModuleType, + func: Callable[Concatenate[str, PS], T], + resolve_target_to_path: Callable[[str], str], + CodeWriter: type[AstTreeProfiler], + target: str, + /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: + cache._debug_output('Calling via {}: `runpy.{}({})`'.format( + THIS_MODULE, + func.__name__, + ', '.join( + [repr(a) for a in (target, *args)] + + [f'{k}={v!r}' for k, v in kwargs.items()] + ), + )) + if cache.rewrite_module: + try: + filename = resolve_target_to_path(target) + profile = os.path.samefile(filename, cache.rewrite_module) + except Exception as e: + cache._debug_output( + f'{THIS_MODULE}: failed to check whether to ' + f'rewrite code loaded in `runpy.{func.__name__}(...)` ' + f'({type(e).__name__}: {e})' + ) + profile = False + else: + profile = False + # If we are about to run the code to be autoprofiled, monkey-patch + # `exec()` into the `runpy` namespace which just rewrites + # `cache.rewrite_module` and executes it + if profile: + runpy.exec = ( # type: ignore[attr-defined] + partial(_exec, cache, CodeWriter) + ) + try: + return func(target, *args, **kwargs) + finally: + try: + del runpy.exec + except AttributeError: + pass + + +def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: + """ + Create a copy of :py:mod:`runpy` which does code rewriting similar + to :py:func:`line_profiler.autoprofile.autoprofile.run` for the + appropriate file as indicated by ``cache``. + """ + runpy = _copy_module('runpy') + for func, resolver, CodeWriter in [ + ('run_path', str, AstTreeProfiler), + ('run_module', modname_to_modpath, AstTreeModuleProfiler), + ]: + impl = getattr(runpy, func) + res = cast(Callable[[str], str], resolver) # Help `mypy` out + wrapper = partial(_run, cache, runpy, impl, res, CodeWriter) + setattr(runpy, func, wrapper) + return runpy From ab8b6a00e145a21636a164bfceb1766a9c60bdb2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 8 Apr 2026 22:53:33 +0200 Subject: [PATCH 019/111] Streamlined debug output in `_child_process_profiling` --- .../_child_process_profiling/cache.py | 8 +++-- .../multiprocessing_patches.py | 13 +++---- .../_child_process_profiling/pth_hook.py | 34 ++++++++++--------- .../_child_process_profiling/runpy_patches.py | 4 +-- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 58801457..7ef9d0f0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -274,9 +274,11 @@ def _debug_log(self) -> Path | None: @cached_property def _debug_message_header(self) -> str: pid = os.getpid() - if self.main_pid == pid: - return f'PID {pid} (main process)' - return f'PID {pid} (from {self.main_pid})' + return 'PID {} ({}): Cache {:#x}'.format( + pid, + 'main process' if self.main_pid == pid else self.main_pid, + id(self), + ) @cached_property def _consistent_with_loaded_instance(self) -> bool: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 28ce9acc..9bfc5abf 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -51,11 +51,7 @@ def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set # up shop here... lp_cache = LineProfilingCache.load() - lp_cache._debug_output(f'cache {id(lp_cache):#x} setting up (mp)...') - has_set_up = _setup_in_child_process(lp_cache) - lp_cache._debug_output('cache {:#x} setup {}'.format( - id(lp_cache), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(lp_cache, False, 'multiprocessing') # ... and we don't care about polluting the `multiprocessing` # namespace either, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): @@ -163,15 +159,14 @@ def replace( setattr(obj, attr, value) if obj_name is None: obj_name = repr(obj) - lp_cache._debug_output('cache {:#x}: patched `{}.{}` -> `{}`'.format( - id(lp_cache), obj_name, attr, value, + lp_cache._debug_output('Patched `{}.{}` -> `{}`'.format( + obj_name, attr, value, )) # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess bootstrap_wrapper = partialmethod( - bootstrap, - Proc._bootstrap, # type: ignore[attr-defined] + bootstrap, Proc._bootstrap, # type: ignore[attr-defined] ) replace( Proc, '_bootstrap', bootstrap_wrapper, diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index e8dafaae..db6daea4 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -118,11 +118,7 @@ def load_pth_hook(ppid): # type: (int) -> None return try: cache = LineProfilingCache.load() - cache._debug_output(f'cache {id(cache):#x} setting up (pth)...') - has_set_up = _setup_in_child_process(cache, True) - cache._debug_output('cache {:#x} setup {}'.format( - id(cache), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(cache, True, 'pth') # _setup_in_child_process(LineProfilingCache.load()) except Exception as e: if DEBUG: @@ -162,42 +158,47 @@ def wrapper(): # type: () -> int forked = cache.copy() if forked._replace_loaded_instance(): forked._debug_output( - f'cache {id(forked):#x} in fork ' - 'superseded cached `.load()`-ed instance' + 'Superseded cached `.load()`-ed instance in forked process' ) # Note: we can reuse the profiler instance in the fork, but it # needs to go through setup so that the separate profiling # results are dumped into another output file - forked._debug_output(f'cache {id(forked):#x} setting up (fork)...') - has_set_up = _setup_in_child_process(forked, prof=cache.profiler) - forked._debug_output('cache {:#x} setup {}'.format( - id(forked), 'done' if has_set_up else 'aborted', - )) + _setup_in_child_process(forked, False, 'fork', cache.profiler) return result os.fork = wrapper cache.add_cleanup(setattr, os, 'fork', fork) -def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): - # type: (LineProfilingCache, bool, LineProfiler | None) -> bool +def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): + # type: (LineProfilingCache, bool, str, LineProfiler | None) -> bool """ Set up shop in a forked/spawned child process so that (line-)profiling can extend therein. Args: - cache (:py:class:`~.LineProfilingCache`): + cache (LineProfilingCache): Cache object wrap_os_fork (bool): Whether to wrap :py:func:`os.fork` which handles profiling; already-forked child processes should set this to false + context (str): + Optional context from which the function is called, to be + used in log messages + prof (LineProfiler | None): + Optional profiler instance to associate with the cache; + if not provided, an instance is created Returns: has_set_up (bool): False is ``cache`` has already been set up prior to calling this function, true otherwise """ + if not context: + context = '...' + cache._debug_output(f'Setting up ({context})...') if cache.profiler is not None: # Already set up + cache._debug_output(f'Setup aborted ({context})') return False import os @@ -225,7 +226,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: - cache._debug_output(f'cache {id(cache):#x} loading preimports...') + cache._debug_output('Loading preimports...') with open(cache.preimports_module, mode='rb') as fobj: code = compile(fobj.read(), cache.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace @@ -251,4 +252,5 @@ def _setup_in_child_process(cache, wrap_os_fork=False, prof=None): # the child process is about to terminate register(cache.cleanup) + cache._debug_output(f'Setup successful ({context})') return True diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 69d38da4..6ebd3496 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -108,8 +108,8 @@ def _run( profile = os.path.samefile(filename, cache.rewrite_module) except Exception as e: cache._debug_output( - f'{THIS_MODULE}: failed to check whether to ' - f'rewrite code loaded in `runpy.{func.__name__}(...)` ' + f'{THIS_MODULE}: Failed to check whether code loaded by ' + f'`runpy.{func.__name__}(...)` is to be rewritten ' f'({type(e).__name__}: {e})' ) profile = False From 5391838410fa4440dbeff2f14c82e47887117918 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 03:56:57 +0200 Subject: [PATCH 020/111] WIP: refactoring `tests/test_child_procs.py` tests/test_child_procs.py test_running_multiproc_literal_code() New test paralleling `test_running_multiproc_{script,module}` to test `kernprof -c ...` test_multiproc_script_sanity_check() - Refactored parameters for better `pytest` output - Added testing for running the code with `python -c ...` - Added CLI argument `--local` to the profiled module to toggle between a locally-defined summing function and an imported one - Refactored how the test modules are injected - Added debugging output to `subprocess.run()` calls - Added provisional support for examining the profiling data --- tests/test_child_procs.py | 375 +++++++++++++++++++++++++++++--------- 1 file changed, 292 insertions(+), 83 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 10be3f78..7d48feab 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,33 +1,61 @@ from __future__ import annotations import os +import shlex import subprocess import sys -from collections.abc import Callable, Generator, Mapping +from collections.abc import ( + Callable, Collection, Generator, Mapping, Sequence, +) +from dataclasses import dataclass +from functools import partial from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent import pytest import ubelt as ub +from _pytest.fixtures import FixtureFunctionDefinition + +from line_profiler.line_profiler import LineStats NUM_NUMBERS = 100 NUM_PROCS = 4 + +EXTERNAL_MODULE_BODY = dedent(""" +from __future__ import annotations + + +def my_external_sum(x: list[int]) -> int: + result: int = 0 # GREP_MARKER[EXT-INVOCATION] + for item in x: + result += item # GREP_MARKER[EXT-LOOP] + return result +""").strip('\n') + TEST_MODULE_BODY = dedent(f""" from __future__ import annotations + from argparse import ArgumentParser +from collections.abc import Callable from multiprocessing import Pool +from my_ext_module import my_external_sum -def my_sum(x: list[int]) -> int: - result: int = 0 - for item in x: - result += item + +def my_local_sum(x: list[int]) -> int: + result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] + # The reversing is to prevent bytecode aliasing with + # `my_external_sum()` (see issue #424, PR #425) + for item in reversed(x): + result += item # GREP_MARKER[LOCAL-LOOP] return result -def sum_in_child_procs(length: int, n: int) -> int: +def sum_in_child_procs( + length: int, n: int, my_sum: Callable[[list[int]], int], +) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] subsums: list[int] @@ -48,8 +76,15 @@ def main(args: list[str] | None = None) -> None: parser = ArgumentParser() parser.add_argument('-l', '--length', type=int, default={NUM_NUMBERS}) parser.add_argument('-n', type=int, default={NUM_PROCS}) + parser.add_argument( + '--local', + action='store_const', + dest='my_sum', + default=my_external_sum, + const=my_local_sum, + ) options = parser.parse_args(args) - print(sum_in_child_procs(options.length, options.n)) + print(sum_in_child_procs(options.length, options.n, options.my_sum)) if __name__ == '__main__': @@ -57,37 +92,79 @@ def main(args: list[str] | None = None) -> None: """).strip('\n') -@pytest.fixture(scope='session') -def test_module() -> Generator[Path, None, None]: - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / 'my_test_module.py' - with my_module.open('w') as fobj: - fobj.write(TEST_MODULE_BODY + '\n') - yield my_module +@dataclass +class _ModuleFixture: + path: Path + monkeypatch: pytest.MonkeyPatch + dependencies: Collection[_ModuleFixture] = () + + def install( + self, *, + local: bool = False, children: bool = False, deps_only: bool = False, + ) -> None: + for dep in self.dependencies: + dep.install(local=local, children=children) + if deps_only: + return + path = str(self.path.parent) + if local: + self.monkeypatch.syspath_prepend(path) + if children: + self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + + @property + def name(self) -> str: + return self.path.stem + + +def _module_path_fixture( + name: str, body: str, **kwargs, +) -> FixtureFunctionDefinition: + def my_fixture_func() -> Generator[Path, None, None]: + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(body) + yield my_module + + my_fixture_func.__name__ = name + return pytest.fixture(my_fixture_func, **kwargs) + + +# Only write the files once per test session +_ext_module = _module_path_fixture( + 'my_ext_module', EXTERNAL_MODULE_BODY, scope='session', +) +_test_module = _module_path_fixture( + 'my_test_module', TEST_MODULE_BODY, scope='session', +) -@pytest.mark.parametrize('as_module', [True, False]) -@pytest.mark.parametrize( - ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], -) -def test_multiproc_script_sanity_check( - test_module: Path, - tmp_path_factory: pytest.TempPathFactory, - nnums: int, - nprocs: int, - as_module: bool, -) -> None: +@pytest.fixture +def ext_module( + _ext_module: Path, monkeypatch: pytest.MonkeyPatch, +) -> Generator[_ModuleFixture, None, None]: """ - Sanity check that the test module functions as expected when run - with vanilla Python. + Yields: + :py:class:`_ModuleFixture` helper object containing the code at + :py:data:`EXTERNAL_MODULE_BODY` """ - _run_test_module( - _run_as_module if as_module else _run_as_script, - test_module, tmp_path_factory, [sys.executable], None, False, - nnums=nnums, nprocs=nprocs, - ) + yield _ModuleFixture(_ext_module, monkeypatch) + + +@pytest.fixture +def test_module( + _test_module: Path, + ext_module: _ModuleFixture, + monkeypatch: pytest.MonkeyPatch, +) -> Generator[_ModuleFixture, None, None]: + """ + Yields: + :py:class:`_ModuleFixture` helper object containing the code at + :py:data:`TEST_MODULE_BODY` + """ + yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) # Note: @@ -99,20 +176,20 @@ def test_multiproc_script_sanity_check( fuzz_invocations = pytest.mark.parametrize( ('runner', 'outfile', 'profile', 'label'), # Dummy argument to make `pytest` output more legible - [ - (['kernprof', '-q'], 'out.prof', False, 'cProfile'), - # Run with `line_profiler` with and w/o profiling targets - (['kernprof', '-q', '-l'], 'out.lprof', False, - 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, - 'line_profiler-active'), - ], + # This is essentially a no-op since it doesn't actually do + # line-profiling, but we check that code path for completeness + [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + # Run line profiling with and w/o profiling targets + + [(['kernprof', '-q', '-l'], 'out.lprof', False, + 'line_profiler-inactive'), + (['kernprof', '-q', '-l'], 'out.lprof', True, + 'line_profiler-active')], ) @fuzz_invocations def test_running_multiproc_script( - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str], outfile: str | None, @@ -123,15 +200,12 @@ def test_running_multiproc_script( Check that `kernprof` can run the test module as a script (`kernprof [...] `). """ - _run_test_module( - _run_as_script, - test_module, tmp_path_factory, runner, outfile, profile, - ) + run_script(test_module, tmp_path_factory, runner, outfile, profile) @fuzz_invocations def test_running_multiproc_module( - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, runner: str | list[str], outfile: str | None, @@ -142,62 +216,150 @@ def test_running_multiproc_module( Check that `kernprof` can run the test module as a module (`kernprof [...] -m `). """ - _run_test_module( - _run_as_module, - test_module, tmp_path_factory, runner, outfile, profile, - ) + run_module(test_module, tmp_path_factory, runner, outfile, profile) + + +@fuzz_invocations +def test_running_multiproc_literal_code( + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str], + outfile: str | None, + profile: bool, + label: str, +) -> None: + """ + Check that `kernprof` can run the test module as literal code + (`kernprof [...] -c "code"`). + """ + run_literal_code(test_module, tmp_path_factory, runner, outfile, profile) def _run_as_script( - runner_args: list[str], test_args: list[str], test_module: Path, **kwargs + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs ) -> subprocess.CompletedProcess: - cmd = runner_args + [str(test_module)] + test_args - return subprocess.run(cmd, **kwargs) + cmd = runner_args + [str(test_module.path)] + test_args + test_module.install(children=True, deps_only=True) + return _run_subproc(cmd, **kwargs) def _run_as_module( - runner_args: list[str], - test_args: list[str], - test_module: Path, - *, + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + ['-m', test_module.name] + test_args + test_module.install(children=True) + return _run_subproc(cmd, **kwargs) + + +def _run_as_literal_code( + runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, + **kwargs +) -> subprocess.CompletedProcess: + cmd = runner_args + ['-c', test_module.path.read_text()] + test_args + test_module.install(children=True, deps_only=True) + return _run_subproc(cmd, **kwargs) + + +def _run_subproc( + cmd: Sequence[str] | str, + /, + *args, env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: - cmd = runner_args + ['-m', test_module.stem] + test_args - env_dict = {**os.environ, **(env or {})} - python_path = env_dict.pop('PYTHONPATH', '') - if python_path: - env_dict['PYTHONPATH'] = '{}:{}'.format( - test_module.parent, python_path, + """ + Wrapper around :py:func:`subprocess.run` which writes debugging + output. + """ + if isinstance(cmd, str): + cmd_str = cmd + else: + cmd_str = shlex.join(cmd) + print('Command:', cmd_str) + if env is not None: + diff: list[str] = [] + for key in set(os.environ).union(env): + old = os.environ.get(key) + new = env.get(key) + if old is not None is new: + item = f'{old!r} -> (deleted)' + elif old is None is not new: + item = f'{new!r} (added)' + else: + if old == new: + continue + item = f'{old!r} -> {new!r}' + diff.append(f'${{{key}}}: {item}') + if diff: + print('Env:', indent('\n'.join(diff), ' '), sep='\n') + print('-- Process start --') + # Note: somehow `mypy` doesn't agree with simply unpacking the + # `*args` into `subprocess.run()`... + status: int | str = '???' + try: + proc = subprocess.run( # type: ignore[call-overload] + cmd, *args, env=env, **kwargs, ) + except Exception: + status = 'error' + raise else: - env_dict['PYTHONPATH'] = str(test_module.parent) - return subprocess.run(cmd, env=env_dict, **kwargs) + status = proc.returncode + return proc + finally: + print(f'-- Process end (return status: {status})--') def _run_test_module( run_helper: Callable[..., subprocess.CompletedProcess], - test_module: Path, + test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, + runner: str | list[str] = 'kernprof', + outfile: str | None = None, + profile: bool = True, *, + profiled_code_is_tempfile: bool = False, nnums: int | None = None, nprocs: int | None = None, check: bool = True, -) -> tuple[subprocess.CompletedProcess, Path | None]: + nhits: Mapping[str, int] | None = None, +) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ - Return - ------ - `(process_running_the_test_module, path_to_profiling_output | None)` + Returns: + process_running_the_test_module (subprocess.CompletedProcess): + Process object + profliing_stats (LineStats | None): + Line-profiling stats (where available) """ + def check_output(output: str, tag: str, nhits: int) -> None: + # The line should be preixed with 5 numbers: + # lineno, nhits, time, time-per-hit, % time + actual_nhits = 0 + for line in output.splitlines(): + if line.endswith(f'# GREP_MARKER[{tag}]'): + try: + _, n, _, _, _, *_ = line.split() + actual_nhits += int(n) + except Exception: + pass + assert actual_nhits == nhits + if isinstance(runner, str): runner_args: list[str] = [runner] else: runner_args = list(runner) - if profile: - runner_args.extend(['--prof-mod', str(test_module)]) + + if not profile: + nhits = None + + if profile and not profiled_code_is_tempfile: + runner_args.extend(['--prof-mod', str(test_module.path)]) + if nhits is not None: + # We need `kernprof` to write the profliing results immediately + # to preserve data from tempfiles (see note below) + runner_args.append('--view') test_args: list[str] = [] if nnums is None: @@ -221,13 +383,60 @@ def _run_test_module( print(f'stdout:\n{indent(proc.stdout, " ")}') print(f'stderr:\n{indent(proc.stderr, " ")}', file=sys.stderr) - assert proc.stdout == f'{nnums * (nnums + 1) // 2}\n' - - prof_result: Path | None = None + # Checks: + # - The result is correctly calculated + assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) + # - Profiling results are written to the specified file + prof_result: LineStats | None = None if outfile is None: assert not list(Path.cwd().iterdir()) else: - prof_result = Path(outfile).resolve() - assert prof_result.exists() - assert prof_result.stat().st_size + assert os.path.exists(outfile) + assert os.stat(outfile).st_size + if profile: + prof_result = LineStats.from_files(outfile) + # - If we're keeping track, the function is called the expected + # number of times and has run the expected # of loops + # (Note: we do it by parsing the output of `kernprof -v` + # instead of reading the `--outfile`, because if the profiled + # code is in a tempfile the profiling data will be dropped in + # the written outfile) + for tag, num in (nhits or {}).items(): + check_output(proc.stdout, tag, num) return proc, prof_result + + +run_module = partial(_run_test_module, _run_as_module) +run_script = partial(_run_test_module, _run_as_script) +run_literal_code = partial( + _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, +) + + +@pytest.mark.parametrize( + ('run_func', + 'label'), # Dummy argument to make `pytest` output more legible + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], +) +@pytest.mark.parametrize( + ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], +) +def test_multiproc_script_sanity_check( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + nnums: int, + nprocs: int, + label: str, +) -> None: + """ + Sanity check that the test module functions as expected when run + with vanilla Python. + """ + run_func( + test_module, tmp_path_factory, + runner=sys.executable, profile=False, + nnums=nnums, nprocs=nprocs, + ) From 3288fdfbfc5e1fac1fa9e1b6090637957de332b9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 06:28:56 +0200 Subject: [PATCH 021/111] Test local funcs in the profiled script tests/test_child_procs.py::test_multiproc_script_sanity_check() Now parametrized to test passing the function defined in the test module itself to `multiprocessing` --- tests/test_child_procs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 7d48feab..26bb14eb 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -321,6 +321,7 @@ def _run_test_module( profile: bool = True, *, profiled_code_is_tempfile: bool = False, + use_local_func: bool = False, nnums: int | None = None, nprocs: int | None = None, check: bool = True, @@ -362,6 +363,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.append('--view') test_args: list[str] = [] + if use_local_func: + test_args.append('--local') if nnums is None: nnums = NUM_NUMBERS else: @@ -414,11 +417,13 @@ def check_output(output: str, tag: str, nhits: int) -> None: @pytest.mark.parametrize( - ('run_func', + ('run_func', 'use_local_func', 'label'), # Dummy argument to make `pytest` output more legible - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')], + [(run_module, True, 'module-local'), (run_module, False, 'module-ext'), + (run_script, True, 'script-local'), (run_script, False, 'script-ext')] + # Python can't pickle things unless they resided in a retrievable + # location (so not the script supplied by `python -c`) + + [(run_literal_code, False, 'literal-code-ext')], ) @pytest.mark.parametrize( ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], @@ -427,6 +432,7 @@ def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, + use_local_func: bool, nnums: int, nprocs: int, label: str, @@ -438,5 +444,6 @@ def test_multiproc_script_sanity_check( run_func( test_module, tmp_path_factory, runner=sys.executable, profile=False, + use_local_func=use_local_func, nnums=nnums, nprocs=nprocs, ) From 7034479ba4dd1e65aade4feec7ccd606151641c8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 9 Apr 2026 06:41:56 +0200 Subject: [PATCH 022/111] Consolidated tests tests/test_child_procs.py test_running_multiproc_{module,literal_code}() Integrated into `test_running_multiproc_script()` test_running_multiproc_script() Extended parametrization --- tests/test_child_procs.py | 112 +++++++++++++++----------------------- 1 file changed, 44 insertions(+), 68 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 26bb14eb..94915fc0 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -167,74 +167,6 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) -# Note: -# Currently code execution in child processes is not properly profiled; -# these tests are just for checking that `kernprof` doesn't impair the -# proper execution of `multiprocessing` code - - -fuzz_invocations = pytest.mark.parametrize( - ('runner', 'outfile', 'profile', - 'label'), # Dummy argument to make `pytest` output more legible - # This is essentially a no-op since it doesn't actually do - # line-profiling, but we check that code path for completeness - [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] - # Run line profiling with and w/o profiling targets - + [(['kernprof', '-q', '-l'], 'out.lprof', False, - 'line_profiler-inactive'), - (['kernprof', '-q', '-l'], 'out.lprof', True, - 'line_profiler-active')], -) - - -@fuzz_invocations -def test_running_multiproc_script( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as a script - (`kernprof [...] `). - """ - run_script(test_module, tmp_path_factory, runner, outfile, profile) - - -@fuzz_invocations -def test_running_multiproc_module( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as a module - (`kernprof [...] -m `). - """ - run_module(test_module, tmp_path_factory, runner, outfile, profile) - - -@fuzz_invocations -def test_running_multiproc_literal_code( - test_module: _ModuleFixture, - tmp_path_factory: pytest.TempPathFactory, - runner: str | list[str], - outfile: str | None, - profile: bool, - label: str, -) -> None: - """ - Check that `kernprof` can run the test module as literal code - (`kernprof [...] -c "code"`). - """ - run_literal_code(test_module, tmp_path_factory, runner, outfile, profile) - - def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -447,3 +379,47 @@ def test_multiproc_script_sanity_check( use_local_func=use_local_func, nnums=nnums, nprocs=nprocs, ) + + +@pytest.mark.parametrize( + ('run_func', + 'label2'), # Dummy argument to make `pytest` output more legible + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')] +) +@pytest.mark.parametrize( + ('runner', 'outfile', 'profile', + 'label'), # Dummy argument to make `pytest` output more legible + # This is essentially a no-op since it doesn't actually do + # line-profiling, but we check that code path for completeness + [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] + # Run line profiling with and w/o profiling targets + + [(['kernprof', '-q', '-l'], 'out.lprof', False, + 'line_profiler-inactive'), + (['kernprof', '-q', '-l'], 'out.lprof', True, + 'line_profiler-active')], +) +def test_running_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + runner: str | list[str], + outfile: str | None, + profile: bool, + label: str, + label2: str, +) -> None: + """ + Check that `kernprof` can RUN the test module in various contexts + (`kernprof [...] `, `kernprof [...] -m `, and + `kernprof [...] -c "code"`). + + Notes: + - See issue #422 for the original motivation. + + - This test does not test the actual profiling, just the + execution of the code and presence of profiling data + thereafter. + """ + run_func(test_module, tmp_path_factory, runner, outfile, profile) From 4f107bf66e07409ceac59dc426a0e5b2b4d3abf4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 00:25:38 +0200 Subject: [PATCH 023/111] Added tests for our `multiprocessing` patches tests/test_child_procs.py test_profiling_multiproc_script() Test parallel to `test_running_multiproc_script()`, checking whether we are correctly profiling the child processes - Added more docs - Updated dummy parameter names _ext_module, _test_module - Refactored how the fixtures are set up - Module names now randomized and clash-proof via `uuid.uuid4()` _run_subproc() - Moved code outputting captured streams from `_run_test_module()` to here - Added timing code --- tests/test_child_procs.py | 224 ++++++++++++++++++++++++++++++++------ 1 file changed, 189 insertions(+), 35 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 94915fc0..970b7cf2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -12,10 +12,11 @@ from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent +from time import monotonic +from uuid import uuid4 import pytest import ubelt as ub -from _pytest.fixtures import FixtureFunctionDefinition from line_profiler.line_profiler import LineStats @@ -34,14 +35,14 @@ def my_external_sum(x: list[int]) -> int: return result """).strip('\n') -TEST_MODULE_BODY = dedent(f""" +TEST_MODULE_TEMPLATE = dedent(""" from __future__ import annotations from argparse import ArgumentParser from collections.abc import Callable from multiprocessing import Pool -from my_ext_module import my_external_sum +from {EXT_MODULE} import my_external_sum def my_local_sum(x: list[int]) -> int: @@ -92,8 +93,15 @@ def main(args: list[str] | None = None) -> None: """).strip('\n') +# ============================== Fixtures ============================== + + @dataclass class _ModuleFixture: + """ + Convenience wrapper around a Python source file which represents an + importable module. + """ path: Path monkeypatch: pytest.MonkeyPatch dependencies: Collection[_ModuleFixture] = () @@ -102,6 +110,20 @@ def install( self, *, local: bool = False, children: bool = False, deps_only: bool = False, ) -> None: + """ + Set the module at :py:attr:`~.path` up to be importable. + + Args: + local (bool): + Make it importable for the CURRENT process (via + :py:data:`sys.path`). + children (bool): + Make it importable for CHILD processes (via + ``os.environ['PYTHONPATH']``). + deps_only (bool): + If true, only does the equivalent setup for + dependencies. + """ for dep in self.dependencies: dep.install(local=local, children=children) if deps_only: @@ -112,33 +134,50 @@ def install( if children: self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + @staticmethod + def propose_name(prefix: str) -> Generator[str, None, None]: + """ + Propose a valid module name that isn't already occupied. + """ + while True: + name = '_'.join([prefix] + str(uuid4()).split('-')) + if name not in sys.modules: + assert name.isidentifier() + yield name + @property def name(self) -> str: return self.path.stem -def _module_path_fixture( - name: str, body: str, **kwargs, -) -> FixtureFunctionDefinition: - def my_fixture_func() -> Generator[Path, None, None]: - with TemporaryDirectory() as mydir_str: - my_dir = Path(mydir_str) - my_dir.mkdir(exist_ok=True) - my_module = my_dir / f'{name}.py' - my_module.write_text(body) - yield my_module - - my_fixture_func.__name__ = name - return pytest.fixture(my_fixture_func, **kwargs) +# Only write the files once per test session -# Only write the files once per test session -_ext_module = _module_path_fixture( - 'my_ext_module', EXTERNAL_MODULE_BODY, scope='session', -) -_test_module = _module_path_fixture( - 'my_test_module', TEST_MODULE_BODY, scope='session', -) +@pytest.fixture(scope='session') +def _ext_module() -> Generator[Path, None, None]: + name = next(_ModuleFixture.propose_name('my_ext_module')) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(EXTERNAL_MODULE_BODY) + yield my_module + + +@pytest.fixture(scope='session') +def _test_module(_ext_module: Path) -> Generator[Path, None, None]: + name = next(_ModuleFixture.propose_name('my_test_module')) + body = TEST_MODULE_TEMPLATE.format( + EXT_MODULE=_ext_module.stem, + NUM_NUMBERS=NUM_NUMBERS, + NUM_PROCS=NUM_PROCS, + ) + with TemporaryDirectory() as mydir_str: + my_dir = Path(mydir_str) + my_dir.mkdir(exist_ok=True) + my_module = my_dir / f'{name}.py' + my_module.write_text(body) + yield my_module @pytest.fixture @@ -162,7 +201,7 @@ def test_module( """ Yields: :py:class:`_ModuleFixture` helper object containing the code at - :py:data:`TEST_MODULE_BODY` + :py:data:`TEST_MODULE_TEMPLATE` """ yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) @@ -230,6 +269,8 @@ def _run_subproc( # Note: somehow `mypy` doesn't agree with simply unpacking the # `*args` into `subprocess.run()`... status: int | str = '???' + proc: subprocess.CompletedProcess | None = None + time = monotonic() try: proc = subprocess.run( # type: ignore[call-overload] cmd, *args, env=env, **kwargs, @@ -241,7 +282,19 @@ def _run_subproc( status = proc.returncode return proc finally: - print(f'-- Process end (return status: {status})--') + time = monotonic() - time + if proc is not None: + for name, captured, stream in [ + ('stdout', proc.stdout, sys.stdout), + ('stderr', proc.stderr, sys.stderr), + ]: + if captured is None: + continue + print(f'{name}:\n{indent(captured, " ")}', file=stream) + print( + f'-- Process end (time elapsed: {time:.2f} s / ' + f'return status: {status})--' + ) def _run_test_module( @@ -309,15 +362,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.extend(['--outfile', outfile]) proc = run_helper( runner_args, test_args, test_module, - text=True, capture_output=True, + text=True, capture_output=True, check=check, ) - try: - if check: - proc.check_returncode() - finally: - print(f'stdout:\n{indent(proc.stdout, " ")}') - print(f'stderr:\n{indent(proc.stderr, " ")}', file=sys.stderr) - # Checks: # - The result is correctly calculated assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) @@ -390,7 +436,7 @@ def test_multiproc_script_sanity_check( ) @pytest.mark.parametrize( ('runner', 'outfile', 'profile', - 'label'), # Dummy argument to make `pytest` output more legible + 'label1'), # Dummy argument to make `pytest` output more legible # This is essentially a no-op since it doesn't actually do # line-profiling, but we check that code path for completeness [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] @@ -407,7 +453,7 @@ def test_running_multiproc_script( runner: str | list[str], outfile: str | None, profile: bool, - label: str, + label1: str, label2: str, ) -> None: """ @@ -423,3 +469,111 @@ def test_running_multiproc_script( thereafter. """ run_func(test_module, tmp_path_factory, runner, outfile, profile) + + +@pytest.mark.parametrize( + ('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')] +) +@pytest.mark.parametrize( + ('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')], +) +@pytest.mark.parametrize( + ('preimports', 'label3'), + [(True, 'with-preimports'), (False, 'no-preimports')], +) +@pytest.mark.parametrize( + ('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], +) +@pytest.mark.parametrize( + # XXX: should we explicitly test the single-proc case? We already + # have quite a lot of subtests tho... + ('nnums', 'nprocs'), [(2000, 3)], +) +def test_profiling_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + nnums: int, + nprocs: int, + # Dummy arguments to make `pytest` output more legible + label1: str, + label2: str, + label3: str, + label4: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts, optionally extending profiling into child processes. + + Note: + This test function is heavily parametrized. Here is why that is + necessary: + + - ``run_func`` tests the different :cmd:`kernprof` modes (see + :py:func:`~.test_running_multiproc_script`). + + - ``use_local_func`` tests that we can consistently set up + profiling in both functions locally-defined in the profiled + code and imported by it. + + - ``preimports`` tests that both mechanisms for setting up + profiling targets work: + + - :py:const:`True`: child processes import the module + generated by + :py:mod:`line_profiler.autoprofile.eager_preimports`, like + the main :py:mod:`kernprof` process does. + + - :py:const:`False`: child processes rewrite the executed code + before passing it to :py:mod:`runpy`, similar to what + :py:mod:`line_profiler.autoprofile.autoprofile` does. + + These code paths go through different + :py:mod:`multiprocessing` components that we have patched and + thus needs separate testing. + + - ``prof_child_procs`` of course toggles whether to do the + patches to set up profiling in child processes. + """ + # How many calls do we expect? + nhits = dict.fromkeys( + ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, + ) + # Make sure we're profiling the right function + tag = 'LOCAL' if use_local_func else 'EXT' + if prof_child_procs: + # - `nprocs` child calls summing the `nnums` numbers + # - One call in the main proc summing the `nprocs` results from + # children + nhits[tag + '-INVOCATION'] = nprocs + 1 + nhits[tag + '-LOOP'] = nnums + nprocs + else: + nhits[tag + '-INVOCATION'] = 1 + nhits[tag + '-LOOP'] = nprocs + + runner = ['kernprof', '-l'] + runner.extend([ + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + '--{}preimports'.format('' if preimports else 'no-'), + ]) + if not use_local_func: + # Also make sure to include the external module in `--prof-mod` + runner.append(f'--prof-mod={ext_module.name}') + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) From e13f5371b875023b593a1c9a975d9770d5e0e2cd Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 05:42:26 +0200 Subject: [PATCH 024/111] WIP: flesh out the child-proc tests more tests/test_child_procs.py TEST_MODULE_BODY, [_]test_module() Added CLI flag to select `multiprocessing` start methods _Params New convenience class for test parametrization test_multiproc_script_sanity_check() - Streamlined parametrization (15 subtests -> 10) - Added subtests for various `multiprocessing` start methods test_multiproc_script_sanity_check() - Streamlined parametrization (24 subtests -> 21) - Added subtests for various `multiprocessing` start methods --- tests/test_child_procs.py | 310 ++++++++++++++++++++++++++++++++++---- 1 file changed, 277 insertions(+), 33 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 970b7cf2..cb87c634 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,18 +1,22 @@ from __future__ import annotations +import enum import os import shlex import subprocess import sys from collections.abc import ( - Callable, Collection, Generator, Mapping, Sequence, + Callable, Collection, Generator, Iterable, Mapping, Sequence, ) from dataclasses import dataclass from functools import partial +from multiprocessing import get_all_start_methods from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic +from typing import Any, Literal, TypeVar, cast, final, overload +from typing_extensions import Self from uuid import uuid4 import pytest @@ -21,8 +25,12 @@ from line_profiler.line_profiler import LineStats +T = TypeVar('T') +C = TypeVar('C', bound=Callable[..., Any]) + NUM_NUMBERS = 100 NUM_PROCS = 4 +START_METHODS = set(get_all_start_methods()) EXTERNAL_MODULE_BODY = dedent(""" from __future__ import annotations @@ -40,7 +48,8 @@ def my_external_sum(x: list[int]) -> int: from argparse import ArgumentParser from collections.abc import Callable -from multiprocessing import Pool +from multiprocessing import get_context, Pool +from typing import Literal from {EXT_MODULE} import my_external_sum @@ -56,6 +65,7 @@ def my_local_sum(x: list[int]) -> int: def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], + start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, ) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] @@ -66,7 +76,11 @@ def sum_in_child_procs( while my_list: sublist, my_list = my_list[:sublength], my_list[sublength:] sublists.append(sublist) - with Pool(n) as pool: + if start_method: + pool = get_context(start_method).Pool(n) + else: + pool = Pool(n) + with pool: subsums = pool.map(my_sum, sublists) pool.close() pool.join() @@ -77,6 +91,10 @@ def main(args: list[str] | None = None) -> None: parser = ArgumentParser() parser.add_argument('-l', '--length', type=int, default={NUM_NUMBERS}) parser.add_argument('-n', type=int, default={NUM_PROCS}) + parser.add_argument( + '-s', '--start-method', + choices=['fork', 'forkserver', 'spawn'], default=None, + ) parser.add_argument( '--local', action='store_const', @@ -206,6 +224,197 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) +# ========================== Helper functions ========================== + + +class _NotSupplied(enum.Enum): + NOT_SUPPLIED = enum.auto() + + +@final +@dataclass +class _Params: + """ + Convenience wrapper around :py:func:`pytest.mark.parametrize`. + """ + params: tuple[str, ...] + values: list[tuple[Any, ...]] + defaults: tuple[Any, ...] + + def __post_init__(self) -> None: + n = len(self.params) + assert all(p.isidentifier() for p in self.params) # Validity + assert len(set(self.params)) == n # Uniqueness + assert len(self.defaults) == n # Consistency + self.values = list(self._unique(self.values)) + assert all(len(v) == n for v in self.values) + + def __mul__(self, other: Self) -> Self: + """ + Form a Cartesian product between the two instances with disjoint + :py:attr:`~.params`, like stacking the + :py:func:`pytest.mark.parametrize `decorators. + + Example: + >>> p1 = _Params.new(('a', 'b'), [(0, 0), (1, 2), (3, 4)], + ... defaults=(1, 2)) + >>> p2 = _Params.new('c', [0, 5, 6]) + >>> p1 * p2 # doctest: +NORMALIZE_WHITESPACE + _Params(params=('a', 'b', 'c'), + values=[(0, 0, 0), (0, 0, 5), (0, 0, 6), + (1, 2, 0), (1, 2, 5), (1, 2, 6), + (3, 4, 0), (3, 4, 5), (3, 4, 6)], + defaults=(1, 2, 0)) + """ + assert not set(self.params) & set(other.params) + return type(self)( + self.params + other.params, + [sv + ov for sv in self.values for ov in other.values], + self.defaults + other.defaults, + ) + + def __add__(self, other: Self) -> Self: + """ + Concatenate two instances: + + - For parameters appearing in both, their lists of values are + concatenated. + + - For parameters appearing in either instance, the missing + values are taken from the other instance's + :py:attr:`~.defaults`. + + Note: + In the case of clashes, the :py:attr:`~.defaults` and the + order of the :py:attr:`~.params` of ``self`` (the left + operand) take precedence. + + Example: + >>> p1 = _Params.new(('a', 'b', 'c'), + ... [(0, 0, 0), # defaults + ... (1, 2, 3), (4, 5, 6)]) + >>> p2 = _Params.new(('c', 'd'), [(7, 8), (9, 10)], + ... defaults=(-1, -1)) + >>> p1 + p2 # doctest: +NORMALIZE_WHITESPACE + _Params(params=('a', 'b', 'c', 'd'), + values=[(0, 0, 0, -1), + (1, 2, 3, -1), + (4, 5, 6, -1), + (0, 0, 7, 8), + (0, 0, 9, 10)], + defaults=(0, 0, 0, -1)) + """ + self_defaults = dict(zip(self.params, self.defaults)) + other_defaults = dict(zip(other.params, other.defaults)) + new_params = tuple(self._unique(self.params + other.params)) + + defaults = {**other_defaults, **self_defaults} + new_defaults_tuple = tuple(defaults[p] for p in new_params) + + new_values: list[tuple[Any, ...]] = [] + for old_values, old_params in [ + (self.values, self.params), (other.values, other.params), + ]: + indices: list[ + tuple[Literal[True], int] | tuple[Literal[False], str] + ] = [ + (True, old_params.index(p)) if p in old_params else (False, p) + for p in new_params + ] + new_values.extend( + tuple( + ( + value[cast(int, index)] + if available else + defaults[cast(str, index)] + ) for available, index in indices + ) + for value in old_values + ) + return type(self)(new_params, new_values, new_defaults_tuple) + + def __call__(self, func: C) -> C: + """ + Mark a callable as with :py:func:`pytest.mark.parametrize`. + """ + # Note: `pytest` automatically assumes single-param values to + # be unpackes, so comply here + if len(self.params) == 1: + marker = pytest.mark.parametrize( + self.params[0], [v[0] for v in self.values], + ) + else: + marker = pytest.mark.parametrize(self.params, self.values) + return marker(func) + + @staticmethod + def _unique(items: Iterable[T]) -> Generator[T, None, None]: + seen: set[T] = set() + for item in items: + if item in seen: + continue + seen.add(item) + yield item + + @overload + @classmethod + def new( + cls, + params: Sequence[str] | str, + values: Sequence[Sequence[Any]], + defaults: Sequence[Any] | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + ... + + @overload + @classmethod + def new( + cls, + params: str, + values: Sequence[Any], + defaults: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + ... + + @classmethod + def new( + cls, + params: Sequence[str] | str, + values: Sequence[Sequence[Any]] | Sequence[Any], + defaults: ( + Sequence[Any] | Any | _NotSupplied + ) = _NotSupplied.NOT_SUPPLIED, + ) -> Self: + """ + Instantiator more akin to :py:func:`pytest.mark.parametrize`: + + - ``params`` can be provided as a comma-separated string + + - Single parameters can be unpacked (singular param-name string + and param-value sequences) + + - If ``defaults`` are not given, it is implicitly set to the + FIRST item in ``values``. + """ + if isinstance(params, str): + param_list: tuple[str, ...] = tuple( + p.strip() for p in params.split(',') + ) + unpacked = len(param_list) == 1 + else: + param_list = tuple(params) + unpacked = False + if defaults == _NotSupplied.NOT_SUPPLIED: + defaults, *_ = values + if unpacked: + default_values: tuple[Any, ...] = defaults, + value_tuple_list: list[tuple[Any, ...]] = [(v,) for v in values] + else: + default_values = tuple(defaults) # type: ignore[arg-type] + value_tuple_list = [tuple(v) for v in values] + return cls(param_list, value_tuple_list, default_values) + + def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -307,6 +516,7 @@ def _run_test_module( *, profiled_code_is_tempfile: bool = False, use_local_func: bool = False, + start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, nnums: int | None = None, nprocs: int | None = None, check: bool = True, @@ -350,6 +560,14 @@ def check_output(output: str, tag: str, nhits: int) -> None: test_args: list[str] = [] if use_local_func: test_args.append('--local') + if start_method: + if start_method in START_METHODS: + test_args.extend(['-s', start_method]) + else: + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) if nnums is None: nnums = NUM_NUMBERS else: @@ -393,27 +611,38 @@ def check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) +# =============================== Tests ================================ -@pytest.mark.parametrize( - ('run_func', 'use_local_func', - 'label'), # Dummy argument to make `pytest` output more legible - [(run_module, True, 'module-local'), (run_module, False, 'module-ext'), - (run_script, True, 'script-local'), (run_script, False, 'script-ext')] + +_fuzz_sanity = ( + _Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) + * _Params.new(('use_local_func', 'label2'), + [(True, 'local'), (False, 'ext')]) # Python can't pickle things unless they resided in a retrievable # location (so not the script supplied by `python -c`) - + [(run_literal_code, False, 'literal-code-ext')], -) -@pytest.mark.parametrize( - ('nnums', 'nprocs'), [(None, None), (None, 3), (200, None)], + + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), + [(run_literal_code, 'literal-code', False, 'ext')]) + # Also fuzz the parallelization-related stuff + + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) + + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], + defaults=(None, None)) ) + + +@_fuzz_sanity def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, - nnums: int, - nprocs: int, - label: str, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int | None, + nprocs: int | None, + # Dummy arguments to make `pytest` output more legible + label1: str, + label2: str, ) -> None: """ Sanity check that the test module functions as expected when run @@ -423,20 +652,20 @@ def test_multiproc_script_sanity_check( test_module, tmp_path_factory, runner=sys.executable, profile=False, use_local_func=use_local_func, + start_method=start_method, nnums=nnums, nprocs=nprocs, ) @pytest.mark.parametrize( - ('run_func', - 'label2'), # Dummy argument to make `pytest` output more legible + ('run_func', 'label1'), [(run_module, 'module'), (run_script, 'script'), (run_literal_code, 'literal-code')] ) @pytest.mark.parametrize( ('runner', 'outfile', 'profile', - 'label1'), # Dummy argument to make `pytest` output more legible + 'label2'), # Dummy argument to make `pytest` output more legible # This is essentially a no-op since it doesn't actually do # line-profiling, but we check that code path for completeness [(['kernprof', '-q', '--no-line'], 'out.prof', False, 'cProfile')] @@ -453,6 +682,7 @@ def test_running_multiproc_script( runner: str | list[str], outfile: str | None, profile: bool, + # Dummy arguments to make `pytest` output more legible label1: str, label2: str, ) -> None: @@ -471,23 +701,29 @@ def test_running_multiproc_script( run_func(test_module, tmp_path_factory, runner, outfile, profile) -@pytest.mark.parametrize( - ('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')] +_fuzz_prof_mp_1 = ( + _Params.new(('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], + defaults=(run_script, 'script')) + + _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) ) -@pytest.mark.parametrize( - ('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')], -) -@pytest.mark.parametrize( - ('preimports', 'label3'), - [(True, 'with-preimports'), (False, 'no-preimports')], -) -@pytest.mark.parametrize( - ('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], +_fuzz_prof_mp_2 = ( + _Params.new(('preimports', 'label3'), + [(True, 'with-preimports'), (False, 'no-preimports')], + defaults=(False, 'no-preimports')) + + _Params.new(('use_local_func', 'label4'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external')) ) + + +@_fuzz_prof_mp_1 +@_fuzz_prof_mp_2 @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... @@ -501,6 +737,7 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible @@ -573,7 +810,14 @@ def test_profiling_multiproc_script( outfile='out.lprof', profile=True, use_local_func=use_local_func, + start_method=start_method, nhits=nhits, nnums=nnums, nprocs=nprocs, ) + + +# TODO: test for profiling under the following circumstances: +# - `os.system()` +# - `subprocess.run()` +# - Nested subprocesses From f5d03c449b064263791cfa1045a9139a8c5b3efa Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 12 Apr 2026 23:17:29 +0200 Subject: [PATCH 025/111] WIP: test when the parallelly-run func fails tests/test_child_procs.py test_module(), ext_module() Updated so that we can toggle for the function sent to `multiprocessing` to raise an error with the `--force-failure` CLI flag _run_test_module() - Now raising a new `ResultMismatch` error class (instead of using base assertions) for: - If `test_module()` writes the wrong number to stdout - If `nhits` are provided and the profiling results differ therefrom - Added argument `fail` for using the aforementioned `--force-failure` flag test_multiproc_script_sanity_check() Now also chceking the cases where the test module is run with `--force-failure` test_profiling_multiproc_script() Now also chceking the cases where the test module is run with `--force-failure` (FIXME: profiling bugged when the function errors out, and doesn't fail with a consistent pattern) --- tests/test_child_procs.py | 199 ++++++++++++++++++++++++++++---------- 1 file changed, 150 insertions(+), 49 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cb87c634..09473e36 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,6 +1,7 @@ from __future__ import annotations import enum +import multiprocessing import os import shlex import subprocess @@ -8,9 +9,9 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) +from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from functools import partial -from multiprocessing import get_all_start_methods from pathlib import Path from tempfile import TemporaryDirectory from textwrap import dedent, indent @@ -30,16 +31,18 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 -START_METHODS = set(get_all_start_methods()) +START_METHODS = set(multiprocessing.get_all_start_methods()) EXTERNAL_MODULE_BODY = dedent(""" from __future__ import annotations -def my_external_sum(x: list[int]) -> int: +def my_external_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[EXT-INVOCATION] for item in x: result += item # GREP_MARKER[EXT-LOOP] + if fail: + raise RuntimeError('forced failure') return result """).strip('\n') @@ -54,18 +57,21 @@ def my_external_sum(x: list[int]) -> int: from {EXT_MODULE} import my_external_sum -def my_local_sum(x: list[int]) -> int: +def my_local_sum(x: list[int], fail: bool = False) -> int: result: int = 0 # GREP_MARKER[LOCAL-INVOCATION] # The reversing is to prevent bytecode aliasing with # `my_external_sum()` (see issue #424, PR #425) for item in reversed(x): result += item # GREP_MARKER[LOCAL-LOOP] + if fail: + raise RuntimeError('forced failure') return result def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, + fail: bool = False, ) -> int: my_list: list[int] = list(range(1, length + 1)) sublists: list[list[int]] = [] @@ -81,10 +87,10 @@ def sum_in_child_procs( else: pool = Pool(n) with pool: - subsums = pool.map(my_sum, sublists) + subsums = pool.starmap(my_sum, [(sl, fail) for sl in sublists]) pool.close() pool.join() - return my_sum(subsums) + return my_sum(subsums, fail) def main(args: list[str] | None = None) -> None: @@ -95,6 +101,7 @@ def main(args: list[str] | None = None) -> None: '-s', '--start-method', choices=['fork', 'forkserver', 'spawn'], default=None, ) + parser.add_argument('-f', '--force-failure', action='store_true') parser.add_argument( '--local', action='store_const', @@ -103,7 +110,11 @@ def main(args: list[str] | None = None) -> None: const=my_local_sum, ) options = parser.parse_args(args) - print(sum_in_child_procs(options.length, options.n, options.my_sum)) + print(sum_in_child_procs( + options.length, options.n, options.my_sum, + start_method=options.start_method, + fail=options.force_failure, + )) if __name__ == '__main__': @@ -231,6 +242,30 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +class ResultMismatch(ValueError): + def __init__( + self, + expected: Any, + actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + ) -> None: + msg = f'expected: {expected}' + if actual != _NotSupplied.NOT_SUPPLIED: + msg = f'{msg}, got {actual}' + super().__init__(msg) + self.expected = expected + self.actual = actual + + @property + def rich_message(self) -> str: + msg = '{}: {}'.format(type(self).__name__, self.args[0]) + if self.__traceback__ is not None: + tb = self.__traceback__ + msg = '{}:{}: {}'.format( + tb.tb_frame.f_code.co_filename, tb.tb_lineno, msg, + ) + return msg + + @final @dataclass class _Params: @@ -516,6 +551,7 @@ def _run_test_module( *, profiled_code_is_tempfile: bool = False, use_local_func: bool = False, + fail: bool = False, start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, nnums: int | None = None, nprocs: int | None = None, @@ -540,7 +576,11 @@ def check_output(output: str, tag: str, nhits: int) -> None: actual_nhits += int(n) except Exception: pass - assert actual_nhits == nhits + if actual_nhits == nhits: + return + raise ResultMismatch( + f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ) if isinstance(runner, str): runner_args: list[str] = [runner] @@ -560,6 +600,8 @@ def check_output(output: str, tag: str, nhits: int) -> None: test_args: list[str] = [] if use_local_func: test_args.append('--local') + if fail: + test_args.append('--force-failure') if start_method: if start_method in START_METHODS: test_args.extend(['-s', start_method]) @@ -580,11 +622,21 @@ def check_output(output: str, tag: str, nhits: int) -> None: runner_args.extend(['--outfile', outfile]) proc = run_helper( runner_args, test_args, test_module, - text=True, capture_output=True, check=check, + text=True, capture_output=True, check=(check and not fail), ) # Checks: - # - The result is correctly calculated - assert proc.stdout.splitlines()[0] == str(nnums * (nnums + 1) // 2) + if fail: + # - The process has failed as expected + if check: + assert proc.returncode + else: + # - The result is correctly calculated + expected = nnums * (nnums + 1) // 2 + output_lines = proc.stdout.splitlines() + if output_lines[0] != str(expected): + raise ResultMismatch( + f'result {expected}', f'output lines: {output_lines}', + ) # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: @@ -614,6 +666,21 @@ def check_output(output: str, tag: str, nhits: int) -> None: # =============================== Tests ================================ +def _get_mp_start_method_fuzzer(label_name: str) -> _Params: + """ + Returns: + :py:class:`_Params` object which does a full Cartesian-product + fuzz between ``fail`` (true or false) and ``start_method`` + ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) + """ + fuzz_fail = _Params.new(('fail', label_name), + [(True, 'failure'), (False, 'success')], + defaults=(False, 'success')) + fuzz_start = _Params.new('start_method', ['fork', 'forkserver', 'spawn'], + defaults=None) + return fuzz_fail * fuzz_start + + _fuzz_sanity = ( _Params.new(('run_func', 'label1'), [(run_module, 'module'), (run_script, 'script')]) @@ -623,9 +690,9 @@ def check_output(output: str, tag: str, nhits: int) -> None: # location (so not the script supplied by `python -c`) + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), [(run_literal_code, 'literal-code', False, 'ext')]) - # Also fuzz the parallelization-related stuff - + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], - defaults=None) + # Also fuzz the parallelization-related stuff, esp. check what + # happens if an exception is raised inside the parallelly-run func + + _get_mp_start_method_fuzzer('label3') + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], defaults=(None, None)) ) @@ -637,12 +704,12 @@ def test_multiproc_script_sanity_check( test_module: _ModuleFixture, tmp_path_factory: pytest.TempPathFactory, use_local_func: bool, + fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int | None, nprocs: int | None, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, + label1: str, label2: str, label3: str, ) -> None: """ Sanity check that the test module functions as expected when run @@ -651,6 +718,7 @@ def test_multiproc_script_sanity_check( run_func( test_module, tmp_path_factory, runner=sys.executable, profile=False, + fail=fail, use_local_func=use_local_func, start_method=start_method, nnums=nnums, nprocs=nprocs, @@ -683,8 +751,7 @@ def test_running_multiproc_script( outfile: str | None, profile: bool, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, + label1: str, label2: str, ) -> None: """ Check that `kernprof` can RUN the test module in various contexts @@ -709,14 +776,13 @@ def test_running_multiproc_script( defaults=(run_script, 'script')) + _Params.new(('prof_child_procs', 'label2'), [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _Params.new('start_method', ['fork', 'forkserver', 'spawn'], - defaults=None) + + _get_mp_start_method_fuzzer('label3') ) _fuzz_prof_mp_2 = ( - _Params.new(('preimports', 'label3'), + _Params.new(('preimports', 'label4'), [(True, 'with-preimports'), (False, 'no-preimports')], defaults=(False, 'no-preimports')) - + _Params.new(('use_local_func', 'label4'), + + _Params.new(('use_local_func', 'label5'), [(True, 'local'), (False, 'external')], defaults=(False, 'external')) ) @@ -737,14 +803,12 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, + fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible - label1: str, - label2: str, - label3: str, - label4: str, + label1: str, label2: str, label3: str, label4: str, label5: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various @@ -757,10 +821,6 @@ def test_profiling_multiproc_script( - ``run_func`` tests the different :cmd:`kernprof` modes (see :py:func:`~.test_running_multiproc_script`). - - ``use_local_func`` tests that we can consistently set up - profiling in both functions locally-defined in the profiled - code and imported by it. - - ``preimports`` tests that both mechanisms for setting up profiling targets work: @@ -777,24 +837,61 @@ def test_profiling_multiproc_script( :py:mod:`multiprocessing` components that we have patched and thus needs separate testing. + - ``use_local_func`` tests that we can consistently set up + profiling in both functions locally-defined in the profiled + code and imported by it. + + - ``fail`` tests that our patches and hook doesn't choke when + exceptions occur in child processes, and profiling data can + still be collected. + + - ``start_method`` tests whether all available + :py:mod:`multiprocessing` start methods are covered. + - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. + + Known bugs: + - When the function sent to :py:mod:`multiprocessing` raises an + exception, profiling results are inconsistently gather from + child processes. + + For now we XFAIL these cases. """ + def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: + xc = info.value + assert isinstance(xc, ResultMismatch) + pytest.xfail(xc.rich_message) + + # XXX: we handle known bugs here... + ctx: AbstractContextManager[pytest.ExceptionInfo | None] + if fail: + ctx = pytest.raises( + ResultMismatch, match='hit.*tagged', + ) + else: + ctx = nullcontext() + # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, ) # Make sure we're profiling the right function tag = 'LOCAL' if use_local_func else 'EXT' + tag_call = tag + '-INVOCATION' + tag_loop = tag + '-LOOP' + if not fail: + # The final sum in the parent process should always be profiled + # unless the child processes failed and we never returned from + # `Pool.starmap()` + nhits[tag_call] += 1 + nhits[tag_loop] += nprocs if prof_child_procs: - # - `nprocs` child calls summing the `nnums` numbers - # - One call in the main proc summing the `nprocs` results from - # children - nhits[tag + '-INVOCATION'] = nprocs + 1 - nhits[tag + '-LOOP'] = nnums + nprocs - else: - nhits[tag + '-INVOCATION'] = 1 - nhits[tag + '-LOOP'] = nprocs + # When profiling extends into child processes, each of them + # invokes the sum function once and when combined they loop thru + # all the items + nhits[tag_call] += nprocs + nhits[tag_loop] += nnums runner = ['kernprof', '-l'] runner.extend([ @@ -804,17 +901,21 @@ def test_profiling_multiproc_script( if not use_local_func: # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, - use_local_func=use_local_func, - start_method=start_method, - nhits=nhits, - nnums=nnums, - nprocs=nprocs, - ) + with ctx as maybe_xc: + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) + if maybe_xc is not None: + xfail_on_result_mismatch(maybe_xc) # TODO: test for profiling under the following circumstances: From d12b26ee14656d5072177839ba7019988e237075 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 04:09:51 +0200 Subject: [PATCH 026/111] More debug output, flexible `multiprocessing` patching --- .../multiprocessing_patches.py | 101 ++++++++++++------ .../_child_process_profiling/pth_hook.py | 1 + 2 files changed, 69 insertions(+), 33 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 9bfc5abf..1e6c1ba2 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,9 +16,9 @@ import multiprocessing.process from collections.abc import Callable -from functools import partial, partialmethod +from functools import partial, wraps from typing import Any, TypeVar -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import ParamSpec from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -58,39 +58,70 @@ def __setstate__(*_) -> None: _apply_mp_patches(lp_cache, _no_op) -def bootstrap( - self: multiprocessing.process.BaseProcess, - vanilla_impl: Callable[ - Concatenate[multiprocessing.process.BaseProcess, PS], T - ], - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> T: +def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: """ - Wrap around - :py:meth:`multiprocessing.process.BaseProcess._bootstrap`, - writing the profiling results after it is run. + Wrap around :py:class:`multiprocessing.process.BaseProcess` methods + like ``._bootstrap()``, writing the profiling results after it is + run. Args: - self (multiprocessing.process.BaseProcess) - :py:class:`~.BaseProcess` vanilla_impl (Callable) - Vanilla :py:meth:`~.BaseProcess._bootstrap` - *args - **kwargs - Passed to :py:meth:`~.BaseProcess._bootstrap` + Vanilla implementation of the method Returns: - Return value of ``vanilla_impl(*args, **kwargs)`` + Wrapper around ``vanilla_impl`` Side effects: - Profiling results are written + Profiling results are written as the wrapper function exits, + before the result of ``vanilla_impl()`` is returned """ - try: - return vanilla_impl(self, *args, **kwargs) - finally: # Write profiling results - LineProfilingCache.load().cleanup() + @wraps(vanilla_impl) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + try: + return vanilla_impl(*args, **kwargs) + finally: # Write profiling results + # FIXME: somehow this finally clause is not consistently and + # fully executed when an error occurs in the function passed + # to `multiprocessing`... maybe the interpreter is being + # actively exited/torned down as we speak + cache = LineProfilingCache.load() + cache._debug_output(f'Calling cleanup hook via: {name}') + cache.cleanup() + + name = vanilla_impl.__name__ + return wrapper + + +def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: + """ + Wrap around :py:class:`multiprocessing.process.BaseProcess` methods + like ``.start()``, setting up profiling before it is run. + + Args: + vanilla_impl (Callable) + Vanilla implementation of the method + + Returns: + Wrapper around ``vanilla_impl`` + + Side effects: + Profiling set up when the wrapper function is called, before + ``vanilla_impl`` itself is invoked + + See also: + :py:func:`cleanup_wrapper` + """ + @wraps(vanilla_impl) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = LineProfilingCache.load() + if cache.profiler is None: + cache._debug_output(f'Calling setup hook via: {name}') + _setup_in_child_process(cache, False, 'multiprocessing') + assert cache.profiler is not None + return vanilla_impl(*args, **kwargs) + + name = vanilla_impl.__name__ + return wrapper def get_preparation_data( @@ -165,13 +196,17 @@ def replace( # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - bootstrap_wrapper = partialmethod( - bootstrap, Proc._bootstrap, # type: ignore[attr-defined] - ) - replace( - Proc, '_bootstrap', bootstrap_wrapper, - f'{Proc.__module__}.{Proc.__qualname__}', - ) + for wrapper_maker, methods in [ + # (setup_wrapper, ['start']), + (setup_wrapper, []), + (cleanup_wrapper, ['_bootstrap']), + ]: + for method in methods: + vanilla = getattr(Proc, method) + replace( + Proc, method, wrapper_maker(vanilla), + f'{Proc.__module__}.{Proc.__qualname__}', + ) # Patch `multiprocessing.spawn` try: diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index db6daea4..93a912ca 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -223,6 +223,7 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) ctx.install() cache.add_cleanup(ctx.uninstall) + cache._debug_output(f'Set up `.profiler` at {id(prof):#x}') # Do the preimports at `cache.preimports_module` where appropriate if cache.preimports_module: From 3529759fb0db36dcec22fc23328ec187de747965 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 05:37:47 +0200 Subject: [PATCH 027/111] Fix attribute-cleanup logic in `multiprocessing_patches.apply()` --- .../_child_process_profiling/multiprocessing_patches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 1e6c1ba2..d2421122 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -184,9 +184,9 @@ def replace( try: old = getattr(obj, attr) except AttributeError: - add_cleanup(setattr, obj, attr, old) - else: add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) setattr(obj, attr, value) if obj_name is None: obj_name = repr(obj) From 4ea789aff2476faf2f2af8cd590eadf196a19417 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 09:41:39 +0200 Subject: [PATCH 028/111] WIP: let child processes fully clean up before terminating them --- .../multiprocessing_patches.py | 224 +++++++++++++++--- 1 file changed, 197 insertions(+), 27 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index d2421122..4d3d3865 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,9 +16,9 @@ import multiprocessing.process from collections.abc import Callable -from functools import partial, wraps +from functools import partial, partialmethod, wraps from typing import Any, TypeVar -from typing_extensions import ParamSpec +from typing_extensions import Concatenate, ParamSpec from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -28,6 +28,7 @@ __all__ = ('apply',) +S = TypeVar('S') T = TypeVar('T') PS = ParamSpec('PS') @@ -49,16 +50,16 @@ def __getstate__(_) -> int: def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set - # up shop here... + # up shop here. lp_cache = LineProfilingCache.load() _setup_in_child_process(lp_cache, False, 'multiprocessing') - # ... and we don't care about polluting the `multiprocessing` - # namespace either, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, _no_op) + _apply_mp_patches(lp_cache, True, True) -def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: +def cleanup_wrapper( + vanilla_impl: Callable[PS, T], name: str | None = None, +) -> Callable[PS, T]: """ Wrap around :py:class:`multiprocessing.process.BaseProcess` methods like ``._bootstrap()``, writing the profiling results after it is @@ -67,6 +68,9 @@ def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Args: vanilla_impl (Callable) Vanilla implementation of the method + name (str | None) + Optional name to use in debug messages; if not provided, use + ``vanilla_impl.__name__`` where available. Returns: Wrapper around ``vanilla_impl`` @@ -74,25 +78,29 @@ def cleanup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Side effects: Profiling results are written as the wrapper function exits, before the result of ``vanilla_impl()`` is returned + + See also: + :py:func:`setup_wrapper` """ @wraps(vanilla_impl) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: try: return vanilla_impl(*args, **kwargs) finally: # Write profiling results - # FIXME: somehow this finally clause is not consistently and - # fully executed when an error occurs in the function passed - # to `multiprocessing`... maybe the interpreter is being - # actively exited/torned down as we speak cache = LineProfilingCache.load() cache._debug_output(f'Calling cleanup hook via: {name}') + for msg in 'FOO', 'BAR', 'BAZ': + cache._debug_output(msg) cache.cleanup() - name = vanilla_impl.__name__ + if name is None: + name = getattr(vanilla_impl, '__name__', '???') return wrapper -def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: +def setup_wrapper( + vanilla_impl: Callable[PS, T], name: str | None = None, +) -> Callable[PS, T]: """ Wrap around :py:class:`multiprocessing.process.BaseProcess` methods like ``.start()``, setting up profiling before it is run. @@ -100,6 +108,9 @@ def setup_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: Args: vanilla_impl (Callable) Vanilla implementation of the method + name (str | None) + Optional name to use in debug messages; if not provided, use + ``vanilla_impl.__name__`` where available. Returns: Wrapper around ``vanilla_impl`` @@ -120,10 +131,112 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: assert cache.profiler is not None return vanilla_impl(*args, **kwargs) - name = vanilla_impl.__name__ + if name is None: + name = getattr(vanilla_impl, '__name__', '???') return wrapper +def get_target_property() -> property: + """ + Returns: + Property object which wraps around the ``._target`` attribute of + (i.e. ``target`` arguemnt to) + :py:class:`multiprocessing.process.BaseProcess` + + Note: + This is a hack to make sure that profiling output is written + ASAP after the call to the target is finished. + + More intuitive solutions that just didn't work are: + + * Wrap ``target`` at initialization time: + By replacing the callable with a wrapper, the whole process + object becomes un-pickle-able. + + * Wrap :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + as :py:mod:`coverage` does: + For currently unclear reasons, if the function set to + :py:mod:`multiprocessing` raises an error, the cleanup clauses + in a try-finally block enclosing the call to the original + ``_bootstrap()`` implementation fails to cleanly, + consistently, and fully execute. Something seems to be + starting process/interpreter teardown prematurely in child + processes... + """ + def getter( + self: multiprocessing.process.BaseProcess, + ) -> Callable[..., Any] | None: + target = vars(self).get(loc) + if target is None: + return None + return cleanup_wrapper(target, name='') + + def setter( + self: multiprocessing.process.BaseProcess, + target: Callable[..., Any] | None, + ) -> None: + vars(self)[loc] = target + + def deleter( + self: multiprocessing.process.BaseProcess, + ) -> None: + vars(self).pop(loc, None) + + loc = '_target' + return property(getter, setter, deleter) + + +def _cache_hook( + vanilla_impl: Callable[PS, T], + get_logging_message: Callable[PS, str], + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + msg = get_logging_message(*args, **kwargs) + LineProfilingCache.load()._debug_output(msg) + return vanilla_impl(*args, **kwargs) + + +def tee_log( + vanilla_impl: Callable[Concatenate[str, PS], None], + marker: str, + /, + msg: str, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around logging functions like + :py:func:`multiprocessing.util.debug` so that we can tee log + messages from the package to our own logs. + """ + def get_msg(msg: str, *_, **__) -> str: + return f'`multiprocessing` logging ({marker}): {msg}' + + _cache_hook( + vanilla_impl, get_msg, # type: ignore[arg-type] + msg, *args, **kwargs, + ) + + +def log_method_call( + self: S, + vanilla_impl: Callable[Concatenate[S, PS], T], + name: str, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + def get_msg(self: S, *_, **__) -> str: + return f'Called: `.{name}()` method of {self!r}' + + return _cache_hook( + vanilla_impl, get_msg, # type: ignore[arg-type] + self, *args, **kwargs, + ) + + def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -172,11 +285,13 @@ def apply(lp_cache: LineProfilingCache) -> None: - Cleanup callbacks registered via `lp_cache.add_cleanup()` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + _apply_mp_patches(lp_cache, False) def _apply_mp_patches( - lp_cache: LineProfilingCache, add_cleanup: Callable[..., None], + lp_cache: LineProfilingCache, + in_child_process: bool, + debug: bool = False, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -194,19 +309,33 @@ def replace( obj_name, attr, value, )) + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup + if in_child_process: + add_cleanup: Callable[..., None] = _no_op + else: + add_cleanup = lp_cache.add_cleanup + # Patch `multiprocessing.process.BaseProcess._bootstrap()` Proc = multiprocessing.process.BaseProcess - for wrapper_maker, methods in [ - # (setup_wrapper, ['start']), - (setup_wrapper, []), - (cleanup_wrapper, ['_bootstrap']), - ]: - for method in methods: - vanilla = getattr(Proc, method) - replace( - Proc, method, wrapper_maker(vanilla), - f'{Proc.__module__}.{Proc.__qualname__}', - ) + if False: + wrapper_maker: Callable[[Callable[PS, T]], Callable[PS, T]] + for wrapper_maker, methods in [ # type: ignore[assignment] + (setup_wrapper, []), + (cleanup_wrapper, ['_bootstrap']), + ]: + for method in methods: + vanilla = getattr(Proc, method) + replace( + Proc, method, wrapper_maker(vanilla), + f'{Proc.__module__}.{Proc.__qualname__}', + ) + else: + # Patch `multiprocessing.process.BaseProcess._target` + replace( + Proc, '_target', get_target_property(), + f'{Proc.__module__}.{Proc.__qualname__}', + ) # Patch `multiprocessing.spawn` try: @@ -226,6 +355,47 @@ def replace( spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, ) + # Log Popen calls + # XXX: these seem to mitigate (but not completely eliminate) the + # issue of incomplete profiling data; the point seems to be deleying + # the call to `Popen.terminate()` in the parent process so that + # "bad" child processes have the chance to complete their cleanup + # calls. Do we have a more robust way of doing this? + if False: + from importlib import import_module + + for submodule in [ + 'popen_fork', 'popen_spawn_posix', 'popen_spawn_win32', + ]: + try: + Popen = import_module('multiprocessing.' + submodule).Popen + except ImportError: + continue + for method in 'kill', 'terminate', 'interrupt', 'close', 'wait': + method_wrapper = partialmethod( + log_method_call, getattr(Popen, method), method, + ) + replace( + Popen, method, method_wrapper, + f'{Popen.__module__}.{Popen.__qualname__}', + ) + + # Intercept `multiprocessing` debug messages + if debug: + from multiprocessing import util + + for logging_func in [ + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', + ]: + try: + vanilla = getattr(util, logging_func) + except AttributeError: + continue + replace( + util, logging_func, partial(tee_log, vanilla, logging_func), + 'multiprocessing.util', + ) + # Mark `multiprocessing` as having been patched setattr(multiprocessing, _PATCHED_MARKER, True) add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) From db3090c486c655706532e3f22aade91661be35ba Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 23:44:07 +0200 Subject: [PATCH 029/111] Clean up `~~.multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py @cleanup_wrapper, @setup_wrapper get_target_property(), log_method_call() Removed _Poller New helper class for polling a callable wrap_{start,terminate}() New method wrappers for wrapping the eponymous methods of `multiprocessing.process.BaseProcess`; this fixes the bug where if the parallel function errors out in the child process, it may be terminated before profiling data can be gathered wrap_bootstrap() Refactored from `bootstrap()` _apply_mp_patches() Cleaned up testing code --- .../multiprocessing_patches.py | 401 +++++++++--------- 1 file changed, 208 insertions(+), 193 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 4d3d3865..fa79cb78 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,11 +14,18 @@ """ from __future__ import annotations -import multiprocessing.process +import multiprocessing +import os from collections.abc import Callable -from functools import partial, partialmethod, wraps +from contextlib import AbstractContextManager, nullcontext +from functools import partial, wraps +from importlib import import_module +from multiprocessing.process import BaseProcess +from pathlib import Path +from tempfile import mkstemp +from time import sleep from typing import Any, TypeVar -from typing_extensions import Concatenate, ParamSpec +from typing_extensions import Concatenate, ParamSpec, Self from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process @@ -33,6 +40,12 @@ PS = ParamSpec('PS') _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' +_PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' +_POLLING_COOLDOWN = 1. / 32 # Seconds +# NOTE: Set this to `None` or `True` to tee the `multiprocessing` +# internal logging messages to the log files; if `None`, logs are only +# written if `LineProfilingCache.load().debug` is set to true. +_INTERCEPT_MP_LOG_MESSAGES = False class PickleHook: @@ -53,137 +66,182 @@ def __setstate__(*_) -> None: # up shop here. lp_cache = LineProfilingCache.load() _setup_in_child_process(lp_cache, False, 'multiprocessing') + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, True, True) + _apply_mp_patches(lp_cache, _no_op) -def cleanup_wrapper( - vanilla_impl: Callable[PS, T], name: str | None = None, -) -> Callable[PS, T]: +class _Poller: """ - Wrap around :py:class:`multiprocessing.process.BaseProcess` methods - like ``._bootstrap()``, writing the profiling results after it is - run. + Poll a callable until it returns true-y. + """ + def __init__( + self, func: Callable[[], Any], cooldown: float | None = None, + ) -> None: + if not (cooldown and cooldown > 0): + cooldown = 0 + self._func: Callable[[], Any] = func + self._cooldown = cooldown + + def sleep(self): + cd = self._cooldown + if cd > 0: + sleep(cd) + + def with_cooldown(self, cooldown: float | None) -> Self: + return type(self)(self._func, cooldown) + + @classmethod + def poll_until( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + if args or kwargs: + func = partial(func, *args, **kwargs) + return cls(func) + + @classmethod + def poll_while( + cls, func: Callable[PS, Any], /, *args: PS.args, **kwargs: PS.kwargs + ) -> Self: + def negated( + func: Callable[PS, Any], *a: PS.args, **k: PS.kwargs + ) -> bool: + return not func(*a, **k) + + return cls(partial(negated, func, *args, **kwargs)) + + def __enter__(self) -> Self: + while not self._func(): + self.sleep() + return self + + def __exit__(self, *_, **__) -> None: + pass - Args: - vanilla_impl (Callable) - Vanilla implementation of the method - name (str | None) - Optional name to use in debug messages; if not provided, use - ``vanilla_impl.__name__`` where available. - Returns: - Wrapper around ``vanilla_impl`` +def _method_wrapper( + wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] +) -> Callable[ + [Callable[Concatenate[S, PS], T]], Callable[Concatenate[S, PS], T] +]: + def inner_wrapper( + vanilla_impl: Callable[Concatenate[S, PS], T], + ) -> Callable[Concatenate[S, PS], T]: + @wraps(vanilla_impl) + def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: + return wrapper(self, vanilla_impl, *a, **k) - Side effects: - Profiling results are written as the wrapper function exits, - before the result of ``vanilla_impl()`` is returned + return wrapped_impl - See also: - :py:func:`setup_wrapper` + for field in 'name', 'qualname', 'doc': + dunder = f'__{field}__' + value = getattr(wrapper, dunder, None) + if value is not None: + setattr(inner_wrapper, dunder, value) + return inner_wrapper + + +@_method_wrapper +def wrap_start( + self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], +) -> None: """ - @wraps(vanilla_impl) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - try: - return vanilla_impl(*args, **kwargs) - finally: # Write profiling results - cache = LineProfilingCache.load() - cache._debug_output(f'Calling cleanup hook via: {name}') - for msg in 'FOO', 'BAR', 'BAZ': - cache._debug_output(msg) - cache.cleanup() - - if name is None: - name = getattr(vanilla_impl, '__name__', '???') - return wrapper - - -def setup_wrapper( - vanilla_impl: Callable[PS, T], name: str | None = None, -) -> Callable[PS, T]: + Wrap around :py:meth:`BaseProcess.start` to specify the location for + a lock file, which is managed by the child process and checked by + the parent. This is to ensure that the child can exit gracefully and + complete any necessary cleanup. """ - Wrap around :py:class:`multiprocessing.process.BaseProcess` methods - like ``.start()``, setting up profiling before it is run. + cache = LineProfilingCache.load() + handle, tempfile = mkstemp( + dir=cache.cache_dir, prefix='process-term-lock-', suffix='.lock', + ) + try: + setattr(self, _PROCESS_TERM_LOCK_LOC, Path(tempfile)) + vanilla_impl(self) + finally: + os.close(handle) - Args: - vanilla_impl (Callable) - Vanilla implementation of the method - name (str | None) - Optional name to use in debug messages; if not provided, use - ``vanilla_impl.__name__`` where available. - Returns: - Wrapper around ``vanilla_impl`` +@_method_wrapper +def wrap_terminate( + self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], +) -> None: + """ + Wrap around :py:meth:`BaseProcess.terminate` to make sure that we + don't actually kill the child (OS-level) process before it has the + chance to properly clean up. This is done by blocking the call as + long as a lock file exists, which is specified by the parent process + and managed by the child. - Side effects: - Profiling set up when the wrapper function is called, before - ``vanilla_impl`` itself is invoked + Note: + We're technically polling in a hot loop, but: - See also: - :py:func:`cleanup_wrapper` - """ - @wraps(vanilla_impl) - def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: - cache = LineProfilingCache.load() - if cache.profiler is None: - cache._debug_output(f'Calling setup hook via: {name}') - _setup_in_child_process(cache, False, 'multiprocessing') - assert cache.profiler is not None - return vanilla_impl(*args, **kwargs) + - We're only calling this when explicitly terminating processes, + which isn't that bad; and - if name is None: - name = getattr(vanilla_impl, '__name__', '???') - return wrapper + - Such calls typically only happen: + - When e.g. the parallel function exectued in child + processes raised an error, so we're already on a "bad" + path; and -def get_target_property() -> property: + - AFTER the performance-critical part of the code (the + parallelly-run function). + + To circumvent this we may use dedicated FS-watching APIs like + :py:mod:`watchdog` (which use syscalls to do this), but we'll + think about introducing extra dependencies when we REALLY have + to. """ - Returns: - Property object which wraps around the ``._target`` attribute of - (i.e. ``target`` arguemnt to) - :py:class:`multiprocessing.process.BaseProcess` + # XXX: why can `coverage` get away with not doing all this lock-file + # hijinks and just patching `BaseProcess._bootstrap()`? + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + if lock_file: + lock: AbstractContextManager[Any] = ( + _Poller.poll_while(lock_file.exists) + .with_cooldown(_POLLING_COOLDOWN) + ) + else: + lock = nullcontext() + with lock: + try: + delattr(self, _PROCESS_TERM_LOCK_LOC) + except AttributeError: + pass + vanilla_impl(self) - Note: - This is a hack to make sure that profiling output is written - ASAP after the call to the target is finished. - - More intuitive solutions that just didn't work are: - - * Wrap ``target`` at initialization time: - By replacing the callable with a wrapper, the whole process - object becomes un-pickle-able. - - * Wrap :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - as :py:mod:`coverage` does: - For currently unclear reasons, if the function set to - :py:mod:`multiprocessing` raises an error, the cleanup clauses - in a try-finally block enclosing the call to the original - ``_bootstrap()`` implementation fails to cleanly, - consistently, and fully execute. Something seems to be - starting process/interpreter teardown prematurely in child - processes... + +@_method_wrapper +def wrap_bootstrap( + self: BaseProcess, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], /, + *args: PS.args, **kwargs: PS.kwargs +) -> T: """ - def getter( - self: multiprocessing.process.BaseProcess, - ) -> Callable[..., Any] | None: - target = vars(self).get(loc) - if target is None: - return None - return cleanup_wrapper(target, name='') - - def setter( - self: multiprocessing.process.BaseProcess, - target: Callable[..., Any] | None, - ) -> None: - vars(self)[loc] = target + Wrap around :py:meth:`BaseProcess._bootstrap` to: - def deleter( - self: multiprocessing.process.BaseProcess, - ) -> None: - vars(self).pop(loc, None) + - Run ``LineProfilingCache.load().cleanup()`` so that profiling + results can be gathered; and - loc = '_target' - return property(getter, setter, deleter) + - Write a lock file before executing ``vanilla_impl()`` and deleted + it thereafter, to ensure that a parant process doesn't prematurely + ``.terminate()`` a failed child before the profiling results can + be gathered. + """ + cache = LineProfilingCache.load() + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + + if lock_file: + lock_file.touch() + cache.add_cleanup(lock_file.unlink, missing_ok=True) + try: + return vanilla_impl(self, *args, **kwargs) + finally: + cache._debug_output( + 'Calling cleanup hook via `BaseProcess._bootstrap`' + ) + cache.cleanup() def _cache_hook( @@ -220,23 +278,6 @@ def get_msg(msg: str, *_, **__) -> str: ) -def log_method_call( - self: S, - vanilla_impl: Callable[Concatenate[S, PS], T], - name: str, - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> T: - def get_msg(self: S, *_, **__) -> str: - return f'Called: `.{name}()` method of {self!r}' - - return _cache_hook( - vanilla_impl, get_msg, # type: ignore[arg-type] - self, *args, **kwargs, - ) - - def get_preparation_data( vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -279,19 +320,26 @@ def apply(lp_cache: LineProfilingCache) -> None: Side effects: - :py:mod:`multiprocessing` marked as having been set up - - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - patched - - :py:func:`multiprocessing.spawn.get_preparation_data` patched - - Cleanup callbacks registered via `lp_cache.add_cleanup()` + + - The following methods and functions patched: + - :py:meth:`multiprocessing.process.BaseProcess.start` + + - :py:meth:`multiprocessing.process.BaseProcess.terminate` + + - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + + - :py:func:`multiprocessing.spawn.get_preparation_data` + + - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, False) + _apply_mp_patches(lp_cache, lp_cache.add_cleanup) def _apply_mp_patches( lp_cache: LineProfilingCache, - in_child_process: bool, - debug: bool = False, + add_cleanup: Callable[..., Any], + debug: bool | None = _INTERCEPT_MP_LOG_MESSAGES, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -309,33 +357,25 @@ def replace( obj_name, attr, value, )) - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup - if in_child_process: - add_cleanup: Callable[..., None] = _no_op - else: - add_cleanup = lp_cache.add_cleanup - - # Patch `multiprocessing.process.BaseProcess._bootstrap()` - Proc = multiprocessing.process.BaseProcess - if False: - wrapper_maker: Callable[[Callable[PS, T]], Callable[PS, T]] - for wrapper_maker, methods in [ # type: ignore[assignment] - (setup_wrapper, []), - (cleanup_wrapper, ['_bootstrap']), - ]: - for method in methods: - vanilla = getattr(Proc, method) - replace( - Proc, method, wrapper_maker(vanilla), - f'{Proc.__module__}.{Proc.__qualname__}', - ) - else: - # Patch `multiprocessing.process.BaseProcess._target` - replace( - Proc, '_target', get_target_property(), - f'{Proc.__module__}.{Proc.__qualname__}', - ) + # Patch `multiprocessing.process.BaseProcess` methods + Method = Callable[Concatenate[S, PS], T] + patches: dict[str, Callable[[Method], Method]] + for submodule, target, patches in [ # type: ignore[assignment] + ('process', 'BaseProcess', { + 'start': wrap_start, + 'terminate': wrap_terminate, + '_bootstrap': wrap_bootstrap, + }), + ]: + try: + mod = import_module('multiprocessing.' + submodule) + except ImportError: + continue + Class = getattr(mod, target) + name = f'{Class.__module__}.{Class.__qualname__}' + for method, method_wrapper in patches.items(): + vanilla = getattr(Class, method) + replace(Class, method, method_wrapper(vanilla), name) # Patch `multiprocessing.spawn` try: @@ -351,36 +391,12 @@ def replace( # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) if hasattr(spawn, 'runpy'): - replace( - spawn, 'runpy', create_runpy_wrapper(lp_cache), spawn.__name__, - ) - - # Log Popen calls - # XXX: these seem to mitigate (but not completely eliminate) the - # issue of incomplete profiling data; the point seems to be deleying - # the call to `Popen.terminate()` in the parent process so that - # "bad" child processes have the chance to complete their cleanup - # calls. Do we have a more robust way of doing this? - if False: - from importlib import import_module - - for submodule in [ - 'popen_fork', 'popen_spawn_posix', 'popen_spawn_win32', - ]: - try: - Popen = import_module('multiprocessing.' + submodule).Popen - except ImportError: - continue - for method in 'kill', 'terminate', 'interrupt', 'close', 'wait': - method_wrapper = partialmethod( - log_method_call, getattr(Popen, method), method, - ) - replace( - Popen, method, method_wrapper, - f'{Popen.__module__}.{Popen.__qualname__}', - ) + runpy_wrapper = create_runpy_wrapper(lp_cache) + replace(spawn, 'runpy', runpy_wrapper, spawn.__name__) # Intercept `multiprocessing` debug messages + if debug is None: + debug = lp_cache.debug if debug: from multiprocessing import util @@ -397,8 +413,7 @@ def replace( ) # Mark `multiprocessing` as having been patched - setattr(multiprocessing, _PATCHED_MARKER, True) - add_cleanup(vars(multiprocessing).pop, _PATCHED_MARKER, None) + replace(multiprocessing, _PATCHED_MARKER, True) def _no_op(*_, **__) -> None: From 67868aa8d214b906eee03f33ec3116e6fab46149 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 13 Apr 2026 23:56:11 +0200 Subject: [PATCH 030/111] Fix `tests/test_child_procs.py` tests/test_child_procs.py::test_profiling_multiproc_script() Removed XFAIL-ing for cases where the profiled function fails (because the bug has been fixed) --- tests/test_child_procs.py | 49 ++++++++++----------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 09473e36..a6effffe 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -9,7 +9,6 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from contextlib import AbstractContextManager, nullcontext from dataclasses import dataclass from functools import partial from pathlib import Path @@ -850,28 +849,7 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. - - Known bugs: - - When the function sent to :py:mod:`multiprocessing` raises an - exception, profiling results are inconsistently gather from - child processes. - - For now we XFAIL these cases. """ - def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: - xc = info.value - assert isinstance(xc, ResultMismatch) - pytest.xfail(xc.rich_message) - - # XXX: we handle known bugs here... - ctx: AbstractContextManager[pytest.ExceptionInfo | None] - if fail: - ctx = pytest.raises( - ResultMismatch, match='hit.*tagged', - ) - else: - ctx = nullcontext() - # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -901,21 +879,18 @@ def xfail_on_result_mismatch(info: pytest.ExceptionInfo) -> None: if not use_local_func: # Also make sure to include the external module in `--prof-mod` runner.append(f'--prof-mod={ext_module.name}') - with ctx as maybe_xc: - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, - use_local_func=use_local_func, - fail=fail, - start_method=start_method, - nhits=nhits, - nnums=nnums, - nprocs=nprocs, - ) - if maybe_xc is not None: - xfail_on_result_mismatch(maybe_xc) + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + ) # TODO: test for profiling under the following circumstances: From cc980e18f31c976078ed44d1601634715edf804c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 01:18:14 +0200 Subject: [PATCH 031/111] Test for non-`multiprocessing` child processes tests/test_child_procs.py::test_profiling_bare_python() New test for checking the profiling of child processes created outside of `multiprocessing` (e.g. `subprocess.run()`, `os.system()`) --- tests/test_child_procs.py | 130 ++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 26 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index a6effffe..9947dbc7 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -32,7 +32,12 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -EXTERNAL_MODULE_BODY = dedent(""" + +def strip(s: str) -> str: + return dedent(s).strip('\n') + + +EXTERNAL_MODULE_BODY = strip(""" from __future__ import annotations @@ -43,9 +48,9 @@ def my_external_sum(x: list[int], fail: bool = False) -> int: if fail: raise RuntimeError('forced failure') return result -""").strip('\n') +""") -TEST_MODULE_TEMPLATE = dedent(""" +TEST_MODULE_TEMPLATE = strip(""" from __future__ import annotations from argparse import ArgumentParser @@ -118,7 +123,7 @@ def main(args: list[str] | None = None) -> None: if __name__ == '__main__': main() -""").strip('\n') +""") # ============================== Fixtures ============================== @@ -564,23 +569,6 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ - def check_output(output: str, tag: str, nhits: int) -> None: - # The line should be preixed with 5 numbers: - # lineno, nhits, time, time-per-hit, % time - actual_nhits = 0 - for line in output.splitlines(): - if line.endswith(f'# GREP_MARKER[{tag}]'): - try: - _, n, _, _, _, *_ = line.split() - actual_nhits += int(n) - except Exception: - pass - if actual_nhits == nhits: - return - raise ResultMismatch( - f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, - ) - if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -652,10 +640,28 @@ def check_output(output: str, tag: str, nhits: int) -> None: # code is in a tempfile the profiling data will be dropped in # the written outfile) for tag, num in (nhits or {}).items(): - check_output(proc.stdout, tag, num) + _check_output(proc.stdout, tag, num) return proc, prof_result +def _check_output(output: str, tag: str, nhits: int) -> None: + # The line should be preixed with 5 numbers: + # lineno, nhits, time, time-per-hit, % time + actual_nhits = 0 + for line in output.splitlines(): + if line.endswith(f'# GREP_MARKER[{tag}]'): + try: + _, n, _, _, _, *_ = line.split() + actual_nhits += int(n) + except Exception: + pass + if actual_nhits == nhits: + return + raise ResultMismatch( + f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ) + + run_module = partial(_run_test_module, _run_as_module) run_script = partial(_run_test_module, _run_as_script) run_literal_code = partial( @@ -893,7 +899,79 @@ def test_profiling_multiproc_script( ) -# TODO: test for profiling under the following circumstances: -# - `os.system()` -# - `subprocess.run()` -# - Nested subprocesses +@pytest.mark.parametrize(('use_subprocess', 'label1'), + [(True, 'subprocess.run'), (False, 'os.system')]) +@pytest.mark.parametrize(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) +@pytest.mark.parametrize(('fail', 'label3'), + [(True, 'failure'), (False, 'success')]) +@pytest.mark.parametrize('n', [200]) +def test_profiling_bare_python( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + fail: bool, + n: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them. + """ + ext_module.install(children=True) + temp_dir = tmp_path_factory.mktemp('mytemp') + + script_path = temp_dir / 'my-script.py' + script_content = strip(""" + from {EXT_MODULE} import my_external_sum + + + if __name__ == '__main__': + numbers = list(range(1, 1 + {N})) + result = my_external_sum(numbers, {FAIL}) + """.format( + EXT_MODULE=ext_module.name, + N=n, + FAIL=fail, + )) + script_path.write_text(script_content) + + out_file = temp_dir / 'out.lprof' + cmd = [ + 'kernprof', '-lv', '--preimports', + f'--prof-mod={ext_module.name}', + f'--outfile={out_file}', + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + ] + sub_cmd = [sys.executable, str(script_path)] + if use_subprocess: + code = strip(f""" + import subprocess + + + subprocess.run({sub_cmd!r}, check=True) + """) + else: + code = strip(""" + import os + + + if os.system({!r}): + raise RuntimeError('called process failed') + """.format(shlex.join(sub_cmd))) + cmd.extend(['-c', code]) + proc = _run_subproc(cmd, text=True, capture_output=True) + + nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} + if not prof_child_procs: + for k in nhits: + nhits[k] = 0 + + # Check that the code errors out when expected + assert bool(fail) == bool(proc.returncode) + # Check that the profiling output is as expected + for tag, num in nhits.items(): + _check_output(proc.stdout, tag, num) From 9070426f135fcece14a16ba92783e371384d3f9c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 02:25:40 +0200 Subject: [PATCH 032/111] Unify tempfile creation line_profiler/_child_process_profiling/ cache.py::LineProfilingCache.make_tempfile() New convenience method for creating tempfiles with `mkstemp()` multiprocessing_patches.py::wrap_start() pth_hook.py::_setup_in_child_process() Simplfied implementations to just use `LineProfilingCache.make_tempfile()` --- .../_child_process_profiling/cache.py | 18 ++++++++++++++++++ .../multiprocessing_patches.py | 13 +++---------- .../_child_process_profiling/pth_hook.py | 9 ++------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 7ef9d0f0..a0d975c1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,6 +15,7 @@ from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL +from tempfile import mkstemp from typing import Any, ClassVar, cast from typing_extensions import Self, ParamSpec @@ -226,6 +227,23 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def make_tempfile(self, **kwargs) -> Path: + """ + Create a fresh tempfile under :py:attr:`~.cache_dir`. The other + arguments are passed as-is to :py:func:`tempfile.mkstemp`. + + Returns: + path (Path): + Path to the created file. + """ + handle, path = mkstemp(dir=self.cache_dir, **kwargs) + try: + path_obj = Path(path) + self._debug_output(f'Created tempfile: {path_obj.name!r}') + return path_obj + finally: + os.close(handle) + def _replace_loaded_instance(self) -> bool: if self._consistent_with_loaded_instance: type(self)._loaded_instance = self diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index fa79cb78..f793affc 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -15,14 +15,12 @@ from __future__ import annotations import multiprocessing -import os from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path -from tempfile import mkstemp from time import sleep from typing import Any, TypeVar from typing_extensions import Concatenate, ParamSpec, Self @@ -153,14 +151,9 @@ def wrap_start( complete any necessary cleanup. """ cache = LineProfilingCache.load() - handle, tempfile = mkstemp( - dir=cache.cache_dir, prefix='process-term-lock-', suffix='.lock', - ) - try: - setattr(self, _PROCESS_TERM_LOCK_LOC, Path(tempfile)) - vanilla_impl(self) - finally: - os.close(handle) + tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') + setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) + vanilla_impl(self) @_method_wrapper diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 93a912ca..69004a05 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -203,7 +203,6 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): import os from atexit import register - from tempfile import mkstemp from ..autoprofile.autoprofile import ( # Note: we need this to equip the profiler with the # `.add_imported_function_or_module()` pseudo-method @@ -234,16 +233,12 @@ def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): # Occupy a tempfile slot in `cache.cache_dir` and set the profiler # up to write thereto when the process terminates - handle, prof_outfile = mkstemp( + prof_outfile = cache.make_tempfile( prefix='child-prof-output-{}-{}-{:#x}-' .format(cache.main_pid, os.getpid(), id(prof)), suffix='.lprof', - dir=cache.cache_dir, ) - try: # Whatever else we do, write the profiling stats first - cache._add_cleanup(prof.dump_stats, -1, prof_outfile) - finally: - os.close(handle) + cache._add_cleanup(prof.dump_stats, -1, prof_outfile) # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) if wrap_os_fork: From 1197fb903331e70bffdb522b55d58ebaa7910bbe Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 03:37:23 +0200 Subject: [PATCH 033/111] Failsafe: timeout the lock-file waiting line_profiler/_child_process_profiling/multiprocessing_patches.py _Poller __init__() - Updated default and typing for `cooldown` - New arguments `timeout` and `on_timeout` for controlling timeout duration and behaviors with_timeout() New method creating a new instance a la `.with_cooldown()` __enter__() Added timeout handling Timeout New `RuntimeError` subclass raised if `timeout` is positive and reached wrap_terminate() Now only allowing the `BaseProcess.terminate()` call to be blocked by at most 1 s by the lock file, before issuing a warning and proceeding anyway tests/test_child_procs.py::test_profiling_multiproc_script() Now timing out the `kernprof` process after 5 s in case the lock files caused a deadlock --- .../multiprocessing_patches.py | 113 +++++++++++++++--- tests/test_child_procs.py | 9 ++ 2 files changed, 106 insertions(+), 16 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index f793affc..0563912c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -15,16 +15,18 @@ from __future__ import annotations import multiprocessing +import warnings from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path -from time import sleep -from typing import Any, TypeVar +from time import sleep, monotonic +from typing import Any, Literal, TypeVar, NoReturn from typing_extensions import Concatenate, ParamSpec, Self +from .. import _diagnostics as diagnostics from .cache import LineProfilingCache from .pth_hook import _setup_in_child_process from .runpy_patches import create_runpy_wrapper @@ -39,11 +41,26 @@ _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' -_POLLING_COOLDOWN = 1. / 32 # Seconds + +# NOTE: polling behaviors for blocking `BaseProcess.terminate()` with +# the lock file: +# - _POLLING_COOLDOWN: +# Seconds between polling for the lock file; set to zero to poll in a +# hot loop (not recommended) +# - _POLLING_TIMEOUT: second to block before timing out anyway; set to +# zero to disable timing out +# - _POLLING_ON_TIMEOUT: what to do on timeout if enabled: +# - 'error': error out with `_Poller.Timeout` +# - 'warn': issue a warning and move on +# - 'ignore': move on without a warning +_POLLING_COOLDOWN = 1. / 32 +_POLLING_TIMEOUT = 1. +_POLLING_ON_TIMEOUT: Literal['error', 'warn', 'ignore'] = 'warn' + # NOTE: Set this to `None` or `True` to tee the `multiprocessing` # internal logging messages to the log files; if `None`, logs are only # written if `LineProfilingCache.load().debug` is set to true. -_INTERCEPT_MP_LOG_MESSAGES = False +_INTERCEPT_MP_LOG_MESSAGES: bool | None = False class PickleHook: @@ -75,20 +92,41 @@ class _Poller: Poll a callable until it returns true-y. """ def __init__( - self, func: Callable[[], Any], cooldown: float | None = None, + self, + func: Callable[[], Any], + cooldown: float = 0, + timeout: float = 0, + on_timeout: Literal['ignore', 'warn', 'error'] = 'error', ) -> None: - if not (cooldown and cooldown > 0): + if cooldown < 0: cooldown = 0 + if timeout < 0: + timeout = 0 self._func: Callable[[], Any] = func self._cooldown = cooldown + self._timeout = timeout + self._on_timeout = on_timeout def sleep(self): cd = self._cooldown if cd > 0: sleep(cd) - def with_cooldown(self, cooldown: float | None) -> Self: - return type(self)(self._func, cooldown) + def with_cooldown(self, cooldown: float) -> Self: + return type(self)( + self._func, cooldown, self._timeout, self._on_timeout, + ) + + def with_timeout( + self, + timeout: float | None = None, + on_timeout: Literal['ignore', 'warn', 'error'] | None = None, + ) -> Self: + if timeout is None: + timeout = self._timeout + if on_timeout is None: + on_timeout = self._on_timeout + return type(self)(self._func, self._cooldown, timeout, on_timeout) @classmethod def poll_until( @@ -110,13 +148,47 @@ def negated( return cls(partial(negated, func, *args, **kwargs)) def __enter__(self) -> Self: - while not self._func(): + def error(msg: str) -> NoReturn: + raise type(self).Timeout(msg) + + def warn(msg: str) -> None: + warnings.warn(msg) + diagnostics.log.warning(msg) + + def ignore(_): + pass + + timeout = self._timeout + callback = self._func + + handle_timeout: Callable[[str], Any] = { + 'error': error, 'warn': warn, 'ignore': ignore, + }[self._on_timeout] + fmt = '.3g' + timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' + + start = monotonic() + while not callback(): + elapsed = monotonic() - start + if timeout and elapsed >= timeout: + handle_timeout( + f'{timeout_msg_header}: ' + f'timed out ({elapsed:{fmt}} s >= {timeout:{fmt}} s) ' + f'waiting for callback {callback!r} to return true' + ) + break self.sleep() return self def __exit__(self, *_, **__) -> None: pass + class Timeout(RuntimeError): + """ + Raised when a :py:class:`_Poller` is timed out when polling. + """ + pass + def _method_wrapper( wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] @@ -187,21 +259,30 @@ def wrap_terminate( think about introducing extra dependencies when we REALLY have to. """ - # XXX: why can `coverage` get away with not doing all this lock-file - # hijinks and just patching `BaseProcess._bootstrap()`? + # XXX: why can `coverage` get away with not doing all these + # lock-file hijinks and just patching `BaseProcess._bootstrap()`? + + def discard_lock() -> None: + assert lock_file is not None + # This should have already happened unless we timed out + lock_file.unlink(missing_ok=True) + try: + delattr(self, _PROCESS_TERM_LOCK_LOC) + except AttributeError: + pass + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) if lock_file: lock: AbstractContextManager[Any] = ( _Poller.poll_while(lock_file.exists) .with_cooldown(_POLLING_COOLDOWN) + .with_timeout(_POLLING_TIMEOUT, _POLLING_ON_TIMEOUT) ) + callback = discard_lock else: - lock = nullcontext() + lock, callback = nullcontext(), _no_op with lock: - try: - delattr(self, _PROCESS_TERM_LOCK_LOC) - except AttributeError: - pass + callback() vanilla_impl(self) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 9947dbc7..ad6a91c5 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -561,6 +561,7 @@ def _run_test_module( nprocs: int | None = None, check: bool = True, nhits: Mapping[str, int] | None = None, + **kwargs ) -> tuple[subprocess.CompletedProcess, LineStats | None]: """ Returns: @@ -610,6 +611,7 @@ def _run_test_module( proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), + **kwargs ) # Checks: if fail: @@ -856,6 +858,12 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. """ + # XXX: owing to the shenanigans in + # `line_profiler._child_process_profiling.multiprocessing_patches`, + # there is a risk that failing child processes are not properly + # `.terminate()`-ed. So just put in a timeout... + timeout = 5 # Seconds + # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -896,6 +904,7 @@ def test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, + timeout=timeout, ) From c0b93a8729b38feb5c0306455e2148b22593c72e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 06:14:13 +0200 Subject: [PATCH 034/111] Added check in tests for .pth-file cleanup --- tests/test_child_procs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ad6a91c5..90456606 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from functools import partial from pathlib import Path +from sysconfig import get_path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic @@ -570,6 +571,11 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ + def get_pth_files(pattern: str = '*.pth') -> set[str]: + return {path.name for path in lib_path.glob(pattern)} + + lib_path = Path(get_path('purelib')) + if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -608,6 +614,7 @@ def _run_test_module( with ub.ChDir(tmp_path_factory.mktemp('mytemp')): if outfile is not None: runner_args.extend(['--outfile', outfile]) + old_pth_files = get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), @@ -626,6 +633,9 @@ def _run_test_module( raise ResultMismatch( f'result {expected}', f'output lines: {output_lines}', ) + # - Temporary `.pth` file(s) created by `~~.pth_hook` has been + # cleaned up + assert get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: From 0aabec48b0e8ac1697d5fc8da196d27e6f7ce2fe Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 14:33:46 +0200 Subject: [PATCH 035/111] Fix kernprof docstring --- kernprof.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 50097fde..5909981d 100755 --- a/kernprof.py +++ b/kernprof.py @@ -138,6 +138,9 @@ def main(): If the script/module profiled is in `--prof-mod`, autoprofile all its imports. Only works with line profiling (`-l`/`--line- by-line`). (Default: False) + --prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + Extend profiling into child Python processes. Only works with + line profiling (`-l`/`--line-by-line`). (Default: False) output options: -o, --outfile OUTFILE @@ -152,7 +155,7 @@ def main(): -q, --quiet Decrease verbosity level (default: 0). At level -1, disable helpful messages (e.g. "Wrote profile results to <...>"); at level -2, silence the stdout; at level -3, silence the stderr. ---rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] + --rich [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Use rich formatting if viewing output. (Default: False; short form: -r) -u, --unit UNIT Output unit (in seconds) in which the timing info is @@ -1059,7 +1062,7 @@ def _write_preimports(prof, options, exclude, keep=False): from line_profiler.autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) - from line_profiler.curated_profiling import ClassifiedPreimportTargets + from line_profiler.curated_profiling import ClassifiedPreimportTargets # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; From 1b1fe3e709c8a5286c46bedd7ba2556bc472d0d6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 17:04:56 +0200 Subject: [PATCH 036/111] Shuffled code inside `~._child_process_profiling` line_profiler/_child_process_profiling/ - The functions `pth_hook.py::_setup_in_child_process()` and `::_wrap_os_fork()` have been relocated to eponymous instance methods of `cache.py::LineProfilingCache` - The implementations of `pth_hook.py::{write,load}_pth_hook()` and `multiprocessing_patches.py::PickleHook.__setstate__()` are updated accordingly --- .../_child_process_profiling/cache.py | 121 +++++++++++++++- .../multiprocessing_patches.py | 3 +- .../_child_process_profiling/pth_hook.py | 135 +----------------- 3 files changed, 124 insertions(+), 135 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index a0d975c1..637cf804 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -4,6 +4,7 @@ """ from __future__ import annotations +import atexit import dataclasses import os try: @@ -11,7 +12,7 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable -from functools import partial, cached_property +from functools import partial, cached_property, wraps from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL @@ -20,6 +21,14 @@ from typing_extensions import Self, ParamSpec from .. import _diagnostics as diagnostics +from ..autoprofile.autoprofile import ( + # Note: we need this to equip the profiler with the + # `.add_imported_function_or_module()` pseudo-method + # (see `kernprof.py::_write_preimports()`), which is required for + # the preimports to work + _extend_line_profiler_for_profiling_imports as upgrade_profiler, +) +from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth @@ -227,6 +236,116 @@ def _debug_output(self, msg: str) -> None: except OSError: # Cache dir may have been rm-ed during cleanup pass + def _setup_in_child_process( + self, + wrap_os_fork: bool = False, + context: str = '', + prof: LineProfiler | None = None, + ) -> bool: + """ + Set up shop in a forked/spawned child process so that + (line-)profiling can extend therein. + + Args: + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles + profiling; already-forked child processes should set + this to false + context (str): + Optional context from which the function is called, to + be used in log messages + prof (LineProfiler | None): + Optional profiler instance to associate with the cache; + if not provided, an instance is created + + Returns: + has_set_up (bool): + False the instance has already been set up prior to + calling this function, true otherwise + """ + if not context: + context = '...' + self._debug_output(f'Setting up ({context})...') + if self.profiler is not None: # Already set up + self._debug_output(f'Setup aborted ({context})') + return False + + # Create a profiler instance and manage it with + # `CuratedProfilerContext` + if prof is None: + prof = LineProfiler() + self.profiler = prof + upgrade_profiler(prof) + ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) + ctx.install() + self.add_cleanup(ctx.uninstall) + self._debug_output(f'Set up `.profiler` at {id(prof):#x}') + + # Do the preimports at `cache.preimports_module` where + # appropriate + if self.preimports_module: + self._debug_output('Loading preimports...') + with open(self.preimports_module, mode='rb') as fobj: + code = compile(fobj.read(), self.preimports_module, 'exec') + exec(code, {}) # Use a fresh, empty namespace + + # Occupy a tempfile slot in `.cache_dir` and set the profiler + # up to write thereto when the process terminates (with high + # priority) + prof_outfile = self.make_tempfile( + prefix='child-prof-output-{}-{}-{:#x}-' + .format(self.main_pid, os.getpid(), id(prof)), + suffix='.lprof', + ) + self._add_cleanup(prof.dump_stats, -1, prof_outfile) + + # Set up `os.fork()` wrapping if needed (i.e. in a spawned + # process) + if wrap_os_fork: + self._wrap_os_fork() + + # Set `.cleanup()` as an atexit hook to handle everything when + # the child process is about to terminate + atexit.register(self.cleanup) + + self._debug_output(f'Setup successful ({context})') + return True + + def _wrap_os_fork(self) -> None: + """ + Create a wrapper around :py:func:`os.fork` which handles + profiling. + + Side effects: + - :py:func:`os.fork` (if available) replaced with the + wrapper + - :py:meth:`~.cleanup` callback registered undoing that + """ + try: + fork = os.fork + except AttributeError: # Can't fork on this platform + return + + @wraps(fork) + def wrapper() -> int: + result = fork() + if result: + return result + # If we're here, we are in the fork + forked = self.copy() # Ditch inherited cleanups + if forked._replace_loaded_instance(): + forked._debug_output( + 'Superseded cached `.load()`-ed instance in forked process' + ) + # Note: we can reuse the profiler instance in the fork, but + # it needs to go through setup so that the separate + # profiling results are dumped into another output file + forked._setup_in_child_process(False, 'fork', self.profiler) + return result + + os.fork = wrapper + self.add_cleanup(setattr, os, 'fork', fork) + def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0563912c..ad84e1b6 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -28,7 +28,6 @@ from .. import _diagnostics as diagnostics from .cache import LineProfilingCache -from .pth_hook import _setup_in_child_process from .runpy_patches import create_runpy_wrapper @@ -80,7 +79,7 @@ def __setstate__(*_) -> None: # We're in a child process created by `multiprocessing`, so set # up shop here. lp_cache = LineProfilingCache.load() - _setup_in_child_process(lp_cache, False, 'multiprocessing') + lp_cache._setup_in_child_process(False, 'multiprocessing') # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 69004a05..ee845a52 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -20,16 +20,11 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from collections.abc import Callable # noqa: F401 from pathlib import Path # noqa: F401 - from typing import Any # noqa: F401 - from ..line_profiler import LineProfiler # noqa: F401 from .cache import LineProfilingCache # noqa: F401 -__all__ = ( - 'write_pth_hook', 'load_pth_hook', '_setup_in_child_process' -) +__all__ = ('write_pth_hook', 'load_pth_hook') INHERITED_PID_ENV_VARNAME = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' @@ -83,7 +78,7 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path finally: # Not closing the handle causes issues on Windows os.close(handle) - _wrap_os_fork(cache) + cache._wrap_os_fork() return fpath @@ -118,8 +113,7 @@ def load_pth_hook(ppid): # type: (int) -> None return try: cache = LineProfilingCache.load() - _setup_in_child_process(cache, True, 'pth') - # _setup_in_child_process(LineProfilingCache.load()) + cache._setup_in_child_process(True, 'pth') except Exception as e: if DEBUG: msg = f'{type(e)}: {e}' @@ -127,126 +121,3 @@ def load_pth_hook(ppid): # type: (int) -> None log.warning(msg) finally: load_pth_hook.called = True # type: ignore[attr-defined] - - -def _wrap_os_fork(cache): # type: (LineProfilingCache) -> None - """ - Create a wrapper around :py:func:`os.fork` which handles profiling. - - Args: - cache (:py:class:`~.LineProfilingCache`): - Cache object - - Side effects: - - :py:func:`os.fork` (if available) replaced with the wrapper - - Cleanup callback registered at ``cache`` undoing that - """ - import os - from functools import wraps - - try: - fork = os.fork - except AttributeError: # Can't fork on this platform - return - - @wraps(fork) - def wrapper(): # type: () -> int - result = fork() - if result: - return result - # If we're here, we are in the fork - forked = cache.copy() - if forked._replace_loaded_instance(): - forked._debug_output( - 'Superseded cached `.load()`-ed instance in forked process' - ) - # Note: we can reuse the profiler instance in the fork, but it - # needs to go through setup so that the separate profiling - # results are dumped into another output file - _setup_in_child_process(forked, False, 'fork', cache.profiler) - return result - - os.fork = wrapper - cache.add_cleanup(setattr, os, 'fork', fork) - - -def _setup_in_child_process(cache, wrap_os_fork=False, context='', prof=None): - # type: (LineProfilingCache, bool, str, LineProfiler | None) -> bool - """ - Set up shop in a forked/spawned child process so that - (line-)profiling can extend therein. - - Args: - cache (LineProfilingCache): - Cache object - wrap_os_fork (bool): - Whether to wrap :py:func:`os.fork` which handles profiling; - already-forked child processes should set this to false - context (str): - Optional context from which the function is called, to be - used in log messages - prof (LineProfiler | None): - Optional profiler instance to associate with the cache; - if not provided, an instance is created - - Returns: - has_set_up (bool): - False is ``cache`` has already been set up prior to calling - this function, true otherwise - """ - if not context: - context = '...' - cache._debug_output(f'Setting up ({context})...') - if cache.profiler is not None: # Already set up - cache._debug_output(f'Setup aborted ({context})') - return False - - import os - from atexit import register - from ..autoprofile.autoprofile import ( - # Note: we need this to equip the profiler with the - # `.add_imported_function_or_module()` pseudo-method - # (see `kernprof.py::_write_preimports()`), which is required - # for the preimports to work - _extend_line_profiler_for_profiling_imports as upgrade_profiler, - ) - from ..curated_profiling import CuratedProfilerContext - from ..line_profiler import LineProfiler # noqa: F811 - - # Create a profiler instance and manage it with - # `CuratedProfilerContext` - if prof is None: - prof = LineProfiler() - cache.profiler = prof - upgrade_profiler(prof) - ctx = CuratedProfilerContext(prof, insert_builtin=cache.insert_builtin) - ctx.install() - cache.add_cleanup(ctx.uninstall) - cache._debug_output(f'Set up `.profiler` at {id(prof):#x}') - - # Do the preimports at `cache.preimports_module` where appropriate - if cache.preimports_module: - cache._debug_output('Loading preimports...') - with open(cache.preimports_module, mode='rb') as fobj: - code = compile(fobj.read(), cache.preimports_module, 'exec') - exec(code, {}) # Use a fresh, empty namespace - - # Occupy a tempfile slot in `cache.cache_dir` and set the profiler - # up to write thereto when the process terminates - prof_outfile = cache.make_tempfile( - prefix='child-prof-output-{}-{}-{:#x}-' - .format(cache.main_pid, os.getpid(), id(prof)), - suffix='.lprof', - ) - cache._add_cleanup(prof.dump_stats, -1, prof_outfile) - - # Set up `os.fork()` wrapping if needed (i.e. in a spawned process) - if wrap_os_fork: - _wrap_os_fork(cache) - - # Set `cache.cleanup()` as an atexit hook to handle everything when - # the child process is about to terminate - register(cache.cleanup) - - cache._debug_output(f'Setup successful ({context})') - return True From 8361ee6612b2f816f148d0f457a1c9e7e5005a04 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 19:08:41 +0200 Subject: [PATCH 037/111] Fix `subprocess.run()` wrapper in tests tests/test_child_procs.py::_run_subproc() - If any of the output streams is captured, we call `subprocess.run(..., check=False)` to get the chance to intercept and print the output, and only call `.check_returncode()` on the `CompletedProcess` afterwards - Fixed bug where if `text=False` we attempt to format the captured stream-content bytes as strings --- tests/test_child_procs.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 90456606..14179c59 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -486,6 +486,7 @@ def _run_subproc( cmd: Sequence[str] | str, /, *args, + check: bool = False, env: Mapping[str, str] | None = None, **kwargs ) -> subprocess.CompletedProcess: @@ -497,6 +498,16 @@ def _run_subproc( cmd_str = cmd else: cmd_str = shlex.join(cmd) + + # If we're capturing outputs, it may be for the best to wait until + # we've processed the output streams to check the return code... + check_rc_in_run = check + for arg in 'stdout', 'stdin': + if kwargs.get(arg) not in {None, subprocess.DEVNULL}: + check_rc_in_run = False + if kwargs.get('capture_output'): + check_rc_in_run = False + print('Command:', cmd_str) if env is not None: diff: list[str] = [] @@ -522,23 +533,29 @@ def _run_subproc( time = monotonic() try: proc = subprocess.run( # type: ignore[call-overload] - cmd, *args, env=env, **kwargs, + cmd, *args, env=env, check=check_rc_in_run, **kwargs, ) except Exception: status = 'error' raise else: + assert proc is not None + if check and not check_rc_in_run: # Perform missing check + proc.check_returncode() status = proc.returncode return proc finally: time = monotonic() - time if proc is not None: + captured: str | bytes | None for name, captured, stream in [ ('stdout', proc.stdout, sys.stdout), ('stderr', proc.stderr, sys.stderr), ]: if captured is None: continue + if isinstance(captured, bytes): # `text=False` + captured = captured.decode() print(f'{name}:\n{indent(captured, " ")}', file=stream) print( f'-- Process end (time elapsed: {time:.2f} s / ' From 08d0b0583c0f42227615bd0c6b49d92791354c1d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 14 Apr 2026 19:30:57 +0200 Subject: [PATCH 038/111] CHANGELOG and help-text update --- CHANGELOG.rst | 2 ++ kernprof.py | 5 +++-- line_profiler/_child_process_profiling/__init__.py | 8 +++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30156f63..118e2fca 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -12,6 +12,8 @@ Changes is available (#427) * FIX: Bytecodes of profiled functions now always labeled to prevent confusion with non-profiled "twins" (#425) +* FEAT: Experimental support for profiling child processes with + ``kernprof --prof-child-procs`` (#431) 5.0.2 diff --git a/kernprof.py b/kernprof.py index 5909981d..e0b91cf2 100755 --- a/kernprof.py +++ b/kernprof.py @@ -140,7 +140,8 @@ def main(): by-line`). (Default: False) --prof-child-procs [Y[es] | N[o] | T[rue] | F[alse] | on | off | 1 | 0] Extend profiling into child Python processes. Only works with - line profiling (`-l`/`--line-by-line`). (Default: False) + line profiling (`-l`/`--line-by-line`). (EXPERIMENTAL; + default: False) output options: -o, --outfile OUTFILE @@ -690,7 +691,7 @@ def _add_core_parser_arguments(parser): action='store_true', help='Extend profiling into child Python processes. ' 'Only works with line profiling (`-l`/`--line-by-line`). ' - f'(Default: {default.conf_dict["prof_child_procs"]})', + f'(EXPERIMENTAL; default: {default.conf_dict["prof_child_procs"]})', ) out_opts = parser.add_argument_group('output options') if default.conf_dict['outfile']: diff --git a/line_profiler/_child_process_profiling/__init__.py b/line_profiler/_child_process_profiling/__init__.py index 4ff89c0a..58e0806f 100644 --- a/line_profiler/_child_process_profiling/__init__.py +++ b/line_profiler/_child_process_profiling/__init__.py @@ -2,7 +2,9 @@ Tooling for profiling child Python processes and gathering their profiling results. -Note: - All contents of this subpackage is to be considered implementation - details. +Notes: + - THIS IS AN EXPERIMENTAL FEATURE. + + - All contents of this subpackage is to be considered implementation + details. """ From ff42d21d602870a7ac4f8ca9ff5130e06a5b1dd7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 15 Apr 2026 09:29:43 +0200 Subject: [PATCH 039/111] Typing fixes --- .../_child_process_profiling/cache.py | 26 +++++++++++++++---- .../multiprocessing_patches.py | 4 +-- .../_child_process_profiling/pth_hook.py | 2 +- .../_child_process_profiling/runpy_patches.py | 16 +++++------- line_profiler/curated_profiling.py | 21 +++++++++++---- line_profiler/line_profiler.py | 6 +++-- 6 files changed, 51 insertions(+), 24 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 637cf804..49837f34 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -166,7 +166,7 @@ def load(cls) -> Self: ) diagnostics.log.debug(msg) instance = cls._from_path(cls._get_filename(cache_dir)) - cls._loaded_instance = instance + instance._replace_loaded_instance(force=True) return instance def dump(self) -> None: @@ -343,7 +343,19 @@ def wrapper() -> int: forked._setup_in_child_process(False, 'fork', self.profiler) return result - os.fork = wrapper + # Note: type checkers have vastly different opinions on + # `os.fork = wrapper`: + # - `ty` wouldn't shut up about shadowing unless we explicitly + # type-annotate the assignment (error type: + # "invalid-assignment") + # - `mypy` is cool with the bare assignment, but complains about + # declaring types in "assignment to non-self-attribute" + # (error type: "misc") + # The only way to satisfy both seems to be either an + # unqualified "type: ignore" comment, or circumventing the + # attribute checks by `setattr()` or assigning to the module + # namespace dict. + os.fork = wrapper # type: ignore self.add_cleanup(setattr, os, 'fork', fork) def make_tempfile(self, **kwargs) -> Path: @@ -363,9 +375,13 @@ def make_tempfile(self, **kwargs) -> Path: finally: os.close(handle) - def _replace_loaded_instance(self) -> bool: - if self._consistent_with_loaded_instance: - type(self)._loaded_instance = self + def _replace_loaded_instance(self, force: bool = False) -> bool: + cls = type(self) + if force or self._consistent_with_loaded_instance: + # Note: `ty` REALLY hates assigning an instance to + # `ClassVar[Self]` (#3274); no choice but to ignore it for + # the time being... + cls._loaded_instance = self # type: ignore return True return False diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ad84e1b6..dffd92e5 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,7 +16,7 @@ import multiprocessing import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module @@ -432,7 +432,7 @@ def replace( # Patch `multiprocessing.process.BaseProcess` methods Method = Callable[Concatenate[S, PS], T] - patches: dict[str, Callable[[Method], Method]] + patches: Mapping[str, Callable[[Method], Method]] for submodule, target, patches in [ # type: ignore[assignment] ('process', 'BaseProcess', { 'start': wrap_start, diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index ee845a52..827bd277 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -120,4 +120,4 @@ def load_pth_hook(ppid): # type: (int) -> None warnings.warn(msg) log.warning(msg) finally: - load_pth_hook.called = True # type: ignore[attr-defined] + load_pth_hook.called = True # type: ignore diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 6ebd3496..659f4de8 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -88,6 +88,7 @@ def _run( cache: LineProfilingCache, runpy: ModuleType, func: Callable[Concatenate[str, PS], T], + name: str, resolve_target_to_path: Callable[[str], str], CodeWriter: type[AstTreeProfiler], target: str, @@ -96,7 +97,7 @@ def _run( ) -> T: cache._debug_output('Calling via {}: `runpy.{}({})`'.format( THIS_MODULE, - func.__name__, + name, ', '.join( [repr(a) for a in (target, *args)] + [f'{k}={v!r}' for k, v in kwargs.items()] @@ -109,7 +110,7 @@ def _run( except Exception as e: cache._debug_output( f'{THIS_MODULE}: Failed to check whether code loaded by ' - f'`runpy.{func.__name__}(...)` is to be rewritten ' + f'`runpy.{name}(...)` is to be rewritten ' f'({type(e).__name__}: {e})' ) profile = False @@ -119,16 +120,13 @@ def _run( # `exec()` into the `runpy` namespace which just rewrites # `cache.rewrite_module` and executes it if profile: - runpy.exec = ( # type: ignore[attr-defined] - partial(_exec, cache, CodeWriter) - ) + # Dodge attr-defined errors and their ilk + vars(runpy)['exec'] = partial(_exec, cache, CodeWriter) try: return func(target, *args, **kwargs) finally: - try: + if hasattr(runpy, 'exec'): del runpy.exec - except AttributeError: - pass def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: @@ -144,6 +142,6 @@ def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: ]: impl = getattr(runpy, func) res = cast(Callable[[str], str], resolver) # Help `mypy` out - wrapper = partial(_run, cache, runpy, impl, res, CodeWriter) + wrapper = partial(_run, cache, runpy, impl, func, res, CodeWriter) setattr(runpy, func, wrapper) return runpy diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index ecad97a9..8ed720c0 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -12,7 +12,7 @@ from collections.abc import Callable, Collection from io import StringIO from textwrap import indent -from typing import Any, TextIO +from typing import Any, TextIO, cast from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler @@ -21,6 +21,7 @@ is_dotted_path, write_eager_import_module, ) from .cli_utils import short_string_path +from .line_profiler import LineProfiler from .profiler_mixin import ByCountProfilerMixin @@ -71,7 +72,10 @@ def write_preimport_module( if not self: return None - write_module_kwargs = { + # Note: `ty` (but not `mypy`) keeps complaining about the our + # splatting this dict; explicitly use `Any` to tell it to shut + # up. + write_module_kwargs: dict[str, Any] = { 'dotted_paths': self.regular, 'recurse': self.recurse, **kwargs, @@ -168,7 +172,14 @@ def __init__( self.insert_builtin = insert_builtin self.builtin_loc = builtin_loc self._installed = False - self._global_install = _global_profiler._kernprof_overwrite + self._kpo = _global_profiler._kernprof_overwrite + + def _global_install(self, prof: ByCountProfilerMixin | None) -> None: + # Wrapper to convince type-checkers it is okay to pass these + # stuff to `._kernprof_overwrite()`. We don't want to patch + # that method's signature because passing non `LineProfiler` + # objects to it should be the exception, not the norm. + self._kpo(cast(LineProfiler, prof)) def install(self) -> None: def del_builtin_profile() -> None: @@ -180,7 +191,7 @@ def set_builtin_profile(old: Any) -> None: if self._installed: return # Overwrite the explicit profiler (`@line_profiler.profile`) - self._global_install(self.prof) # type: ignore[arg-type] + self._global_install(self.prof) # type: ignore # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: try: @@ -205,7 +216,7 @@ def uninstall(self) -> None: for _i in range(getattr(self.prof, 'enable_count', 0)): self.prof.disable_by_count() # Restore the state of the global `@line_profiler.profile` - self._global_install(None) # type: ignore[arg-type] + self._global_install(None) self._installed = False def __enter__(self) -> None: diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 03899261..50ef0d88 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -375,8 +375,10 @@ def get_empty_instance(cls) -> Self: instance (LineStats): New instance without any profiling data. """ - unit = LineProfiler().timer_unit # type: ignore[attr-defined] - return cls({}, unit) + prof = LineProfiler() + if TYPE_CHECKING: + assert hasattr(prof, 'timer_unit') + return cls({}, cast(float, prof.timer_unit)) @classmethod def from_files( From 102f8eb1167c90af54e47b6bb371abc7b8e18093 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 02:10:23 +0200 Subject: [PATCH 040/111] Cache refactoring + logging improvement kernprof.py _manage_profiler.__exit__() Now gathering the debug logs of the child processes and writing them to the main logger _prepare_child_profiling_cache() Simplified implementation line_profiler/_child_process_profiling/cache.py _CallbackRepr New `reprlib.Repr` subclass for handling reprs of the cleanup callbacks (mostly for truncating the repr of `os.environ`, and that of objects it appears in) LineProfilingCache [_add_]cleanup() Updated to use `_CallbackRepr` to represent the callbacks _dump_debug_logs() New method for gathering debug log files from child processes _debug_output() Added timestamps to the messages _setup_in_main_process() New method consisting mostly of code relocated from `kernprof.py::_prepare_child_profiling_cache()` line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() No longer wrapping `os.fork()` (relocated to `LineProfilingCache._setup_in_main_process()`) --- kernprof.py | 39 +-- .../_child_process_profiling/cache.py | 262 +++++++++++++++++- .../_child_process_profiling/pth_hook.py | 4 - 3 files changed, 264 insertions(+), 41 deletions(-) diff --git a/kernprof.py b/kernprof.py index e0b91cf2..70ae3a59 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1225,6 +1225,9 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: + if self.cache.debug: + # Recover debug output from child processes + self.cache._dump_debug_logs() try: extra_stats = self.cache.gather_stats() finally: @@ -1340,10 +1343,6 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): Handle the (line-)profiling of spawned/forked child Python processes. """ - from line_profiler._child_process_profiling import ( - pth_hook, multiprocessing_patches, - ) - # Create the cache dir and cache file here; the cache instance will # be responsible for managing their lifetimes, while derivative # instances in child processes will merely inherit and use them @@ -1356,11 +1355,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): insert_builtin=options.builtin, debug=options.debug, ) - clean_up = functools.partial(cache.add_cleanup, _remove) + clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - clean_up(cache.cache_dir, recursive=True, missing_ok=True) - cache.dump() - clean_up(cache.filename, missing_ok=True) + clean_up(cache.cache_dir, recursive=True) + clean_up(cache.filename) # This file is handed to us at the end of # `_manage_profiler.__enter__()`; @@ -1368,29 +1366,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # child-process profiling is used, it is to persist for the lifetime # of the cache (so that child processes can do the same preimports) if not (preimports_file is None or diagnostics.KEEP_TEMPDIRS): - clean_up(preimports_file, missing_ok=True) - - # Note: the following functions/methods all clean up after - # themselves, so there is no need to explicitly call - # `cache.add_cleanup()` - - # Inject environment variables so that child Python processes can - # inherit them and resume profiling - cache.inject_env_vars() - - # Create the .pth file which allows for setting up profiling in all - # the child Python processes (e.g. those created by `os.system()` or - # `subprocess.run()` - pth_hook.write_pth_hook(cache) - - # Patch `multiprocessing` so that child processes are properly - # handled across all "start methods" - multiprocessing_patches.apply(cache) + clean_up(preimports_file) - # Misc. setup: - # - Set up the cache instance to be the default one `.load()`-ed - cache._replace_loaded_instance() - # - Set `cache.profiler` + # Handle various setup tasks (see docs thereof) + cache._setup_in_main_process() cache.profiler = prof return cache diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 49837f34..4b06993e 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -12,13 +12,18 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import Collection, Callable +from datetime import datetime from functools import partial, cached_property, wraps +from importlib import import_module from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL +from reprlib import Repr from tempfile import mkstemp -from typing import Any, ClassVar, cast -from typing_extensions import Self, ParamSpec +from textwrap import indent +from types import MethodType +from typing import Any, ClassVar, TypeVar, TypedDict, cast +from typing_extensions import Self, ParamSpec, Unpack from .. import _diagnostics as diagnostics from ..autoprofile.autoprofile import ( @@ -41,15 +46,183 @@ PS = ParamSpec('PS') +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) INHERITED_CACHE_ENV_VARNAME_PREFIX = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' ) CACHE_FILENAME = 'line_profiler_cache.pkl' +_DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' + + +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`_CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class _CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> + >>> + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(_CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = _CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + Bound-method formatting: + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + >>> r.indent = None + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + for k, v in kwargs.items(): + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return _indent_with_prefix(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in ptl.args] + args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) + args.insert(0, get(ptl.func)) + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_items(args, (name + '(', ')'), self.maxargs + 1) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + if self.indent is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', self.indent) for item in items), end, + ]) + + @property + def indent(self) -> str | None: + return self._indent + + @indent.setter + def indent(self, indent: str | int | None) -> None: + if indent is None or isinstance(indent, str): + self._indent = indent + return + self._indent = ' ' * indent + + +_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr @dataclasses.dataclass class LineProfilingCache: + """ + Helper object for coordinating a line-profiling session, caching the + info required to make profiling persist into child processes. + """ cache_dir: os.PathLike[str] | str profiling_targets: Collection[str] = dataclasses.field( default_factory=list, @@ -78,12 +251,13 @@ def cleanup(self) -> None: callbacks = self._cleanup_stacks.pop(priority) while callbacks: callback = callbacks.pop() + callback_repr = _CALLBACK_REPR(callback) try: callback() except Exception as e: - msg = f'failed: {callback}: {type(e).__name__}: {e}' + msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' else: - msg = f'succeeded: {callback}' + msg = f'succeeded: {callback_repr}' self._debug_output('Cleanup ' + msg) def add_cleanup( @@ -105,7 +279,7 @@ def _add_cleanup( header = 'Cleanup callback added' if priority: header = f'{header} (priority: {priority})' - self._debug_output(f'{header}: {callback}') + self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') def copy( self, *, @@ -199,6 +373,23 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: return LineStats.get_empty_instance() return LineStats.from_files(*fnames, on_defective='ignore') + def _dump_debug_logs(self) -> None: + """ + Gather the debug logfiles in child processes and write them to + the logger; to be called in the main process. + """ + pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid='*', + ) + for log in sorted(Path(self.cache_dir).glob(pattern)): + if log == self._debug_log: # Don't double dip + continue + *_, child_pid = log.stem.rpartition('_') + msg = 'Cache log messages from child process {}:\n{}'.format( + child_pid, indent(log.read_text(), ' '), + ) + diagnostics.log.debug(msg) + def inject_env_vars( self, env: dict[str, str] | None = None, ) -> None: @@ -232,10 +423,56 @@ def _debug_output(self, msg: str) -> None: return try: with self._debug_log.open(mode='a') as fobj: - print(msg, file=fobj) + prefix = self._debug_message_timestamp + ' ' + print(_indent_with_prefix(msg, prefix), file=fobj) except OSError: # Cache dir may have been rm-ed during cleanup pass + def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: + """ + Set up shop in the main process so that (line-)profiling can + extend into child processes. + + Args: + wrap_os_fork (bool): + Whether to wrap :py:func:`os.fork` which handles + profiling + + Side effects: + + - Instance data written to :py:attr:`~.cache_dir` + + - Environment variables injected + (see :py:meth:`~.inject_env_vars()`) + + - A ``.pth`` file written so that child processes + automaticaly runs setup code (see + :py:func:`line_profiler._child_process_hook.pth_hook.\ +write_pth_hook`) + + - :py:func:`os.fork` wrapped so that profiling set up in + forked processes is properly handled (if + ``wrap_os_fork=True``) + + - :py:mod:`multiprocessing` wrapped so that child processes + managed by the package are properly handled + + - Instance to be returned if :py:func:`~.load()` is called + from now on + """ + this_subpkg, *_, _ = (lambda: None).__module__.rpartition('.') + + self.dump() + self.inject_env_vars() + pth_hook = import_module(this_subpkg + '.pth_hook') + pth_hook.write_pth_hook(self) + + self._wrap_os_fork() + mp_patches = import_module(this_subpkg + '.multiprocessing_patches') + mp_patches.apply(self) + + self._replace_loaded_instance() + def _setup_in_child_process( self, wrap_os_fork: bool = False, @@ -421,9 +658,15 @@ def filename(self) -> str: def _debug_log(self) -> Path | None: if not self.debug: return None - fname = f'debug_log_{self.main_pid}_{os.getpid()}.log' + fname = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid=os.getpid(), + ) return Path(self.cache_dir) / fname + @property + def _debug_message_timestamp(self) -> str: + return f'[cache-debug-log {datetime.now()} DEBUG]' + @cached_property def _debug_message_header(self) -> str: pid = os.getpid() @@ -436,3 +679,8 @@ def _debug_message_header(self) -> str: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() + + +def _indent_with_prefix(string: str, prefix: str, fill_char: str = ' ') -> str: + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 827bd277..3065f52d 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -48,8 +48,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - To be called in the main process. - The ``cache`` is responsible for deleting the written .pth file via the registered cleanup callback. - - For convenience, we also wrap :py:func:`os.fork` when this - function is called. """ import os from pathlib import Path # noqa: F811 @@ -78,8 +76,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path finally: # Not closing the handle causes issues on Windows os.close(handle) - cache._wrap_os_fork() - return fpath From 234dbe5dbc666f89979cba9d7687a91d8a38626b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 03:53:01 +0200 Subject: [PATCH 041/111] Factor out common utils line_profiler/_child_process_profiling/ cache.py, pth_hook.py Relocated shared code snippets to `misc_utils` misc_utils.py New submodules for util functions --- .../_child_process_profiling/cache.py | 21 +++------- .../_child_process_profiling/misc_utils.py | 40 +++++++++++++++++++ .../_child_process_profiling/pth_hook.py | 10 ++--- 3 files changed, 49 insertions(+), 22 deletions(-) create mode 100644 line_profiler/_child_process_profiling/misc_utils.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 4b06993e..1c997ec0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -19,7 +19,6 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from reprlib import Repr -from tempfile import mkstemp from textwrap import indent from types import MethodType from typing import Any, ClassVar, TypeVar, TypedDict, cast @@ -35,6 +34,7 @@ ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -176,7 +176,7 @@ def repr_method(self, method: MethodType, level: int) -> str: func = getattr(method.__func__, '__qualname__', '?') prefix, suffix = f'' # Take care of possible multi-line reprs - return _indent_with_prefix(instance, prefix) + suffix + return block_indent(instance, prefix) + suffix def repr_partial(self, ptl: partial, level: int) -> str: get: Callable[[Any], str] = partial(self.repr1, level=level-1) @@ -424,7 +424,7 @@ def _debug_output(self, msg: str) -> None: try: with self._debug_log.open(mode='a') as fobj: prefix = self._debug_message_timestamp + ' ' - print(_indent_with_prefix(msg, prefix), file=fobj) + print(block_indent(msg, prefix), file=fobj) except OSError: # Cache dir may have been rm-ed during cleanup pass @@ -604,13 +604,9 @@ def make_tempfile(self, **kwargs) -> Path: path (Path): Path to the created file. """ - handle, path = mkstemp(dir=self.cache_dir, **kwargs) - try: - path_obj = Path(path) - self._debug_output(f'Created tempfile: {path_obj.name!r}') - return path_obj - finally: - os.close(handle) + path = make_tempfile(dir=self.cache_dir, **kwargs) + self._debug_output(f'Created tempfile: {path.name!r}') + return path def _replace_loaded_instance(self, force: bool = False) -> bool: cls = type(self) @@ -679,8 +675,3 @@ def _debug_message_header(self) -> str: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() - - -def _indent_with_prefix(string: str, prefix: str, fill_char: str = ' ') -> str: - width = len(prefix) - return prefix + indent(string, fill_char * width)[width:] diff --git a/line_profiler/_child_process_profiling/misc_utils.py b/line_profiler/_child_process_profiling/misc_utils.py new file mode 100644 index 00000000..050380bf --- /dev/null +++ b/line_profiler/_child_process_profiling/misc_utils.py @@ -0,0 +1,40 @@ +""" +Misc. utility functions used by the subpackage. +""" +import os +from pathlib import Path +from tempfile import mkstemp +from textwrap import indent + + +__all__ = ('block_indent', 'make_tempfile') + + +def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: + r""" + Example: + >>> string = 'foo\nbar\nbaz' + >>> print(string) + foo + bar + baz + >>> print(block_indent(string, '++++', '-')) + ++++foo + ----bar + ----baz + """ + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] + + +def make_tempfile(**kwargs) -> Path: + """ + Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding + and closing the integer handle (which if left unattended causes + problems on some platforms). + """ + handle, fname = mkstemp(**kwargs) + try: + return Path(fname) + finally: + os.close(handle) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 3065f52d..5a19cbce 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -50,20 +50,18 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path file via the registered cleanup callback. """ import os - from pathlib import Path # noqa: F811 from sysconfig import get_path - from tempfile import mkstemp + from .misc_utils import make_tempfile if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) - handle, fname = mkstemp( + fpath = make_tempfile( prefix='_line_profiler_profiling_hook_', suffix='.pth', dir=get_path('purelib'), ) - fpath = Path(fname) try: pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( (lambda: None).__module__, cache.main_pid, @@ -71,10 +69,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path fpath.write_text(pth_content) cache.add_cleanup(fpath.unlink, missing_ok=True) except Exception: - os.remove(fpath) + fpath.unlink(missing_ok=True) raise - finally: # Not closing the handle causes issues on Windows - os.close(handle) return fpath From 908780da1b3ba2e7fcc7efc560b23c07f3b02a3c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 16:58:20 +0200 Subject: [PATCH 042/111] Refactored `LineProfilingCache` logging line_profiler/_child_process_profiling/_cache_logging.py ::CacheLoggingEntry New object for IO to/from `LineProfilingCache._debug_log` line_profiler/_child_process_profiling/cache.py::LineProfilingCache cleanup() More verbose logging messages to help with: - Distinguishing cases where no callback has been registered - Indicating whether the cleanup callbacks have been successfully exhausted (e.g. may not be the case for child processes managed by `multiprocessing`) _gather_debug_log_entries() New method for reading in `CacheLoggingEntry`-s from all log files of the session _debug_message_{header,timestamp} Refactored away (functionalities absorbed into `CacheLoggingEntry`) --- .../_cache_logging.py | 338 ++++++++++++++++++ .../_child_process_profiling/cache.py | 89 +++-- 2 files changed, 398 insertions(+), 29 deletions(-) create mode 100644 line_profiler/_child_process_profiling/_cache_logging.py diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py new file mode 100644 index 00000000..91b0635f --- /dev/null +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -0,0 +1,338 @@ +""" +Logging utilities. +""" +from __future__ import annotations + +import os +import re +from collections.abc import Generator +from datetime import datetime +from itertools import pairwise +from pathlib import Path +from string import Formatter as StringParser +from textwrap import dedent +from typing import TYPE_CHECKING, NamedTuple, TextIO, overload +from typing_extensions import Self + +from .. import _diagnostics as diagnostics +from .misc_utils import block_indent + + +__all__ = ('CacheLoggingEntry',) + + +FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +TIMESTAMP_PATTERN = '[cache-debug-log {timestamp} DEBUG]' +HEADER_PATTERN = 'PID {current_pid} ({main_pid}): Cache {obj_id:#x}' + +TIMESTAMP_FORMAT = '%Y-%m-%d %H:%M:%S' +TIMESTAMP_MICROSECOND_SEP = ',' +TIMESTAMP_MICROSECOND_PLACES = 3 +TIMESTAMP_SPACING = ' ' + +HEADER_SEP = ': ' +HEADER_MAIN_INDICATOR = 'main process' + + +def get_logger_header(current_pid: int, main_pid: int, obj_id: int) -> str: + """ + Returns: + msg_header (str): + Message header, to be prefixed to messages sent to + :py:data:`line_profiler._diagnostics.log`. + """ + return HEADER_PATTERN.format( + current_pid=current_pid, + main_pid=( + HEADER_MAIN_INDICATOR if main_pid == current_pid else main_pid + ), + obj_id=obj_id, + ) + + +def format_timestamp(ts: datetime) -> str: + """ + Replicate the :py:mod:`logging`'s default formatting for timestamps. + + Example: + >>> ts = datetime(2000, 1, 23, 4, 5, 6, 789000) + >>> as_str = format_timestamp(ts) + >>> print(as_str) + 2000-01-23 04:05:06,789 + >>> assert parse_timestamp(as_str) == ts + """ + return '{}{}{:0{}d}'.format( + ts.strftime(TIMESTAMP_FORMAT), + TIMESTAMP_MICROSECOND_SEP, + int(ts.microsecond / 1000), + TIMESTAMP_MICROSECOND_PLACES, + ) + + +def parse_timestamp(ts: str) -> datetime: + """ + Turn a formatted string timestamp back to a + :py:class:`datetime.datetime` object. + """ + assert TIMESTAMP_MICROSECOND_SEP in ts + base, _, fractional = ts.rpartition(TIMESTAMP_MICROSECOND_SEP) + # The microsecond field %f must be 6 digits long + if len(fractional) < 6: + fractional = f'{fractional:<06}' + else: + fractional = fractional[:6] + parse_format = f'{TIMESTAMP_FORMAT}{TIMESTAMP_MICROSECOND_SEP}%f' + ts = f'{base}{TIMESTAMP_MICROSECOND_SEP}{fractional}' + return datetime.strptime(ts, parse_format) + + +def add_timestamp(msg: str, timestamp: datetime | None = None) -> str: + """ + Returns: + msg_with_timestamp (str): + (Block-indented) message with timestamp, to be written to + the :py:attr:`LineProfilingCache._debug_log`. + """ + if timestamp is None: + timestamp = datetime.now() + ts_formatted = TIMESTAMP_PATTERN.format( + timestamp=format_timestamp(timestamp), + ) + return block_indent(msg, ts_formatted + TIMESTAMP_SPACING) + + +def parse_id(uint: str) -> int: + """ + Example: + >>> n = 123456 + >>> for formatter in str, bin, oct, hex: + ... assert parse_id(formatter(n)) == n + """ + for prefix, base in ('0b', 2), ('0o', 8), ('0x', 16): + if uint.startswith(prefix): + return int(uint[len(prefix):], base=base) + return int(uint) + + +@overload +def fmt_to_regex(fmt: str, /, *auto_numbered_fields: str) -> str: + ... + + +@overload +def fmt_to_regex(fmt: str, /, **named_fields: str) -> str: + ... + + +def fmt_to_regex( + fmt: str, /, *auto_numbered_fields: str, **named_fields: str +) -> str: + """ + Example: + >>> import re + + Simple case: + + >>> pattern = fmt_to_regex( + ... '{func}({args})', func=r'[_\\w][_\\w\\d]+', args='.*', + ... ) + >>> print(pattern) + (?P[_\\w][_\\w\\d]+)\\((?P.*)\\) + >>> regex = re.compile('^' + pattern, re.MULTILINE) + >>> assert not regex.search('0(1)') + >>> match = regex.search(' \\nint(-1.5)') + >>> assert match.group('func', 'args') == ('int', '-1.5') + + Repeated fields: + + >>> palindrome_5l = re.compile(fmt_to_regex( + ... '{first}{second}{third}{second}{first}', + ... first='.', second='.', third='.', + ... )) + >>> print(palindrome_5l.pattern) + (?P.)(?P.)(?P.)(?P=second)(?P=first) + >>> assert not palindrome_5l.match('abbbe') + >>> match = palindrome_5l.match('aBcBa') + >>> assert match.group('first', 'second', 'third') == ( + ... 'a', 'B', 'c', + ... ) + + Auto-numbered fields: + + >>> print(fmt_to_regex( + ... '[{} {}-{}-{} {}:{}:{},{} {}]', + ... # Logger name + ... '.+', + ... # Date + ... r'\\d\\d', r'\\d\\d', r'\\d\\d', + ... # Time + milliseconds + ... r'\\d\\d', r'\\d\\d', r'\\d\\d', r'\\d\\d\\d', + ... # Category + ... 'DEBUG|INFO|WARNING|ERROR|CRITICAL', + ... )) + \\[(.+)\\ (\\d\\d)\\-(\\d\\d)\\-(\\d\\d)\\ \ +(\\d\\d):(\\d\\d):(\\d\\d),(\\d\\d\\d)\\ \ +(DEBUG|INFO|WARNING|ERROR|CRITICAL)\\] + """ + chunks: list[str] = [] + seen_fields: set[str] = set() + for i, (prefix, field, *_) in enumerate(StringParser().parse(fmt)): + chunks.append(re.escape(prefix)) + if field is None: + break # Suffix -> we're done + if field: # Named fields + assert field.isidentifier() + if field in seen_fields: + chunks.append(f'(?P={field})') + else: + chunks.append(f'(?P<{field}>{named_fields[field]})') + seen_fields.add(field) + else: # Auto-numbered fields + chunks.append(f'({auto_numbered_fields[i]})') + return ''.join(chunks) + + +class CacheLoggingEntry(NamedTuple): + """ + Logging entry written to a log file by + :py:meth:`LineProfilingCache._debug_output`. + + Example: + >>> from datetime import datetime + >>> + >>> + >>> entry = CacheLoggingEntry( + ... datetime(1900, 1, 1, 0, 0, 0, 0), + ... 12345, + ... 12345, + ... 12345678, + ... 'This is a log message;\\nit has multiple lines', + ... ) + >>> print(entry.to_text()) + [cache-debug-log 1900-01-01 00:00:00,000 DEBUG] PID 12345 \ +(main process): Cache 0xbc614e: This is a log message; + it has \ +multiple lines + >>> another_entry = CacheLoggingEntry( + ... datetime(2000, 12, 31, 12, 34, 56, 789000), + ... 12345, + ... 54321, + ... 87654321, + ... 'FOO BAR BAZ', + ... ) + >>> print(another_entry.to_text()) + [cache-debug-log 2000-12-31 12:34:56,789 DEBUG] PID 54321 \ +(12345): Cache 0x5397fb1: FOO BAR BAZ + >>> log_text = '\\n'.join([ + ... e.to_text() for e in [entry, another_entry] + ... ]) + >>> assert CacheLoggingEntry.from_text(log_text) == [ + ... entry, another_entry, + ... ] + """ + timestamp: datetime + main_pid: int + current_pid: int + cache_id: int + msg: str + + def to_text(self) -> str: + return add_timestamp(self._get_header() + self.msg, self.timestamp) + + def _get_header(self) -> str: + return get_logger_header( + self.current_pid, self.main_pid, self.cache_id, + ) + HEADER_SEP + + def write(self, tee: os.PathLike[str] | str | None = None) -> None: + log_msg = self._get_header() + self.msg + diagnostics.log.debug(log_msg) + if tee is None: + return + with Path(tee).open(mode='a') as fobj: + print(add_timestamp(log_msg, self.timestamp), file=fobj) + + @classmethod + def new(cls, main_pid: int, cache_id: int, msg: str) -> Self: + return cls(datetime.now(), main_pid, os.getpid(), cache_id, msg) + + @classmethod + def from_file(cls, file: os.PathLike[str] | str | TextIO) -> list[Self]: + try: + path = Path(file) # type: ignore + except TypeError: # File object + # If we're here, `file` is a file object + if TYPE_CHECKING: + assert isinstance(file, TextIO) + content = file.read() + else: + content = path.read_text() + return cls.from_text(content) + + @classmethod + def from_text(cls, text: str) -> list[Self]: + def gen_timestamps(text: str) -> Generator[re.Match, None, None]: + last_ts_match: re.Match | None = None + while True: + ts_match = timestamp_regex.search( + text, last_ts_match.end() if last_ts_match else 0, + ) + if ts_match: + yield ts_match + last_ts_match = ts_match + else: + return + + def gen_message_blocks(text: str) -> Generator[ + tuple[datetime, re.Match, str], None, None + ]: + timestamps = list(gen_timestamps(text)) + if not timestamps: + return + + # Handle all the entries up till the 2nd-to-last one + for this_match, next_match in pairwise(timestamps): + ts = parse_timestamp(this_match.group('timestamp')) + # The -1 is for stripping the trailing newline + text_block = text[this_match.start():next_match.start() - 1] + yield (ts, this_match, text_block) + # Handle the last entry + last_match = timestamps[-1] + yield ( + parse_timestamp(last_match.group('timestamp')), + last_match, + text[last_match.start():], + ) + + def get_entries(text: str) -> Generator[Self, None, None]: + for timestamp, ts_match, text_block in gen_message_blocks(text): + # Strip the block indent + ts_text = ts_match.group(0) + assert text_block.startswith(ts_text) + ts_width = len(ts_text) + text_block = dedent(' ' * ts_width + text_block[ts_width:]) + # Strip the header and parse the relevant info from it + header_match = header_regex.match(text_block) + assert header_match + current_pid = int(header_match.group('current_pid')) + main_pid_ = header_match.group('main_pid') + if main_pid_ == HEADER_MAIN_INDICATOR: + main_pid = current_pid + else: + main_pid = int(main_pid_) + cache_id = parse_id(header_match.group('obj_id')) + # The rest of the block is the message proper + msg = text_block[header_match.end():] + yield cls(timestamp, main_pid, current_pid, cache_id, msg) + + timestamp_pattern = fmt_to_regex( + f'{TIMESTAMP_PATTERN}{TIMESTAMP_SPACING}', timestamp='.+?', + ) + timestamp_regex = re.compile('^' + timestamp_pattern, re.MULTILINE) + header_regex = re.compile(fmt_to_regex( + HEADER_PATTERN + HEADER_SEP, + current_pid=r'\d+', + main_pid=r'\d+|' + re.escape(HEADER_MAIN_INDICATOR), + obj_id='.+?', + )) + return list(get_entries(text)) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 1c997ec0..95ef42bd 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -11,8 +11,7 @@ import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable -from datetime import datetime +from collections.abc import Collection, Callable, Iterable from functools import partial, cached_property, wraps from importlib import import_module from operator import setitem @@ -34,6 +33,7 @@ ) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from ._cache_logging import CacheLoggingEntry from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth @@ -247,8 +247,18 @@ def cleanup(self) -> None: Pop all the cleanup callbacks from the internal stack added via :py:meth:`~.add_cleanup` and call them in order. """ - for priority in sorted(self._cleanup_stacks): - callbacks = self._cleanup_stacks.pop(priority) + stacks = self._cleanup_stacks + num_callbacks = sum(len(stack) for stack in stacks.values()) + if not num_callbacks: + self._debug_output('Cleanup aborted (no registered callbacks)') + return + # Bookend the cleanup loop with log messages to help detect if + # child processes are prematurely terminated + self._debug_output( + f'Starting cleanup ({num_callbacks} callback(s))...', + ) + for priority in sorted(stacks): + callbacks = stacks.pop(priority) while callbacks: callback = callbacks.pop() callback_repr = _CALLBACK_REPR(callback) @@ -258,7 +268,10 @@ def cleanup(self) -> None: msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' else: msg = f'succeeded: {callback_repr}' - self._debug_output('Cleanup ' + msg) + self._debug_output('- Cleanup ' + msg) + self._debug_output( + f'... cleanup completed ({num_callbacks} callback(s))', + ) def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, @@ -375,13 +388,18 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: def _dump_debug_logs(self) -> None: """ - Gather the debug logfiles in child processes and write them to - the logger; to be called in the main process. + Gather the debug logfiles in child processes and write their + contents to the logger + (:py:data:`line_profiler._diagnostics.log`). + + Notes: + - The content of each child-process log file is not + re-parsed and is written to the logger as a single + multi-line message. + + - To be called in the main process. """ - pattern = _DEBUG_LOG_FILENAME_PATTERN.format( - main_pid=self.main_pid, current_pid='*', - ) - for log in sorted(Path(self.cache_dir).glob(pattern)): + for log in sorted(self._get_debug_logfiles()): if log == self._debug_log: # Don't double dip continue *_, child_pid = log.stem.rpartition('_') @@ -390,6 +408,34 @@ def _dump_debug_logs(self) -> None: ) diagnostics.log.debug(msg) + def _gather_debug_log_entries( + self, chronological: bool = False, + ) -> list[CacheLoggingEntry]: + """ + Gather and return all entries from debug logfiles sorted by + timestamps. + """ + log_files: Iterable[Path] = self._get_debug_logfiles() + if chronological: # Sorting on the entries -> chronological + to_list: Callable[ + [Iterable[CacheLoggingEntry]], list[CacheLoggingEntry] + ] = sorted + else: + # Otherwise, just sort by filename (entries in each file are + # still chronological) + log_files = sorted(log_files) + to_list = list + return to_list( + entry for log in log_files + for entry in CacheLoggingEntry.from_file(log) + ) + + def _get_debug_logfiles(self) -> Iterable[Path]: + pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + main_pid=self.main_pid, current_pid='*', + ) + return Path(self.cache_dir).glob(pattern) + def inject_env_vars( self, env: dict[str, str] | None = None, ) -> None: @@ -417,14 +463,8 @@ def inject_env_vars( env[name] = value def _debug_output(self, msg: str) -> None: - msg = f'{self._debug_message_header}: {msg}' - diagnostics.log.debug(msg) - if not self._debug_log: - return try: - with self._debug_log.open(mode='a') as fobj: - prefix = self._debug_message_timestamp + ' ' - print(block_indent(msg, prefix), file=fobj) + self._make_debug_entry(msg).write(self._debug_log) except OSError: # Cache dir may have been rm-ed during cleanup pass @@ -659,18 +699,9 @@ def _debug_log(self) -> Path | None: ) return Path(self.cache_dir) / fname - @property - def _debug_message_timestamp(self) -> str: - return f'[cache-debug-log {datetime.now()} DEBUG]' - @cached_property - def _debug_message_header(self) -> str: - pid = os.getpid() - return 'PID {} ({}): Cache {:#x}'.format( - pid, - 'main process' if self.main_pid == pid else self.main_pid, - id(self), - ) + def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: + return partial(CacheLoggingEntry.new, self.main_pid, id(self)) @cached_property def _consistent_with_loaded_instance(self) -> bool: From aca4e2cebf84c40369eaf29536be53273b53e574 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 16 Apr 2026 20:19:31 +0200 Subject: [PATCH 043/111] WIP: write combined debug logs for debugging kernprof.py _add_core_parser_arguments() Added temporary/undocumented flag `--debug-log=...` for gathering the cache logs and writing them to one place _manage_profiler Now registering a delayed cleanup callback for writing the debug log on `.__enter__()`, so that the gathering of log entries happens as late as possible (right before the cache dir is wiped) and includes most of the cleanup in the main process _prepare_child_profiling_cache() Now explicitly deferring the deletion of the cache dir to the very end line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.cleanup() Made debug log messages more verbose, indicating the number of callbacks made as the result of each is reported tests/test_child_procs.py ::test_profiling_{bare_python,multiproc_script}() Now gathering cache logs via `--debug-log` and printing them out XXX: do we drop these functionalities when/if the bugs are fixed? --- kernprof.py | 35 +++++- .../_child_process_profiling/cache.py | 20 ++-- tests/test_child_procs.py | 103 +++++++++++------- 3 files changed, 107 insertions(+), 51 deletions(-) diff --git a/kernprof.py b/kernprof.py index 70ae3a59..6fc4e725 100755 --- a/kernprof.py +++ b/kernprof.py @@ -202,7 +202,7 @@ def main(): import shutil import tempfile import time -from argparse import ArgumentParser +from argparse import ArgumentParser, SUPPRESS from operator import methodcaller from runpy import run_module from pathlib import Path @@ -237,6 +237,10 @@ def main(): DIAGNOSITICS_VERBOSITY = 2 +CLEANUP_PRIORITIES = { # Bigger number -> more delayed + 'rm_cache_dir': 1024, + 'gather_logs': 1, +} def execfile(filename, globals=None, locals=None): @@ -783,6 +787,8 @@ def _add_core_parser_arguments(parser): 'Minimum value (and the value implied if the bare option ' f'is given) is 1 s. (Default: {def_out_int})', ) + # Hidden option for dumping the debug logs to a desinated location + add_argument(out_opts, '--debug-log', help=SUPPRESS) def _build_parsers(args=None): @@ -1219,16 +1225,24 @@ def __enter__(self): self.cache = _prepare_child_profiling_cache( self.options, self.prof, preimports_file, script_file, ) + # Add a deferred callback for gathering debug logfiles + # (should run right before `.cache.cache_dir` is wiped) + if self.options.debug_log: + self.cache._add_cleanup( + self._gather_debug_log, + CLEANUP_PRIORITIES['gather_logs'], + self.options.debug_log, + ) return self.prof, script_file def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: - if self.cache.debug: - # Recover debug output from child processes - self.cache._dump_debug_logs() try: + if self.cache.debug: + # Recover debug output from child processes + self.cache._dump_debug_logs() extra_stats = self.cache.gather_stats() finally: self.cache.cleanup() @@ -1236,6 +1250,11 @@ def __exit__(self, *_, **__): finally: self._ctx.uninstall() + def _gather_debug_log(self, logfile): + with open(logfile, mode='w') as fobj: + for entry in self.cache._gather_debug_log_entries(): + print(entry.to_text(), file=fobj) + @property def set_up_child_profiling(self): return bool( @@ -1353,11 +1372,15 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): profile_imports=options.prof_imports, preimports_module=preimports_file, insert_builtin=options.builtin, - debug=options.debug, + debug=bool(options.debug or options.debug_log), ) clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - clean_up(cache.cache_dir, recursive=True) + # Defer the scrubbing of the cache dir + cache._add_cleanup( + _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, + recursive=True, + ) clean_up(cache.filename) # This file is handed to us at the end of diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 95ef42bd..330b400a 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -248,29 +248,35 @@ def cleanup(self) -> None: :py:meth:`~.add_cleanup` and call them in order. """ stacks = self._cleanup_stacks - num_callbacks = sum(len(stack) for stack in stacks.values()) - if not num_callbacks: + ncallbacks_total = sum(len(stack) for stack in stacks.values()) + if not ncallbacks_total: self._debug_output('Cleanup aborted (no registered callbacks)') return # Bookend the cleanup loop with log messages to help detect if # child processes are prematurely terminated self._debug_output( - f'Starting cleanup ({num_callbacks} callback(s))...', + f'Starting cleanup ({ncallbacks_total} callback(s))...', ) + ncallbacks_run = 0 for priority in sorted(stacks): callbacks = stacks.pop(priority) while callbacks: callback = callbacks.pop() callback_repr = _CALLBACK_REPR(callback) + ncallbacks_run += 1 try: callback() except Exception as e: - msg = f'failed: {callback_repr}: {type(e).__name__}: {e}' + state = 'failed' + msg = f'{callback_repr}: {type(e).__name__}: {e}' else: - msg = f'succeeded: {callback_repr}' - self._debug_output('- Cleanup ' + msg) + state, msg = 'succeeded', f'{callback_repr}' + self._debug_output( + f'- Cleanup {state} ' + f'({ncallbacks_run}/{ncallbacks_total}): {msg}', + ) self._debug_output( - f'... cleanup completed ({num_callbacks} callback(s))', + f'... cleanup completed ({ncallbacks_total} callback(s))', ) def add_cleanup( diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 14179c59..16352b3d 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -32,6 +32,7 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) +_WRITE_DEBUG_LOGS = True def strip(s: str) -> str: @@ -578,6 +579,7 @@ def _run_test_module( nnums: int | None = None, nprocs: int | None = None, check: bool = True, + debug_log: str | None = None, nhits: Mapping[str, int] | None = None, **kwargs ) -> tuple[subprocess.CompletedProcess, LineStats | None]: @@ -631,45 +633,54 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: with ub.ChDir(tmp_path_factory.mktemp('mytemp')): if outfile is not None: runner_args.extend(['--outfile', outfile]) + if debug_log: + runner_args.extend(['--debug-log', debug_log]) old_pth_files = get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), **kwargs ) - # Checks: - if fail: - # - The process has failed as expected - if check: - assert proc.returncode - else: - # - The result is correctly calculated - expected = nnums * (nnums + 1) // 2 - output_lines = proc.stdout.splitlines() - if output_lines[0] != str(expected): - raise ResultMismatch( - f'result {expected}', f'output lines: {output_lines}', - ) - # - Temporary `.pth` file(s) created by `~~.pth_hook` has been - # cleaned up - assert get_pth_files() == old_pth_files - # - Profiling results are written to the specified file - prof_result: LineStats | None = None - if outfile is None: - assert not list(Path.cwd().iterdir()) - else: - assert os.path.exists(outfile) - assert os.stat(outfile).st_size - if profile: - prof_result = LineStats.from_files(outfile) - # - If we're keeping track, the function is called the expected - # number of times and has run the expected # of loops - # (Note: we do it by parsing the output of `kernprof -v` - # instead of reading the `--outfile`, because if the profiled - # code is in a tempfile the profiling data will be dropped in - # the written outfile) - for tag, num in (nhits or {}).items(): - _check_output(proc.stdout, tag, num) + try: + # Checks: + if fail: + # - The process has failed as expected + if check: + assert proc.returncode + else: + # - The result is correctly calculated + expected = nnums * (nnums + 1) // 2 + output_lines = proc.stdout.splitlines() + if output_lines[0] != str(expected): + raise ResultMismatch( + f'result {expected}', f'output lines: {output_lines}', + ) + # - Temporary `.pth` file(s) created by `~~.pth_hook` has + # been cleaned up + assert get_pth_files() == old_pth_files + # - Profiling results are written to the specified file + prof_result: LineStats | None = None + if outfile is None: + assert not list(Path.cwd().iterdir()) + else: + assert os.path.exists(outfile) + assert os.stat(outfile).st_size + if profile: + prof_result = LineStats.from_files(outfile) + # - If we're keeping track, the function is called the + # expected number of times and has run the expected # of + # loops (Note: we do it by parsing the output of + # `kernprof -v` instead of reading the `--outfile`, + # because if the profiled code is in a tempfile the + # profiling data will be dropped in the written outfile) + for tag, num in (nhits or {}).items(): + _check_output(proc.stdout, tag, num) + finally: + if debug_log is not None: + with open(debug_log) as fobj: + print('-- Combined debug logs --', file=sys.stderr) + print(indent(fobj.read(), ' '), end='', file=sys.stderr) + print('-- End of debug logs --', file=sys.stderr) return proc, prof_result @@ -932,6 +943,9 @@ def test_profiling_multiproc_script( nnums=nnums, nprocs=nprocs, timeout=timeout, + debug_log=( + 'debug.log' if prof_child_procs and _WRITE_DEBUG_LOGS else None + ), ) @@ -976,12 +990,16 @@ def test_profiling_bare_python( script_path.write_text(script_content) out_file = temp_dir / 'out.lprof' + debug_log_file = temp_dir / 'debug.log' + write_debug = _WRITE_DEBUG_LOGS and prof_child_procs cmd = [ 'kernprof', '-lv', '--preimports', f'--prof-mod={ext_module.name}', f'--outfile={out_file}', '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), ] + if write_debug: + cmd.append(f'--debug-log={debug_log_file}') sub_cmd = [sys.executable, str(script_path)] if use_subprocess: code = strip(f""" @@ -1006,8 +1024,17 @@ def test_profiling_bare_python( for k in nhits: nhits[k] = 0 - # Check that the code errors out when expected - assert bool(fail) == bool(proc.returncode) - # Check that the profiling output is as expected - for tag, num in nhits.items(): - _check_output(proc.stdout, tag, num) + try: + # Check that the code errors out when expected + assert bool(fail) == bool(proc.returncode) + # Check that the profiling output is as expected + for tag, num in nhits.items(): + _check_output(proc.stdout, tag, num) + finally: + if write_debug: + print('-- Combined debug logs --', file=sys.stderr) + print( + indent(debug_log_file.read_text(), ' '), + end='', file=sys.stderr, + ) + print('-- End of debug logs --', file=sys.stderr) From 25e11f97655a9e95f505ac771997a552570c04fb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 17 Apr 2026 00:00:04 +0200 Subject: [PATCH 044/111] Compatibility fix line_profiler/_child_process_profiling/cache.py::_CallbackRepr __doc__ Updated so that the parts that test `.indent` is skipped on Python < 3.12 indent No longer a property; private methods now use `._get_indent()` to indirectly access it _get_indent() Wrapper around `.indent`; falls back to `None` on legacy versions (< 3.12) without the attribute --- .../_child_process_profiling/cache.py | 59 ++++++++++++------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 330b400a..01bf03e5 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,6 +7,7 @@ import atexit import dataclasses import os +import sys try: import _pickle as pickle except ImportError: @@ -86,8 +87,8 @@ class _CallbackRepr(Repr): Example: >>> from functools import partial - >>> - >>> + >>> from sys import version_info + >>> class MyEnviron(dict): ... def some_method(self) -> None: ... ... @@ -117,6 +118,28 @@ class _CallbackRepr(Repr): Partial-object formatting: + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) >>> r.indent = 2 >>> callback_1 = partial(int, base=8) >>> print(r.repr(callback_1)) @@ -137,8 +160,6 @@ class _CallbackRepr(Repr): ----..., ) - Bound-method formatting: - >>> r.indent = ' ' >>> r.maxenv = 2 >>> print(r.repr(my_env.some_method)) @@ -147,10 +168,6 @@ class _CallbackRepr(Repr): 'bar': '2', ..., })> - >>> r.indent = None - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - """ def __init__( self, @@ -196,22 +213,24 @@ def _format_items( start, end = delims if maxlen is not None and len(items) > maxlen: items = list(items)[:maxlen] + ['...'] - if self.indent is None or not items: + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: return '{}{}{}'.format(start, ', '.join(items), end) return '\n'.join([ - start, *(indent(item + ',', self.indent) for item in items), end, + start, *(indent(item + ',', indent_prefix) for item in items), end, ]) - @property - def indent(self) -> str | None: - return self._indent - - @indent.setter - def indent(self, indent: str | int | None) -> None: - if indent is None or isinstance(indent, str): - self._indent = indent - return - self._indent = ' ' * indent + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None _CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr From 48cc7d1ee4da7ef34214111acffb1513f9dcc6e6 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 17 Apr 2026 01:16:09 +0200 Subject: [PATCH 045/111] Compat fix (test) tests/test_child_procs.py::{_run_subproc,test_profiling_bare_python}() Fixed bug where command lines are malformed on Windows owing to the use of `shlex.join()`; the (undocumented but stable and correct) `subprocess.list2cmdline()` is used instead if we're on the platform --- tests/test_child_procs.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 16352b3d..802f2c4f 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -456,6 +456,19 @@ def new( return cls(param_list, value_tuple_list, default_values) +# `shlex.join()` doesn't work properly on Windows, so use +# `subprocess.list2cmdline()` instead; +# though an "intentionally" undocumented API (cpython issue #10308), +# it's been around since 2.4, seems stable enough, and does exactly what +# is needed +if sys.platform == 'win32': + concat_command_line: Callable[ + [Sequence[str]], str + ] = subprocess.list2cmdline +else: + concat_command_line = shlex.join + + def _run_as_script( runner_args: list[str], test_args: list[str], test_module: _ModuleFixture, **kwargs @@ -498,7 +511,7 @@ def _run_subproc( if isinstance(cmd, str): cmd_str = cmd else: - cmd_str = shlex.join(cmd) + cmd_str = concat_command_line(cmd) # If we're capturing outputs, it may be for the best to wait until # we've processed the output streams to check the return code... @@ -1015,7 +1028,7 @@ def test_profiling_bare_python( if os.system({!r}): raise RuntimeError('called process failed') - """.format(shlex.join(sub_cmd))) + """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) proc = _run_subproc(cmd, text=True, capture_output=True) From 4630938f4467d88968b5ce286bcec217996ff6a9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 01:52:27 +0200 Subject: [PATCH 046/111] Better handling of `multiprocessing` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_{start,terminate,bootstrap}() Wrapped methods now write debug-log entries when: - Before the wrapped call is made - After the wrapped call returns/errors out wrap_terminate() Now always calling `vanilla_impl()` (`= Process.terminate()`) regardless of whether the preceding polling and callback errors out or not, to avoid orphaning child processes --- .../multiprocessing_patches.py | 85 +++++++++++++------ 1 file changed, 60 insertions(+), 25 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index dffd92e5..11accc69 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,14 +16,14 @@ import multiprocessing import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable from contextlib import AbstractContextManager, nullcontext from functools import partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from pathlib import Path from time import sleep, monotonic -from typing import Any, Literal, TypeVar, NoReturn +from typing import Any, Generic, Literal, Protocol, TypeVar, NoReturn, cast from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics @@ -34,7 +34,6 @@ __all__ = ('apply',) -S = TypeVar('S') T = TypeVar('T') PS = ParamSpec('PS') @@ -62,6 +61,11 @@ _INTERCEPT_MP_LOG_MESSAGES: bool | None = False +class _Wrapper(Protocol, Generic[PS, T]): + def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: + ... + + class PickleHook: """ Object which, when unpickled, sets up profiling in the @@ -190,16 +194,41 @@ class Timeout(RuntimeError): def _method_wrapper( - wrapper: Callable[Concatenate[S, Callable[Concatenate[S, PS], T], PS], T] -) -> Callable[ - [Callable[Concatenate[S, PS], T]], Callable[Concatenate[S, PS], T] -]: - def inner_wrapper( - vanilla_impl: Callable[Concatenate[S, PS], T], - ) -> Callable[Concatenate[S, PS], T]: + wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], +) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: @wraps(vanilla_impl) - def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: - return wrapper(self, vanilla_impl, *a, **k) + def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = LineProfilingCache.load() + debug = cache._debug_output + + arg_reprs: list[str] = [repr(arg) for arg in args] + arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) + formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) + + debug(f'Wrapped method call made: {formatted_call}...') + try: + result = wrapper(cache, vanilla_impl, *args, **kwargs) + except Exception as e: + debug( + 'Wrapped method call failed: ' + f'{formatted_call} -> {type(e).__name__}: {e}', + ) + raise + else: + debug( + 'Wrapped method call succeeded: ' + f'{formatted_call} -> {result!r}', + ) + return result + + if ( + hasattr(vanilla_impl, '__module__') + and hasattr(vanilla_impl, '__qualname__') + ): + name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) + else: + name = f'' return wrapped_impl @@ -213,7 +242,9 @@ def wrapped_impl(self: S, *a: PS.args, **k: PS.kwargs) -> T: @_method_wrapper def wrap_start( - self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], + cache: LineProfilingCache, + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, ) -> None: """ Wrap around :py:meth:`BaseProcess.start` to specify the location for @@ -221,7 +252,6 @@ def wrap_start( the parent. This is to ensure that the child can exit gracefully and complete any necessary cleanup. """ - cache = LineProfilingCache.load() tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) vanilla_impl(self) @@ -229,7 +259,9 @@ def wrap_start( @_method_wrapper def wrap_terminate( - self: BaseProcess, vanilla_impl: Callable[[BaseProcess], None], + cache: LineProfilingCache, # TODO: config timeouts + vanilla_impl: Callable[[BaseProcess], None], + self: BaseProcess, ) -> None: """ Wrap around :py:meth:`BaseProcess.terminate` to make sure that we @@ -281,14 +313,18 @@ def discard_lock() -> None: else: lock, callback = nullcontext(), _no_op with lock: - callback() - vanilla_impl(self) + try: + callback() + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) @_method_wrapper def wrap_bootstrap( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], self: BaseProcess, - vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], /, + /, *args: PS.args, **kwargs: PS.kwargs ) -> T: """ @@ -302,7 +338,6 @@ def wrap_bootstrap( ``.terminate()`` a failed child before the profiling results can be gathered. """ - cache = LineProfilingCache.load() lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) if lock_file: @@ -431,13 +466,13 @@ def replace( )) # Patch `multiprocessing.process.BaseProcess` methods - Method = Callable[Concatenate[S, PS], T] - patches: Mapping[str, Callable[[Method], Method]] - for submodule, target, patches in [ # type: ignore[assignment] + # Note: the type checkers seem to need some help figuring the + # `patches` out... so do explicit `cast()`s + for submodule, target, patches in [ ('process', 'BaseProcess', { - 'start': wrap_start, - 'terminate': wrap_terminate, - '_bootstrap': wrap_bootstrap, + 'start': cast(_Wrapper[..., None], wrap_start), + 'terminate': cast(_Wrapper[..., None], wrap_terminate), + '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), }), ]: try: From 8ccba54b4badc40999d2c77a605efac645e467b9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 02:35:19 +0200 Subject: [PATCH 047/111] Cache the active config line_profiler/_child_process_profiling/cache.py::LineProfilingCache Added init field/attribute `config` for storing the config source line_profiler/toml_config.py::ConfigSource.get_subconfig() Fixed bug where the `copy` argument is silently ignored --- kernprof.py | 1 + line_profiler/_child_process_profiling/cache.py | 1 + line_profiler/toml_config.py | 8 ++++++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kernprof.py b/kernprof.py index 6fc4e725..110c88a3 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1367,6 +1367,7 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): # instances in child processes will merely inherit and use them cache = LineProfilingCache( cache_dir=tempfile.mkdtemp(), + config=options.config, profiling_targets=options.prof_mod, rewrite_module=script_file, profile_imports=options.prof_imports, diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 01bf03e5..04819f37 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -243,6 +243,7 @@ class LineProfilingCache: info required to make profiling persist into child processes. """ cache_dir: os.PathLike[str] | str + config: os.PathLike[str] | str | None = None profiling_targets: Collection[str] = dataclasses.field( default_factory=list, ) diff --git a/line_profiler/toml_config.py b/line_profiler/toml_config.py index 781dc60d..236e86da 100644 --- a/line_profiler/toml_config.py +++ b/line_profiler/toml_config.py @@ -106,7 +106,10 @@ def get_subconfig( get_subtable(self.conf_dict, headers, allow_absence=allow_absence), ) new_subtable = [*self.subtable, *headers] - return type(self)(new_dict, self.path, new_subtable) + new_instance = type(self)(new_dict, self.path, new_subtable) + if copy: + new_instance = new_instance.copy() + return new_instance @classmethod def from_default(cls, *, copy: bool = True) -> ConfigSource: @@ -355,7 +358,8 @@ def iter_configs(dir_path): def get_subtable( - table: Mapping[K, Mapping], keys: Sequence[K], *, allow_absence: bool = True + table: Mapping[K, Mapping], keys: Sequence[K], *, + allow_absence: bool = True, ) -> Mapping: """ Arguments: From 04023697be90f340845d51e00b456fed2645a150 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 05:14:29 +0200 Subject: [PATCH 048/111] Move config for `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_terminate() Now configuring the `_Poller` object based on `cache.config` wrap_get_preparation_data() Refactored from `get_preparation_data()`; now also emitting extra debug output like the other `wrap_*()` functions _apply_mp_patches(debug=None) Instead of using `lp_cache.debug`, now using `cache.config` to decide on whether to intercept the `multiprocessing` logs line_profiler/rc/line_profiler.toml ::[tool.line_profiler.multiprocessing] New private subtable for configuring the `multiprocessing_patches` behavior (poller timeouts, logging, etc.) --- .../multiprocessing_patches.py | 130 ++++++++++++------ line_profiler/rc/line_profiler.toml | 33 +++++ 2 files changed, 118 insertions(+), 45 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 11accc69..03eba201 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,17 +16,22 @@ import multiprocessing import warnings -from collections.abc import Callable +from collections.abc import Callable, Mapping from contextlib import AbstractContextManager, nullcontext -from functools import partial, wraps +from functools import lru_cache, partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess +from os import PathLike from pathlib import Path from time import sleep, monotonic -from typing import Any, Generic, Literal, Protocol, TypeVar, NoReturn, cast +from types import MappingProxyType +from typing import ( + Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, +) from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics +from ..toml_config import ConfigSource from .cache import LineProfilingCache from .runpy_patches import create_runpy_wrapper @@ -40,26 +45,6 @@ _PATCHED_MARKER = '_line_profiler_patched_multiprocessing' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' -# NOTE: polling behaviors for blocking `BaseProcess.terminate()` with -# the lock file: -# - _POLLING_COOLDOWN: -# Seconds between polling for the lock file; set to zero to poll in a -# hot loop (not recommended) -# - _POLLING_TIMEOUT: second to block before timing out anyway; set to -# zero to disable timing out -# - _POLLING_ON_TIMEOUT: what to do on timeout if enabled: -# - 'error': error out with `_Poller.Timeout` -# - 'warn': issue a warning and move on -# - 'ignore': move on without a warning -_POLLING_COOLDOWN = 1. / 32 -_POLLING_TIMEOUT = 1. -_POLLING_ON_TIMEOUT: Literal['error', 'warn', 'ignore'] = 'warn' - -# NOTE: Set this to `None` or `True` to tee the `multiprocessing` -# internal logging messages to the log files; if `None`, logs are only -# written if `LineProfilingCache.load().debug` is set to true. -_INTERCEPT_MP_LOG_MESSAGES: bool | None = False - class _Wrapper(Protocol, Generic[PS, T]): def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: @@ -193,6 +178,27 @@ class Timeout(RuntimeError): pass +def _get_config( + config: PathLike[str] | str | bool | None = None, +) -> Mapping[str, Any]: + if config not in (True, False, None): + config = str(config) + return _get_config_cached(cast(Union[str, bool, None], config)) + + +@lru_cache() +def _get_config_cached( + config: PathLike[str] | str | bool | None = None, +) -> Mapping[str, Any]: + cd = dict( + ConfigSource.from_config(config) + .get_subconfig('multiprocessing', copy=True) + .conf_dict + ) + assert isinstance(cd.get('polling'), Mapping) + return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) + + def _method_wrapper( wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: @@ -206,18 +212,18 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) - debug(f'Wrapped method call made: {formatted_call}...') + debug(f'Wrapped call made: {formatted_call}...') try: result = wrapper(cache, vanilla_impl, *args, **kwargs) except Exception as e: debug( - 'Wrapped method call failed: ' + 'Wrapped call failed: ' f'{formatted_call} -> {type(e).__name__}: {e}', ) raise else: debug( - 'Wrapped method call succeeded: ' + 'Wrapped call succeeded: ' f'{formatted_call} -> {result!r}', ) return result @@ -302,21 +308,53 @@ def discard_lock() -> None: except AttributeError: pass - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - if lock_file: - lock: AbstractContextManager[Any] = ( - _Poller.poll_while(lock_file.exists) - .with_cooldown(_POLLING_COOLDOWN) - .with_timeout(_POLLING_TIMEOUT, _POLLING_ON_TIMEOUT) - ) - callback = discard_lock - else: - lock, callback = nullcontext(), _no_op - with lock: + def get_poller_args( + config: PathLike[str] | str | bool | None = None, + ) -> tuple[float, float, str | None]: + values = _get_config(config)['polling'] + try: + cooldown = max(float(values['cooldown']), 0) + except (TypeError, ValueError): + cooldown = 0 + try: + timeout = max(float(values['timeout']), 0) + except (TypeError, ValueError): + timeout = 0 try: + on_timeout: str | None = values['on_timeout'].lower() + except Exception: # Fallback (use `_Poller`'s default) + on_timeout = None + return cooldown, timeout, on_timeout + + def wait_for_deletion( + path: Path, config: PathLike[str] | str | None = None, + ) -> _Poller: + cooldown, timeout, on_timeout = get_poller_args(config) + # `False` -> no resolution, force loading the vanilla file + *_, default_on_timeout = get_poller_args(False) + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = default_on_timeout + return ( + _Poller.poll_while(path.exists) + .with_cooldown(cooldown) + .with_timeout( + timeout, cast(Literal['ignore', 'warn', 'error'], on_timeout), + ) + ) + + try: + lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) + if lock_file: + lock: AbstractContextManager[Any] = wait_for_deletion( + lock_file, cache.config, + ) + callback = discard_lock + else: + lock, callback = nullcontext(), _no_op + with lock: callback() - finally: # Always call `Process.terminate()` to avoid orphans - vanilla_impl(self) + finally: # Always call `Process.terminate()` to avoid orphans + vanilla_impl(self) @_method_wrapper @@ -386,7 +424,11 @@ def get_msg(msg: str, *_, **__) -> str: ) -def get_preparation_data( +@_method_wrapper +def wrap_get_preparation_data( + # We don't use the cache here, but `@_method_wrapper` expects it in + # the signature (and we want the debug output) + _, vanilla_impl: Callable[PS, dict[str, Any]], /, *args: PS.args, @@ -447,7 +489,7 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, add_cleanup: Callable[..., Any], - debug: bool | None = _INTERCEPT_MP_LOG_MESSAGES, + debug: bool | None = None, ) -> None: def replace( obj: Any, attr: str, value: Any, obj_name: str | None = None, @@ -492,9 +534,7 @@ def replace( pass else: # Patch `get_preparation_data()` - gpd_wrapper = partial( # type: ignore[call-arg] - get_preparation_data, spawn.get_preparation_data, - ) + gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) @@ -504,7 +544,7 @@ def replace( # Intercept `multiprocessing` debug messages if debug is None: - debug = lp_cache.debug + debug = _get_config(lp_cache.config)['intercept_logs'] if debug: from multiprocessing import util diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index a5b7740c..5b39f5cb 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -209,3 +209,36 @@ hits = 9 time = 12 perhit = 8 percent = 8 + +# XXX: --- Start of implementation details --- +# `line_profiler._child_process_profiling.multiprocessing_patches` +# settings + +[tool.line_profiler.multiprocessing] + +# - `intercept_logs` (bool): +# Whether to patch the `multiprocessing.util` logging funcions so that +# the internal log messages are captured and writtent to the debug +# logs +intercept_logs = false + +[tool.line_profiler.multiprocessing.polling] + +# - `polling.cooldown` (float): +# Cooldown time (seconds) before successive polls on lock files (set +# to <= 0 to disable cooldowns) +cooldown = 0.03125 # 1/32-nd of a second +# - `polling.timeout` (float): +# Time (seconds) before the main process disregards the existence of +# lock file and unblocks calls to `.terminate()` a child process +# anyway (set to <= 0 to disable timeouts) +timeout = 1 +# - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) +# What to do when the above timeout is exhausted, before actually +# `.terminate()`-ing the child process: +# - 'error': raise an error +# - `warn`: issue a warning +# - `ignore`: nothing +on_timeout = 'error' + +# XXX: --- End of implementation details --- From eb5deb64e26c6bf50c996e5d1277b2c7eb744b0d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 07:17:53 +0200 Subject: [PATCH 049/111] Streamlined profiler installation line_profiler/curated_profiling.py::CuratedProfilerContext.install() Now responsible for equiping `.prof` with the `.add_imported_function_or_module()` pseudo-method kernprof.py::_write_preimports() line_profiler/_child_process_profiling/cache.py::LineProfilingCache Removed code doing the same --- kernprof.py | 4 ---- line_profiler/_child_process_profiling/cache.py | 8 -------- line_profiler/curated_profiling.py | 6 ++++++ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/kernprof.py b/kernprof.py index 110c88a3..96888468 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1066,15 +1066,11 @@ def _write_preimports(prof, options, exclude, keep=False): Called by :py:func:`main()` to handle eager pre-imports; not to be invoked on its own. """ - from line_profiler.autoprofile.autoprofile import ( - _extend_line_profiler_for_profiling_imports as upgrade_profiler, - ) from line_profiler.curated_profiling import ClassifiedPreimportTargets # We could've done everything in-memory with `io.StringIO` and `exec()`, # but that results in indecipherable tracebacks should anything goes wrong; # so we write to a tempfile and `execfile()` it - upgrade_profiler(prof) temp_mod_path = _touch_tempfile( dir=options.tmpdir, prefix='kernprof-eager-preimports-', suffix='.py' ) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 04819f37..6cebe16f 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -25,13 +25,6 @@ from typing_extensions import Self, ParamSpec, Unpack from .. import _diagnostics as diagnostics -from ..autoprofile.autoprofile import ( - # Note: we need this to equip the profiler with the - # `.add_imported_function_or_module()` pseudo-method - # (see `kernprof.py::_write_preimports()`), which is required for - # the preimports to work - _extend_line_profiler_for_profiling_imports as upgrade_profiler, -) from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ._cache_logging import CacheLoggingEntry @@ -578,7 +571,6 @@ def _setup_in_child_process( if prof is None: prof = LineProfiler() self.profiler = prof - upgrade_profiler(prof) ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) ctx.install() self.add_cleanup(ctx.uninstall) diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 8ed720c0..6e21ffe8 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -16,6 +16,9 @@ from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler +from .autoprofile.autoprofile import ( + _extend_line_profiler_for_profiling_imports as upgrade_profiler, +) from .autoprofile.util_static import modpath_to_modname from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, @@ -190,6 +193,9 @@ def set_builtin_profile(old: Any) -> None: if self._installed: return + # Equip the profiler instance with the + # `.add_imported_function_or_module()` pseudo-method + upgrade_profiler(self.prof) # Overwrite the explicit profiler (`@line_profiler.profile`) self._global_install(self.prof) # type: ignore # Set up hooks to deal with inserting `.prof` as a builtin name From 6057c4a4740d1cee82911e4f4d00a6965a3018e2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 08:18:12 +0200 Subject: [PATCH 050/111] WIP: boost test coverage line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__doc__ Added doctest tests/test_child_procs.py test_module_clone New fixture which is identical in content to `test_module` but backed by a different physical file create_cache() New fixture function for creating a `LineProfilingCache` instance curated_profiler New fixture for a `LineProfiler` managed by a `CuratedProfilerContext` test_runpy_patches() New test for `~._child_process_profiling.runpy_patches.create_runpy_wrapper()` --- .../multiprocessing_patches.py | 28 +++ tests/test_child_procs.py | 186 +++++++++++++++++- 2 files changed, 212 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 03eba201..24029022 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -78,6 +78,34 @@ def __setstate__(*_) -> None: class _Poller: """ Poll a callable until it returns true-y. + + Example: + >>> from itertools import count + >>> from typing import Iterator + >>> + >>> + >>> def count_until(limit: int) -> bool: + ... def counter_is_big_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) >= limit + ... + ... return _Poller.poll_until( + ... counter_is_big_enough, count(), limit, + ... ) + ... + >>> + >>> with count_until(10).with_cooldown(.01).with_timeout(.25): + ... print('We counted up to 10') + We counted up to 10 + >>> with count_until(30).with_cooldown(.01).with_timeout(.25): + ... print('We counted up to 30') \ +# doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + ... + line_profiler..._Poller.Timeout: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_big_enough... to return true """ def __init__( self, diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 802f2c4f..91548298 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -11,6 +11,7 @@ ) from dataclasses import dataclass from functools import partial +from io import StringIO from pathlib import Path from sysconfig import get_path from tempfile import TemporaryDirectory @@ -23,7 +24,12 @@ import pytest import ubelt as ub -from line_profiler.line_profiler import LineStats +from line_profiler._child_process_profiling.cache import LineProfilingCache +from line_profiler._child_process_profiling.runpy_patches import ( + create_runpy_wrapper, +) +from line_profiler.curated_profiling import CuratedProfilerContext +from line_profiler.line_profiler import LineProfiler, LineStats T = TypeVar('T') @@ -241,6 +247,80 @@ def test_module( yield _ModuleFixture(_test_module, monkeypatch, [ext_module]) +@pytest.fixture +def test_module_clone( + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, + _test_module: Path, + ext_module: _ModuleFixture, +) -> Generator[_ModuleFixture, None, None]: + """ + Yields: + :py:class:`_ModuleFixture` helper object containing the same + code as :py:data:`test_module` + """ + tmpdir = tmp_path_factory.mktemp('my_path') + name = next(_ModuleFixture.propose_name('my_cloned_module')) + path = tmpdir / f'{name}.py' + path.write_text(_test_module.read_text()) + yield _ModuleFixture(path, monkeypatch, [ext_module]) + + +@pytest.fixture +def create_cache( + tmp_path_factory: pytest.TempPathFactory, + curated_profiler: LineProfiler, +) -> Generator[Callable[..., LineProfilingCache], None, None]: + """ + Wrapper around the :py:class:`LineProfilingCache` instantiator + which: + + - Automatically creates a tempdir and provides it as the + :py:attr:`LineProfilingCache.cache_dir`, + + - Automatically creates an instance of :py:class:`LineProfiler` that + is curated by a :py:class:`CuratedProfilerContext` and provides it + as the :py:attr:`LineProfilingCache.profiler`, and + + - At teardown: + + - Restores the value of the class' internal reference to the + :py:meth:`LineProfilingCache.load`-ed instance. + + - Calls the `.cleanup()` method of each instance created. + """ + def instantiate(**kwargs) -> LineProfilingCache: + tmpdir = tmp_path_factory.mktemp('my_cache_dir') + cache = LineProfilingCache(tmpdir, **kwargs) + cache.profiler = curated_profiler + instances.append(cache) + return cache + + old_value = LineProfilingCache._loaded_instance + instances: list[LineProfilingCache] = [] + try: + yield instantiate + finally: + LineProfilingCache._loaded_instance = old_value # type: ignore + for cache in instances: + try: + cache.cleanup() + except Exception: + pass + + +@pytest.fixture +def curated_profiler() -> Generator[LineProfiler, None, None]: + """ + Yields: + Fresh instance of :py:class:`LineProfiler` that is managed by a + :py:class:`CuratedProfilerContext` + """ + prof = LineProfiler() + with CuratedProfilerContext(prof, insert_builtin=True): + yield prof + + # ========================== Helper functions ========================== @@ -721,7 +801,109 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) -# =============================== Tests ================================ +# ============================= Unit tests ============================= + + +@pytest.mark.parametrize( + ('run_profiled_code', 'label1'), + [(True, 'run-profiled'), (False, 'run-unrelated')]) +@pytest.mark.parametrize( + ('as_module', 'label2'), + [(True, 'run_module'), (False, 'run_path')]) +@pytest.mark.parametrize( + ('debug', 'label3'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_runpy_patches( + capsys: pytest.CaptureFixture[str], + ext_module: _ModuleFixture, + test_module: _ModuleFixture, + test_module_clone: _ModuleFixture, + create_cache: Callable[..., LineProfilingCache], + run_profiled_code: bool, + as_module: bool, + debug: bool, + label1: str, label2: str, label3: str, +) -> None: + """ + Test that the :py:mod:`runpy` clone created by + :py:func:`line_profiler._child_process_profiling\ +.create_runpy_wrapper` + correctly sets up profiling when its ``run_*()`` functions are + called. + """ + class restore_argv: + def __enter__(self) -> None: + self.argv = list(sys.argv) + + def __exit__(self, *_, **__) -> None: + sys.argv[:] = self.argv + + cache = create_cache( + rewrite_module=test_module.path, + profiling_targets=[str(ext_module.path)], + profile_imports=True, + debug=debug, + ) + assert cache.profiler is not None + runpy = create_runpy_wrapper(cache) + + nnums = 42 + nprocs = 2 + # If we're running some unrelated code, the profiler should not be + # involved + if run_profiled_code: + module = test_module + num_invocations, num_loops = 1, nprocs + expected_funcs: list[str] = ['my_external_sum'] + else: + module = test_module_clone + num_invocations, num_loops = 0, 0 + expected_funcs = [] + if as_module: + first_arg = module.name + runner = partial(runpy.run_module, alter_sys=True) + called_func = 'run_module' + else: + first_arg = str(module.path) + runner = runpy.run_path + called_func = 'run_path' + + # Check that the code is run + module.install(local=True, deps_only=not as_module) + with restore_argv(): + sys.argv[:] = [first_arg, f'--length={nnums}', '-n', str(nprocs)] + runner(first_arg, run_name='__main__') + stdout = capsys.readouterr().out + assert stdout.rstrip('\n') == str(nnums * (nnums + 1) // 2) + + # Check that profiler has received the appropriate targets + funcs = [func.__name__ for func in getattr(cache.profiler, 'functions')] + assert funcs == expected_funcs + + # Check that calls in the current process are profiled iif the + # correct file is executed + with StringIO() as sio: + cache.profiler.print_stats(sio) + stats = sio.getvalue() + _check_output(stats, 'EXT-INVOCATION', num_invocations) + _check_output(stats, 'EXT-LOOP', num_loops) + + # Check the debug-log entries are correctly gathered + log = [entry.msg.lower() for entry in cache._gather_debug_log_entries()] + if debug: + assert ( + sum(1 for msg in log if called_func in msg if 'call' in msg) + == 1 + ), log + assert ( + sum(1 for msg in log if 'exec' in msg if 'call' in msg) + == int(run_profiled_code) + ), log + else: + assert not log # No logs written + + +# ========================= Integration tests ========================== def _get_mp_start_method_fuzzer(label_name: str) -> _Params: From a59bfd077fab62fa079f5c159c7c664231834df3 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 16:37:07 +0200 Subject: [PATCH 051/111] Minor refactoring line_profiler/_child_process_profiling/cache.py::LineProfilingCache __repr__() Now hiding cluttering fields from the repr _setup_in_main_process() Fixed bug where `wrap_os_fork` is not respected _write_pth_hook() Now using `.patch()` instead of manually overriding `os.fork()` patch() New convenience method for replacing/setting an attribute on an object line_profiler/_child_process_profiling/multiprocessing_patches.py _apply_mp_patches() - Updated call signature - Simplified implementation to use `lp_cache.patch()` --- .../_child_process_profiling/cache.py | 46 ++++++++++++------- .../multiprocessing_patches.py | 42 ++++++----------- 2 files changed, 43 insertions(+), 45 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 6cebe16f..3d2484b7 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -249,9 +249,11 @@ class LineProfilingCache: insert_builtin: bool = True debug: bool = diagnostics.DEBUG - profiler: LineProfiler | None = dataclasses.field(default=None, init=False) + profiler: LineProfiler | None = dataclasses.field( + default=None, init=False, repr=False, + ) _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( - default_factory=dict, init=False, + default_factory=dict, init=False, repr=False, ) _loaded_instance: ClassVar[Self | None] = None @@ -526,7 +528,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: pth_hook = import_module(this_subpkg + '.pth_hook') pth_hook.write_pth_hook(self) - self._wrap_os_fork() + if wrap_os_fork: + self._wrap_os_fork() mp_patches = import_module(this_subpkg + '.multiprocessing_patches') mp_patches.apply(self) @@ -638,20 +641,29 @@ def wrapper() -> int: forked._setup_in_child_process(False, 'fork', self.profiler) return result - # Note: type checkers have vastly different opinions on - # `os.fork = wrapper`: - # - `ty` wouldn't shut up about shadowing unless we explicitly - # type-annotate the assignment (error type: - # "invalid-assignment") - # - `mypy` is cool with the bare assignment, but complains about - # declaring types in "assignment to non-self-attribute" - # (error type: "misc") - # The only way to satisfy both seems to be either an - # unqualified "type: ignore" comment, or circumventing the - # attribute checks by `setattr()` or assigning to the module - # namespace dict. - os.fork = wrapper # type: ignore - self.add_cleanup(setattr, os, 'fork', fork) + self.patch(os, 'fork', wrapper, name='os') + + def patch( + self, obj: Any, attr: str, value: Any, *, + name: str | None = None, cleanup: bool = True, + ) -> None: + """ + Patch ``attr`` on ``obj`` with ``value``. If ``cleanup`` is + true, register a cleanup callback to either reset or delete the + attribute. + """ + add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) + setattr(obj, attr, value) + if name is None: + name = repr(obj) + msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) + self._debug_output(msg) def make_tempfile(self, **kwargs) -> Path: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 24029022..ca7be7cf 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -42,7 +42,7 @@ T = TypeVar('T') PS = ParamSpec('PS') -_PATCHED_MARKER = '_line_profiler_patched_multiprocessing' +_PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' @@ -72,7 +72,7 @@ def __setstate__(*_) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, _no_op) + _apply_mp_patches(lp_cache, add_cleanup=False) class _Poller: @@ -293,7 +293,7 @@ def wrap_start( @_method_wrapper def wrap_terminate( - cache: LineProfilingCache, # TODO: config timeouts + cache: LineProfilingCache, vanilla_impl: Callable[[BaseProcess], None], self: BaseProcess, ) -> None: @@ -511,29 +511,15 @@ def apply(lp_cache: LineProfilingCache) -> None: - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, lp_cache.add_cleanup) + _apply_mp_patches(lp_cache) def _apply_mp_patches( lp_cache: LineProfilingCache, - add_cleanup: Callable[..., Any], + add_cleanup: bool = True, debug: bool | None = None, ) -> None: - def replace( - obj: Any, attr: str, value: Any, obj_name: str | None = None, - ) -> None: - try: - old = getattr(obj, attr) - except AttributeError: - add_cleanup(delattr, obj, attr) - else: - add_cleanup(setattr, obj, attr, old) - setattr(obj, attr, value) - if obj_name is None: - obj_name = repr(obj) - lp_cache._debug_output('Patched `{}.{}` -> `{}`'.format( - obj_name, attr, value, - )) + replace = partial(lp_cache.patch, cleanup=add_cleanup) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the @@ -551,9 +537,10 @@ def replace( continue Class = getattr(mod, target) name = f'{Class.__module__}.{Class.__qualname__}' + patch_class = partial(replace, Class, name=name) for method, method_wrapper in patches.items(): vanilla = getattr(Class, method) - replace(Class, method, method_wrapper(vanilla), name) + patch_class(method, method_wrapper(vanilla)) # Patch `multiprocessing.spawn` try: @@ -561,14 +548,15 @@ def replace( except ImportError: # Incompatible platforms pass else: + patch_spawn = partial(replace, spawn, name=spawn.__name__) # Patch `get_preparation_data()` gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) - replace(spawn, 'get_preparation_data', gpd_wrapper, spawn.__name__) + patch_spawn('get_preparation_data', gpd_wrapper) # Patch `runpy` (do it locally instead of tempering with the # global `runpy` mmodule) if hasattr(spawn, 'runpy'): runpy_wrapper = create_runpy_wrapper(lp_cache) - replace(spawn, 'runpy', runpy_wrapper, spawn.__name__) + patch_spawn('runpy', runpy_wrapper) # Intercept `multiprocessing` debug messages if debug is None: @@ -576,6 +564,7 @@ def replace( if debug: from multiprocessing import util + patch_util = partial(replace, util, name=util.__name__) for logging_func in [ 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', ]: @@ -583,13 +572,10 @@ def replace( vanilla = getattr(util, logging_func) except AttributeError: continue - replace( - util, logging_func, partial(tee_log, vanilla, logging_func), - 'multiprocessing.util', - ) + patch_util(logging_func, partial(tee_log, vanilla, logging_func)) # Mark `multiprocessing` as having been patched - replace(multiprocessing, _PATCHED_MARKER, True) + replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') def _no_op(*_, **__) -> None: From 4683c05460103a7079f32e0e378ee38c8c386a9e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 18 Apr 2026 22:54:57 +0200 Subject: [PATCH 052/111] More in-process unit tests for `test_child_procs` tests/test_child_procs.py create_cache() - Now permits toggling whether the `curated_profiler` is supplied by a private argument - Now permits passing a boolean to `preimports_module`, processing the `profiling_targets` to write a preimports-module tempfile on-the-fly - Added callbacks for outputting the `.profiler` state, `.gather_stats()` results, and debug logs test_*() - Added more robust protection of global states (e.g. for modules prone to being patched) - Refactored debug-log output matching test_cache_dump_load() New unit test for `line_profiler._child_process_profiling.cache .LineProfilingCache.dump()` and `.load()` test_cache_cleanup_order() New unit test for `~~.LineProfilingCache.cleanup()` test_cache_setup_{main,child}_process() New unit tests for `~~.LineProfilingCache._setup_in_{main,child}_process()` test_apply_mp_patches() New unit test for `line_profiler._child_process_profiling .multiprocessing_patches.apply()` TODO: One subtest in `test_apply_mp_patches()` involving `forkserver` runs fine in isolation and errors out otherwise; investigate --- tests/test_child_procs.py | 739 +++++++++++++++++++++++++++++++++++--- 1 file changed, 689 insertions(+), 50 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 91548298..42c6e12b 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,24 +1,31 @@ from __future__ import annotations +import ast +import dataclasses import enum -import multiprocessing +import inspect +import multiprocessing.pool import os +import re import shlex import subprocess import sys +import sysconfig +from abc import ABC, abstractmethod from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from dataclasses import dataclass -from functools import partial +from contextlib import ExitStack +from functools import lru_cache, partial, wraps from io import StringIO +from importlib import import_module from pathlib import Path -from sysconfig import get_path +from runpy import run_path from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from typing import Any, Literal, TypeVar, cast, final, overload -from typing_extensions import Self +from typing import Any, Generic, Literal, TypeVar, cast, final, overload +from typing_extensions import Self, ParamSpec from uuid import uuid4 import pytest @@ -28,11 +35,15 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler.curated_profiling import CuratedProfilerContext +from line_profiler.curated_profiling import ( + CuratedProfilerContext, ClassifiedPreimportTargets, +) from line_profiler.line_profiler import LineProfiler, LineStats T = TypeVar('T') +TCtx_ = TypeVar('TCtx_') +PS = ParamSpec('PS') C = TypeVar('C', bound=Callable[..., Any]) NUM_NUMBERS = 100 @@ -137,7 +148,7 @@ def main(args: list[str] | None = None) -> None: # ============================== Fixtures ============================== -@dataclass +@dataclasses.dataclass class _ModuleFixture: """ Convenience wrapper around a Python source file which represents an @@ -269,7 +280,7 @@ def test_module_clone( @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, - curated_profiler: LineProfiler, + request: pytest.FixtureRequest, ) -> Generator[Callable[..., LineProfilingCache], None, None]: """ Wrapper around the :py:class:`LineProfilingCache` instantiator @@ -278,35 +289,121 @@ def create_cache( - Automatically creates a tempdir and provides it as the :py:attr:`LineProfilingCache.cache_dir`, - - Automatically creates an instance of :py:class:`LineProfiler` that - is curated by a :py:class:`CuratedProfilerContext` and provides it - as the :py:attr:`LineProfilingCache.profiler`, and + - Extends the argument ``preimports_module`` to allow for taking + boolean values: + + - ``True``: a temporary preimports module is automatically written + based on ``profiling_targets`` and supplied to the base + constructor. + + - ``False``: equivalent to ``None``. + + - Unless the argument ``_use_curated_profiler: bool = True`` is set + to :py:const`True`, automatically creates an instance of + :py:class:`LineProfiler` that is curated by a + :py:class:`CuratedProfilerContext` and provides it as the + :py:attr:`LineProfilingCache.profiler`, and - At teardown: + - Removes tempdirs and tempfiles generated. + - Restores the value of the class' internal reference to the :py:meth:`LineProfilingCache.load`-ed instance. - Calls the `.cleanup()` method of each instance created. + + - Prints these diagnostics for each instance: + + - The stats on the ``.profiler`` associated with each instance + (if any) + + - The stats gathered by + :py:meth:`LineProfilingCache.gather_stats()` + + - The debug logs (if ``.debug`` is true) """ - def instantiate(**kwargs) -> LineProfilingCache: + def instantiate( + *, + profiling_targets: Collection[str] = (), + preimports_module: os.PathLike[str] | str | bool | None = None, + _use_curated_profiler: bool = True, + **kwargs + ) -> LineProfilingCache: tmpdir = tmp_path_factory.mktemp('my_cache_dir') - cache = LineProfilingCache(tmpdir, **kwargs) - cache.profiler = curated_profiler + pim: os.PathLike[str] | str | None + if preimports_module in (True, False): + if preimports_module: + targets = ( + ClassifiedPreimportTargets.from_targets(profiling_targets) + ) + if targets: + pim = tmpdir / 'preimports.py' + with pim.open(mode='w') as fobj: + targets.write_preimport_module(fobj) + else: + pim = None + else: + pim = None + else: + # The type checker needs some convincing... + assert not isinstance(preimports_module, bool) + pim = preimports_module + cache = LineProfilingCache( + tmpdir, + profiling_targets=profiling_targets, + preimports_module=pim, + **kwargs, + ) + if _use_curated_profiler: + cache.profiler = request.getfixturevalue('curated_profiler') instances.append(cache) return cache - old_value = LineProfilingCache._loaded_instance + def print_result( + cache: LineProfilingCache, topic: str, result: str, + ) -> None: + header = f'{topic} (cache instance {id(cache):#x}' + print(header, indent(result, ' '), sep='\n') + + def print_profiler_stats(cache: LineProfilingCache) -> None: + if cache.profiler is None: + result = '' + else: + with StringIO() as sio: + cache.profiler.print_stats(sio) + result = sio.getvalue() + print_result(cache, 'Native profiler stats', result) + + def print_gathered_stats(cache: LineProfilingCache) -> None: + with StringIO() as sio: + cache.gather_stats().print(sio) + result = sio.getvalue() + print_result(cache, 'Gathered profiler stats', result) + + def print_debug_logs(cache: LineProfilingCache) -> None: + result = '\n'.join( + entry.to_text() for entry in cache._gather_debug_log_entries() + ) + print_result(cache, 'Gathered debug logs', result) + instances: list[LineProfilingCache] = [] + handlers: list[Callable[[LineProfilingCache], None]] + handlers = [print_profiler_stats, print_gathered_stats, print_debug_logs] try: - yield instantiate + with _preserve_obj_attributes( + LineProfilingCache, ['_loaded_instance'], + ): + yield instantiate finally: - LineProfilingCache._loaded_instance = old_value # type: ignore for cache in instances: - try: - cache.cleanup() - except Exception: - pass + callbacks: list[Callable[[], Any]] = [cache.cleanup] + callbacks.extend(partial(func, cache) for func in handlers) + for callback in callbacks: + try: + callback() + except Exception: + pass @pytest.fixture @@ -353,7 +450,7 @@ def rich_message(self) -> str: @final -@dataclass +@dataclasses.dataclass class _Params: """ Convenience wrapper around :py:func:`pytest.mark.parametrize`. @@ -536,6 +633,207 @@ def new( return cls(param_list, value_tuple_list, default_values) +class _CallableContextManager(ABC, Generic[TCtx_]): + @abstractmethod + def __enter__(self) -> TCtx_: + ... + + @abstractmethod + def __exit__(self, *a, **k) -> Any: + ... + + def __call__(self, func: Callable[PS, T]) -> Callable[PS, T]: + """ + Wrap ``func()`` so that its calls always happen in the context + of the instance. + """ + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with self: + return func(*args, **kwargs) + + return wrapper + + +class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): + def __init__(self, obj: Any, attrs: Collection[str]) -> None: + self.obj = obj + self.attrs = set(attrs) + self._callbacks: list[Callable[[], None]] = [] + + def __enter__(self) -> dict[str, Any]: + def delete(obj: Any, attr: str) -> None: + try: + delattr(obj, attr) + except AttributeError: + pass + + result: dict[str, Any] = {} + for attr in self.attrs: + old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) + if old is _NotSupplied.NOT_SUPPLIED: + callback = partial(delete, self.obj, attr) + else: + callback = partial(setattr, self.obj, attr, old) + result[attr] = old + self._callbacks.append(callback) + return result + + def __exit__(self, *_, **__) -> None: + for callback in self._callbacks[::-1]: + try: + callback() + except Exception: + pass + + +class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): + """ + Example: + >>> from functools import wraps + >>> from line_profiler.curated_profiling import ( + ... CuratedProfilerContext, + ... ) + >>> from line_profiler import line_profiler + + >>> assert not hasattr(CuratedProfilerContext, 'foo') + >>> old_main = line_profiler.main + >>> + >>> + >>> def foo(_) -> None: + ... pass + ... + >>> + >>> @wraps(old_main) + ... def main(*a, **k): + ... return old_main(*a, **k) + ... + >>> + >>> with _preserve_attributes({ + ... 'line_profiler.curated_profiling' + ... '.CuratedProfilerContext': {'foo'}, + ... 'line_profiler.line_profiler': {'main'}, + ... }) as old_values: + ... assert old_values == { + ... 'line_profiler.curated_profiling' + ... '.CuratedProfilerContext': { + ... 'foo': _NotSupplied.NOT_SUPPLIED, + ... }, + ... 'line_profiler.line_profiler': {'main': old_main}, + ... } + ... CuratedProfilerContext.foo = foo + ... line_profiler.main = main + ... print('ok') + ... + ok + >>> assert not hasattr(CuratedProfilerContext, 'foo') + >>> assert main is not line_profiler.main is old_main + """ + def __init__(self, targets: Mapping[str, Collection[str]]) -> None: + self.targets = { + target: set(attrs) for target, attrs in targets.items() + } + self._stacks: list[ExitStack] = [] + + def __enter__(self) -> dict[str, dict[str, Any]]: + stack = ExitStack() + self._stacks.append(stack) + result: dict[str, Any] = {} + for target, attrs in self.targets.items(): + result[target] = stack.enter_context(_preserve_obj_attributes( + _import_target(target), attrs, + )) + return result + + def __exit__(self, *_, **__) -> None: + self._stacks.pop().close() + + +class _preserve_pth_files(_CallableContextManager[frozenset[str]]): + def __enter__(self) -> frozenset[str]: + self.old = self.get_pth_files() + return self.old + + def __exit__(self, *_, **__) -> None: + for new_pth_file in self.get_pth_files() - self.old: + (self._get_path() / new_pth_file).unlink(missing_ok=True) + del self.old + + @classmethod + def get_pth_files(cls, name_only: bool = True) -> frozenset[str]: + return frozenset( + pth.name if name_only else str(pth) + for pth in cls._get_path().glob('*.pth') + ) + + @staticmethod + def _get_path() -> Path: + return Path(sysconfig.get_path('purelib')) + + +def _import_target(target: str) -> Any: + try: + return import_module(target) + except ImportError: # Not a module + assert '.' in target + module, _, attr = target.rpartition('.') + return getattr(import_module(module), attr) + + +def _search_cache_logs( + cache: LineProfilingCache, + expecting_logs: bool, + patterns: Mapping[str, bool] | Collection[str], + match_individual_messages: bool = False, + flags: int = 0, +) -> None: + entries = cache._gather_debug_log_entries() + if bool(entries) != expecting_logs: + raise ResultMismatch( + 'logs' if expecting_logs else 'no logs', + repr(entries) if entries else 'nothing' + ) + if not expecting_logs: + return + text_chunks: list[str] = [entry.to_text() for entry in entries] + if not match_individual_messages: + text_chunks = ['\n'.join(text_chunks)] + if isinstance(patterns, Mapping): + to_match: dict[str, bool] = { + str(pat): bool(should_match) + for pat, should_match in patterns.items() + } + else: + to_match = dict.fromkeys(patterns, True) + for pat, should_match in to_match.items(): + pattern = re.compile(pat, flags) + if any(pattern.search(chunk) for chunk in text_chunks) == should_match: + continue + raise ResultMismatch( + f'pattern {pattern!r} to {"" if should_match else "not "}match ' + f'{cache!r}\'s logs: {text_chunks!r}' + ) + + +@lru_cache() +def _find_return_lines(func: str) -> list[int]: + class FindReturns(ast.NodeVisitor): + def __init__(self) -> None: + self.found: set[int] = set() + + def visit_Return(self, node: ast.Return) -> None: + self.found.add(node.lineno) + self.generic_visit(node) + + func_obj = _import_target(func) + assert inspect.isfunction(func_obj) + lines, start = inspect.getsourcelines(func_obj) + tree = ast.parse(''.join(lines)) + finder = FindReturns() + finder.visit(tree) + return sorted(lineno + start - 1 for lineno in finder.found) + + # `shlex.join()` doesn't work properly on Windows, so use # `subprocess.list2cmdline()` instead; # though an "intentionally" undocumented API (cpython issue #10308), @@ -683,11 +981,6 @@ def _run_test_module( profliing_stats (LineStats | None): Line-profiling stats (where available) """ - def get_pth_files(pattern: str = '*.pth') -> set[str]: - return {path.name for path in lib_path.glob(pattern)} - - lib_path = Path(get_path('purelib')) - if isinstance(runner, str): runner_args: list[str] = [runner] else: @@ -728,7 +1021,7 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: runner_args.extend(['--outfile', outfile]) if debug_log: runner_args.extend(['--debug-log', debug_log]) - old_pth_files = get_pth_files() + old_pth_files = _preserve_pth_files.get_pth_files() proc = run_helper( runner_args, test_args, test_module, text=True, capture_output=True, check=(check and not fail), @@ -750,7 +1043,7 @@ def get_pth_files(pattern: str = '*.pth') -> set[str]: ) # - Temporary `.pth` file(s) created by `~~.pth_hook` has # been cleaned up - assert get_pth_files() == old_pth_files + assert _preserve_pth_files.get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None if outfile is None: @@ -803,16 +1096,33 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # ============================= Unit tests ============================= - -@pytest.mark.parametrize( - ('run_profiled_code', 'label1'), - [(True, 'run-profiled'), (False, 'run-unrelated')]) -@pytest.mark.parametrize( - ('as_module', 'label2'), - [(True, 'run_module'), (False, 'run_path')]) -@pytest.mark.parametrize( - ('debug', 'label3'), - [(True, 'with-debug'), (False, 'no-debug')]) +# XXX: Tests in this section concerns implementation details, and the +# tested APIs and behaviors MUST NOT be relied upon by end-users. + + +_MP_PATCHED_NAMES = { + 'multiprocessing.process.BaseProcess': frozenset({ + '_bootstrap', 'start', 'terminate' + }), + 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), +} +# NOTE: we need a function which isn't used by the codebase itself +# (esp. during cache cleanup); otherwise the profiling results may +# be skewed +_SAFE_TARGET = 'calendar.weekday' +_SAFE_TARGET_ARGS = [ + (1970, 1, 1), + (2000, 12, 31), + (2008, 9, 16), # Where the repo started +] + + +@pytest.mark.parametrize(('run_profiled_code', 'label1'), + [(True, 'run-profiled'), (False, 'run-unrelated')]) +@pytest.mark.parametrize(('as_module', 'label2'), + [(True, 'run_module'), (False, 'run_path')]) +@pytest.mark.parametrize(('debug', 'label3'), + [(True, 'with-debug'), (False, 'no-debug')]) def test_runpy_patches( capsys: pytest.CaptureFixture[str], ext_module: _ModuleFixture, @@ -889,19 +1199,348 @@ def __exit__(self, *_, **__) -> None: _check_output(stats, 'EXT-LOOP', num_loops) # Check the debug-log entries are correctly gathered - log = [entry.msg.lower() for entry in cache._gather_debug_log_entries()] + _search_cache_logs( + cache, + debug, + { + rf'calling .*{called_func}\(': True, + r'calling .*exec\(': run_profiled_code, + }, + match_individual_messages=True, + flags=re.IGNORECASE, + ) + + +def test_cache_dump_load( + create_cache: Callable[..., LineProfilingCache], +) -> None: + """ + Test that: + + - We can round-trip the cache via :py:meth:`LineProfilingCache.dump` + and :py:meth:`LineProfilingCache.load` + + - The same instance is :py:meth:`LineProfilingCache.load`-ed in + subsequent calls + """ + original = create_cache( + profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, + ) + envvars: set[str] = set(os.environ) + try: + original.inject_env_vars() # Needed for `.load()` + try: + # Env vars should be inserted + assert set(os.environ) == envvars.union(original.environ) > envvars + original.dump() + loaded = original.load() + reloaded = original.load() + assert original is not loaded is reloaded + # Compare init fields + for field in dataclasses.fields(LineProfilingCache): + if not field.init: + continue + assert ( + getattr(original, field.name) + == getattr(loaded, field.name) + ) + finally: # Explicitly cleanup + original.cleanup() + finally: # Env vars restored after cleanup + assert set(os.environ) == envvars + + +@pytest.mark.parametrize(('debug', 'label'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_cleanup_order( + create_cache: Callable[..., LineProfilingCache], debug: bool, label: str, +) -> None: + """ + Test that :py:meth`LineProfilingCache.cleanup` executes the cleanup + callback stacks in order. + """ + strings: list[str] = [] + cache = create_cache(debug=debug) + + cache.add_cleanup(strings.append, 'first') + # Decreased priority + cache._add_cleanup(strings.append, 1, 'second') + # Increased priority + cache._add_cleanup(strings.append, -1, 'third') + cache.add_cleanup(strings.append, 'fourth') + + cache.cleanup() + assert strings == ['third', 'fourth', 'first', 'second'] + + pattern = '\n'.join( + rf'.*succeeded \({i + 1}/{len(strings)}\): .*append.*{string!r}.*' + for i, string in enumerate(strings) + ) + _search_cache_logs(cache, debug, [pattern]) + + +@pytest.mark.parametrize(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) +@pytest.mark.parametrize(('debug', 'label2'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_setup_main_process( + create_cache: Callable[..., LineProfilingCache], + wrap_os_fork: bool, + debug: bool, + label1: str, label2: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_main_process` works + as expected. + """ + cache = create_cache(debug=debug) + patches: dict[str, dict[str, bool]] = {'os': {'fork': wrap_os_fork}} + patches.update( + (target, dict.fromkeys(attrs, True)) + for target, attrs in _MP_PATCHED_NAMES.items() + ) + targets: dict[str, Any] = { + target: _import_target(target) for target in patches + } + with ExitStack() as stack: + patched = stack.enter_context(_preserve_attributes(patches)) + original_pths = stack.enter_context(_preserve_pth_files()) + cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) + # There should be exactly one extra `.pth` file + new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths + # Check whether the patches are applied + for target, maybe_patches in patches.items(): + obj = targets[target] + for attr, is_patched in maybe_patches.items(): + assert ( + getattr(obj, attr) is patched[target][attr] + ) != is_patched + # Check whether the patches are reversed + cache.cleanup() + for target, orig_attrs in patched.items(): + obj = targets[target] + for attr, orig_value in orig_attrs.items(): + if orig_value is _NotSupplied.NOT_SUPPLIED: + assert not hasattr(obj, attr) + else: + assert getattr(obj, attr) is orig_value + # Check that the instance is set as the `.load()`-ed one + assert cache is cache.load() + + # Check the debug-log output + patterns: dict[str, bool] = dict.fromkeys( + [ + r'\(main process\)', + r'Injecting env var.*\$\{LINE_PROFILER_\w+\}', + re.escape(new_pth_hook), + ], + True, + ) + for target, maybe_patches in patches.items(): + patterns.update( + ('Patched.*' + re.escape(f'{target}.{attr}'), is_patched) + for attr, is_patched in maybe_patches.items() + ) + _search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) +@pytest.mark.parametrize(('preimports', 'label2'), + [(True, 'with-preimports'), (False, 'no-preimports')]) +@pytest.mark.parametrize(('new_profiler', 'label3'), + [(True, 'no-profiler'), (False, 'with-profiler')]) +@pytest.mark.parametrize(('debug', 'label4'), + [(True, 'with-debug'), (False, 'no-debug')]) +def test_cache_setup_child( + create_cache: Callable[..., LineProfilingCache], + curated_profiler: LineProfiler, + wrap_os_fork: bool, + preimports: bool, + new_profiler: bool, + debug: bool, + label1: str, label2: str, label3: str, label4: str, +) -> None: + """ + Test that :py:meth:`LineProfilingCache._setup_in_child_process` + works as expected. + """ + def list_profiled_funcs() -> list[str]: + return [ + f'{func.__module__}.{func.__qualname__}' + for func in getattr(cache.profiler, 'functions', []) + ] + + # Make sure we get a different PID from the current process + curr_pid = os.getpid() + main_pid = (curr_pid - 42) % (2 * 16) + assert main_pid != curr_pid + + cache = create_cache( + profiling_targets=[_SAFE_TARGET], + preimports_module=preimports, + _use_curated_profiler=not new_profiler, + main_pid=main_pid, + debug=debug, + ) + assert (cache.profiler is None) == new_profiler + + seen_funcs = list_profiled_funcs() + if preimports: + preimport_targets = list(cache.profiling_targets) + else: + preimport_targets = [] + + with _preserve_obj_attributes(os, ['fork']) as preserved: + old_fork = preserved['fork'] + # Check that we're only setting up if there isn't already a + # profiler + assert cache._setup_in_child_process( + wrap_os_fork=wrap_os_fork, context='test_cache_setup_child', + ) == new_profiler + assert cache.profiler + if not new_profiler: + return + + # Check that the profiler has been presented with the profiling + # target + assert list_profiled_funcs() == (seen_funcs + preimport_targets) + + # Check that on cache cleanup: + # - Profiling data is collected + # - `os.fork()` is restored + _import_target(_SAFE_TARGET)(*_SAFE_TARGET_ARGS[0]) + stats = cache.profiler.get_stats() + for callback, has_prof_data, fork_patched in [ + (lambda: None, False, wrap_os_fork), + (cache.cleanup, preimports, False), + ]: + callback() + gathered = cache.gather_stats() + assert any(gathered.timings.values()) == has_prof_data, gathered + assert (os.fork is not old_fork) == fork_patched + + # Check that profiling results have been written to the cache + # directory + stats_file, = Path(cache.cache_dir).glob('*.lprof') + assert LineStats.from_files(stats_file) == stats == gathered + + # Check the debug-log output + patterns = { + f'Set up .*profiler.* {id(cache.profiler):#x}': True, + 'Loading preimports': preimports, + 'Created .*' + re.escape(stats_file.name): True, + 'Cleanup succeeded.*: .*dump_stats': True, + 'Loading results .*' + re.escape(stats_file.name): True, + } + _search_cache_logs(cache, debug, patterns) + + +@pytest.mark.parametrize('start_method', + ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('debug', 'label'), + [(True, 'with-debug'), (False, 'no-debug')]) +@_preserve_pth_files() +@_preserve_attributes(_MP_PATCHED_NAMES) +def test_apply_mp_patches( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + debug: bool, + label: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected. + """ + def gpd_wrapper(name: str) -> dict[str, Any]: + impl = _import_target('multiprocessing.spawn.get_preparation_data') + return impl(name) + + def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: + try: + LineStats.from_files(path, on_defective='error') + except Exception: + return False + return True + + config: Path | None = None if debug: - assert ( - sum(1 for msg in log if called_func in msg if 'call' in msg) - == 1 - ), log - assert ( - sum(1 for msg in log if 'exec' in msg if 'call' in msg) - == int(run_profiled_code) - ), log + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + config.write_text( + '[tool.line_profiler.multiprocessing]\n' + 'intercept_logs = true' + ) + + cache = create_cache( + profiling_targets=[_SAFE_TARGET], + preimports_module=True, + config=config, + debug=True, + ) + old_data = gpd_wrapper('my_name') + # Note: + # - The reversibility of the patches have already been tested in + # `test_cache_setup_main_process()`, so we just actually test the + # patched-in components themselves here. + # - `._setup_in_main_process()` doesn't include actually doing the + # preimports. To may the results more consistent between + # `start_method='dummy'` and the others, manually do them below. + cache._setup_in_main_process() # This calls `apply()` + assert cache.preimports_module is not None + run_path(str(cache.preimports_module), {'profile': cache.profiler}) + + # Check for the insertion of the `PickleHook` by the patched + # `get_preparation_data()` + new_data = gpd_wrapper('my_name') + inserted_key, = set(new_data) - set(old_data) + assert type(new_data[inserted_key]).__name__ == 'PickleHook' + + func = _import_target(_SAFE_TARGET) + return_lines = _find_return_lines(_SAFE_TARGET) + Pool: Callable[..., multiprocessing.pool.Pool] + if start_method == 'dummy': + Pool = _import_target('multiprocessing.dummy.Pool') + elif start_method not in START_METHODS: + pytest.skip( + f'`multiprocessing` start method {start_method!r} ' + 'not available on the platform' + ) else: - assert not log # No logs written + Pool = multiprocessing.get_context(start_method).Pool + + with Pool(2) as pool: + pool.starmap(func, _SAFE_TARGET_ARGS) + pool.close() + pool.join() + + # Check that calls in children are traced + if start_method == 'dummy': + assert cache.profiler is not None + stats = cache.profiler.get_stats() + else: + stats = cache.gather_stats() + line_entries = stats.timings[ + inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, + ] + num_returns = sum( + nhits for lineno, nhits, _ in line_entries if lineno in return_lines + ) + assert num_returns == len(_SAFE_TARGET_ARGS) + + # Check the debug logs to see if we have done everything right, esp. + # the logging interception part not covered by other tests + patterns: dict[str, bool] = { + 'Cleanup succeeded.*: .*dump_stats.*' + re.escape(path.name): True + for path in Path(cache.cache_dir).glob('*.lprof') + if is_valid_stats_file(path) + } + patterns[re.escape('`multiprocessing` logging (debug)')] = debug + _search_cache_logs(cache, True, patterns) + +# XXX: End of tests for implementation details # ========================= Integration tests ========================== From 8e9cd0ab72275ce7311c6ec103ca1e86c4e6f624 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 00:03:26 +0200 Subject: [PATCH 053/111] WIP debug the new unit tests tests/test_child_procs.py Added debugging output to the private context managers for preserving attributes and purging stray .pth files --- tests/test_child_procs.py | 81 ++++++++++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 19 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 42c6e12b..c309973c 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -49,7 +49,7 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -_WRITE_DEBUG_LOGS = True +_DEBUG = True def strip(s: str) -> str: @@ -363,7 +363,7 @@ def instantiate( def print_result( cache: LineProfilingCache, topic: str, result: str, ) -> None: - header = f'{topic} (cache instance {id(cache):#x}' + header = f'{topic} (cache instance {id(cache):#x}):' print(header, indent(result, ' '), sep='\n') def print_profiler_stats(cache: LineProfilingCache) -> None: @@ -382,9 +382,12 @@ def print_gathered_stats(cache: LineProfilingCache) -> None: print_result(cache, 'Gathered profiler stats', result) def print_debug_logs(cache: LineProfilingCache) -> None: - result = '\n'.join( - entry.to_text() for entry in cache._gather_debug_log_entries() - ) + if cache.debug: + result = '\n'.join( + entry.to_text() for entry in cache._gather_debug_log_entries() + ) + else: + result = '' print_result(cache, 'Gathered debug logs', result) instances: list[LineProfilingCache] = [] @@ -634,6 +637,8 @@ def new( class _CallableContextManager(ABC, Generic[TCtx_]): + debug: bool + @abstractmethod def __enter__(self) -> TCtx_: ... @@ -654,27 +659,53 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: return wrapper + def _debug(self, msg: str, **kwargs) -> None: + if not self.debug: + return + header = f'{os.environ["PYTEST_CURRENT_TEST"]}: {type(self).__name__}' + print(f'{header}: {msg}', **kwargs) + class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): - def __init__(self, obj: Any, attrs: Collection[str]) -> None: + def __init__( + self, obj: Any, attrs: Collection[str], debug: bool = _DEBUG, + ) -> None: self.obj = obj self.attrs = set(attrs) self._callbacks: list[Callable[[], None]] = [] + self.debug = debug def __enter__(self) -> dict[str, Any]: - def delete(obj: Any, attr: str) -> None: + def get_repr(attr: str) -> str: try: - delattr(obj, attr) + value = getattr(self.obj, attr) + except ValueError: + return '' + else: + return repr(value) + + def delete(attr: str) -> None: + try: + self._debug('Deleted attr `.{} = {}` on `{!r}`'.format( + attr, get_repr(attr), self.obj, + )) + delattr(self.obj, attr) except AttributeError: pass + def reset(attr: str, value: Any) -> None: + self._debug('Reset attr `.{} = {} -> {!r}` on `{!r}`'.format( + attr, get_repr(attr), value, self.obj, + )) + setattr(self.obj, attr, value) + result: dict[str, Any] = {} for attr in self.attrs: old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) if old is _NotSupplied.NOT_SUPPLIED: - callback = partial(delete, self.obj, attr) + callback = partial(delete, attr) else: - callback = partial(setattr, self.obj, attr, old) + callback = partial(reset, attr, old) result[attr] = old self._callbacks.append(callback) return result @@ -709,12 +740,13 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): ... return old_main(*a, **k) ... >>> - >>> with _preserve_attributes({ + >>> preserved = { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': {'foo'}, ... 'line_profiler.line_profiler': {'main'}, - ... }) as old_values: - ... assert old_values == { + ... } + >>> with _preserve_attributes(preserved, debug=False) as old: + ... assert old == { ... 'line_profiler.curated_profiling' ... '.CuratedProfilerContext': { ... 'foo': _NotSupplied.NOT_SUPPLIED, @@ -727,13 +759,19 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): ... ok >>> assert not hasattr(CuratedProfilerContext, 'foo') - >>> assert main is not line_profiler.main is old_main + >>> assert old_main is \ +old['line_profiler.line_profiler']['main'] + >>> assert old_main is line_profiler.main + >>> assert main is not line_profiler.main """ - def __init__(self, targets: Mapping[str, Collection[str]]) -> None: + def __init__( + self, targets: Mapping[str, Collection[str]], debug: bool = _DEBUG, + ) -> None: self.targets = { target: set(attrs) for target, attrs in targets.items() } self._stacks: list[ExitStack] = [] + self.debug = debug def __enter__(self) -> dict[str, dict[str, Any]]: stack = ExitStack() @@ -741,7 +779,7 @@ def __enter__(self) -> dict[str, dict[str, Any]]: result: dict[str, Any] = {} for target, attrs in self.targets.items(): result[target] = stack.enter_context(_preserve_obj_attributes( - _import_target(target), attrs, + _import_target(target), attrs, debug=self.debug, )) return result @@ -750,12 +788,16 @@ def __exit__(self, *_, **__) -> None: class _preserve_pth_files(_CallableContextManager[frozenset[str]]): + def __init__(self, debug: bool = _DEBUG) -> None: + self.debug = debug + def __enter__(self) -> frozenset[str]: self.old = self.get_pth_files() return self.old def __exit__(self, *_, **__) -> None: for new_pth_file in self.get_pth_files() - self.old: + self._debug(f'Deleting stray .pth file: {new_pth_file!r}') (self._get_path() / new_pth_file).unlink(missing_ok=True) del self.old @@ -1511,9 +1553,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: Pool = multiprocessing.get_context(start_method).Pool with Pool(2) as pool: - pool.starmap(func, _SAFE_TARGET_ARGS) + par_result = pool.starmap(func, _SAFE_TARGET_ARGS) pool.close() pool.join() + assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] # Check that calls in children are traced if start_method == 'dummy': @@ -1778,7 +1821,7 @@ def test_profiling_multiproc_script( nprocs=nprocs, timeout=timeout, debug_log=( - 'debug.log' if prof_child_procs and _WRITE_DEBUG_LOGS else None + 'debug.log' if prof_child_procs and _DEBUG else None ), ) @@ -1825,7 +1868,7 @@ def test_profiling_bare_python( out_file = temp_dir / 'out.lprof' debug_log_file = temp_dir / 'debug.log' - write_debug = _WRITE_DEBUG_LOGS and prof_child_procs + write_debug = _DEBUG and prof_child_procs cmd = [ 'kernprof', '-lv', '--preimports', f'--prof-mod={ext_module.name}', From 7fc09e926eb6135859f12f87b9b8d826b9ad4b95 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 03:56:12 +0200 Subject: [PATCH 054/111] Restart the fork-server process after each session line_profiler/_child_process_profiling/multiprocessing_patches.py ::apply() Now registering a cleanup callback to restart the stored `multiprocessing.forkserver.ForkServer` used globally, fixing the bug where child processes started by the `forkserver` start method cannot don't have profiling set up from the second profiling session/`LineProfilingCache` instance onwards --- .../multiprocessing_patches.py | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ca7be7cf..4ae261f4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -69,10 +69,8 @@ def __setstate__(*_) -> None: # up shop here. lp_cache = LineProfilingCache.load() lp_cache._setup_in_child_process(False, 'multiprocessing') - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, add_cleanup=False) + _apply_mp_patches(lp_cache, main_process=False) class _Poller: @@ -509,6 +507,19 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:func:`multiprocessing.spawn.get_preparation_data` - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + + Note: + When ``lp_cache.cleanup()`` is run, the global + :py:class:`multiprocessing.forkserver.ForkServer` object will be + rebooted. This is necessary because the server process staticly + inherits the environment when it is first spun up + (see :py:func:`multiprocessing.forkserver.ensure_running`). + Thus, if in the same Python process we ever start up two + separate profliing sessions managed by different caches, the + child processes forked from the server will fail to inherit the + updated environment variables injected by the newer cache + instance, leading to the setup code in this subpackage not being + loaded. """ if not getattr(multiprocessing, _PATCHED_MARKER, False): _apply_mp_patches(lp_cache) @@ -516,10 +527,12 @@ def apply(lp_cache: LineProfilingCache) -> None: def _apply_mp_patches( lp_cache: LineProfilingCache, - add_cleanup: bool = True, + main_process: bool = True, debug: bool | None = None, ) -> None: - replace = partial(lp_cache.patch, cleanup=add_cleanup) + # In a child process, we don't care about polluting the + # `multiprocessing` namespace, so don't bother with cleanup + replace = partial(lp_cache.patch, cleanup=main_process) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the @@ -574,6 +587,21 @@ def _apply_mp_patches( continue patch_util(logging_func, partial(tee_log, vanilla, logging_func)) + # Stop the current `ForkServer` server process as a part of cache + # cleanup (this uses `ForkServer._stop()` which is private API, but + # it's the same hack used in Python's own test suite -- see the + # comment to said method) + if main_process: + try: + from multiprocessing import forkserver + except ImportError: # Incompatible platform + pass + else: + server_instance: forkserver.ForkServer = forkserver._forkserver + stop = getattr(server_instance, '_stop', None) + assert callable(stop) # Appease the type checker + lp_cache.add_cleanup(stop) + # Mark `multiprocessing` as having been patched replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') From 11893a131b4f73dc3eda04b2b439d010d3d5c858 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 05:45:37 +0200 Subject: [PATCH 055/111] Simplify `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_start() Removed wrap_terminate() Now polling on the `Process.Popen` object instead of a lock file, simplifying the implementation wrap_bootstrap() Simplified implementation apply() No longer wrapping `BaseProcess.start` line_profiler/rc/line_profiler.toml ::[tool.line_profiler.multiprocessing] - Shortened `timeout` to .25 s - Set `on_timeout` to `'warn'` --- .../multiprocessing_patches.py | 120 +++++------------- line_profiler/rc/line_profiler.toml | 8 +- 2 files changed, 36 insertions(+), 92 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 4ae261f4..1b37720e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,12 +17,10 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from contextlib import AbstractContextManager, nullcontext from functools import lru_cache, partial, wraps from importlib import import_module from multiprocessing.process import BaseProcess from os import PathLike -from pathlib import Path from time import sleep, monotonic from types import MappingProxyType from typing import ( @@ -41,9 +39,9 @@ T = TypeVar('T') PS = ParamSpec('PS') +_OnTimeout = Literal['ignore', 'warn', 'error'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' -_PROCESS_TERM_LOCK_LOC = '_line_profiler_process_terminate_lock' class _Wrapper(Protocol, Generic[PS, T]): @@ -110,7 +108,7 @@ def __init__( func: Callable[[], Any], cooldown: float = 0, timeout: float = 0, - on_timeout: Literal['ignore', 'warn', 'error'] = 'error', + on_timeout: _OnTimeout = 'error', ) -> None: if cooldown < 0: cooldown = 0 @@ -134,7 +132,7 @@ def with_cooldown(self, cooldown: float) -> Self: def with_timeout( self, timeout: float | None = None, - on_timeout: Literal['ignore', 'warn', 'error'] | None = None, + on_timeout: _OnTimeout | None = None, ) -> Self: if timeout is None: timeout = self._timeout @@ -272,23 +270,6 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: return inner_wrapper -@_method_wrapper -def wrap_start( - cache: LineProfilingCache, - vanilla_impl: Callable[[BaseProcess], None], - self: BaseProcess, -) -> None: - """ - Wrap around :py:meth:`BaseProcess.start` to specify the location for - a lock file, which is managed by the child process and checked by - the parent. This is to ensure that the child can exit gracefully and - complete any necessary cleanup. - """ - tempfile = cache.make_tempfile(prefix='process-term-lock-', suffix='.lock') - setattr(self, _PROCESS_TERM_LOCK_LOC, tempfile) - vanilla_impl(self) - - @_method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -298,42 +279,16 @@ def wrap_terminate( """ Wrap around :py:meth:`BaseProcess.terminate` to make sure that we don't actually kill the child (OS-level) process before it has the - chance to properly clean up. This is done by blocking the call as - long as a lock file exists, which is specified by the parent process - and managed by the child. + chance to properly clean up. Note: - We're technically polling in a hot loop, but: - - - We're only calling this when explicitly terminating processes, - which isn't that bad; and - - - Such calls typically only happen: - - - When e.g. the parallel function exectued in child - processes raised an error, so we're already on a "bad" - path; and - - - AFTER the performance-critical part of the code (the - parallelly-run function). - - To circumvent this we may use dedicated FS-watching APIs like - :py:mod:`watchdog` (which use syscalls to do this), but we'll - think about introducing extra dependencies when we REALLY have - to. + We're technically polling in a loop, but it isn't actually + *that* bad: typically ``.terminate()`` is only called when we're + on the bad path (e.g. the parallel workload errored out), and + after the performance-critical part of the code (said workload). """ # XXX: why can `coverage` get away with not doing all these # lock-file hijinks and just patching `BaseProcess._bootstrap()`? - - def discard_lock() -> None: - assert lock_file is not None - # This should have already happened unless we timed out - lock_file.unlink(missing_ok=True) - try: - delattr(self, _PROCESS_TERM_LOCK_LOC) - except AttributeError: - pass - def get_poller_args( config: PathLike[str] | str | bool | None = None, ) -> tuple[float, float, str | None]: @@ -352,8 +307,21 @@ def get_poller_args( on_timeout = None return cooldown, timeout, on_timeout - def wait_for_deletion( - path: Path, config: PathLike[str] | str | None = None, + def process_has_returned(proc: BaseProcess, timeout: float) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result + + def wait_for_return( + config: PathLike[str] | str | None = None, ) -> _Poller: cooldown, timeout, on_timeout = get_poller_args(config) # `False` -> no resolution, force loading the vanilla file @@ -361,24 +329,13 @@ def wait_for_deletion( if on_timeout not in ('ignore', 'warn', 'error'): on_timeout = default_on_timeout return ( - _Poller.poll_while(path.exists) - .with_cooldown(cooldown) - .with_timeout( - timeout, cast(Literal['ignore', 'warn', 'error'], on_timeout), - ) + _Poller.poll_until(process_has_returned, self, cooldown) + .with_timeout(timeout, cast(_OnTimeout, on_timeout)) ) try: - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - if lock_file: - lock: AbstractContextManager[Any] = wait_for_deletion( - lock_file, cache.config, - ) - callback = discard_lock - else: - lock, callback = nullcontext(), _no_op - with lock: - callback() + with wait_for_return(cache.config): + pass finally: # Always call `Process.terminate()` to avoid orphans vanilla_impl(self) @@ -392,27 +349,15 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`BaseProcess._bootstrap` to: - - - Run ``LineProfilingCache.load().cleanup()`` so that profiling - results can be gathered; and - - - Write a lock file before executing ``vanilla_impl()`` and deleted - it thereafter, to ensure that a parant process doesn't prematurely - ``.terminate()`` a failed child before the profiling results can - be gathered. + Wrap around :py:meth:`BaseProcess._bootstrap` to run + ``LineProfilingCache.load().cleanup()`` so that profiling results + can be gathered. """ - lock_file: Path | None = getattr(self, _PROCESS_TERM_LOCK_LOC, None) - - if lock_file: - lock_file.touch() - cache.add_cleanup(lock_file.unlink, missing_ok=True) try: return vanilla_impl(self, *args, **kwargs) finally: - cache._debug_output( - 'Calling cleanup hook via `BaseProcess._bootstrap`' - ) + msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' + cache._debug_output(msg) cache.cleanup() @@ -539,7 +484,6 @@ def _apply_mp_patches( # `patches` out... so do explicit `cast()`s for submodule, target, patches in [ ('process', 'BaseProcess', { - 'start': cast(_Wrapper[..., None], wrap_start), 'terminate': cast(_Wrapper[..., None], wrap_terminate), '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), }), diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 5b39f5cb..a420aad8 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -230,15 +230,15 @@ intercept_logs = false cooldown = 0.03125 # 1/32-nd of a second # - `polling.timeout` (float): # Time (seconds) before the main process disregards the existence of -# lock file and unblocks calls to `.terminate()` a child process -# anyway (set to <= 0 to disable timeouts) -timeout = 1 +# lock file and unblocks calls to `.terminate()` a (most probably +# errored-out) child process anyway (set to <= 0 to disable timeouts) +timeout = 0.25 # - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) # What to do when the above timeout is exhausted, before actually # `.terminate()`-ing the child process: # - 'error': raise an error # - `warn`: issue a warning # - `ignore`: nothing -on_timeout = 'error' +on_timeout = 'warn' # XXX: --- End of implementation details --- From 34adbabb87f46353baa2b2ee68d6a9d71639dbd4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 20 Apr 2026 06:31:39 +0200 Subject: [PATCH 056/111] iMisc. test fixes tests/test_child_procs.py test_cache_setup_main_process() Fixed bug where we checked `BaseProcess.start()`, which is no longer patched test_cache_setup_{main,child}() Fixed bug where we directly check for `os.fork()` without accounting for its nonexistence on Windows test_apply_mp_patches() Fixed bug where the expected number of function calls is wrong when `multiprocessing.dummy` is used --- tests/test_child_procs.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index c309973c..3094c244 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1144,7 +1144,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _MP_PATCHED_NAMES = { 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', 'start', 'terminate' + '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), } @@ -1354,9 +1354,11 @@ def test_cache_setup_main_process( for target, maybe_patches in patches.items(): obj = targets[target] for attr, is_patched in maybe_patches.items(): - assert ( - getattr(obj, attr) is patched[target][attr] - ) != is_patched + orig_value = patched[target][attr] + if orig_value is _NotSupplied.NOT_SUPPLIED: + assert not hasattr(obj, attr) + else: + assert (getattr(obj, attr) is orig_value) != is_patched # Check whether the patches are reversed cache.cleanup() for target, orig_attrs in patched.items(): @@ -1460,7 +1462,10 @@ def list_profiled_funcs() -> list[str]: callback() gathered = cache.gather_stats() assert any(gathered.timings.values()) == has_prof_data, gathered - assert (os.fork is not old_fork) == fork_patched + if hasattr(os, 'fork'): + assert (os.fork is not old_fork) == fork_patched + else: # E.g. Windows + assert old_fork == _NotSupplied.NOT_SUPPLIED # Check that profiling results have been written to the cache # directory @@ -1530,6 +1535,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: # preimports. To may the results more consistent between # `start_method='dummy'` and the others, manually do them below. cache._setup_in_main_process() # This calls `apply()` + assert cache.profiler is not None assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) @@ -1544,6 +1550,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: Pool: Callable[..., multiprocessing.pool.Pool] if start_method == 'dummy': Pool = _import_target('multiprocessing.dummy.Pool') + # Twice the counted calls because we're also collecting the + # checking calls in this process + expected_ncalls = len(_SAFE_TARGET_ARGS) * 2 + get_stats: Callable[[], LineStats] = cache.profiler.get_stats elif start_method not in START_METHODS: pytest.skip( f'`multiprocessing` start method {start_method!r} ' @@ -1551,6 +1561,8 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: ) else: Pool = multiprocessing.get_context(start_method).Pool + expected_ncalls = len(_SAFE_TARGET_ARGS) + get_stats = cache.gather_stats with Pool(2) as pool: par_result = pool.starmap(func, _SAFE_TARGET_ARGS) @@ -1559,18 +1571,13 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] # Check that calls in children are traced - if start_method == 'dummy': - assert cache.profiler is not None - stats = cache.profiler.get_stats() - else: - stats = cache.gather_stats() - line_entries = stats.timings[ + line_entries = get_stats().timings[ inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, ] num_returns = sum( nhits for lineno, nhits, _ in line_entries if lineno in return_lines ) - assert num_returns == len(_SAFE_TARGET_ARGS) + assert num_returns == expected_ncalls # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From 711292e326974e72d0438ac9f4927a02839f394d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 00:24:17 +0200 Subject: [PATCH 057/111] Fix thread profiling + refactoring line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._method_wrapper New decorator (relocated from `~.multiprocessing_patches`) line_profiler/_child_process_profiling/multiprocessing_patches.py Replaced uses of `@_method_wrapper` with `@LineProfilingCache._method_wrapper` line_profiler/_child_process_profiling/threading_patches.py::apply() New function for patching `threading.Thread` to set up thread profiling: when the thread object `.run()`s the target function, the profiler's `.enable_count` is synced to that of the starting thread. This fixes the bug where calls to a profiled function on new threads may not be profiled despite profiling being currently on tests/test_child_procs.py create_cache() Now also writing the profiler `id()` in the output test_cache_setup_main_process() Updated tested list of patched attributes test_apply_mp_patches() Updated preserved attributes --- .../_child_process_profiling/cache.py | 121 +++++++++++++++++- .../multiprocessing_patches.py | 61 +-------- .../threading_patches.py | 108 ++++++++++++++++ tests/test_child_procs.py | 26 ++-- 4 files changed, 248 insertions(+), 68 deletions(-) create mode 100644 line_profiler/_child_process_profiling/threading_patches.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 3d2484b7..5ccca6fe 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -21,8 +21,8 @@ from reprlib import Repr from textwrap import indent from types import MethodType -from typing import Any, ClassVar, TypeVar, TypedDict, cast -from typing_extensions import Self, ParamSpec, Unpack +from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload +from typing_extensions import Concatenate, ParamSpec, Self, Unpack from .. import _diagnostics as diagnostics from ..curated_profiling import CuratedProfilerContext @@ -39,6 +39,7 @@ __all__ = ('LineProfilingCache',) +T = TypeVar('T') PS = ParamSpec('PS') # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) @@ -515,8 +516,9 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: forked processes is properly handled (if ``wrap_os_fork=True``) - - :py:mod:`multiprocessing` wrapped so that child processes - managed by the package are properly handled + - :py:mod:`multiprocessing` and :py:mod:`threading` patched + so that child processes and threads managed thereby are + properly handled - Instance to be returned if :py:func:`~.load()` is called from now on @@ -532,6 +534,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: self._wrap_os_fork() mp_patches = import_module(this_subpkg + '.multiprocessing_patches') mp_patches.apply(self) + th_patches = import_module(this_subpkg + '.threading_patches') + th_patches.apply(self) self._replace_loaded_instance() @@ -704,6 +708,115 @@ def _get_init_args(self) -> dict[str, Any]: def _get_filename(cache_dir: os.PathLike[str] | str) -> str: return os.path.join(cache_dir, CACHE_FILENAME) + @overload + @classmethod + def _method_wrapper( + cls, + wrapper: Callable[Concatenate[Self, Callable[PS, T], PS], T], + debug: bool | None = None, + ) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + ... + + @overload + @classmethod + def _method_wrapper( + cls, wrapper: None = None, debug: bool | None = None, + ) -> Callable[ + [Callable[Concatenate[Self, Callable[PS, T], PS], T]], + Callable[[Callable[PS, T]], Callable[PS, T]] + ]: + ... + + @classmethod + def _method_wrapper( + cls, + wrapper: ( + Callable[Concatenate[Self, Callable[PS, T], PS], T] | None + ) = None, + debug: bool | None = None, + ) -> ( + Callable[ + [Callable[Concatenate[Self, Callable[PS, T], PS], T]], + Callable[[Callable[PS, T]], Callable[PS, T]] + ] + | Callable[[Callable[PS, T]], Callable[PS, T]] + ): + """ + Convenience wrapper decorator for functions which use the + :py:meth:`load`-ed session instance and wrap another callable. + + Args: + wrapper (Callable[..., T]) + Callable with the call signature + ``(cache, vanilla_impl, *args, **kwargs) -> retval``; + ``*args``, ``**kwargs``, and ``retval`` should be + consistent with that of ``vanilla_impl()``'s. + debug (bool | None) + Whether to format and write debug messages before and + after the call to the ``wrapper`` callable; + if ``debug`` is not set, it will be taken from the + session instance. + + Returns: + inner_wrapper (Callable[[Callable[PS, T]], Callable[PS, T]]) + Wrapper(-maker) which takes the ``vanilla_impl`` and + return a wrapper around it. + """ + if wrapper is None: + # `ty` doesn't quite support `partial` yet, see issue #1536 + return cast( + Callable[[Callable[PS, T]], Callable[PS, T]], + partial(cls._method_wrapper, debug=debug), + ) + + def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: + @wraps(vanilla_impl) + def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: + cache = cls.load() + write = cache._debug_output + debug_: bool | None = debug + if debug_ is None: + debug_ = cache.debug + + if debug_: + arg_reprs: list[str] = [repr(arg) for arg in args] + arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) + formatted_call = f'{name}({", ".join(arg_reprs)})' + write(f'Wrapped call made: {formatted_call}...') + try: + result = wrapper(cache, vanilla_impl, *args, **kwargs) + except Exception as e: + if debug_: + write( + 'Wrapped call failed: ' + f'{formatted_call} -> {type(e).__name__}: {e}', + ) + raise + else: + if debug_: + write( + 'Wrapped call succeeded: ' + f'{formatted_call} -> {result!r}', + ) + return result + + if ( + hasattr(vanilla_impl, '__module__') + and hasattr(vanilla_impl, '__qualname__') + ): + name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) + else: + name = f'' + + return wrapped_impl + + for field in 'name', 'qualname', 'doc': + dunder = f'__{field}__' + value = getattr(wrapper, dunder, None) + if value is not None: + setattr(inner_wrapper, dunder, value) + return inner_wrapper + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 1b37720e..ef76ca45 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -17,7 +17,7 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from functools import lru_cache, partial, wraps +from functools import lru_cache, partial from importlib import import_module from multiprocessing.process import BaseProcess from os import PathLike @@ -223,54 +223,7 @@ def _get_config_cached( return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) -def _method_wrapper( - wrapper: Callable[Concatenate[LineProfilingCache, Callable[PS, T], PS], T], -) -> Callable[[Callable[PS, T]], Callable[PS, T]]: - def inner_wrapper(vanilla_impl: Callable[PS, T]) -> Callable[PS, T]: - @wraps(vanilla_impl) - def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: - cache = LineProfilingCache.load() - debug = cache._debug_output - - arg_reprs: list[str] = [repr(arg) for arg in args] - arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) - formatted_call = '{}({})'.format(name, ', '.join(arg_reprs)) - - debug(f'Wrapped call made: {formatted_call}...') - try: - result = wrapper(cache, vanilla_impl, *args, **kwargs) - except Exception as e: - debug( - 'Wrapped call failed: ' - f'{formatted_call} -> {type(e).__name__}: {e}', - ) - raise - else: - debug( - 'Wrapped call succeeded: ' - f'{formatted_call} -> {result!r}', - ) - return result - - if ( - hasattr(vanilla_impl, '__module__') - and hasattr(vanilla_impl, '__qualname__') - ): - name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) - else: - name = f'' - - return wrapped_impl - - for field in 'name', 'qualname', 'doc': - dunder = f'__{field}__' - value = getattr(wrapper, dunder, None) - if value is not None: - setattr(inner_wrapper, dunder, value) - return inner_wrapper - - -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, vanilla_impl: Callable[[BaseProcess], None], @@ -340,7 +293,7 @@ def wait_for_return( vanilla_impl(self) -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_bootstrap( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], @@ -395,10 +348,11 @@ def get_msg(msg: str, *_, **__) -> str: ) -@_method_wrapper +@LineProfilingCache._method_wrapper def wrap_get_preparation_data( - # We don't use the cache here, but `@_method_wrapper` expects it in - # the signature (and we want the debug output) + # We don't use the cache here, but + # `@LineProfilingCache._method_wrapper` expects it in the signature + # (and we want the debug output) _, vanilla_impl: Callable[PS, dict[str, Any]], /, @@ -443,7 +397,6 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:mod:`multiprocessing` marked as having been set up - The following methods and functions patched: - - :py:meth:`multiprocessing.process.BaseProcess.start` - :py:meth:`multiprocessing.process.BaseProcess.terminate` diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_child_process_profiling/threading_patches.py new file mode 100644 index 00000000..d5a2e94d --- /dev/null +++ b/line_profiler/_child_process_profiling/threading_patches.py @@ -0,0 +1,108 @@ +""" +Patch :py:mod:`threading` so that profiling extends into processes +it creates. +""" +from __future__ import annotations + +import threading +from collections.abc import Callable +from functools import wraps +from typing import TYPE_CHECKING, Any, TypeVar +from typing_extensions import ParamSpec + +from ..line_profiler import LineProfiler +from .cache import LineProfilingCache + + +__all__ = ('apply',) + + +T = TypeVar('T') +PS = ParamSpec('PS') + +_PATCHED_MARKER = '__line_profiler_patched_threading__' + + +def make_syncing_wrapper( + func: Callable[PS, T], prof: LineProfiler, enable_count: int, +) -> Callable[PS, T]: + """ + Wrap the callable ``func`` so that when we spin up a new thread, we + sync the + :py:attr:`line_profiler.line_profiler.LineProfiler.enable_count` of + the active profiler (stored at the cache instance loaded from + :py:meth:`LineProfilingCache.load`) with ``enable_count``. + """ + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + if TYPE_CHECKING: + assert hasattr(prof, 'enable_count') + assert isinstance(prof.enable_count, int) + # Note: `prof.enable_count` is most likely to be zero on the new + # thread + thread_enable_count: int = prof.enable_count + for _ in range(enable_count - thread_enable_count): + prof.enable_by_count() + try: + return func(*args, **kwargs) + finally: + # Reset enable counts to avoid problems if the thread id is + # ever reused + for _ in range(prof.enable_count - thread_enable_count): + prof.disable_by_count() + + return wrapper + + +# Threads are supposed to be lightweight, so don't waste time formatting +# debug messages during startup + + +@LineProfilingCache._method_wrapper(debug=False) +def wrap_init( + cache: LineProfilingCache, + vanilla_impl: Callable[..., None], + self: threading.Thread, + group: None = None, + target: Callable[..., Any] | None = None, + *a, **k +) -> None: + """ + Wrap the initializer of :py:class:`threading.Thread` so that the + profiler's :py:attr:`LineProfiler.enable_count` is synced up on + newly spun-up threads. + """ + prof = cache.profiler + enable_count: int | None = getattr(prof, 'enable_count', None) + if target is not None and enable_count: + if TYPE_CHECKING: + assert prof is not None + target = make_syncing_wrapper(target, prof, enable_count) + vanilla_impl(self, group, target, *a, **k) + + +def apply(lp_cache: LineProfilingCache) -> None: + """ + Set up profiling in threads started by :py:mod:`threading` by + applying patches to the module. + + Args: + lp_cache (LineProfilingCache) + Cache instance governing the profiling run + + Side effects: + - :py:mod:`threading` marked as having been set up + + - The following methods and functions patched: + + - :py:meth:`threading.Thread.__init__` + + - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + """ + if getattr(threading, _PATCHED_MARKER, False): + return + init_wrapper = wrap_init(threading.Thread.__init__) + lp_cache.patch( + threading.Thread, '__init__', init_wrapper, name='threading.Thread', + ) + lp_cache.patch(threading, _PATCHED_MARKER, True, name='threading') diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 3094c244..68ad3045 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -361,19 +361,23 @@ def instantiate( return cache def print_result( - cache: LineProfilingCache, topic: str, result: str, + cache: LineProfilingCache, topic: str, result: str, *notes: str, ) -> None: - header = f'{topic} (cache instance {id(cache):#x}):' + header = '{} ({}):'.format( + topic, '; '.join([f'cache instance {id(cache):#x}', *notes]), + ) print(header, indent(result, ' '), sep='\n') def print_profiler_stats(cache: LineProfilingCache) -> None: if cache.profiler is None: result = '' + notes = [] else: with StringIO() as sio: cache.profiler.print_stats(sio) result = sio.getvalue() - print_result(cache, 'Native profiler stats', result) + notes = [f'profiler instance {id(cache.profiler):#x}'] + print_result(cache, 'Native profiler stats', result, *notes) def print_gathered_stats(cache: LineProfilingCache) -> None: with StringIO() as sio: @@ -1142,11 +1146,13 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # tested APIs and behaviors MUST NOT be relied upon by end-users. -_MP_PATCHED_NAMES = { +_GLOBAL_PATCHES = { 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), + 'threading.Thread': frozenset({'__init__'}), + 'os': frozenset({'fork'}), } # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may @@ -1336,11 +1342,11 @@ def test_cache_setup_main_process( as expected. """ cache = create_cache(debug=debug) - patches: dict[str, dict[str, bool]] = {'os': {'fork': wrap_os_fork}} - patches.update( - (target, dict.fromkeys(attrs, True)) - for target, attrs in _MP_PATCHED_NAMES.items() - ) + patches: dict[str, dict[str, bool]] = { + target: dict.fromkeys(attrs, True) + for target, attrs in _GLOBAL_PATCHES.items() + } + patches['os']['fork'] = wrap_os_fork targets: dict[str, Any] = { target: _import_target(target) for target in patches } @@ -1488,7 +1494,7 @@ def list_profiled_funcs() -> list[str]: @pytest.mark.parametrize(('debug', 'label'), [(True, 'with-debug'), (False, 'no-debug')]) @_preserve_pth_files() -@_preserve_attributes(_MP_PATCHED_NAMES) +@_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], From bfbef7a4ac1cbe30026aa6415533c4094f5e9fee Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 02:25:12 +0200 Subject: [PATCH 058/111] Partial rollback of `threading` fix line_profiler/_child_process_profiling/threading_patches.py SHOULD_PATCH_THREADING Alias to `line_profiler._line_profiler.USE_LEGACY_TRACE` apply() Now a no-op if `SHOULD_PATCH_THREADING` is false (because the fix only seems to work when using `LINE_PROFILER_CORE=ctrace`, and causes flakiness otherwise) tests/test_child_procs.py test_cache_setup_main_process() Now only checking for `threading` patches if `SHOULD_PATCH_THREADING` is true test_apply_mp_patches() Now only protecting against `threading` patches if `SHOULD_PATCH_THREADING` is true --- .../_child_process_profiling/threading_patches.py | 15 ++++++++++++++- tests/test_child_procs.py | 8 ++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_child_process_profiling/threading_patches.py index d5a2e94d..c8a5b62c 100644 --- a/line_profiler/_child_process_profiling/threading_patches.py +++ b/line_profiler/_child_process_profiling/threading_patches.py @@ -10,11 +10,14 @@ from typing import TYPE_CHECKING, Any, TypeVar from typing_extensions import ParamSpec +from .._line_profiler import ( # type: ignore + USE_LEGACY_TRACE as SHOULD_PATCH_THREADING, +) from ..line_profiler import LineProfiler from .cache import LineProfilingCache -__all__ = ('apply',) +__all__ = ('apply', 'SHOULD_PATCH_THREADING') T = TypeVar('T') @@ -32,6 +35,10 @@ def make_syncing_wrapper( :py:attr:`line_profiler.line_profiler.LineProfiler.enable_count` of the active profiler (stored at the cache instance loaded from :py:meth:`LineProfilingCache.load`) with ``enable_count``. + + Note: + This only seems to work as intended when using the legacy trace + system... """ @wraps(func) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: @@ -98,7 +105,13 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`threading.Thread.__init__` - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + + Note: + This is a no-op when using :py:mod:`sys.monitoring`-based + profiling. """ + if not SHOULD_PATCH_THREADING: + return if getattr(threading, _PATCHED_MARKER, False): return init_wrapper = wrap_init(threading.Thread.__init__) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 68ad3045..90d51562 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -35,6 +35,9 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) +from line_profiler._child_process_profiling.threading_patches import ( + SHOULD_PATCH_THREADING, +) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) @@ -1145,15 +1148,16 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # XXX: Tests in this section concerns implementation details, and the # tested APIs and behaviors MUST NOT be relied upon by end-users. - _GLOBAL_PATCHES = { 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), - 'threading.Thread': frozenset({'__init__'}), 'os': frozenset({'fork'}), } +if SHOULD_PATCH_THREADING: + _GLOBAL_PATCHES['threading.Thread'] = frozenset({'__init__'}) + # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may # be skewed From 6e60a6af54f5f91e0a973d88e05bc6c24c405458 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 08:02:33 +0200 Subject: [PATCH 059/111] Fix forkserver profiling in newer Pythons line_profiler/_child_process_profiling/cache.py::LineProfilingCache _setup_in_main_process() Refactored part into `._setup_common()` _setup_in_child_process() Now calling `._setup_common()`, making sure that the fork-server process is also set up for profiling _wrap_os_hook() `os.fork()` wrapper now writing debug-log messages indicating the parent and child PIDs line_profiler/_child_process_profiling/multiprocessing_patches.py PickleHook, wrap_get_preparation_data() Removed no-unused code tee_log() Updated call signature apply() Added argument `reboot_forkserver` _apply_mp_patches() - Updated call signature - Refactored internals - Now rebooting the fork-server process to make sure that it is set up for profiling tests/test_child_procs.py test_cache_setup_main_process() No longer checking patching/restoration of `multiprocessing.spawn.get_preparation_data()` because it is no longer patched test_apply_mp_patches() No longer checking the output `multiprocessing.spawn.get_preparation_data()` because it is no longer patched FIXME: performance regression in newer Python versions... --- .../_child_process_profiling/cache.py | 40 ++-- .../multiprocessing_patches.py | 214 ++++++++---------- tests/test_child_procs.py | 13 +- 3 files changed, 116 insertions(+), 151 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 5ccca6fe..51a464b3 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -20,7 +20,7 @@ from pickle import HIGHEST_PROTOCOL from reprlib import Repr from textwrap import indent -from types import MethodType +from types import MethodType, ModuleType from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload from typing_extensions import Concatenate, ParamSpec, Self, Unpack @@ -44,6 +44,7 @@ # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) +_THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_DIR' ) @@ -51,6 +52,10 @@ _DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +def _import_sibling(submodule: str) -> ModuleType: + return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') + + class _ReprAttributes(TypedDict, total=False): """ Note: @@ -523,20 +528,10 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: - Instance to be returned if :py:func:`~.load()` is called from now on """ - this_subpkg, *_, _ = (lambda: None).__module__.rpartition('.') - self.dump() self.inject_env_vars() - pth_hook = import_module(this_subpkg + '.pth_hook') - pth_hook.write_pth_hook(self) - - if wrap_os_fork: - self._wrap_os_fork() - mp_patches = import_module(this_subpkg + '.multiprocessing_patches') - mp_patches.apply(self) - th_patches = import_module(this_subpkg + '.threading_patches') - th_patches.apply(self) - + _import_sibling('pth_hook').write_pth_hook(self) + self._setup_common(wrap_os_fork, reboot_forkserver=True) self._replace_loaded_instance() def _setup_in_child_process( @@ -601,10 +596,8 @@ def _setup_in_child_process( ) self._add_cleanup(prof.dump_stats, -1, prof_outfile) - # Set up `os.fork()` wrapping if needed (i.e. in a spawned - # process) - if wrap_os_fork: - self._wrap_os_fork() + # Various setups + self._setup_common(wrap_os_fork, reboot_forkserver=False) # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate @@ -613,6 +606,16 @@ def _setup_in_child_process( self._debug_output(f'Setup successful ({context})') return True + def _setup_common( + self, wrap_os_fork: bool, reboot_forkserver: bool, + ) -> None: + if wrap_os_fork: + self._wrap_os_fork() + _import_sibling('multiprocessing_patches').apply( + self, reboot_forkserver, + ) + _import_sibling('threading_patches').apply(self) + def _wrap_os_fork(self) -> None: """ Create a wrapper around :py:func:`os.fork` which handles @@ -630,11 +633,14 @@ def _wrap_os_fork(self) -> None: @wraps(fork) def wrapper() -> int: + ppid = os.getpid() result = fork() if result: return result # If we're here, we are in the fork + pid = os.getpid() forked = self.copy() # Ditch inherited cleanups + forked._debug_output(f'Forked: {ppid} -> {pid}') if forked._replace_loaded_instance(): forked._debug_output( 'Superseded cached `.load()`-ed instance in forked process' diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index ef76ca45..c5561fa4 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -49,28 +49,6 @@ def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: ... -class PickleHook: - """ - Object which, when unpickled, sets up profiling in the - :py:mod:`multiprocessing`-created process. - - See also: - :py:class:`coverage.multiproc.Stowaway` - """ - def __getstate__(_) -> int: - # Cannot return `None`, or nothing will be pickled and - # `.__getstate__()` will not be invoked in the child - return 1 - - def __setstate__(*_) -> None: - # We're in a child process created by `multiprocessing`, so set - # up shop here. - lp_cache = LineProfilingCache.load() - lp_cache._setup_in_child_process(False, 'multiprocessing') - if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, main_process=False) - - class _Poller: """ Poll a callable until it returns true-y. @@ -327,8 +305,8 @@ def _cache_hook( def tee_log( - vanilla_impl: Callable[Concatenate[str, PS], None], marker: str, + vanilla_impl: Callable[Concatenate[str, PS], None], /, msg: str, *args: PS.args, @@ -348,50 +326,21 @@ def get_msg(msg: str, *_, **__) -> str: ) -@LineProfilingCache._method_wrapper -def wrap_get_preparation_data( - # We don't use the cache here, but - # `@LineProfilingCache._method_wrapper` expects it in the signature - # (and we want the debug output) - _, - vanilla_impl: Callable[PS, dict[str, Any]], - /, - *args: PS.args, - **kwargs: PS.kwargs -) -> dict[str, Any]: - """ - Wrap around :py:func:`multiprocessing.spawn.get_preparation_data`, - slipping a :py:class:`PickleHook` into the returned dictionary so - that profiling is triggered upon unpickling. - - Args: - vanilla_impl - Vanilla - :py:func:`multiprocessing.spawn.get_preparation_data` - *args - **kwargs - Passed to - :py:func:`multiprocessing.spawn.get_preparation_data` - - Returns - Dictionary returned by - ``get_preparation_data(*args, **kwargs)`` with an extra key - """ - key = 'line_profiler_pickle_hook' # Doesn't matter - data = vanilla_impl(*args, **kwargs) - assert key not in data - data[key] = PickleHook() - return data - - -def apply(lp_cache: LineProfilingCache) -> None: +def apply( + lp_cache: LineProfilingCache, reboot_forkserver: bool = True, +) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - lp_cache (LineProfilingCache) + lp_cache (LineProfilingCache): Cache instance governing the profiling run + reboot_forkserver (bool): + Whether to reboot the global + :py:class`multiprocessing.forkserver.ForkServer` instance + so as to ensure that profiling happens on processes forked + therefrom (see Note) Side effects: - :py:mod:`multiprocessing` marked as having been set up @@ -402,93 +351,112 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` - - :py:func:`multiprocessing.spawn.get_preparation_data` + - If ``reboot_forkserver=True``, fork-server process rebooted: + + - Immediately + + - When ``lp_cache.cleanup()`` is run - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` Note: - When ``lp_cache.cleanup()`` is run, the global - :py:class:`multiprocessing.forkserver.ForkServer` object will be - rebooted. This is necessary because the server process staticly - inherits the environment when it is first spun up + Rebooting the fork server is necessary because its process + staticly inherits the environment when it is first spun up (see :py:func:`multiprocessing.forkserver.ensure_running`). - Thus, if in the same Python process we ever start up two - separate profliing sessions managed by different caches, the - child processes forked from the server will fail to inherit the - updated environment variables injected by the newer cache - instance, leading to the setup code in this subpackage not being - loaded. + Thus, without the reboots: + + - If in the same Python process we ever start up two separate + profliing sessions managed by different caches, the child + processes forked from the server will fail to inherit the + updated environment variables injected by the newer cache + instance, leading to the setup code in this subpackage not + being loaded. + + - Since 3.13.8 and 3.14.1, the bug where the ``main_path`` + argument to :py:func:`multiprocessing.forkserver.main` is + unused has been fixed (see ``cpython`` issue `GH-126631`_). + This causes ``sys.modules['__main__']`` to be set up in the + fork-server process, meaning that children forked therefrom + will NOT redo the setup. Thus, the fork-server process itself + will also need to be properly set up for profiling. + + .. _GH-126631: https://github.com/python/cpython/issues/126631 """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache) + _apply_mp_patches(lp_cache, reboot_forkserver) + + +def _apply_patches_generic( + lp_cache: LineProfilingCache, + submodule: str, + targets: Mapping[str, Mapping[str, Callable[[Any], Any]]], + cleanup: bool = True, +) -> None: + submod_name = 'multiprocessing.' + submodule + try: + mod = import_module(submod_name) + except ImportError: + return + for target, patches in targets.items(): + if target: + try: + obj: Any = getattr(mod, target) + except AttributeError: + continue + name = f'{submod_name}.{target}' + else: + obj, name = mod, submod_name + replace = partial(lp_cache.patch, obj, cleanup=cleanup, name=name) + for method, method_wrapper in patches.items(): + try: + vanilla = getattr(obj, method) + except AttributeError: + continue + replace(method, method_wrapper(vanilla)) def _apply_mp_patches( lp_cache: LineProfilingCache, - main_process: bool = True, + reboot_forkserver: bool = True, debug: bool | None = None, ) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup - replace = partial(lp_cache.patch, cleanup=main_process) - + apply_patches = partial(_apply_patches_generic, lp_cache) # Patch `multiprocessing.process.BaseProcess` methods # Note: the type checkers seem to need some help figuring the # `patches` out... so do explicit `cast()`s - for submodule, target, patches in [ - ('process', 'BaseProcess', { - 'terminate': cast(_Wrapper[..., None], wrap_terminate), - '_bootstrap': cast(_Wrapper[..., Any], wrap_bootstrap), - }), - ]: - try: - mod = import_module('multiprocessing.' + submodule) - except ImportError: - continue - Class = getattr(mod, target) - name = f'{Class.__module__}.{Class.__qualname__}' - patch_class = partial(replace, Class, name=name) - for method, method_wrapper in patches.items(): - vanilla = getattr(Class, method) - patch_class(method, method_wrapper(vanilla)) - + apply_patches( + 'process', + {'BaseProcess': {'terminate': wrap_terminate, + '_bootstrap': wrap_bootstrap}}, + ) # Patch `multiprocessing.spawn` try: from multiprocessing import spawn - except ImportError: # Incompatible platforms + except ImportError: pass else: - patch_spawn = partial(replace, spawn, name=spawn.__name__) - # Patch `get_preparation_data()` - gpd_wrapper = wrap_get_preparation_data(spawn.get_preparation_data) - patch_spawn('get_preparation_data', gpd_wrapper) - # Patch `runpy` (do it locally instead of tempering with the - # global `runpy` mmodule) if hasattr(spawn, 'runpy'): - runpy_wrapper = create_runpy_wrapper(lp_cache) - patch_spawn('runpy', runpy_wrapper) - + lp_cache.patch( + spawn, 'runpy', create_runpy_wrapper(lp_cache), + name='multiprocessing.spawn', + ) # Intercept `multiprocessing` debug messages if debug is None: debug = _get_config(lp_cache.config)['intercept_logs'] if debug: - from multiprocessing import util - - patch_util = partial(replace, util, name=util.__name__) - for logging_func in [ - 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', - ]: - try: - vanilla = getattr(util, logging_func) - except AttributeError: - continue - patch_util(logging_func, partial(tee_log, vanilla, logging_func)) - - # Stop the current `ForkServer` server process as a part of cache - # cleanup (this uses `ForkServer._stop()` which is private API, but - # it's the same hack used in Python's own test suite -- see the - # comment to said method) - if main_process: + lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] + lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} + apply_patches('util', {'': lpatches}) + # Stop the current `ForkServer` server process: + # - Now, so that the (rebooted) fork-server process has profiling + # set up; and + # - Also as a part of cache cleanup + # (this uses `ForkServer._stop()` which is private API, but it's the + # same hack used in Python's own test suite -- see the comment to + # said method) + if reboot_forkserver: try: from multiprocessing import forkserver except ImportError: # Incompatible platform @@ -497,10 +465,12 @@ def _apply_mp_patches( server_instance: forkserver.ForkServer = forkserver._forkserver stop = getattr(server_instance, '_stop', None) assert callable(stop) # Appease the type checker + stop() lp_cache.add_cleanup(stop) - # Mark `multiprocessing` as having been patched - replace(multiprocessing, _PATCHED_MARKER, True, name='multiprocessing') + lp_cache.patch( + multiprocessing, _PATCHED_MARKER, True, name='multiprocessing', + ) def _no_op(*_, **__) -> None: diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 90d51562..ba1061a8 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1152,7 +1152,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), - 'multiprocessing.spawn': frozenset({'get_preparation_data', 'runpy'}), + 'multiprocessing.spawn': frozenset({'runpy'}), 'os': frozenset({'fork'}), } if SHOULD_PATCH_THREADING: @@ -1511,10 +1511,6 @@ def test_apply_mp_patches( .multiprocessing_patches.apply` works as expected. """ - def gpd_wrapper(name: str) -> dict[str, Any]: - impl = _import_target('multiprocessing.spawn.get_preparation_data') - return impl(name) - def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: LineStats.from_files(path, on_defective='error') @@ -1536,7 +1532,6 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: config=config, debug=True, ) - old_data = gpd_wrapper('my_name') # Note: # - The reversibility of the patches have already been tested in # `test_cache_setup_main_process()`, so we just actually test the @@ -1549,12 +1544,6 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - # Check for the insertion of the `PickleHook` by the patched - # `get_preparation_data()` - new_data = gpd_wrapper('my_name') - inserted_key, = set(new_data) - set(old_data) - assert type(new_data[inserted_key]).__name__ == 'PickleHook' - func = _import_target(_SAFE_TARGET) return_lines = _find_return_lines(_SAFE_TARGET) Pool: Callable[..., multiprocessing.pool.Pool] From d2203a0c4217ebe54a44d2190308bf8b7017a094 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 22 Apr 2026 18:08:09 +0200 Subject: [PATCH 060/111] Fix test on Windows --- tests/test_child_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ba1061a8..cadf145e 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1350,7 +1350,7 @@ def test_cache_setup_main_process( target: dict.fromkeys(attrs, True) for target, attrs in _GLOBAL_PATCHES.items() } - patches['os']['fork'] = wrap_os_fork + patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') targets: dict[str, Any] = { target: _import_target(target) for target in patches } From 4ae963fe20ad6894211e46ebc3d24efb4d71b8bf Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 25 Apr 2026 04:08:57 +0200 Subject: [PATCH 061/111] Added `conftest.py` conftest.py Installs the marker `@pytest.mark.retry(...)` so that certain tests can be retried TODO: tests --- tests/conftest.py | 380 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 380 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..41deeb64 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,380 @@ +""" +A simple :py:deco:`pytest.mark.retry` decorator. +Function-scoped fixtures are re-fetched bewteen retries. +""" +import dataclasses +from collections.abc import ( + Callable, Collection, Generator, Hashable, Iterable, Mapping, +) +from functools import cached_property, partial +from operator import contains +from pathlib import Path +from typing import ( + TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeVar, cast, final, +) + +import pytest +from _pytest.nodes import Node +from _pytest.unittest import TestCaseFunction +try: + from pytest import TerminalReporter +except ImportError: # pytest < ~8.4 + from _pytest.terminal import TerminalReporter + + +_Status = Literal['passed', 'failed', 'skipped'] +F = TypeVar('F', bound=TestCaseFunction) +FCls = TypeVar('FCls', bound=type[TestCaseFunction]) +T = TypeVar('T') + + +class _PyfuncCallImpl(Protocol): + def __call__(self, *, pyfuncitem: pytest.Function) -> Any: + ... + + +@final +@dataclasses.dataclass +class _RetryEntry: + func: pytest.Function + retries: int + status: _Status + + def _get_name(self, with_params: bool) -> str: + path, prefix = self._name_prefixes + if with_params: + name = self.func.name + else: + name = self.func.originalname + if prefix: + name = f'{prefix}.{name}' + if path: + name = f'{path}::{name}' + return name + + @classmethod + def add_entry(cls, func: pytest.Function, *args, **kwargs) -> _RetryEntry: + assert func.config is not None + entry = cls(func, *args, **kwargs) + entry.get_entries(func.config).append(entry) + return entry + + @staticmethod + def get_entries(config: pytest.Config) -> list[_RetryEntry]: + return config.stash.setdefault(_RETRY_ENTRIES_KEY, []) + + @property + def full_name(self) -> str: + return self._get_name(True) + + @property + def full_original_name(self) -> str: + return self._get_name(False) + + @cached_property + def _name_prefixes(self) -> tuple[str, str]: + chunks: list[str] = [] + node: Node | None = self.func.parent + path = '' + seen: set[int] = set() + while True: + if id(node) in seen: + break + else: + seen.add(id(node)) + if isinstance(node, (pytest.Module, pytest.Package)): + if node.path: + path = str(node.path.relative_to(Path.cwd())) + else: + path = repr(node) + break + name: str | None = getattr(node, 'name', None) + if not name: + break + chunks.append(name) + return path, '.'.join(reversed(chunks)) + + +_RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() + + +@final +@dataclasses.dataclass +class _RetryHelper: + retries: int = 1 + exceptions: type[Exception] | tuple[type[Exception], ...] = () + reset_fixtures: bool | Collection[str] = True + name: ClassVar[str] = 'retry' + + def __post_init__(self) -> None: + if not (int(self.retries) == self.retries > 0): + raise TypeError( + f'.entries = {self.retries!r}: expected a positive integer' + ) + if isinstance(self.exceptions, tuple): + xc: tuple[type[Exception], ...] = self.exceptions + else: + xc = self.exceptions, + if not all(issubclass(X, Exception) for X in xc): + raise TypeError( + f'.exceptions = {self.exceptions!r}: ' + 'expected an exception type or a tuple thereof' + ) + + def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: + """ + Manage the call(s) to a function. + + Args: + impl (Callable): + Implementation of + ``pytest_pyfunc_call(pyfuncitem: pytest.Function) \ +-> Any`` + func (pytest.Function): + Test function item + + Returns: + Value of the first successful call to + ``pytest_pyfunc_call(pyfuncitem=func)`` + """ + check_fixture_name: Callable[[str], bool] + reset_fixtures: Callable[[pytest.Function], None] + if self.reset_fixtures: + if isinstance(self.reset_fixtures, Collection): + check_fixture_name = partial(contains, self.reset_fixtures) + else: + check_fixture_name = lambda _: True # noqa: E731 + reset_fixtures = partial( + self._reset_between_retries, + reset_fixtures=True, + should_reset=check_fixture_name, + ) + else: + reset_fixtures = self._reset_between_retries + + result: Any = None + xc: Exception | None = None + for i in range(1 + self.retries): + if i: + reset_fixtures(func) + try: + result = impl(pyfuncitem=func) + except self.exceptions as e: + # `ty` doesn't agree that `e` is an exception... + xc = cast(Exception, e) + except Exception as e: # Uncaught exc. -> break to raise + xc = e + break + else: # Correct execution -> break to return + xc = None + break + if i: + if xc is None: + status: _Status = 'passed' + elif isinstance(xc, pytest.skip.Exception): + status = 'skipped' + else: + status = 'failed' + _RetryEntry.add_entry(func, i, status) + if xc is None: + return result + else: + raise xc + + @staticmethod + def _reset_between_retries( + func: pytest.Function, + reset_fixtures: bool = False, + should_reset: Callable[[str], bool] = lambda _: False, + ) -> None: + def clear_fixture_cache(fixture_def: pytest.FixtureDef[Any]) -> None: + if ( + fixture_def.scope == 'function' + and should_reset(fixture_def.argname) + and hasattr(fixture_def, 'cached_result') + ): + fixture_def.cached_result = None + + def unique( + items: Iterable[T], key: Callable[[T], Hashable] = id, + ) -> Generator[T, None, None]: + seen: set[Hashable] = set() + for item in items: + hashed = key(item) + if hashed in seen: + continue + seen.add(hashed) + yield item + + def iter_all_fixture_defs( + func: pytest.Function, + ) -> Generator[pytest.FixtureDef[Any], None, None]: + fdef_mapping: Mapping[Any, Iterable[pytest.FixtureDef[Any]]] + # Somehow `mypy` doesn't trust the below but `ty` does... + for fdef_mapping in [ # type:ignore[assignment] + func._fixtureinfo.name2fixturedefs, + func._request._arg2fixturedefs, + func.session._fixturemanager._arg2fixturedefs, + ]: + for fixture_defs in fdef_mapping.values(): + yield from fixture_defs + + if reset_fixtures: + # Beside clearing `.funcargs`, `._initrequest()` also resets + # the `TopRequest` instance that `func` has (`._request`) + func._initrequest() + for fixture_def in unique(iter_all_fixture_defs(func)): + clear_fixture_cache(fixture_def) + else: + # Fixture values will naturally refill, possibly from caches + func.funcargs.clear() + func.setup() + + @classmethod + def pytest_pyfunc_call(cls, pyfuncitem: pytest.Function) -> Any: + """ + Run the :py:class:`pytest.Function` object with the requisite + number of retries if necessary. + """ + pm = pyfuncitem.config.pluginmanager + impl: _PyfuncCallImpl = pm.subset_hook_caller( + 'pytest_pyfunc_call', [cls], + ) + helper = cls.get_helper(pyfuncitem) + if helper: + return helper.manage_call(impl, pyfuncitem) + return impl(pyfuncitem=pyfuncitem) + + @classmethod + def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: + retries: int = 0 + xc: set[type[Exception]] = set() + reset_fixtures: bool | set[str] = True + for mark in pyfuncitem.iter_markers(): + if mark.name != cls.name: + continue + instance = cls(*mark.args, **mark.kwargs) + retries += instance.retries + if isinstance(instance.exceptions, tuple): + xc.update(instance.exceptions) + else: + xc.add(instance.exceptions) + if ( + reset_fixtures not in (True, False) + and instance.reset_fixtures not in (True, False) + ): # Both collections of fixture names + if TYPE_CHECKING: + assert not isinstance(reset_fixtures, bool) + assert not isinstance(instance.reset_fixtures, bool) + reset_fixtures.update(instance.reset_fixtures) + elif instance.reset_fixtures not in (True, False): + if TYPE_CHECKING: + assert not isinstance(instance.reset_fixtures, bool) + reset_fixtures = set(instance.reset_fixtures) + else: + reset_fixtures = bool(instance.reset_fixtures) + if not retries: + return None + if not xc: + xc = {Exception} + return cls(retries, tuple(xc), reset_fixtures) + + +def _pluralize(noun: str, count: int, plural: str | None = None) -> str: + if plural is None: + plural = noun + 's' + return f'{count} {noun if count == 1 else plural}' + + +def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: + """ + Register :py:class:`_RetryHelper` as a plugin so that its + :py:meth:`pytest_pyfunc_call` method can safely call other + implementations without recursing to itself. + """ + pluginmanager.register(_RetryHelper) + + +def pytest_configure(config: pytest.Config) -> None: + """ + Register the :py:deco:`pytest.mark.retry` marker. + """ + help_text = ' '.join(""" + retry[(retries=1, exceptions=Exception, reset_fixtures=True)]: + + mark the test for retrying upon failure. + + Args: + retries (int): + Max number of retries for the (sub-)test; + exceptions (type[Exception] | tuple[type[Exception], ...]): + Error types which trigger a retry when caught; + reset_fixtures (bool | Collection[str]): + Names of function-scoped fixtures to reset between retries, + `True` reset all such fixtures, + `False` none thereof + """.split()) + config.addinivalue_line('markers', help_text) + + +# TODO: do we need to call `pytest_runtest_teardown()` between retries +# to make sure that the reset fixtures are properly cleaned up?` + + +def pytest_terminal_summary( + terminalreporter: TerminalReporter, config: pytest.Config, +) -> None: + """ + Write a summary section about rerun tests. + """ + def get_summary(status: _Status, entries: list[_RetryEntry]) -> str: + return f'{_pluralize("test", len(entries))} {status} with retries' + + def group_subtests( + entries: list[_RetryEntry] + ) -> dict[str, list[_RetryEntry]]: + result: dict[str, list[_RetryEntry]] = {} + for entry in entries: + result.setdefault(entry.full_original_name, []).append(entry) + return result + + formatting = {'yellow': True} + write_line: Callable[[str], None] = partial( + terminalreporter.write_line, **formatting # type: ignore + ) + write_header = partial( + terminalreporter.write_sep, '=', 'retries summary', **formatting + ) + write_newline = partial(write_line, '') + try: + verbosity: int = config.get_verbosity() # type: ignore + except AttributeError: # pytest < 8.0 + verbosity = int(config.option.verbose) + retry_entries: dict[_Status, list[_RetryEntry]] = {} + for entry in _RetryEntry.get_entries(config): + retry_entries.setdefault(entry.status, []).append(entry) + if not retry_entries: + return + if verbosity > 0: + write_newline() + write_header() + write_newline() + for status, entries in retry_entries.items(): + write_line(get_summary(status, entries) + ':') + for entry in entries: + write_line( + f' {entry.full_name}: ' + f'retried {_pluralize("time", entry.retries)}' + ) + write_newline() + else: + write_header() + for status, entries in retry_entries.items(): + tests: list[str] = [] + for name, children in group_subtests(entries).items(): + if len(children) == 1: + tests.append(children[0].full_name) + else: + msg = f'{name} ({_pluralize("subtest", len(children))})' + tests.append(msg) + write_line(f'{get_summary(status, entries)}: {", ".join(tests)}') From a6f5e2966d5d006d76c7977a60b84df2fef27c8e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 00:12:42 +0200 Subject: [PATCH 062/111] Test for `@pytest.mark.retry` tests/conftest.py::pytest_configure() Fixed formatting of the line added by `config.addinivalue_line()` so that usage of the marker does not result in a `PytestUnknownMarkWarning` tests/test_retry_tests.py test_fixture_scoping() Test that `@pytest.mark.retry(reset_fixtures=...)` is handled correctly test_fixture_teardown() Test that stale fixtures created by reruns are torn down correctly test_exception_restrictions() Test that `@pytest.mark.retry(exceptions=...)` is handled correctly --- tests/conftest.py | 6 +- tests/test_retry_tests.py | 599 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 603 insertions(+), 2 deletions(-) create mode 100644 tests/test_retry_tests.py diff --git a/tests/conftest.py b/tests/conftest.py index 41deeb64..76ebd9f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ A simple :py:deco:`pytest.mark.retry` decorator. Function-scoped fixtures are re-fetched bewteen retries. """ +from __future__ import annotations + import dataclasses from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, @@ -300,11 +302,11 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry[(retries=1, exceptions=Exception, reset_fixtures=True)]: + retry(retries=1, exceptions=Exception, reset_fixtures=True): mark the test for retrying upon failure. - Args: + Args (all optional): retries (int): Max number of retries for the (sub-)test; exceptions (type[Exception] | tuple[type[Exception], ...]): diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py new file mode 100644 index 00000000..0b6ee307 --- /dev/null +++ b/tests/test_retry_tests.py @@ -0,0 +1,599 @@ +""" +Tests to make sure that our :py:deco:`pytest.mark.retry` decorator +works. +""" +from __future__ import annotations + +import re +import textwrap +from collections.abc import Callable, Generator +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from shutil import rmtree +from typing import Literal +from typing_extensions import Self + +import pytest + + +pytest_plugins = ('pytester',) + +_Status = Literal['passed', 'failed', 'skipped'] +_RunPytest_Method = Literal[ + 'runpytest', 'runpytest_inprocess', 'runpytest_subprocess', +] +_RunPytest = Callable[..., pytest.RunResult] +_RunnerGetter = Callable[[str, str], _RunPytest] + + +TEST_COUNTERS = """ +from __future__ import annotations +from itertools import count +from typing import Literal + +import pytest + + +@pytest.fixture +def func_scoped_counter() -> count: + return count() + + +@pytest.fixture(scope='module') +def module_scoped_counter() -> count: + return count() + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 2), # This passes with 2 retries + ('func', 6), # This fails with 3 retries + ('module', 4), # This fails with 3 retries (counter now at 3) + ('module', 5)] # This passes with 1 retry (counter now at 5) +) +@pytest.mark.retry(3, reset_fixtures=False) +def test_dynamic_fixtures_persisted( + request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, +) -> None: + ''' + Test counter fixtures that are requested dynamically via the + ``request`` fixture; function-scoped fixtures persist between + test retries. + ''' + counter = request.getfixturevalue(scope + '_scoped_counter') + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 3), # This passes with 3 retries + ('func', 4), # This fails with 3 retries + ('module', 4), # This passes (counter now at 6) + ('module', 9)] # This passes with 2 retries (counter now at 9) +) +@pytest.mark.retry(3, reset_fixtures=False) +def test_static_fixtures_persisted( + func_scoped_counter: Iterable[int], + module_scoped_counter: Iterable[int], + scope: Literal['func', 'module'], + n: int, +) -> None: + ''' + Test counter fixtures that are requested by name; function-scoped + fixtures persist between test retries. + ''' + if scope == 'func': + counter = func_scoped_counter + else: + counter = module_scoped_counter + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 1), # This fails with 1 retry + ('module', 11)] # This passes with 1 retry (counter now at 11) +) +@pytest.mark.retry # Counters reset between retries +def test_dynamic_fixtures_reset( + request: pytest.FixtureRequest, scope: Literal['func', 'module'], n: int, +) -> None: + ''' + Test counter fixtures that are requested dynamically via the + ``request`` fixture; function-scoped fixtures are reset between + test retries. + ''' + counter = request.getfixturevalue(scope + '_scoped_counter') + assert next(counter) >= n + + +@pytest.mark.parametrize( + ('scope', 'n'), + [('func', 0), # This passes + ('func', 1), # This fails with 2 retries + ('module', 14)] # This passes with 2 retries (counter now at 14) +) +@pytest.mark.retry(2) # Ditto above +def test_static_fixtures_reset( + func_scoped_counter: Iterable[int], + module_scoped_counter: Iterable[int], + scope: Literal['func', 'module'], + n: int, +) -> None: + ''' + Test counter fixtures that are requested by name; function-scoped + fixtures are reset between test retries. + ''' + if scope == 'func': + counter = func_scoped_counter + else: + counter = module_scoped_counter + assert next(counter) >= n +""" +TEST_TEARDOWN = """ +from __future__ import annotations + +import os +import tempfile +from collections.abc import Callable, Generator +from pathlib import Path + +import pytest + + +@pytest.fixture(scope='module') +def my_temp_dir(pytestconfig: pytest.Config) -> Generator[Path, None, None]: + path: Path | None = getattr(pytestconfig.option, 'my_temp_dir', None) + if path is None: + with tempfile.TemporaryDirectory() as tmpdir: + yield Path(tmpdir) + else: + yield path + + +def _tempfile(*args, **kwargs) -> Path: + handle, path = tempfile.mkstemp(*args, **kwargs) + try: + return Path(path) + finally: + os.close(handle) + + +@pytest.fixture +def maketemp( + my_temp_dir: Path, +) -> Generator[Callable[..., Path], None, None]: + paths: list[Path] = [] + try: + def _maketemp(*args, **kwargs) -> Path: + path = _tempfile(*args, **kwargs) + paths.append(path) + print(f'maketemp() @ {id(_maketemp):#x}: created tempfile {path}') + return path + + yield _maketemp + finally: + for path in paths: + path.unlink(missing_ok=True) + print(f'maketemp() @ {id(_maketemp):#x}: removed tempfile {path}') + + +@pytest.mark.retry(reset_fixtures=True) +def test_no_fixture_reset( + my_temp_dir: Path, maketemp: Callable[..., Path], +) -> None: + path = maketemp(dir=my_temp_dir) + assert False + + +@pytest.mark.retry(2, reset_fixtures=False) +def test_with_fixture_reset( + my_temp_dir: Path, maketemp: Callable[..., Path], +) -> None: + path = maketemp(dir=my_temp_dir) + assert False +""" +TEST_EXCEPTIONS = """ +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +import pytest + + +@pytest.fixture +def items() -> Iterable[Any]: + return iter(['1', None, '', '-1']) + + +@pytest.mark.retry(3, reset_fixtures=('foo',)) # Not resetting `items` +def test_all_xc_types(items: Iterable[Any]) -> None: + ''' + This should pass after 3 retries because the last item fulfills the + criterion. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry(3, exceptions=AssertionError, reset_fixtures=()) +def test_one_xc_type(items: Iterable[Any]) -> None: + ''' + This should fail after 1 retry because the second item triggers a + :py:class:`TypeError`. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry(reset_fixtures=False) +@pytest.mark.retry(exceptions=TypeError) +@pytest.mark.retry(exceptions=AssertionError) +def test_two_xc_types(items: Iterable[Any]) -> None: + ''' + This should fail after 2 retries because the third item triggers a + :py:class:`ValueError`. + + Note: + The three decorators stack to give 3 retries and to accept both + :py:class:`AssertionError` and :py:class:`TypeError`. + ''' + assert int(next(items)) < 0 + + +@pytest.mark.retry( + 3, + exceptions=(AssertionError, TypeError, ValueError), + reset_fixtures=False, +) +def test_three_xc_types(items: Iterable[Any]) -> None: + ''' + This should pass after 3 retries because the last item fulfills the + criterion, and the preceding errors are all included in the + ``exceptions`` argument to the wrapper. + ''' + assert int(next(items)) < 0 +""" + + +@dataclass +class _TestOutcome: + name: str = '' + status: _Status = 'passed' + retries: int = 0 + + def subtest( + self, + *params: str, + status: _Status | None = None, + retries: int | None = None, + ) -> Self: + if status is None: + status = self.status + if retries is None: + retries = self.retries + name = f'{self.name}[{"-".join(params)}]' + return type(self)(name, status, retries) + + +@dataclass +class _TestModule: + """ + Helper object for running a test module. + """ + name: str + content: str + expected_outcomes: dict[str, list[_TestOutcome]] + pytester: pytest.Pytester + conftest: str | None = None + + def __post_init__(self) -> None: + self.content = self._strip(self.content) + if self.conftest: + self.conftest = self._strip(self.conftest) + + def run( + self, + *args: str, + check_results: bool = False, + check_summary: Literal['verbose', 'concise'] | None = None, + check_warnings: int | None = None, + runner: _RunPytest_Method = 'runpytest', + ) -> pytest.RunResult: + """ + Args: + *args (str): + Passed to :py:meth:`pytester.Pytester.runpytest` + check_results (bool): + If true, check that the test outcomes are as expected + using :py:meth:`pytester.Pytester.assert_outcomes` + check_summary (bool): + If true, check that the 'retries summary' report section + is written with the expected content indicating test + results and number of retries + check_warnings (int | None): + If an integer and if ``check_results`` is true, also + check that the number of captured warnings match + runner (Literal['runpytest', 'runpytest_inprocess', \ +'runpytest_subprocess']): + The :py:class:`pytest.Pytester` method used to run the + test module + + Returns: + :py:class:`pytest.RunResult` object returned by the + :py:class:`pytest.Pytester` method + """ + tempfiles: list[Path] = [] + tempdirs: list[Path] = [] + try: + conftests: list[str] = [self._get_proj_conftest().read_text()] + if self.conftest: + conftests.append(self.conftest) + # Create separate conftest.py in nested subdirs to avoid + # hook-func implementations stepping oer one another + path = self.pytester.path + for i, conftest in enumerate(conftests): + if i: + path /= 'nested' + path.mkdir() + tempdirs.append(path) + conftest_file = path / 'conftest.py' + conftest_file.write_text(conftest) + tempfiles.append(conftest_file) + module = path / f'{self.name}.py' + module.write_text(self.content) + tempfiles.append(module) + result = getattr(self.pytester, runner)(*args, str(module)) + if check_results: + self.check_results(result, check_warnings) + if check_summary is not None: + if check_summary == 'verbose': + self.check_verbose_summary(result) + else: + self.check_concise_summary(result) + return result + finally: + for path in tempfiles: + try: + path.unlink(missing_ok=True) + except OSError: + pass + else: + print('Removed temppath', path) + for path in reversed(tempdirs): + try: + rmtree(path) + except OSError: + pass + else: + print('Removed tempdir', path) + + def check_results( + self, result: pytest.RunResult, warnings: int | None = None, + ) -> None: + counts: dict[_Status, int] = {} + for outcomes in self.expected_outcomes.values(): + for outcome in outcomes: + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + result.assert_outcomes(warnings=warnings, **counts) + + def check_verbose_summary(self, result: pytest.RunResult) -> None: + lines: list[str] = [] + counts: dict[_Status, int] = {} + for outcomes in self.expected_outcomes.values(): + for outcome in outcomes: + lines.append( + f'.*::{re.escape(outcome.name)} +{outcome.status.upper()}', + ) + if not outcome.retries: + continue + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + lines.append(r'.*{}.*retried {} time{}'.format( + re.escape(outcome.name), + outcome.retries, + '' if outcome.retries == 1 else 's', + )) + lines.extend( + self._format_header(status, n) for status, n in counts.items() + ) + + print(f'Expecting these lines in the output: {lines!r}...') + result.stdout.re_match_lines_random(lines) + + def check_concise_summary(self, result: pytest.RunResult) -> None: + lines: list[str] = [] + counts: dict[_Status, int] = {} + test_names: dict[_Status, dict[str, set[str]]] = {} + consolidated_names: dict[_Status, set[str]] = {} + for parent_test, outcomes in self.expected_outcomes.items(): + for outcome in outcomes: + if outcome.status == 'failed': + lines.append( + f'{outcome.status.upper()} +' + f'.*::{re.escape(outcome.name)}', + ) + if not outcome.retries: + continue + counts[outcome.status] = counts.get(outcome.status, 0) + 1 + ( + test_names + .setdefault(outcome.status, {}) + .setdefault(parent_test, set()) + .add(outcome.name) + ) + + for status, tests in test_names.items(): + for parent_test, subtests in tests.items(): + names = consolidated_names.setdefault(status, set()) + n = len(subtests) + if n == 1: + names.add(*subtests) + else: + names.add('{} ({} subtest{})'.format( + parent_test, n, '' if n == 1 else 's', + )) + + print(f'Expecting these lines in the output: {lines!r}...') + result.stdout.re_match_lines_random(lines) + + for status, n in counts.items(): + header = self._format_header(status, n) + names = consolidated_names[status] + print(f'Expecting line in the output: "{header}: <...>"...') + print(f'Expecting these names in said line: {names!r}...') + line = self._find_line(header + ':', str(result.stdout)) + for test_name in names: + assert test_name in line + + @staticmethod + def _find_line(pattern: str, text: str) -> str: + pattern = f'^.*{pattern}.*' + maybe_match = re.search(pattern, text, re.MULTILINE) + if not maybe_match: + raise ValueError(f'Cannot find {pattern!r} in {text!r}') + return maybe_match.group() + + @staticmethod + def _format_header(status: _Status, n: int) -> str: + return '{} test{} {} with retries'.format( + n, '' if n == 1 else 's', status, + ) + + @staticmethod + def _get_proj_conftest() -> Path: + return Path(__file__).parent / 'conftest.py' + + @staticmethod + def _strip(text: str) -> str: + return textwrap.dedent(text).strip('\n') + + +@pytest.fixture +def counters_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + dynamic_p = _TestOutcome('test_dynamic_fixtures_persisted').subtest + static_p = _TestOutcome('test_static_fixtures_persisted').subtest + dynamic_r = _TestOutcome('test_dynamic_fixtures_reset').subtest + static_r = _TestOutcome('test_static_fixtures_reset').subtest + outcomes = { + 'test_dynamic_fixtures_persisted': [ + dynamic_p('func-0'), + dynamic_p('func-2', retries=2), + dynamic_p('func-6', status='failed', retries=3), + dynamic_p('module-4', status='failed', retries=3), + dynamic_p('module-5', retries=1), + ], + 'test_static_fixtures_persisted': [ + static_p('func-3', retries=3), + static_p('func-4', status='failed', retries=3), + static_p('module-4'), + static_p('module-9', retries=2), + ], + 'test_dynamic_fixtures_reset': [ + dynamic_r('func-0'), + dynamic_r('func-1', status='failed', retries=1), + dynamic_r('module-11', retries=1), + ], + 'test_static_fixtures_reset': [ + static_r('func-0'), + static_r('func-1', status='failed', retries=2), + static_r('module-14', retries=2), + ], + } + yield _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) + + +@pytest.fixture +def teardown_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + yield _TestModule( + 'test_teardown', + TEST_TEARDOWN, + { + 'test_no_fixture_reset': + [_TestOutcome('test_no_fixture_reset', 'failed', 1)], + 'test_with_fixture_reset': + [_TestOutcome('test_with_fixture_reset', 'failed', 2)], + }, + pytester, + conftest=""" + from __future__ import annotations + + from pathlib import Path + + import pytest + + + def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + '--my-temp-dir', + type=Path, + help=f'persisted tempdir location for {__file__!r}', + ) + """, + ) + + +@pytest.fixture +def exceptions_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + yield _TestModule( + 'test_exceptions', + TEST_EXCEPTIONS, + { + 'test_all_xc_types': + [_TestOutcome('test_all_xc_types', retries=3)], + 'test_one_xc_type': + [_TestOutcome('test_one_xc_type', 'failed', 1)], + 'test_two_xc_types': + [_TestOutcome('test_two_xc_types', 'failed', 2)], + 'test_three_xc_types': + [_TestOutcome('test_three_xc_types', retries=3)], + }, + pytester, + ) + + +@pytest.mark.parametrize('verbose', [True, False]) +def test_fixture_scoping(counters_module: _TestModule, verbose: bool) -> None: + """ + Test that the decorator correctly handles scoped fixtures. + """ + run = partial(counters_module.run, check_results=True, check_warnings=0) + if verbose: + run('--verbose', check_summary='verbose') + else: + run(check_summary='concise') + + +def test_fixture_teardown( + tmp_path_factory: pytest.TempPathFactory, teardown_module: _TestModule, +) -> None: + """ + Test that the decorator correctly handles teardown for additional + fixture copies incurred by retries. + """ + tempdir = tmp_path_factory.mktemp('my_temp') + print(tempdir) + teardown_module.run( + '--verbose', f'--my-temp-dir={tempdir}', + check_results=True, check_summary='verbose', check_warnings=0, + ) + files = {path.name for path in tempdir.iterdir()} + assert not files + + +def test_exception_restrictions(exceptions_module: _TestModule) -> None: + """ + Test that the decorator correctly handles failures owing to + different exception classes. + """ + exceptions_module.run( + '--verbose', + check_results=True, check_summary='verbose', check_warnings=0, + ) From 68e783f544af1c7746ced83e909298581e6006e8 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 07:31:57 +0200 Subject: [PATCH 063/111] Check retried fixture teardown order tests/conftest.py::_RetryHelper._reset_between_retries() Now calling `FixtureDef.finish()` on the cleaned-up function-scoped fixtures tests/test_retry_tests.py::test_fixture_teardown() Added check that the teardown events happen in the expected order (old fixture value torn down before the new value is used) --- tests/conftest.py | 58 +++++++++++++--- tests/test_retry_tests.py | 141 +++++++++++++++++++++++++++++++++----- 2 files changed, 170 insertions(+), 29 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 76ebd9f5..f1f0ce19 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ from __future__ import annotations import dataclasses +import warnings from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, ) @@ -16,7 +17,10 @@ ) import pytest +from _pytest.compat import NOTSET +from _pytest.fixtures import SubRequest from _pytest.nodes import Node +from _pytest.scope import Scope from _pytest.unittest import TestCaseFunction try: from pytest import TerminalReporter @@ -29,6 +33,8 @@ FCls = TypeVar('FCls', bound=type[TestCaseFunction]) T = TypeVar('T') +_FUNCTION_SCOPE = Scope.Function + class _PyfuncCallImpl(Protocol): def __call__(self, *, pyfuncitem: pytest.Function) -> Any: @@ -98,6 +104,7 @@ def _name_prefixes(self) -> tuple[str, str]: _RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() +_NEXTITEMS_KEY = pytest.StashKey[dict[pytest.Item, pytest.Item | None]]() @final @@ -189,13 +196,46 @@ def _reset_between_retries( reset_fixtures: bool = False, should_reset: Callable[[str], bool] = lambda _: False, ) -> None: - def clear_fixture_cache(fixture_def: pytest.FixtureDef[Any]) -> None: - if ( - fixture_def.scope == 'function' - and should_reset(fixture_def.argname) - and hasattr(fixture_def, 'cached_result') + """ + Note: + This makes HEAVY use of :py:mod`_pytest` internals. + """ + def cleanup_fixture(fdef: pytest.FixtureDef[Any]) -> None: + if not ( + fdef.scope == 'function' + and should_reset(fdef.argname) + and getattr(fdef, 'cached_result', None) is not None ): - fixture_def.cached_result = None + return + fdef.cached_result = None + finalize(fdef) + + def finalize(fdef: pytest.FixtureDef[Any]) -> None: + assert fdef.scope == 'function' + + # Plagiarized code from + # `FixtureRequest._get_active_fixture_def()` + try: + callspec = func.callspec + except AttributeError: + callspec = None + if callspec is not None and fdef.argname in callspec.params: + value = callspec.params[fdef.argname] + index = callspec.indices[fdef.argname] + else: + value, index = NOTSET, 0 + + with warnings.catch_warnings(): + warnings.simplefilter( + 'ignore', pytest.PytestDeprecationWarning, + ) + fdef.finish(SubRequest( + request=func._request, + scope=_FUNCTION_SCOPE, + param=value, + param_index=index, + fixturedef=fdef, + )) def unique( items: Iterable[T], key: Callable[[T], Hashable] = id, @@ -226,7 +266,7 @@ def iter_all_fixture_defs( # the `TopRequest` instance that `func` has (`._request`) func._initrequest() for fixture_def in unique(iter_all_fixture_defs(func)): - clear_fixture_cache(fixture_def) + cleanup_fixture(fixture_def) else: # Fixture values will naturally refill, possibly from caches func.funcargs.clear() @@ -319,10 +359,6 @@ def pytest_configure(config: pytest.Config) -> None: config.addinivalue_line('markers', help_text) -# TODO: do we need to call `pytest_runtest_teardown()` between retries -# to make sure that the reset fixtures are properly cleaned up?` - - def pytest_terminal_summary( terminalreporter: TerminalReporter, config: pytest.Config, ) -> None: diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 0b6ee307..b18df4e0 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -5,13 +5,15 @@ from __future__ import annotations import re +import pprint import textwrap -from collections.abc import Callable, Generator +from collections.abc import Callable, Generator, Iterable from dataclasses import dataclass from functools import partial +from operator import attrgetter from pathlib import Path from shutil import rmtree -from typing import Literal +from typing import Any, Literal, cast from typing_extensions import Self import pytest @@ -139,6 +141,7 @@ def test_static_fixtures_reset( import os import tempfile from collections.abc import Callable, Generator +from functools import partial from pathlib import Path import pytest @@ -153,6 +156,11 @@ def my_temp_dir(pytestconfig: pytest.Config) -> Generator[Path, None, None]: else: yield path +@pytest.fixture(scope='module') +def my_log(pytestconfig: pytest.Config) -> Path | None: + path: Path | None = getattr(pytestconfig.option, 'my_log', None) + return path + def _tempfile(*args, **kwargs) -> Path: handle, path = tempfile.mkstemp(*args, **kwargs) @@ -164,25 +172,41 @@ def _tempfile(*args, **kwargs) -> Path: @pytest.fixture def maketemp( - my_temp_dir: Path, + my_temp_dir: Path, my_log: Path | None, ) -> Generator[Callable[..., Path], None, None]: paths: list[Path] = [] - try: - def _maketemp(*args, **kwargs) -> Path: - path = _tempfile(*args, **kwargs) - paths.append(path) - print(f'maketemp() @ {id(_maketemp):#x}: created tempfile {path}') - return path + def _maketemp(*args, **kwargs) -> Path: + path = _tempfile(*args, **kwargs) + paths.append(path) + log(f'created tempfile {path}') + return path + + log = partial(_log, _maketemp, my_log) + try: yield _maketemp finally: for path in paths: path.unlink(missing_ok=True) - print(f'maketemp() @ {id(_maketemp):#x}: removed tempfile {path}') + log(f'removed tempfile {path}') + + +def _log(maketemp: Any, my_log: Path | None, msg: str) -> None: + chunks: list[str] = [ + os.environ['PYTEST_CURRENT_TEST'], + f'maketemp() @ {id(maketemp):#x}', + msg, + ] + msg = ': '.join(chunks) + print(msg) + if my_log is None: + return + with my_log.open(mode='a') as fobj: + print(msg, file=fobj) @pytest.mark.retry(reset_fixtures=True) -def test_no_fixture_reset( +def test_with_fixture_reset( my_temp_dir: Path, maketemp: Callable[..., Path], ) -> None: path = maketemp(dir=my_temp_dir) @@ -190,7 +214,7 @@ def test_no_fixture_reset( @pytest.mark.retry(2, reset_fixtures=False) -def test_with_fixture_reset( +def test_no_fixture_reset( my_temp_dir: Path, maketemp: Callable[..., Path], ) -> None: path = maketemp(dir=my_temp_dir) @@ -470,6 +494,34 @@ def _strip(text: str) -> str: return textwrap.dedent(text).strip('\n') +def _identical_items_are_adjacent(items: Iterable[Any]) -> bool: + """ + Example: + >>> _identical_items_are_adjacent([]) + True + >>> _identical_items_are_adjacent([1]) + True + >>> _identical_items_are_adjacent([1, 10]) + True + >>> _identical_items_are_adjacent([1, 10, 1]) + False + >>> _identical_items_are_adjacent('AAcCb') + True + >>> _identical_items_are_adjacent('AcCAb') + False + """ + past: set[Any] = set() + sentinel = object() + last: Any = sentinel + for item in items: + if last is not sentinel and last != item: + past.add(last) + if item in past: + return False + last = item + return True + + @pytest.fixture def counters_module( pytester: pytest.Pytester, @@ -515,9 +567,9 @@ def teardown_module( TEST_TEARDOWN, { 'test_no_fixture_reset': - [_TestOutcome('test_no_fixture_reset', 'failed', 1)], + [_TestOutcome('test_no_fixture_reset', 'failed', 2)], 'test_with_fixture_reset': - [_TestOutcome('test_with_fixture_reset', 'failed', 2)], + [_TestOutcome('test_with_fixture_reset', 'failed', 1)], }, pytester, conftest=""" @@ -534,6 +586,11 @@ def pytest_addoption(parser: pytest.Parser) -> None: type=Path, help=f'persisted tempdir location for {__file__!r}', ) + parser.addoption( + '--my-log', + type=Path, + help=f'log file location for tempfile creation/deletion', + ) """, ) @@ -576,16 +633,64 @@ def test_fixture_teardown( ) -> None: """ Test that the decorator correctly handles teardown for additional - fixture copies incurred by retries. + fixture copies incurred by retries; in particular, superseded + function-scoped fixtures should be torn down before their + replacements are set up. """ + Stage = Literal['setup', 'call', 'teardown'] + + @dataclass + class LogEntry: + test: str + stage: Stage + fixture_id: int + msg: str + + @classmethod + def parse_line(cls, line: str) -> Self: + test, ident, *remainder = line.split(': ') + msg = ': '.join(remainder) + test_match = re.fullmatch( + r'(.+) +\((setup|call|teardown)\)', test, + ) + assert test_match + test, stage = test_match.group(1, 2) + assert stage in ('setup', 'call', 'teardown') + ident_match = re.fullmatch( + r'maketemp\(\) @ 0x([0-9a-f]+)', ident, + ) + assert ident_match + fixture_id = int(ident_match.group(1), base=16) + return cls(test, cast(Stage, stage), fixture_id, msg) + tempdir = tmp_path_factory.mktemp('my_temp') - print(tempdir) + log = tempdir / 'tempfiles.log' teardown_module.run( - '--verbose', f'--my-temp-dir={tempdir}', + '--verbose', f'--my-temp-dir={tempdir}', f'--my-log={log}', check_results=True, check_summary='verbose', check_warnings=0, ) + + # Check that all the tempfiles ahve been wiped files = {path.name for path in tempdir.iterdir()} - assert not files + assert not (files - {log.name}) + + # Check that tempfiles are deleted as soon as the fixture value + # that created them went obsolete, before the next rerun; + # we can verify that by checking that the ids of the `makefile()` + # fixtures appear in contiguous blocks + with log.open() as fobj: + entries = [LogEntry.parse_line(line.rstrip('\n')) for line in fobj] + pprint.pprint(entries) + field: tuple[str, ...] | str + for field in ('test', 'stage'), 'fixture_id': + if isinstance(field, str): + getter: Callable[[LogEntry], Any] = attrgetter(field) + else: + getter = attrgetter(*field) + values = [getter(entry) for entry in entries] + assert _identical_items_are_adjacent(values), ( + f'Inconsistency in {field} order: {values!r}' + ) def test_exception_restrictions(exceptions_module: _TestModule) -> None: From 962242d9742dbae478a6993280a15f54a8dc655e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 08:21:36 +0200 Subject: [PATCH 064/111] Update `test_child_procs.py` tests/test_child_procs.py ResultMismatch compare() New helper method for raising an error pytest_runtest_makereport() New helper method for truncating the traceback so that it stops at the frame calling `.compare()` test_apply_mp_patches() test_profiling_multiproc_script() Added `@pytest.mark.retry` decorators to deal with flakiness test_profiling_bare_python() - Added `@pytest.mark.retry` decorators to deal with flakiness - Added timeout (5 s) to the `subprocess.run()` call --- tests/test_child_procs.py | 118 +++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 22 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cadf145e..e03e41d2 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -5,6 +5,7 @@ import enum import inspect import multiprocessing.pool +import operator import os import re import shlex @@ -24,6 +25,7 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic +from types import TracebackType from typing import Any, Generic, Literal, TypeVar, cast, final, overload from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -38,6 +40,9 @@ from line_profiler._child_process_profiling.threading_patches import ( SHOULD_PATCH_THREADING, ) +from line_profiler._child_process_profiling.multiprocessing_patches import ( + _Poller, +) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) @@ -45,6 +50,8 @@ T = TypeVar('T') +T1 = TypeVar('T1') +T2 = TypeVar('T2') TCtx_ = TypeVar('TCtx_') PS = ParamSpec('PS') C = TypeVar('C', bound=Callable[..., Any]) @@ -52,6 +59,13 @@ NUM_NUMBERS = 100 NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) + +# XXX: owing to the shenanigans in +# `line_profiler._child_process_profiling.multiprocessing_patches`, +# there is a risk that failing child processes are not properly +# `.terminate()`-ed. So just put in a timeout... +_NUM_RETRIES = 2 +_SUBPROC_TIMEOUT = 5 # Seconds _DEBUG = True @@ -428,6 +442,18 @@ def curated_profiler() -> Generator[LineProfiler, None, None]: yield prof +@pytest.fixture(autouse=True) +def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: + """ + Truncate the traceback of raised :py:class`ResultMismatch` for more + useful error attribution. + """ + try: + pytestconfig.pluginmanager.register(ResultMismatch) + except ValueError: # Already registered + pass + + # ========================== Helper functions ========================== @@ -435,11 +461,13 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +@final class ResultMismatch(ValueError): def __init__( self, expected: Any, actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, + _trunc_tb: int = 0, ) -> None: msg = f'expected: {expected}' if actual != _NotSupplied.NOT_SUPPLIED: @@ -447,6 +475,54 @@ def __init__( super().__init__(msg) self.expected = expected self.actual = actual + self._trunc_tb = max(0, _trunc_tb) + + @classmethod + def compare( + cls, expected_: T1, actual_: T2, /, *, + comparator: Callable[[T1, T2], bool] = operator.eq, + expected: str | None = None, + actual: str | None = None, + ) -> None: + if comparator(expected_, actual_): + return + raise cls( + expected_ if expected is None else expected, + actual_ if actual is None else actual, + _trunc_tb=1, + ) + + @classmethod + def pytest_runtest_makereport( + cls, item: pytest.Item, call: pytest.CallInfo, + ) -> Any: + """ + Truncate the tracebacks of instances so that pytest outputs are + more useful and actually stops at the frame where the comparends + are shown. + """ + impl: Callable[..., Any] + impl = item.config.pluginmanager.subset_hook_caller( + 'pytest_runtest_makereport', [cls], + ) + make_report = partial(impl, item=item, call=call) + + xc = call.excinfo + if xc is None: + return make_report() + if not (isinstance(xc.value, cls) and xc.value._trunc_tb): + return make_report() + + tb_stack: list[TracebackType] = [xc.tb] + while tb_stack[-1].tb_next: + tb_stack.append(tb_stack[-1].tb_next) + if len(tb_stack) <= xc.value._trunc_tb: + return make_report() + tb_stack[-(xc.value._trunc_tb + 1)].tb_next = None + + del tb_stack # Help the GC + call.excinfo = xc.from_exception(xc.value.with_traceback(xc.tb)) + return make_report(call=call) @property def rich_message(self) -> str: @@ -837,11 +913,11 @@ def _search_cache_logs( flags: int = 0, ) -> None: entries = cache._gather_debug_log_entries() - if bool(entries) != expecting_logs: - raise ResultMismatch( - 'logs' if expecting_logs else 'no logs', - repr(entries) if entries else 'nothing' - ) + ResultMismatch.compare( + expecting_logs, bool(entries), + expected='logs' if expecting_logs else 'no logs', + actual=repr(entries) if entries else 'nothing', + ) if not expecting_logs: return text_chunks: list[str] = [entry.to_text() for entry in entries] @@ -1086,10 +1162,7 @@ def _run_test_module( # - The result is correctly calculated expected = nnums * (nnums + 1) // 2 output_lines = proc.stdout.splitlines() - if output_lines[0] != str(expected): - raise ResultMismatch( - f'result {expected}', f'output lines: {output_lines}', - ) + ResultMismatch.compare(str(expected), output_lines[0]) # - Temporary `.pth` file(s) created by `~~.pth_hook` has # been cleaned up assert _preserve_pth_files.get_pth_files() == old_pth_files @@ -1130,10 +1203,9 @@ def _check_output(output: str, tag: str, nhits: int) -> None: actual_nhits += int(n) except Exception: pass - if actual_nhits == nhits: - return - raise ResultMismatch( - f'{nhits} hit(s) on line(s) tagged with {tag!r}', actual_nhits, + ResultMismatch.compare( + nhits, actual_nhits, + expected=f'{nhits} hit(s) on line(s) tagged with {tag!r}', ) @@ -1406,6 +1478,7 @@ def test_cache_setup_main_process( [(True, 'no-profiler'), (False, 'with-profiler')]) @pytest.mark.parametrize(('debug', 'label4'), [(True, 'with-debug'), (False, 'no-debug')]) +@_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], curated_profiler: LineProfiler, @@ -1493,6 +1566,7 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) +@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) @pytest.mark.parametrize(('debug', 'label'), @@ -1576,7 +1650,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: num_returns = sum( nhits for lineno, nhits, _ in line_entries if lineno in return_lines ) - assert num_returns == expected_ncalls + ResultMismatch.compare(expected_ncalls, num_returns) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests @@ -1718,6 +1792,8 @@ def test_running_multiproc_script( @_fuzz_prof_mp_1 @_fuzz_prof_mp_2 +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... @@ -1779,12 +1855,6 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. """ - # XXX: owing to the shenanigans in - # `line_profiler._child_process_profiling.multiprocessing_patches`, - # there is a risk that failing child processes are not properly - # `.terminate()`-ed. So just put in a timeout... - timeout = 5 # Seconds - # How many calls do we expect? nhits = dict.fromkeys( ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, @@ -1825,13 +1895,15 @@ def test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=timeout, + timeout=_SUBPROC_TIMEOUT, debug_log=( 'debug.log' if prof_child_procs and _DEBUG else None ), ) +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize(('use_subprocess', 'label1'), [(True, 'subprocess.run'), (False, 'os.system')]) @pytest.mark.parametrize(('prof_child_procs', 'label2'), @@ -1900,7 +1972,9 @@ def test_profiling_bare_python( raise RuntimeError('called process failed') """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) - proc = _run_subproc(cmd, text=True, capture_output=True) + proc = _run_subproc( + cmd, text=True, capture_output=True, timeout=_SUBPROC_TIMEOUT, + ) nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} if not prof_child_procs: From 5b61e60a6aca963029cd02d58b568e7249c8549d Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 16:58:36 +0200 Subject: [PATCH 065/111] Test fixes line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__doc__ Increased timeout threshold on the passing test statement because something in CI seems to be slowing code execution of each polling loop down tests/conftest.py::_RetryEntry._name_prefixes Now gracefully handling when the curdir isn't a parent of the file the test function is defined in --- .../_child_process_profiling/multiprocessing_patches.py | 5 ++++- tests/conftest.py | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c5561fa4..eacd0746 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -69,7 +69,10 @@ class _Poller: ... ) ... >>> - >>> with count_until(10).with_cooldown(.01).with_timeout(.25): + >>> with count_until(10).with_cooldown(.01).with_timeout(.5): + ... # Note: we shouldn't really need that much time, but + ... # something in CI seems to be slowing down the polling + ... # loop... ... print('We counted up to 10') We counted up to 10 >>> with count_until(30).with_cooldown(.01).with_timeout(.25): diff --git a/tests/conftest.py b/tests/conftest.py index f1f0ce19..a11bb3aa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -92,7 +92,12 @@ def _name_prefixes(self) -> tuple[str, str]: seen.add(id(node)) if isinstance(node, (pytest.Module, pytest.Package)): if node.path: - path = str(node.path.relative_to(Path.cwd())) + npath = node.path + try: + npath = npath.relative_to(Path.cwd()) + except ValueError: # Not a subpath + pass + path = str(npath) else: path = repr(node) break From 245d08a640d618b08960f985d9e9e0f0bfb40d90 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 18:42:09 +0200 Subject: [PATCH 066/111] WIP: improve coverage line_profiler/_child_process_profiling/multiprocessing_patches.py _Poller __doc__ - Lengthened timeout in passing case (again) since it is slow in CI - Extended example to test these components: - `.poll_while()` - `.with_timeout(None, ...)` - `on_timeout='ignore'` - `on_timeout='warn'` __init__() Minor refactoring to hack coverage __enter__() Now issuing a `_Poller.TimeoutWarning` TimeoutWarning New warning subclass wrap_bootstrap() `# nocover` since `coverage` is having trouble tracing calls in child processes _apply_patches_generic(), _apply_mp_patches() Added `# nocover` to unlikely exception branches --- .../multiprocessing_patches.py | 99 ++++++++++++++----- 1 file changed, 76 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index eacd0746..97dd803c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -54,35 +54,79 @@ class _Poller: Poll a callable until it returns true-y. Example: + >>> import warnings + >>> from contextlib import ExitStack + >>> from functools import partial >>> from itertools import count - >>> from typing import Iterator - >>> - >>> - >>> def count_until(limit: int) -> bool: + >>> from typing import Iterator, Literal + + >>> def count_until( + ... limit: int, mode: Literal['until', 'while'] = 'until', + ... ) -> bool: ... def counter_is_big_enough( ... counter: Iterator[int], limit: int, ... ) -> bool: ... return next(counter) >= limit ... - ... return _Poller.poll_until( - ... counter_is_big_enough, count(), limit, - ... ) + ... def counter_is_small_enough( + ... counter: Iterator[int], limit: int, + ... ) -> bool: + ... return next(counter) < limit ... - >>> - >>> with count_until(10).with_cooldown(.01).with_timeout(.5): + ... # The branches are ultimately equal in results, but we + ... # want to explicitly test both `.poll_until()` and + ... # `.poll_while()` + ... if mode == 'until': + ... get_poller = partial( + ... _Poller.poll_until, counter_is_big_enough, + ... ) + ... else: + ... get_poller = partial( + ... _Poller.poll_while, counter_is_small_enough, + ... ) + ... return get_poller(count(), limit) + + >>> with count_until(10).with_cooldown(.01).with_timeout(1): ... # Note: we shouldn't really need that much time, but ... # something in CI seems to be slowing down the polling ... # loop... ... print('We counted up to 10') We counted up to 10 - >>> with count_until(30).with_cooldown(.01).with_timeout(.25): - ... print('We counted up to 30') \ -# doctest: +NORMALIZE_WHITESPACE + + >>> with ( + ... count_until(100) + ... .with_cooldown(.01) + ... .with_timeout(.5) # `[on_]timeout` separately supplied + ... .with_timeout(on_timeout='ignore') + ... ): + ... print("We probably didn't count up to 100 but whatever") + We probably didn't count up to 100 but whatever + + >>> with ( # doctest: +NORMALIZE_WHITESPACE + ... count_until(30).with_cooldown(.01).with_timeout(.25) + ... ): + ... print('We counted up to 30') Traceback (most recent call last): ... line_profiler..._Poller.Timeout: ... timed out (... s >= 0.25 s) waiting for callback ...counter_is_big_enough... to return true + + >>> with ExitStack() as stack: # doctest: +NORMALIZE_WHITESPACE + ... enter = stack.enter_context + ... enter(warnings.catch_warnings()) + ... warnings.simplefilter('error', _Poller.TimeoutWarning) + ... enter( + ... count_until(30, 'while') + ... .with_cooldown(.01) + ... .with_timeout(.25, 'warn') + ... ) + ... print('We counted up to 30 again') + Traceback (most recent call last): + ... + line_profiler..._Poller.TimeoutWarning: ... + timed out (... s >= 0.25 s) waiting for + callback ...counter_is_small_enough... to return true """ def __init__( self, @@ -91,13 +135,9 @@ def __init__( timeout: float = 0, on_timeout: _OnTimeout = 'error', ) -> None: - if cooldown < 0: - cooldown = 0 - if timeout < 0: - timeout = 0 self._func: Callable[[], Any] = func - self._cooldown = cooldown - self._timeout = timeout + self._cooldown = max(0, cooldown) + self._timeout = max(0, timeout) self._on_timeout = on_timeout def sleep(self): @@ -145,7 +185,7 @@ def error(msg: str) -> NoReturn: raise type(self).Timeout(msg) def warn(msg: str) -> None: - warnings.warn(msg) + warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) diagnostics.log.warning(msg) def ignore(_): @@ -182,6 +222,12 @@ class Timeout(RuntimeError): """ pass + class TimeoutWarning(Timeout, UserWarning): + """ + Issued when a :py:class:`_Poller` is timed out when polling. + """ + pass + def _get_config( config: PathLike[str] | str | bool | None = None, @@ -274,7 +320,7 @@ def wait_for_return( vanilla_impl(self) -@LineProfilingCache._method_wrapper +@LineProfilingCache._method_wrapper # nocover def wrap_bootstrap( cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[BaseProcess, PS], T], @@ -286,6 +332,13 @@ def wrap_bootstrap( Wrap around :py:meth:`BaseProcess._bootstrap` to run ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. + + Note: + This is only invoked in child processes, and :py:mod:`coverage` + seem to be having trouble with them in the current setup, + probably due to issues with .pth file precendence causing + :py:mod:`line_profiler` to be loaded before it. Hence the + ``# nocover``. """ try: return vanilla_impl(self, *args, **kwargs) @@ -398,13 +451,13 @@ def _apply_patches_generic( submod_name = 'multiprocessing.' + submodule try: mod = import_module(submod_name) - except ImportError: + except ImportError: # nocover return for target, patches in targets.items(): if target: try: obj: Any = getattr(mod, target) - except AttributeError: + except AttributeError: # nocover continue name = f'{submod_name}.{target}' else: @@ -462,7 +515,7 @@ def _apply_mp_patches( if reboot_forkserver: try: from multiprocessing import forkserver - except ImportError: # Incompatible platform + except ImportError: # Incompatible platform # nocover pass else: server_instance: forkserver.ForkServer = forkserver._forkserver From 4224ccee30ace66fa234458fd180fe4ed904c98c Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 21:30:53 +0200 Subject: [PATCH 067/111] WIP: refactor cleanup functionalities line_profiler/cleanup.py::Cleanup New base class for objects that do cleanup TODO: migrate `CuratedProfileContext` and `LineProfilingCache` to this class line_profiler/line_profiler_utils.py Migrated `block_indent()` and `make_tempfile()` from `line_profiler/_child_process_profiling/misc_utils.py` to here --- line_profiler/cleanup.py | 582 +++++++++++++++++++++++++++ line_profiler/line_profiler_utils.py | 43 +- 2 files changed, 621 insertions(+), 4 deletions(-) create mode 100644 line_profiler/cleanup.py diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py new file mode 100644 index 00000000..6794e203 --- /dev/null +++ b/line_profiler/cleanup.py @@ -0,0 +1,582 @@ +""" +Utilities for cleaning up after ourselves. +""" +from __future__ import annotations + +import sys +import os +from collections.abc import ( + Callable, Generator, Iterable, Mapping, MutableMapping, Collection, +) +from functools import partial +from operator import setitem +from reprlib import Repr +from textwrap import indent +from pathlib import Path +from types import MethodType +from typing import Any, TypeVar, TypedDict, cast +from typing_extensions import ParamSpec, Self, Unpack + +from .line_profiler_utils import block_indent, make_tempfile +from . import _diagnostics as diagnostics + + +__all__ = ('Cleanup',) + +PS = ParamSpec('PS') +K = TypeVar('K') +V = TypeVar('V') +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) +_Stacks = dict[float, list[Callable[[], Any]]] +_StackContexts = list[_Stacks] + + +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`_CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class _CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> from sys import version_info + + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(_CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = _CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + valid_kwargs = ( + _ReprAttributes.__optional_keys__ + | _ReprAttributes.__required_keys__ + ) + for k, v in kwargs.items(): + if k in valid_kwargs: + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return block_indent(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in ptl.args] + args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) + args.insert(0, get(ptl.func)) + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_items(args, (name + '(', ')'), self.maxargs + 1) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', indent_prefix) for item in items), end, + ]) + + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None + + +_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr + + +class Cleanup: + """ + Object which holds cleanup callbacks. Also provides convenience + methods for creating tempfiles, updating mappings, and setting + attributes on objects. + """ + def __init__(self, *_, **__) -> None: + self._contexts: _StackContexts = [] + + def __enter__(self) -> Self: + """ + Returns: + The instance + + Note: + This context manager is reentrant; entering the context + create a new set of cleanup stacks, which is then cleaned up + on :py:meth:`~.__exit__`. + + Example: + >>> strings = [] + >>> add = strings.append + >>> with Cleanup() as cleanup: + ... cleanup.add_cleanup(add, 'one') + ... # Increased priority + ... cleanup.add_cleanup_with_priority(add, 1, 'two') + ... add('three') + ... with cleanup: + ... # Decreased priority + ... cleanup.add_cleanup_with_priority( + ... add, -1, 'four', + ... ) + ... cleanup.add_cleanup(add, 'five') + ... add('six') + ... add('seven') + ... # Increased priority + ... cleanup.add_cleanup_with_priority(add, 1, 'eight') + ... + >>> strings # doctest: +NORMALIZE_WHITESPACE + ['three', 'six', 'five', 'four', 'seven', 'eight', 'two', + 'one'] + """ + self._contexts.append({}) + return self + + def __exit__(self, *_, **__) -> Any: + """ + Call ``~.cleanup(1)``, clearing the level of cleanup stacks we + previously :py:meth:`~.__enter__`-ed into. + """ + self.cleanup(1) + + # Cleanup methods + + def cleanup(self, levels: int | None = None) -> None: + """ + Pop cleanup callbacks from the internal stacks added via + :py:meth:`~.add_cleanup` etc. and call them in order. + + Args: + levels (int | None): + Number of stack levels to clear; passing :py:const`None` + clears the entire stack of callback stacks + """ + def pop_all_contexts( + contexts: _StackContexts, + ) -> Generator[_Stacks, None, None]: + while contexts: + yield contexts.pop() + + def pop_n_levels_of_contexts( + contexts: _StackContexts, n: int, + ) -> Generator[_Stacks, None, None]: + for _ in range(n): + try: + yield contexts.pop() + except IndexError: # Ran out of levels + return + + pop_contexts: Iterable[_Stacks] + if levels is None: + pop_contexts = pop_all_contexts(self._contexts) + else: + pop_contexts = pop_n_levels_of_contexts(self._contexts, levels) + cleanup = partial(self._cleanup, self._debug_output) + for stacks in pop_contexts: + cleanup(stacks) + + @staticmethod + def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: + ncallbacks_total = sum(len(stack) for stack in stacks.values()) + if not ncallbacks_total: + log('Cleanup aborted (no registered callbacks)') + return + # Bookend the cleanup loop with log messages to help detect if + # child processes are prematurely terminated + log(f'Starting cleanup ({ncallbacks_total} callback(s))...') + ncallbacks_run = 0 + for priority in sorted(stacks, reverse=True): + callbacks = stacks.pop(priority) + while callbacks: + callback = callbacks.pop() + callback_repr = _CALLBACK_REPR(callback) + ncallbacks_run += 1 + try: + callback() + except Exception as e: + state = 'failed' + msg = f'{callback_repr}: {type(e).__name__}: {e}' + else: + state, msg = 'succeeded', f'{callback_repr}' + log( + f'- Cleanup {state} ' + f'({ncallbacks_run}/{ncallbacks_total}): {msg}', + ) + log(f'... cleanup completed ({ncallbacks_total} callback(s))') + + def add_cleanup( + self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Shorthand for calling :py:meth:`~.add_cleanup_with_priority` + with ``priority=0``, which should be considered the default. + """ + self.add_cleanup_with_priority(callback, 0, *args, **kwargs) + + def add_cleanup_with_priority( + self, callback: Callable[PS, Any], priority: float, /, + *args: PS.args, **kwargs: PS.kwargs, + ) -> None: + """ + Add a cleanup callback to the internal stacks. + + Args: + callback (Callable[..., Any]): + Callback to be called at cleanup + priority (float): + Numeric priority value; callbacks with a HIGHER value + are invoked BEFORE those with bigger values + *args, **kwargs: + Arguments ``callback`` should be called with + + Example: + >>> strings = [] + >>> cleanup = Cleanup() + >>> # Default priority + >>> cleanup.add_cleanup(strings.append, 'first') + >>> # Decreased priority + >>> cleanup.add_cleanup_with_priority( + ... strings.append, -1, 'second', + ... ) + >>> # Increased priority + >>> cleanup.add_cleanup_with_priority( + ... strings.append, 1, 'third', + ... ) + >>> cleanup.add_cleanup(strings.append, 'fourth') + >>> assert not strings + >>> cleanup.cleanup() + >>> strings + ['third', 'fourth', 'first', 'second'] + """ + if args or kwargs: + callback = partial(callback, *args, **kwargs) + self._current_context.setdefault(priority, []).append(callback) + header = 'Cleanup callback added' + if priority: + header = f'{header} (priority: {priority})' + self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') + + # Convenience methods + + def update_mapping( + self, + mapping: MutableMapping[K, V], + updates: Mapping[K, V], + *, + _format_debug_msg: Callable[[Mapping[K, V], K, str], str] = ( + lambda mapping, key, change: 'Update {}[{!r}]: {}'.format( + object.__repr__(mapping), key, change, + ) + ), + ) -> None: + """ + Update a mapping with another and add cleanup callbacks to + reverse them. + + Args: + mapping (MutableMapping[K, V]): + Mapping to be updated + updates (Mapping[K, V]): + Mapping containing the updates + + Example: + >>> d1 = {1: 2, 3: 4} + >>> d2 = d1.copy() + >>> updates = {0: -1, 3: 5} + >>> with Cleanup() as cleanup: + ... cleanup.update_mapping(d1, updates) + ... for key, value in updates.items(): + ... assert d1[key] == value + ... + >>> assert d1 == d2 + """ + for key, value in updates.items(): + try: + old = mapping[key] + except KeyError: + self.add_cleanup(mapping.pop, key, None) + change = f'{value!r} (new)' + else: + self.add_cleanup(setitem, mapping, key, old) + change = f'{old!r} -> {value!r}' + self._debug_output(_format_debug_msg(mapping, key, change)) + mapping[key] = value + + def make_tempfile( + self, *, + delete: bool = True, + priority: float = 0, + _format_debug_msg: Callable[[Path], str] = ( + lambda path: f'Created tempfile: {path.name!r}' + ), + **kwargs + ) -> Path: + """ + Create a fresh tempfile with :py:func:`tempfile.mkstemp`. + + Args: + delete (bool): + Whether to remove the file on cleanup + priority (float): + Cleanup priority (see + :py:meth:`~.add_cleanup_with_priority`) + **kwargs: + Passed to :py:func:`tempfile.mkstemp` + + Returns: + path (Path): + Path to the created file. + + Example: + >>> prefix, suffix = 'my_file_', '.txt' + >>> with Cleanup() as cleanup: + ... path = cleanup.make_tempfile( + ... prefix=prefix, suffix=suffix, + ... ) + ... assert path.exists() + ... assert path.name.startswith(prefix) + ... assert path.name.endswith(suffix) + ... + >>> assert not path.exists() + """ + path = make_tempfile(**kwargs) + self._debug_output(_format_debug_msg(path)) + if delete: + self.add_cleanup_with_priority( + path.unlink, priority, missing_ok=True, + ) + return path + + def patch( + self, obj: Any, attr: str, value: Any, *, + name: str | None = None, + cleanup: bool = True, + priority: float = 0, + ) -> None: + """ + Patch an attribute on an object. + + Args: + obj (Any): + Object to be patched + attr (str): + Name of the attribute + value (Any): + Value to be assigned to said attribute of ``obj`` + name (str | None): + Optional name for ``obj`` to be used in debug messages + cleanup (bool): + Whether to reverse the patch (by resetting or deleting + the attribute) on cleanup + priority (float): + Cleanup priority (see + :py:meth:`~.add_cleanup_with_priority`) + + Example: + >>> class Object(object): + ... pass # Allow setting arbitrary attributes + ... + >>> + >>> obj = Object() + >>> obj.foo = 1 + >>> with Cleanup() as cleanup: + ... cleanup.patch(obj, 'foo', 2) + ... cleanup.patch(obj, 'bar', 3) + ... assert obj.foo == 2 + ... assert obj.bar == 3 + ... + >>> assert obj.foo == 1 + >>> assert not hasattr(obj, 'bar') + """ + add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + try: + old = getattr(obj, attr) + except AttributeError: + add_cleanup(delattr, obj, attr) + else: + add_cleanup(setattr, obj, attr, old) + setattr(obj, attr, value) + if name is None: + name = self._get_name(obj) + msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) + self._debug_output(msg) + + # Helper methods + + @staticmethod + def _get_name(obj: Any, /) -> str: + """ + Get an appropriate name for an arbitrary object. + + Example: + >>> import textwrap + >>> + >>> + >>> Cleanup._get_name(textwrap) + 'textwrap' + >>> Cleanup._get_name(textwrap.dedent) + 'textwrap.dedent' + >>> Cleanup._get_name(str) + 'str' + >>> Cleanup._get_name(print) + 'print' + >>> Cleanup._get_name(object()) # doctest: +ELLIPSIS + '' + """ + if hasattr(obj, '__qualname__'): + name = obj.__qualname__ + elif hasattr(obj, '__name__'): + name = obj.__name__ + else: + return repr(obj) + if hasattr(obj, '__module__'): + if obj.__module__ not in ('builtins', '__builtins__'): + name = f'{obj.__module__}.{name}' + return str(name) + + def _debug_output(self, msg: str, /) -> None: + """ + Write debugging output. + + Note: + This default implementation just writes to the logger. + """ + diagnostics.log.debug(msg) + + @property + def _current_context(self) -> _Stacks: + try: + return self._contexts[-1] + except IndexError: + ctx: _Stacks = {} + self._contexts.append(ctx) + return ctx diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py index 887cdd55..ea86695e 100644 --- a/line_profiler/line_profiler_utils.py +++ b/line_profiler/line_profiler_utils.py @@ -5,10 +5,15 @@ from __future__ import annotations import enum -import typing +import os +from pathlib import Path +from tempfile import mkstemp +from textwrap import indent +from typing import TYPE_CHECKING +from typing_extensions import Self -if typing.TYPE_CHECKING: - from typing_extensions import Self + +__all__ = ('StringEnum', 'block_indent', 'make_tempfile') class _StrEnumBase(str, enum.Enum): @@ -49,7 +54,7 @@ def __str__(self) -> str: try: from enum import StrEnum as _StrEnum except ImportError: - if not typing.TYPE_CHECKING: # Don't confuse the typechecker + if not TYPE_CHECKING: # Don't confuse the typechecker _StrEnum = _StrEnumBase @@ -89,3 +94,33 @@ def _missing_(cls, value: object) -> Self | None: for name, instance in cls.__members__.items() } return members.get(value.casefold()) + + +def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: + r""" + Example: + >>> string = 'foo\nbar\nbaz' + >>> print(string) + foo + bar + baz + >>> print(block_indent(string, '++++', '-')) + ++++foo + ----bar + ----baz + """ + width = len(prefix) + return prefix + indent(string, fill_char * width)[width:] + + +def make_tempfile(**kwargs) -> Path: + """ + Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding + and closing the integer handle (which if left unattended causes + problems on some platforms). + """ + handle, fname = mkstemp(**kwargs) + try: + return Path(fname) + finally: + os.close(handle) From fe74af6571401ea68eca5d98578ba0ae8bafc358 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 22:54:52 +0200 Subject: [PATCH 068/111] Move files around line_profiler/_threading_patches.py Moved from `line_profiler/_child_process_profiling/threading_patches.py` because the patch is needed for correctness even if not profiling child-process-based parallelism TODO: fix and adapt and use it with `CuratedProfilerContext` --- .../threading_patches.py => _threading_patches.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename line_profiler/{_child_process_profiling/threading_patches.py => _threading_patches.py} (100%) diff --git a/line_profiler/_child_process_profiling/threading_patches.py b/line_profiler/_threading_patches.py similarity index 100% rename from line_profiler/_child_process_profiling/threading_patches.py rename to line_profiler/_threading_patches.py From 7c80b1f86bf99667861f4bfcb4a604b2bf105db3 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 26 Apr 2026 23:38:41 +0200 Subject: [PATCH 069/111] Patch `threading` with `CuratedProfilerContext` line_profiler/_threading_patches.py make_thread_init_wrapper() Refactored from `wrap_init()` apply() Changed call signature (non-backwards compatible) line_profiler/curated_profiling.py::CuratedProfilerContext Now inheriting from `~.cleanup.Cleanup` install(), uninstall() - Simplified implementations - Added handling for `threading` patching __enter__() Now returning the instance to be consistent with the base class line_profiler/_child_process_profiling/cache.py Removed code applying (the now-nonexistent) `~.threading_patches.apply()` --- .../_child_process_profiling/cache.py | 6 +- line_profiler/_threading_patches.py | 73 ++++++++++--------- line_profiler/curated_profiling.py | 58 +++++++-------- 3 files changed, 66 insertions(+), 71 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 51a464b3..b70807f1 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -521,9 +521,8 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: forked processes is properly handled (if ``wrap_os_fork=True``) - - :py:mod:`multiprocessing` and :py:mod:`threading` patched - so that child processes and threads managed thereby are - properly handled + - :py:mod:`multiprocessing` patched so that child processes + managed thereby are properly handled - Instance to be returned if :py:func:`~.load()` is called from now on @@ -614,7 +613,6 @@ def _setup_common( _import_sibling('multiprocessing_patches').apply( self, reboot_forkserver, ) - _import_sibling('threading_patches').apply(self) def _wrap_os_fork(self) -> None: """ diff --git a/line_profiler/_threading_patches.py b/line_profiler/_threading_patches.py index c8a5b62c..32f13471 100644 --- a/line_profiler/_threading_patches.py +++ b/line_profiler/_threading_patches.py @@ -1,6 +1,6 @@ """ -Patch :py:mod:`threading` so that profiling extends into processes -it creates. +Patch :py:mod:`threading` so that profiling extends consistenly into +processes it creates. """ from __future__ import annotations @@ -8,13 +8,13 @@ from collections.abc import Callable from functools import wraps from typing import TYPE_CHECKING, Any, TypeVar -from typing_extensions import ParamSpec +from typing_extensions import ParamSpec, Concatenate -from .._line_profiler import ( # type: ignore +from ._line_profiler import ( # type: ignore USE_LEGACY_TRACE as SHOULD_PATCH_THREADING, ) -from ..line_profiler import LineProfiler -from .cache import LineProfilingCache +from .line_profiler import LineProfiler +from .cleanup import Cleanup __all__ = ('apply', 'SHOULD_PATCH_THREADING') @@ -61,41 +61,46 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: return wrapper -# Threads are supposed to be lightweight, so don't waste time formatting -# debug messages during startup - - -@LineProfilingCache._method_wrapper(debug=False) -def wrap_init( - cache: LineProfilingCache, - vanilla_impl: Callable[..., None], - self: threading.Thread, - group: None = None, - target: Callable[..., Any] | None = None, - *a, **k -) -> None: +def make_thread_init_wrapper( + prof: LineProfiler, + vanilla_impl: Callable[ + Concatenate[threading.Thread, None, Callable[..., Any] | None, PS], + None + ], +) -> Callable[ + Concatenate[threading.Thread, None, Callable[..., Any] | None, PS], None +]: """ Wrap the initializer of :py:class:`threading.Thread` so that the profiler's :py:attr:`LineProfiler.enable_count` is synced up on newly spun-up threads. """ - prof = cache.profiler - enable_count: int | None = getattr(prof, 'enable_count', None) - if target is not None and enable_count: - if TYPE_CHECKING: - assert prof is not None - target = make_syncing_wrapper(target, prof, enable_count) - vanilla_impl(self, group, target, *a, **k) + @wraps(vanilla_impl) + def wrapper( + self: threading.Thread, + group: None = None, + target: Callable[..., Any] | None = None, + *args: PS.args, + **kwargs: PS.kwargs + ) -> None: + enable_count: int | None = getattr(prof, 'enable_count', None) + if target is not None and enable_count: + if TYPE_CHECKING: + assert prof is not None + target = make_syncing_wrapper(target, prof, enable_count) + vanilla_impl(self, group, target, *args, **kwargs) + + return wrapper -def apply(lp_cache: LineProfilingCache) -> None: +def apply(cleanup: Cleanup, prof: LineProfiler) -> None: """ Set up profiling in threads started by :py:mod:`threading` by applying patches to the module. Args: - lp_cache (LineProfilingCache) - Cache instance governing the profiling run + cleanup (Cleanup) + Cleanup instance managing the profiling session Side effects: - :py:mod:`threading` marked as having been set up @@ -104,7 +109,7 @@ def apply(lp_cache: LineProfilingCache) -> None: - :py:meth:`threading.Thread.__init__` - - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + - Cleanup callbacks registered via ``cleanup.add_cleanup()`` Note: This is a no-op when using :py:mod:`sys.monitoring`-based @@ -114,8 +119,6 @@ def apply(lp_cache: LineProfilingCache) -> None: return if getattr(threading, _PATCHED_MARKER, False): return - init_wrapper = wrap_init(threading.Thread.__init__) - lp_cache.patch( - threading.Thread, '__init__', init_wrapper, name='threading.Thread', - ) - lp_cache.patch(threading, _PATCHED_MARKER, True, name='threading') + init_wrapper = make_thread_init_wrapper(prof, threading.Thread.__init__) + cleanup.patch(threading.Thread, '__init__', init_wrapper) + cleanup.patch(threading, _PATCHED_MARKER, True) diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 6e21ffe8..3b4c8ac4 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -6,16 +6,16 @@ import builtins import dataclasses -import functools import os import warnings -from collections.abc import Callable, Collection +from collections.abc import Collection from io import StringIO from textwrap import indent from typing import Any, TextIO, cast from typing_extensions import Self from . import _diagnostics as diagnostics, profile as _global_profiler +from ._threading_patches import apply as apply_threading_patches from .autoprofile.autoprofile import ( _extend_line_profiler_for_profiling_imports as upgrade_profiler, ) @@ -23,6 +23,7 @@ from .autoprofile.eager_preimports import ( is_dotted_path, write_eager_import_module, ) +from .cleanup import Cleanup from .cli_utils import short_string_path from .line_profiler import LineProfiler from .profiler_mixin import ByCountProfilerMixin @@ -150,20 +151,27 @@ def from_targets( return cls(filtered_targets, recurse_targets, invalid_targets) -class CuratedProfilerContext: +class CuratedProfilerContext(Cleanup): """ Context manager for handling various bookkeeping tasks when setting up and tearing down profiling: - Slipping ``prof`` into the builtin namespace (if ``insert_builtin`` is true) and :py::deco:`~.profile` + - Patch :py:class:`threading.Thread` so that line-profiling is + enabled on new threads if it is on the spawning threads - At exit, clearing the ``enable_count`` of ``prof``, properly disabling it - Note: - The attributes on this object are to be considered - implementation details, but not its methods and their - signatures. + Notes: + + - The attributes on this object are to be considered + implementation details, but not its methods and their + signatures. + + - In contrast to the base class (:py:class:`Cleanup`), while + this context manager is still reentrant, reentering in nested + `with: ...` statements is a no-op. """ def __init__( self, @@ -171,6 +179,7 @@ def __init__( insert_builtin: bool = False, builtin_loc: str = 'profile', ) -> None: + super().__init__() self.prof = prof self.insert_builtin = insert_builtin self.builtin_loc = builtin_loc @@ -185,48 +194,33 @@ def _global_install(self, prof: ByCountProfilerMixin | None) -> None: self._kpo(cast(LineProfiler, prof)) def install(self) -> None: - def del_builtin_profile() -> None: - delattr(builtins, self.builtin_loc) - - def set_builtin_profile(old: Any) -> None: - setattr(builtins, self.builtin_loc, old) - if self._installed: return # Equip the profiler instance with the # `.add_imported_function_or_module()` pseudo-method upgrade_profiler(self.prof) # Overwrite the explicit profiler (`@line_profiler.profile`) - self._global_install(self.prof) # type: ignore + self._global_install(self.prof) + self.add_cleanup(self._global_install, None) + # Patch `threading` + if isinstance(self.prof, LineProfiler): + apply_threading_patches(self, self.prof) # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: - try: - old = getattr(builtins, self.builtin_loc) - except AttributeError: - self._restore: Callable[[], None] = del_builtin_profile - else: - self._restore = functools.partial(set_builtin_profile, old) - set_builtin_profile(self.prof) - self._installed = True + self.patch(builtins, self.builtin_loc, self.prof) + self.patch(self, '_installed', True) def uninstall(self) -> None: if not self._installed: return - # Restore the `builtins` namespace - if ( - self.insert_builtin - and getattr(builtins, self.builtin_loc, None) is self.prof - ): - self._restore() # Fully disable the profiler for _i in range(getattr(self.prof, 'enable_count', 0)): self.prof.disable_by_count() - # Restore the state of the global `@line_profiler.profile` - self._global_install(None) - self._installed = False + self.cleanup() - def __enter__(self) -> None: + def __enter__(self) -> Self: self.install() + return self def __exit__(self, *_, **__) -> None: self.uninstall() From 5729e4aef70bbdb473a19b18c10cb99e6d529181 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 00:44:44 +0200 Subject: [PATCH 070/111] Continue migration line_profiler/cleanup.py::Cleanup.make_tempfile() Updated default formatting for the debug-logging message line_profiler/_child_process_profiling/ _cache_logging.py Updated import location of `block_indent()` cache.py - Removed duplicate code (previously migrated to `line_profiler/cleanup.py`) - Made `LineProfilingCache` a `@final` class and a `line_profiler.cleanup.Cleanup` subclass - Updated signature of `LineProfilingCache.inject_env_vars()` - Removed `LineProfilingCache._add_cleanup()` pth_hook.py Minor internal refactoring --- .../_cache_logging.py | 2 +- .../_child_process_profiling/cache.py | 324 ++---------------- .../_child_process_profiling/pth_hook.py | 4 +- line_profiler/cleanup.py | 2 +- 4 files changed, 35 insertions(+), 297 deletions(-) diff --git a/line_profiler/_child_process_profiling/_cache_logging.py b/line_profiler/_child_process_profiling/_cache_logging.py index 91b0635f..992f68d0 100644 --- a/line_profiler/_child_process_profiling/_cache_logging.py +++ b/line_profiler/_child_process_profiling/_cache_logging.py @@ -15,7 +15,7 @@ from typing_extensions import Self from .. import _diagnostics as diagnostics -from .misc_utils import block_indent +from ..line_profiler_utils import block_indent __all__ = ('CacheLoggingEntry',) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index b70807f1..d7978f84 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,28 +7,25 @@ import atexit import dataclasses import os -import sys try: import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable, Iterable +from collections.abc import Collection, Callable, MutableMapping, Iterable from functools import partial, cached_property, wraps from importlib import import_module -from operator import setitem from pathlib import Path from pickle import HIGHEST_PROTOCOL -from reprlib import Repr from textwrap import indent -from types import MethodType, ModuleType -from typing import Any, ClassVar, TypeVar, TypedDict, cast, overload -from typing_extensions import Concatenate, ParamSpec, Self, Unpack +from types import ModuleType +from typing import Any, ClassVar, TypeVar, cast, final, overload +from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics +from ..cleanup import Cleanup from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ._cache_logging import CacheLoggingEntry -from .misc_utils import block_indent, make_tempfile # Note: this should have been defined here in this file, but we moved it # over to `~._child_process_hook` because that module contains the .pth # hook, which must run with minimal overhead when a Python process isn't @@ -56,187 +53,9 @@ def _import_sibling(submodule: str) -> ModuleType: return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') -class _ReprAttributes(TypedDict, total=False): - """ - Note: - We use this typed dict instead of directly supplying them in the - :py:meth:`_CallbackRepr.__init__()` signature, because we don't - want to bother with the default values there. - """ - maxlevel: int - maxtuple: int - maxlist: int - maxarray: int - maxdict: int - maxset: int - maxfrozenset: int - maxdeque: int - maxstring: int - maxlog: int - maxother: int - fillvalue: str - indent: str | int | None - - -class _CallbackRepr(Repr): - """ - :py:class:`reprlib.Repr` subclass to help with representing cleanup - callbacks, special-casing certain relevant object types (see - examples below). - - Example: - >>> from functools import partial - >>> from sys import version_info - - >>> class MyEnviron(dict): - ... def some_method(self) -> None: - ... ... - ... - >>> - >>> class MyRepr(_CallbackRepr): - ... # Since we can't instantiate a new `os._Environ`, test - ... # the relevant method with a mock - ... repr_MyEnviron = _CallbackRepr.repr__Environ - ... - >>> - >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) - - Environ-dict formatting: - - >>> my_env = MyEnviron( - ... foo='1', - ... bar='2', - ... this_varname_is_long_but_isnt_truncated=( - ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" - ... ), - ... baz='4', - ... ) - >>> print(r.repr(my_env)) - environ({'foo': '1', 'bar': '2', \ -'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) - - Partial-object formatting: - - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - - - Bound-method formatting: - - >>> r.maxargs = 0 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial(, ...) - - Indentation (Python 3.12+): - - >>> if version_info < (3, 12): - ... from pytest import skip - ... - ... skip( - ... '`Repr.indent` not available on {}.{},{}' - ... .format(*sys.version_info) - ... ) - - >>> r = MyRepr(maxenv=2, maxargs=4) - >>> r.indent = 2 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial( - , - base=8, - ) - - >>> callback_2 = partial(min, 5, 4, 3, 2, 1) - >>> r.indent = '----' - >>> print(r.repr(callback_2)) - functools.partial( - ----, - ----5, - ----4, - ----3, - ----2, - ----..., - ) - - >>> r.indent = ' ' - >>> r.maxenv = 2 - >>> print(r.repr(my_env.some_method)) - - """ - def __init__( - self, - *, - maxargs: int = 5, - maxenv: int = 3, - **kwargs: Unpack[_ReprAttributes] - ) -> None: - super().__init__() # kwargs are 3.12+ - for k, v in kwargs.items(): - setattr(self, k, v) - self.maxargs = maxargs - self.maxenv = maxenv - - def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: - get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) - # Truncate envvar values, but not their names - envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] - return self._format_items(envvars, ('environ({', '})'), self.maxenv) - - def repr_method(self, method: MethodType, level: int) -> str: - instance = self.repr1(method.__self__, level-1) - func = getattr(method.__func__, '__qualname__', '?') - prefix, suffix = f'' - # Take care of possible multi-line reprs - return block_indent(instance, prefix) + suffix - - def repr_partial(self, ptl: partial, level: int) -> str: - get: Callable[[Any], str] = partial(self.repr1, level=level-1) - args = [get(arg) for arg in ptl.args] - args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) - args.insert(0, get(ptl.func)) - name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) - # The +1 is to account for `ptl.func` - return self._format_items(args, (name + '(', ')'), self.maxargs + 1) - - def _format_items( - self, - items: Collection[str], - delims: tuple[str, str], - maxlen: int | None = None, - ) -> str: - start, end = delims - if maxlen is not None and len(items) > maxlen: - items = list(items)[:maxlen] + ['...'] - indent_prefix: str | None = self._get_indent() - if indent_prefix is None or not items: - return '{}{}{}'.format(start, ', '.join(items), end) - return '\n'.join([ - start, *(indent(item + ',', indent_prefix) for item in items), end, - ]) - - if sys.version_info >= (3, 12): - # Note: `.indent` only available since 3.12 - def _get_indent(self) -> str | None: - indent = self.indent - if indent is None or isinstance(indent, str): - return indent - return ' ' * indent - else: - @staticmethod - def _get_indent() -> None: - return None - - -_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr - - +@final @dataclasses.dataclass -class LineProfilingCache: +class LineProfilingCache(Cleanup): """ Helper object for coordinating a line-profiling session, caching the info required to make profiling persist into child processes. @@ -261,65 +80,10 @@ class LineProfilingCache: _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( default_factory=dict, init=False, repr=False, ) - _loaded_instance: ClassVar[Self | None] = None + _loaded_instance: ClassVar[LineProfilingCache | None] = None - def cleanup(self) -> None: - """ - Pop all the cleanup callbacks from the internal stack added via - :py:meth:`~.add_cleanup` and call them in order. - """ - stacks = self._cleanup_stacks - ncallbacks_total = sum(len(stack) for stack in stacks.values()) - if not ncallbacks_total: - self._debug_output('Cleanup aborted (no registered callbacks)') - return - # Bookend the cleanup loop with log messages to help detect if - # child processes are prematurely terminated - self._debug_output( - f'Starting cleanup ({ncallbacks_total} callback(s))...', - ) - ncallbacks_run = 0 - for priority in sorted(stacks): - callbacks = stacks.pop(priority) - while callbacks: - callback = callbacks.pop() - callback_repr = _CALLBACK_REPR(callback) - ncallbacks_run += 1 - try: - callback() - except Exception as e: - state = 'failed' - msg = f'{callback_repr}: {type(e).__name__}: {e}' - else: - state, msg = 'succeeded', f'{callback_repr}' - self._debug_output( - f'- Cleanup {state} ' - f'({ncallbacks_run}/{ncallbacks_total}): {msg}', - ) - self._debug_output( - f'... cleanup completed ({ncallbacks_total} callback(s))', - ) - - def add_cleanup( - self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, - ) -> None: - """ - Add a cleanup callback to the internal stack; they can be later - called by :py:meth:`~.cleanup`. - """ - self._add_cleanup(callback, 0, *args, **kwargs) - - def _add_cleanup( - self, callback: Callable[PS, Any], priority: float, - *args: PS.args, **kwargs: PS.kwargs, - ) -> None: - if args or kwargs: - callback = partial(callback, *args, **kwargs) - self._cleanup_stacks.setdefault(priority, []).append(callback) - header = 'Cleanup callback added' - if priority: - header = f'{header} (priority: {priority})' - self._debug_output(f'{header}: {_CALLBACK_REPR(callback)}') + def __post_init__(self) -> None: + super().__init__() def copy( self, *, @@ -369,7 +133,9 @@ def load(cls) -> Self: If a previously :py:meth:`.~.load`-ed instance exists, it is returned instead of a new instance. """ - instance = cls._loaded_instance + # `ty` needs some help here, evenif we've marked the class to be + # `@final` + instance = cast(Self | None, cls._loaded_instance) if instance is None: pid = os.environ[INHERITED_PID_ENV_VARNAME] cache_varname = f'{INHERITED_CACHE_ENV_VARNAME_PREFIX}_{pid}' @@ -464,32 +230,28 @@ def _get_debug_logfiles(self) -> Iterable[Path]: return Path(self.cache_dir).glob(pattern) def inject_env_vars( - self, env: dict[str, str] | None = None, + self, env: MutableMapping[str, str] | None = None, ) -> None: """ Inject the :py:attr:`~.environ` variables into ``env`` and add cleanup callbacks to reverse them. Args: - env (dict[str, str] | None): + env (MutableMapping[str, str] | None): Dictionary in the format of :py:data:`os.environ`; default is to use that """ - if env is None: - env = cast(dict[str, str], os.environ) - for name, value in self.environ.items(): - try: - old = env[name] - except KeyError: - self.add_cleanup(env.pop, name, None) - change = f'{value!r} (new)' - else: - self.add_cleanup(setitem, env, name, old) - change = f'{old!r} -> {value!r}' - self._debug_output(f'Injecting env var ${{{name}}}: {change}') - env[name] = value + self.update_mapping( + os.environ if env is None else env, + self.environ, + _format_debug_msg='Injecting env var ${{{1}}}: {2}'.format, + ) def _debug_output(self, msg: str) -> None: + """ + Beside writing to the logger, also write to the + :py:attr:`~._debug_log`. + """ try: self._make_debug_entry(msg).write(self._debug_log) except OSError: # Cache dir may have been rm-ed during cleanup @@ -592,8 +354,9 @@ def _setup_in_child_process( prefix='child-prof-output-{}-{}-{:#x}-' .format(self.main_pid, os.getpid(), id(prof)), suffix='.lprof', + delete=False, ) - self._add_cleanup(prof.dump_stats, -1, prof_outfile) + self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) # Various setups self._setup_common(wrap_os_fork, reboot_forkserver=False) @@ -651,28 +414,6 @@ def wrapper() -> int: self.patch(os, 'fork', wrapper, name='os') - def patch( - self, obj: Any, attr: str, value: Any, *, - name: str | None = None, cleanup: bool = True, - ) -> None: - """ - Patch ``attr`` on ``obj`` with ``value``. If ``cleanup`` is - true, register a cleanup callback to either reset or delete the - attribute. - """ - add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) - try: - old = getattr(obj, attr) - except AttributeError: - add_cleanup(delattr, obj, attr) - else: - add_cleanup(setattr, obj, attr, old) - setattr(obj, attr, value) - if name is None: - name = repr(obj) - msg = 'Patched `{}.{}` -> `{}`'.format(name, attr, value) - self._debug_output(msg) - def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other @@ -682,17 +423,16 @@ def make_tempfile(self, **kwargs) -> Path: path (Path): Path to the created file. """ - path = make_tempfile(dir=self.cache_dir, **kwargs) - self._debug_output(f'Created tempfile: {path.name!r}') - return path + kwargs.setdefault('dir', self.cache_dir) + kwargs.setdefault( + '_format_debug_msg', 'Created tempfile: {0.name!r}'.format, + ) + return super().make_tempfile(**kwargs) def _replace_loaded_instance(self, force: bool = False) -> bool: cls = type(self) if force or self._consistent_with_loaded_instance: - # Note: `ty` REALLY hates assigning an instance to - # `ClassVar[Self]` (#3274); no choice but to ignore it for - # the time being... - cls._loaded_instance = self # type: ignore + cls._loaded_instance = self return True return False diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 5a19cbce..fa3ee9dd 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -51,13 +51,12 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path """ import os from sysconfig import get_path - from .misc_utils import make_tempfile if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) - fpath = make_tempfile( + fpath = cache.make_tempfile( prefix='_line_profiler_profiling_hook_', suffix='.pth', dir=get_path('purelib'), @@ -67,7 +66,6 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path (lambda: None).__module__, cache.main_pid, ) fpath.write_text(pth_content) - cache.add_cleanup(fpath.unlink, missing_ok=True) except Exception: fpath.unlink(missing_ok=True) raise diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 6794e203..97b6b05a 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -436,7 +436,7 @@ def make_tempfile( delete: bool = True, priority: float = 0, _format_debug_msg: Callable[[Path], str] = ( - lambda path: f'Created tempfile: {path.name!r}' + 'Created tempfile: {}'.format ), **kwargs ) -> Path: From ac4351bf620bfbd5299ce81f770a29c7a0df4631 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 01:31:12 +0200 Subject: [PATCH 071/111] Finish migration kernprof.py _touch_tempfile() Now an imported alias for `line_profiler.line_profiler_utils.make_tempfile()` _manage_profiler.__enter__(), .__exit__() Simplified implementations making use of `CuratedProfilerContext.[add_]cleanup()` line_profiler/_child_process_profiling/misc_utils.py Deleted because its functionalities have been migrated to `line_profiler.line_profiler_utils` tests/test_child_procs.py test_cache_cleanup_order() Removed test because it is covered by `Cleanup.add_cleanup_with_priority()`'s doctest test_cache_setup_main_process() No longer checking the patching of `threading` because it is no longer handled by `LineProfilingCache._setup_in_main_process()` test_cache_setup_child() No longer protecting against the patching of `threading` because it is no longer handled by `LineProfilingCache._setup_in_main_process()` --- kernprof.py | 51 ++++++++----------- .../_child_process_profiling/misc_utils.py | 40 --------------- tests/test_child_procs.py | 34 ------------- 3 files changed, 21 insertions(+), 104 deletions(-) delete mode 100644 line_profiler/_child_process_profiling/misc_utils.py diff --git a/kernprof.py b/kernprof.py index 96888468..0e1a4b77 100755 --- a/kernprof.py +++ b/kernprof.py @@ -230,6 +230,9 @@ def main(): positive_float, short_string_path, ) +from line_profiler.line_profiler_utils import ( + make_tempfile as _touch_tempfile, # Compatibility +) from line_profiler.profiler_mixin import ByCountProfilerMixin from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._logger import Logger @@ -237,9 +240,9 @@ def main(): DIAGNOSITICS_VERBOSITY = 2 -CLEANUP_PRIORITIES = { # Bigger number -> more delayed - 'rm_cache_dir': 1024, - 'gather_logs': 1, +CLEANUP_PRIORITIES = { # More negative number -> more delayed + 'rm_cache_dir': -1024, + 'gather_logs': -1, } @@ -1012,21 +1015,6 @@ def main(args=None, *, exit_on_error=True): cleanup() -def _touch_tempfile(*args, **kwargs): - """ - Wrapper around :py:func:`tempfile.mkstemp()` which drops and closes - the integer handle (which we don't need and may cause issues on some - platforms). - """ - handle, path = tempfile.mkstemp(*args, **kwargs) - try: - os.close(handle) - except Exception: - os.remove(path) - raise - return path - - def _write_tempfile(source, content, options): """ Called by :py:func:`main()` to handle :command:`kernprof -c` and @@ -1218,13 +1206,20 @@ def __enter__(self): keep_preimports_file=self.set_up_child_profiling, ) if self.set_up_child_profiling: - self.cache = _prepare_child_profiling_cache( + self.cache = cache = _prepare_child_profiling_cache( self.options, self.prof, preimports_file, script_file, ) - # Add a deferred callback for gathering debug logfiles - # (should run right before `.cache.cache_dir` is wiped) + self._ctx.add_cleanup(cache.cleanup) + # Add deferred callbacks for gathering debug logfiles + # (should run right before `.cache.cache_dir` is wiped): + # - Write the debug logs to the `._diagnostics` logger + if cache.debug: + cache.add_cleanup_with_priority( + cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], + ) + # - Write the debug logs to a specific file if self.options.debug_log: - self.cache._add_cleanup( + cache.add_cleanup_with_priority( self._gather_debug_log, CLEANUP_PRIORITIES['gather_logs'], self.options.debug_log, @@ -1235,15 +1230,11 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: - try: - if self.cache.debug: - # Recover debug output from child processes - self.cache._dump_debug_logs() - extra_stats = self.cache.gather_stats() - finally: - self.cache.cleanup() + extra_stats = self.cache.gather_stats() _post_profile(self.options, self.prof, extra_stats) finally: + # This also calls its `.cleanup()`, and in turn that of + # `.cache`'s self._ctx.uninstall() def _gather_debug_log(self, logfile): @@ -1374,7 +1365,7 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: # Defer the scrubbing of the cache dir - cache._add_cleanup( + cache.add_cleanup_with_priority( _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, recursive=True, ) diff --git a/line_profiler/_child_process_profiling/misc_utils.py b/line_profiler/_child_process_profiling/misc_utils.py deleted file mode 100644 index 050380bf..00000000 --- a/line_profiler/_child_process_profiling/misc_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Misc. utility functions used by the subpackage. -""" -import os -from pathlib import Path -from tempfile import mkstemp -from textwrap import indent - - -__all__ = ('block_indent', 'make_tempfile') - - -def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: - r""" - Example: - >>> string = 'foo\nbar\nbaz' - >>> print(string) - foo - bar - baz - >>> print(block_indent(string, '++++', '-')) - ++++foo - ----bar - ----baz - """ - width = len(prefix) - return prefix + indent(string, fill_char * width)[width:] - - -def make_tempfile(**kwargs) -> Path: - """ - Convenience wrapper around :py:func:`tempfile.mkstemp`, discarding - and closing the integer handle (which if left unattended causes - problems on some platforms). - """ - handle, fname = mkstemp(**kwargs) - try: - return Path(fname) - finally: - os.close(handle) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index e03e41d2..5330535a 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -37,9 +37,6 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler._child_process_profiling.threading_patches import ( - SHOULD_PATCH_THREADING, -) from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, ) @@ -1227,8 +1224,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'multiprocessing.spawn': frozenset({'runpy'}), 'os': frozenset({'fork'}), } -if SHOULD_PATCH_THREADING: - _GLOBAL_PATCHES['threading.Thread'] = frozenset({'__init__'}) # NOTE: we need a function which isn't used by the codebase itself # (esp. during cache cleanup); otherwise the profiling results may @@ -1374,35 +1369,6 @@ def test_cache_dump_load( assert set(os.environ) == envvars -@pytest.mark.parametrize(('debug', 'label'), - [(True, 'with-debug'), (False, 'no-debug')]) -def test_cache_cleanup_order( - create_cache: Callable[..., LineProfilingCache], debug: bool, label: str, -) -> None: - """ - Test that :py:meth`LineProfilingCache.cleanup` executes the cleanup - callback stacks in order. - """ - strings: list[str] = [] - cache = create_cache(debug=debug) - - cache.add_cleanup(strings.append, 'first') - # Decreased priority - cache._add_cleanup(strings.append, 1, 'second') - # Increased priority - cache._add_cleanup(strings.append, -1, 'third') - cache.add_cleanup(strings.append, 'fourth') - - cache.cleanup() - assert strings == ['third', 'fourth', 'first', 'second'] - - pattern = '\n'.join( - rf'.*succeeded \({i + 1}/{len(strings)}\): .*append.*{string!r}.*' - for i, string in enumerate(strings) - ) - _search_cache_logs(cache, debug, [pattern]) - - @pytest.mark.parametrize(('wrap_os_fork', 'label1'), [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) @pytest.mark.parametrize(('debug', 'label2'), From a64f01d7e2c4a9393d007df76cf13a5c46f6a5ce Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 02:26:37 +0200 Subject: [PATCH 072/111] Tweak config layout line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes] New subtable for options related to child-processing profiling: - [pth_files] New subtable for options for .pth-file creation - [multiprocessing] Migrated from `[tool.line_profiler.multiprocessing]` line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() Now loading affixes to the .pth-filename stem from `cache.config` line_profiler/_child_process_profiling/multiprocessing_patches.py _get_config_cached() Updated load location of config options _apply_mp_patches() Renamed (currently not explicitly used) argument `debug` -> `intercept_mp_logs` tests/test_child_procs.py::test_apply_mp_patches() Updated .toml file written to align with the new config locations --- .../multiprocessing_patches.py | 19 ++++++-------- .../_child_process_profiling/pth_hook.py | 10 ++++++-- line_profiler/rc/line_profiler.toml | 25 +++++++++++++------ tests/test_child_procs.py | 2 +- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 97dd803c..b2822f49 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -243,7 +243,7 @@ def _get_config_cached( ) -> Mapping[str, Any]: cd = dict( ConfigSource.from_config(config) - .get_subconfig('multiprocessing', copy=True) + .get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) assert isinstance(cd.get('polling'), Mapping) @@ -474,7 +474,7 @@ def _apply_patches_generic( def _apply_mp_patches( lp_cache: LineProfilingCache, reboot_forkserver: bool = True, - debug: bool | None = None, + intercept_mp_logs: bool | None = None, ) -> None: # In a child process, we don't care about polluting the # `multiprocessing` namespace, so don't bother with cleanup @@ -494,14 +494,11 @@ def _apply_mp_patches( pass else: if hasattr(spawn, 'runpy'): - lp_cache.patch( - spawn, 'runpy', create_runpy_wrapper(lp_cache), - name='multiprocessing.spawn', - ) + lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages - if debug is None: - debug = _get_config(lp_cache.config)['intercept_logs'] - if debug: + if intercept_mp_logs is None: + intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] + if intercept_mp_logs: lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} apply_patches('util', {'': lpatches}) @@ -524,9 +521,7 @@ def _apply_mp_patches( stop() lp_cache.add_cleanup(stop) # Mark `multiprocessing` as having been patched - lp_cache.patch( - multiprocessing, _PATCHED_MARKER, True, name='multiprocessing', - ) + lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) def _no_op(*_, **__) -> None: diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index fa3ee9dd..b6197600 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -51,14 +51,20 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path """ import os from sysconfig import get_path + from ..toml_config import ConfigSource if not os.path.exists(cache.filename): cache.dump() assert os.path.exists(cache.filename) + pth_config = ( + ConfigSource.from_config(cache.config) + .get_subconfig('child_processes', 'pth_files') + .conf_dict + ) fpath = cache.make_tempfile( - prefix='_line_profiler_profiling_hook_', - suffix='.pth', + prefix=pth_config['prefix'], + suffix=pth_config['suffix'] + '.pth', dir=get_path('purelib'), ) try: diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index a420aad8..5c53aadd 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -211,29 +211,38 @@ perhit = 8 percent = 8 # XXX: --- Start of implementation details --- -# `line_profiler._child_process_profiling.multiprocessing_patches` -# settings +# `line_profiler._child_process_profiling` settings -[tool.line_profiler.multiprocessing] +[tool.line_profiler.child_processes.pth_files] -# - `intercept_logs` (bool): +# - `pth_files.prefix`, `.suffix` (str): +# Affixes to use for the stem of the name of the .pth file created +# Note: may be useful to tweak this in case of issues with .pth file +# precedence. +prefix = '_line_profiler-profiling-hook-' +suffix = '' + +[tool.line_profiler.child_processes.multiprocessing] + +# - `multiprocessing.intercept_logs` (bool): # Whether to patch the `multiprocessing.util` logging funcions so that # the internal log messages are captured and writtent to the debug # logs intercept_logs = false -[tool.line_profiler.multiprocessing.polling] +[tool.line_profiler.child_processes.multiprocessing.polling] -# - `polling.cooldown` (float): +# - `multiprocessing.polling.cooldown` (float): # Cooldown time (seconds) before successive polls on lock files (set # to <= 0 to disable cooldowns) cooldown = 0.03125 # 1/32-nd of a second -# - `polling.timeout` (float): +# - `multiprocessing.polling.timeout` (float): # Time (seconds) before the main process disregards the existence of # lock file and unblocks calls to `.terminate()` a (most probably # errored-out) child process anyway (set to <= 0 to disable timeouts) timeout = 0.25 -# - `polling.on_timeout` (Literal['error', 'warn', 'ignore']) +# - `multiprocessing.polling.on_timeout` +# (Literal['error', 'warn', 'ignore']): # What to do when the above timeout is exhausted, before actually # `.terminate()`-ing the child process: # - 'error': raise an error diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5330535a..6a2ed112 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1562,7 +1562,7 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: if debug: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' config.write_text( - '[tool.line_profiler.multiprocessing]\n' + '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) From f3ab34aee3cc2712ef39c949b217e3219790cddc Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 10:22:41 +0200 Subject: [PATCH 073/111] WIP: increase test coverage line_profiler/_child_process_profiling/pth_hook.py::load_pth_hook() - Added `# nocover` directive to unlikely `except` clause - Now patching `load_pth_hook.called` with the `cache` where possible tests/test_child_procs.py ext_module_object New fixture for actually loading the module object represented by the `ext_module` fixture another_pid Convenience fixture for a dummy PID _preserve_attributes .fetch_current_values(), .compare_with_current_values() New convenience methods for comparing the preserved vs. current attributes of multiple objects test_cache_setup_main_process() Simplified implementation with the `_preserve_attributes` methods test_cache_setup_child() Now using the new `ext_module_object` fixture for the profiled function test_load_pth_hook() New unit test for `load_pth_hook()` to boost coverage of `pth_hook.py` test_apply_mp_patches() Refactored to: - Use the new `ext_module_object` fixture for the profiled function - Test cases where the parallel workload fails TODO: `@LineProfilingCache._method_wrapper` and `wrap_terminate()` still not covered; try to tweak`test_apply_mp_patches()` to cover them --- .../_child_process_profiling/pth_hook.py | 5 +- tests/test_child_procs.py | 291 +++++++++++++----- 2 files changed, 214 insertions(+), 82 deletions(-) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index b6197600..4fa6b9ef 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -110,10 +110,11 @@ def load_pth_hook(ppid): # type: (int) -> None try: cache = LineProfilingCache.load() cache._setup_in_child_process(True, 'pth') - except Exception as e: + except Exception as e: # nocover if DEBUG: msg = f'{type(e)}: {e}' warnings.warn(msg) log.warning(msg) - finally: load_pth_hook.called = True # type: ignore + else: + cache.patch(load_pth_hook, 'called', True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 6a2ed112..cc3c3f0e 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1,6 +1,5 @@ from __future__ import annotations -import ast import dataclasses import enum import inspect @@ -16,8 +15,8 @@ from collections.abc import ( Callable, Collection, Generator, Iterable, Mapping, Sequence, ) -from contextlib import ExitStack -from functools import lru_cache, partial, wraps +from contextlib import AbstractContextManager, ExitStack, nullcontext +from functools import partial, wraps from io import StringIO from importlib import import_module from pathlib import Path @@ -25,7 +24,7 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from types import TracebackType +from types import ModuleType, TracebackType from typing import Any, Generic, Literal, TypeVar, cast, final, overload from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -37,6 +36,7 @@ from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) +from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, ) @@ -200,6 +200,21 @@ def install( if children: self.monkeypatch.setenv('PYTHONPATH', path, prepend=os.pathsep) + def _import_module_helper(self) -> Generator[ModuleType, None, None]: + def iter_module_names( + module: _ModuleFixture, + ) -> Generator[str, None, None]: + yield module.name + for dep in module.dependencies: + yield from iter_module_names(dep) + + self.install(local=True, children=True) + try: + yield import_module(self.name) + finally: + for name in set(iter_module_names(self)): + sys.modules.pop(name, None) + @staticmethod def propose_name(prefix: str) -> Generator[str, None, None]: """ @@ -291,6 +306,19 @@ def test_module_clone( yield _ModuleFixture(path, monkeypatch, [ext_module]) +@pytest.fixture +def ext_module_object( + ext_module: _ModuleFixture, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`EXTERNAL_MODULE_BODY`, and is torn down at the end of + the test + """ + yield from ext_module._import_module_helper() + + @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, @@ -439,6 +467,17 @@ def curated_profiler() -> Generator[LineProfiler, None, None]: yield prof +@pytest.fixture +def another_pid() -> int: + """ + Get a PID which is distinct from the current one. + """ + curr_pid = os.getpid() + pid = (curr_pid - 42) % (2 * 16) + assert pid != curr_pid + return pid + + @pytest.fixture(autouse=True) def _trim_mismatch_traceback(pytestconfig: pytest.Config) -> None: """ @@ -866,6 +905,43 @@ def __enter__(self) -> dict[str, dict[str, Any]]: def __exit__(self, *_, **__) -> None: self._stacks.pop().close() + @staticmethod + def fetch_current_values( + targets: Mapping[str, Collection[str]], + ) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + na = _NotSupplied.NOT_SUPPLIED + for target, attrs in targets.items(): + obj = _import_target(target) + result[target] = {attr: getattr(obj, attr, na) for attr in attrs} + return result + + @classmethod + def compare_with_current_values( + cls, + old: Mapping[str, Mapping[str, Any]], + comparator: Callable[[Any, Any], bool] = operator.is_, + assert_true: bool = True, + ) -> dict[str, dict[str, bool]]: + result: dict[str, dict[str, bool]] = {} + new = cls.fetch_current_values(old) + for target, old_values in old.items(): + new_values = new[target] + cmp_results = result[target] = {} + for attr, old_value in old_values.items(): + print(f'Checking: {target}.{attr}') + new_value = new_values[attr] + if assert_true: + assert cmp_results.setdefault( + attr, comparator(new_value, old_value), + ) + else: + # There's probably a more concise way to write this, + # but we want more info to be available in the + # traceback should the above assertion fail... + cmp_results[attr] = comparator(new_value, old_value) + return result + class _preserve_pth_files(_CallableContextManager[frozenset[str]]): def __init__(self, debug: bool = _DEBUG) -> None: @@ -937,25 +1013,6 @@ def _search_cache_logs( ) -@lru_cache() -def _find_return_lines(func: str) -> list[int]: - class FindReturns(ast.NodeVisitor): - def __init__(self) -> None: - self.found: set[int] = set() - - def visit_Return(self, node: ast.Return) -> None: - self.found.add(node.lineno) - self.generic_visit(node) - - func_obj = _import_target(func) - assert inspect.isfunction(func_obj) - lines, start = inspect.getsourcelines(func_obj) - tree = ast.parse(''.join(lines)) - finder = FindReturns() - finder.visit(tree) - return sorted(lineno + start - 1 for lineno in finder.found) - - # `shlex.join()` doesn't work properly on Windows, so use # `subprocess.list2cmdline()` instead; # though an "intentionally" undocumented API (cpython issue #10308), @@ -1225,16 +1282,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: 'os': frozenset({'fork'}), } -# NOTE: we need a function which isn't used by the codebase itself -# (esp. during cache cleanup); otherwise the profiling results may -# be skewed -_SAFE_TARGET = 'calendar.weekday' -_SAFE_TARGET_ARGS = [ - (1970, 1, 1), - (2000, 12, 31), - (2008, 9, 16), # Where the repo started -] - @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @@ -1389,33 +1436,21 @@ def test_cache_setup_main_process( for target, attrs in _GLOBAL_PATCHES.items() } patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') - targets: dict[str, Any] = { - target: _import_target(target) for target in patches - } with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) + compare_patched = partial( + _preserve_attributes.compare_with_current_values, patched, + ) original_pths = stack.enter_context(_preserve_pth_files()) cache._setup_in_main_process(wrap_os_fork=wrap_os_fork) # There should be exactly one extra `.pth` file new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths # Check whether the patches are applied - for target, maybe_patches in patches.items(): - obj = targets[target] - for attr, is_patched in maybe_patches.items(): - orig_value = patched[target][attr] - if orig_value is _NotSupplied.NOT_SUPPLIED: - assert not hasattr(obj, attr) - else: - assert (getattr(obj, attr) is orig_value) != is_patched + patch_summary = compare_patched(operator.is_not, assert_true=False) + assert patch_summary == patches # Check whether the patches are reversed cache.cleanup() - for target, orig_attrs in patched.items(): - obj = targets[target] - for attr, orig_value in orig_attrs.items(): - if orig_value is _NotSupplied.NOT_SUPPLIED: - assert not hasattr(obj, attr) - else: - assert getattr(obj, attr) is orig_value + compare_patched() # Check that the instance is set as the `.load()`-ed one assert cache is cache.load() @@ -1444,14 +1479,18 @@ def test_cache_setup_main_process( [(True, 'no-profiler'), (False, 'with-profiler')]) @pytest.mark.parametrize(('debug', 'label4'), [(True, 'with-debug'), (False, 'no-debug')]) +@pytest.mark.parametrize('n', [100]) @_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], curated_profiler: LineProfiler, + ext_module_object: ModuleType, + another_pid: int, wrap_os_fork: bool, preimports: bool, new_profiler: bool, debug: bool, + n: int, label1: str, label2: str, label3: str, label4: str, ) -> None: """ @@ -1464,16 +1503,12 @@ def list_profiled_funcs() -> list[str]: for func in getattr(cache.profiler, 'functions', []) ] - # Make sure we get a different PID from the current process - curr_pid = os.getpid() - main_pid = (curr_pid - 42) % (2 * 16) - assert main_pid != curr_pid - + func = ext_module_object.my_external_sum cache = create_cache( - profiling_targets=[_SAFE_TARGET], + profiling_targets=[f'{func.__module__}.{func.__qualname__}'], preimports_module=preimports, _use_curated_profiler=not new_profiler, - main_pid=main_pid, + main_pid=another_pid, debug=debug, ) assert (cache.profiler is None) == new_profiler @@ -1502,7 +1537,7 @@ def list_profiled_funcs() -> list[str]: # Check that on cache cleanup: # - Profiling data is collected # - `os.fork()` is restored - _import_target(_SAFE_TARGET)(*_SAFE_TARGET_ARGS[0]) + assert func(range(1, n + 1)) == n * (n + 1) // 2 stats = cache.profiler.get_stats() for callback, has_prof_data, fork_patched in [ (lambda: None, False, wrap_os_fork), @@ -1532,19 +1567,91 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) +@_preserve_attributes({ + **_GLOBAL_PATCHES, + 'line_profiler._child_process_profiling.pth_hook.load_pth_hook': + frozenset({'called'}), +}) +@pytest.mark.parametrize('ppid_should_match', [True, False, None]) +def test_load_pth_hook( + create_cache: Callable[..., LineProfilingCache], + another_pid: int, + ppid_should_match: bool | None, +) -> None: + """ + Simulate calling :py:func:`line_profiler._child_process_profiling\ +.pth_hook.load_pth_hook()` in a child process. + + Notes: + + - The function is CALLED in the .pth file, but we don't actually + NEED a .pth file to call and test it. + + - The counterpart :py:func:`line_profiler\ +._child_process_profiling.pth_hook.write_pth_hook()` + is implicitly tested in + :py:func:`test_cache_setup_main_process()`. + """ + # This test is mostly here to hack coverage; since the function is + # only to be called in child processes, `coverage` seems to have + # trouble getting data on it... + + # We basically only need this cache instance to set up the + # environment variables and the requisite files... + cache = create_cache(main_pid=another_pid) + if ppid_should_match is not None: + cache.inject_env_vars() + if ppid_should_match: + call_ppid = another_pid + else: # On a PPID mismatch, the function bails after checking + call_ppid = another_pid + 10 + else: + # Without the requisite envvars, the hook should bail very + # quickly (due to the `environ` lookup erroring out), regardless + # of the provided PPID + call_ppid = 0 + cache.dump() + + compare = _preserve_attributes.compare_with_current_values + with _preserve_attributes(_GLOBAL_PATCHES) as patched: + load_pth_hook(call_ppid) + # Check that the patches are applied where appropriate + assert ( + getattr(load_pth_hook, 'called', False) == bool(ppid_should_match) + ) + if ppid_should_match: + compare(patched, operator.is_not) + else: # no-op + compare(patched) + return + # Check that calling `load_pth_hook()` again is a no-op + with _preserve_attributes(_GLOBAL_PATCHES) as re_patched: + load_pth_hook(call_ppid) + compare(re_patched) + # Check that the patches are reversed + LineProfilingCache.load().cleanup() # 'Current' instance + compare(patched) + + @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('debug', 'label'), +@pytest.mark.parametrize(('debug', 'label1'), [(True, 'with-debug'), (False, 'no-debug')]) +@pytest.mark.parametrize(('fail', 'label2'), + [(True, 'failure'), (False, 'success')]) +@pytest.mark.parametrize('n', [[100, 200, 1000]]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], debug: bool, - label: str, + fail: bool, + n: list[int], + label1: str, label2: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -1558,6 +1665,15 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: return False return True + def get_lineno(path: os.PathLike[str] | str, query: str) -> int: + with Path(path).open() as fobj: + for i, line in enumerate(fobj): + if query in line: + return 1 + i + raise RuntimeError( + f'Did not find line containing {query!r} in {path!r}', + ) + config: Path | None = None if debug: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' @@ -1566,8 +1682,10 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: 'intercept_logs = true' ) + func = ext_module_object.my_external_sum + func_name = f'{func.__module__}.{func.__qualname__}' cache = create_cache( - profiling_targets=[_SAFE_TARGET], + profiling_targets=[func_name], preimports_module=True, config=config, debug=True, @@ -1584,14 +1702,24 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - func = _import_target(_SAFE_TARGET) - return_lines = _find_return_lines(_SAFE_TARGET) + ranges = [range(1, x + 1) for x in n] + expected_nloops = sum(n) + timing_key = ( + inspect.getfile(func), + inspect.getsourcelines(func)[1], + func.__qualname__, + ) + assert ext_module_object.__file__ + loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') Pool: Callable[..., multiprocessing.pool.Pool] + + if fail: + xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + else: + xc_context = nullcontext() + if start_method == 'dummy': Pool = _import_target('multiprocessing.dummy.Pool') - # Twice the counted calls because we're also collecting the - # checking calls in this process - expected_ncalls = len(_SAFE_TARGET_ARGS) * 2 get_stats: Callable[[], LineStats] = cache.profiler.get_stats elif start_method not in START_METHODS: pytest.skip( @@ -1600,23 +1728,26 @@ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: ) else: Pool = multiprocessing.get_context(start_method).Pool - expected_ncalls = len(_SAFE_TARGET_ARGS) get_stats = cache.gather_stats - with Pool(2) as pool: - par_result = pool.starmap(func, _SAFE_TARGET_ARGS) - pool.close() - pool.join() - assert par_result == [func(*args) for args in _SAFE_TARGET_ARGS] + with ExitStack() as stack: + pool = stack.enter_context(Pool(2)) + # Check for the expected error (if `fail`) + stack.enter_context(xc_context) + try: + par_result = pool.starmap(func, [(r, fail) for r in ranges]) + finally: + pool.close() + pool.join() + + # Check correctness of the results + if not fail: + assert par_result == [x * (x + 1) // 2 for x in n] # Check that calls in children are traced - line_entries = get_stats().timings[ - inspect.getfile(func), inspect.getsourcelines(func)[1], func.__name__, - ] - num_returns = sum( - nhits for lineno, nhits, _ in line_entries if lineno in return_lines - ) - ResultMismatch.compare(expected_ncalls, num_returns) + entries = get_stats().timings[timing_key] + nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) + ResultMismatch.compare(expected_nloops, nloops) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From 97a7dd7216302bf41e266cf9b60aad3bf6ce1eae Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 27 Apr 2026 22:12:48 +0200 Subject: [PATCH 074/111] Updated test code path for coverage tests/test_child_procs.py TEST_MODULE_TEMPLATE, test_module Updated code to allow for calling `sum_in_child_procs()` with the 'dummy' argument and switching to `multiprocessing.dummy.Pool` test_module_object `ModuleType` fixture representing the content of the above test_apply_mp_patches() - New parametrization `nprocs` for the number of child processes/threads created - Updated implementation to use `test_module_object.sum_in_child_procs()` After the changes, `coverage` seem to fare much better in tracing `multiprocessing_patches.py` and `cache.py`. --- tests/test_child_procs.py | 95 +++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 38 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index cc3c3f0e..340dd963 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -88,7 +88,7 @@ def my_external_sum(x: list[int], fail: bool = False) -> int: from argparse import ArgumentParser from collections.abc import Callable -from multiprocessing import get_context, Pool +from multiprocessing import dummy, get_context, Pool from typing import Literal from {EXT_MODULE} import my_external_sum @@ -107,7 +107,9 @@ def my_local_sum(x: list[int], fail: bool = False) -> int: def sum_in_child_procs( length: int, n: int, my_sum: Callable[[list[int]], int], - start_method: Literal['fork', 'forkserver', 'spawn'] | None = None, + start_method: Literal[ + 'fork', 'forkserver', 'spawn', 'dummy' + ] | None = None, fail: bool = False, ) -> int: my_list: list[int] = list(range(1, length + 1)) @@ -119,7 +121,9 @@ def sum_in_child_procs( while my_list: sublist, my_list = my_list[:sublength], my_list[sublength:] sublists.append(sublist) - if start_method: + if start_method == 'dummy': + pool = dummy.Pool(n) + elif start_method: pool = get_context(start_method).Pool(n) else: pool = Pool(n) @@ -319,6 +323,19 @@ def ext_module_object( yield from ext_module._import_module_helper() +@pytest.fixture +def test_module_object( + test_module: _ModuleFixture, ext_module_object: ModuleType, +) -> Generator[ModuleType, None, None]: + """ + Yields: + :py:class:`ModuleType` object containing the code at + :py:data:`TEST_MODULE_TEMPLATE`, and is torn down at the end of + the test + """ + yield from test_module._import_module_helper() + + @pytest.fixture def create_cache( tmp_path_factory: pytest.TempPathFactory, @@ -1640,17 +1657,19 @@ def test_load_pth_hook( [(True, 'with-debug'), (False, 'no-debug')]) @pytest.mark.parametrize(('fail', 'label2'), [(True, 'failure'), (False, 'success')]) -@pytest.mark.parametrize('n', [[100, 200, 1000]]) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) def test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, + test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], debug: bool, fail: bool, - n: list[int], + n: int, + nprocs: int, label1: str, label2: str, ) -> None: """ @@ -1674,16 +1693,29 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: f'Did not find line containing {query!r} in {path!r}', ) - config: Path | None = None + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + cfg_chunks: list[str] = [] if debug: - config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - config.write_text( + cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) + cfg_chunks.append( + '[tool.line_profiler.child_processes.multiprocessing.polling]\n' + 'on_timeout = "error"' + ) + config.write_text('\n\n'.join(cfg_chunks)) - func = ext_module_object.my_external_sum - func_name = f'{func.__module__}.{func.__qualname__}' + profiled_func = ext_module_object.my_external_sum + called_func = partial( + test_module_object.sum_in_child_procs, + n=nprocs, + my_sum=profiled_func, + start_method=start_method, + fail=fail, + ) + + func_name = f'{profiled_func.__module__}.{profiled_func.__qualname__}' cache = create_cache( profiling_targets=[func_name], preimports_module=True, @@ -1702,52 +1734,39 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert cache.preimports_module is not None run_path(str(cache.preimports_module), {'profile': cache.profiler}) - ranges = [range(1, x + 1) for x in n] - expected_nloops = sum(n) timing_key = ( - inspect.getfile(func), - inspect.getsourcelines(func)[1], - func.__qualname__, + inspect.getfile(profiled_func), + inspect.getsourcelines(profiled_func)[1], + profiled_func.__qualname__, ) assert ext_module_object.__file__ loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') - Pool: Callable[..., multiprocessing.pool.Pool] if fail: xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + nloops_expected = n else: xc_context = nullcontext() + nloops_expected = n + nprocs - if start_method == 'dummy': - Pool = _import_target('multiprocessing.dummy.Pool') - get_stats: Callable[[], LineStats] = cache.profiler.get_stats - elif start_method not in START_METHODS: + if start_method not in ('dummy', *START_METHODS): pytest.skip( f'`multiprocessing` start method {start_method!r} ' 'not available on the platform' ) - else: - Pool = multiprocessing.get_context(start_method).Pool - get_stats = cache.gather_stats - - with ExitStack() as stack: - pool = stack.enter_context(Pool(2)) - # Check for the expected error (if `fail`) - stack.enter_context(xc_context) - try: - par_result = pool.starmap(func, [(r, fail) for r in ranges]) - finally: - pool.close() - pool.join() - # Check correctness of the results - if not fail: - assert par_result == [x * (x + 1) // 2 for x in n] + with xc_context: + result = called_func(n) + # Check correctness of the results + if not fail: + assert result == n * (n + 1) // 2 # Check that calls in children are traced - entries = get_stats().timings[timing_key] + stats = cache.profiler.get_stats() + stats += cache.gather_stats() + entries = stats.timings[timing_key] nloops = sum(nhits for ln, nhits, _ in entries if ln == loop_line) - ResultMismatch.compare(expected_nloops, nloops) + ResultMismatch.compare(nloops_expected, nloops) # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests From 100be5ad63dcc0b6aa205072dc54a1f7bbdc09e7 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 28 Apr 2026 23:27:12 +0200 Subject: [PATCH 075/111] Fixed test pollution line_profiler/_child_process_profiling/multiprocessing_patches.py ::_apply_mp_patches() Refactored internals; existence of `multiprocessing.spawn` and `multiprocessing.forkserver` now checked at import time tests/test_child_procs.py _preserve_attributes.compare_with_current_values() - Argument `assert_true` can now take a mapping to which the result is compared - Now explicitly constructing an error message when the assertion is false, instead of relying on `pytest`'s assertion rewriting _GLOBAL_PATCHES Now protecting against changes to additional attributes patched_attributes Wrapper fixture around `_GLOBAL_PATCHES` which excludes nonexistent names (e.g. `os.fork()` on Windows) test_cache_dump_load() test_load_pth_hook() Now making sure that all cache instances created are cleaned up, not just those created by the `create_cache()` fixture test_cache_setup_main_process() Minor refactoring --- .../multiprocessing_patches.py | 54 +++++--- tests/test_child_procs.py | 127 +++++++++++++----- 2 files changed, 127 insertions(+), 54 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index b2822f49..c9631ad2 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -28,6 +28,21 @@ ) from typing_extensions import Concatenate, ParamSpec, Self +try: + from multiprocessing import spawn +except ImportError: + _CAN_USE_SPAWN = False +else: + _CAN_USE_SPAWN = True +try: + from multiprocessing import forkserver +except ImportError: + _CAN_USE_FORKSERVER = False +else: + _CAN_USE_FORKSERVER = ( + 'forkserver' in multiprocessing.get_all_start_methods() + ) + from .. import _diagnostics as diagnostics from ..toml_config import ConfigSource from .cache import LineProfilingCache @@ -488,13 +503,8 @@ def _apply_mp_patches( '_bootstrap': wrap_bootstrap}}, ) # Patch `multiprocessing.spawn` - try: - from multiprocessing import spawn - except ImportError: - pass - else: - if hasattr(spawn, 'runpy'): - lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) + if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): + lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages if intercept_mp_logs is None: intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] @@ -506,23 +516,25 @@ def _apply_mp_patches( # - Now, so that the (rebooted) fork-server process has profiling # set up; and # - Also as a part of cache cleanup - # (this uses `ForkServer._stop()` which is private API, but it's the - # same hack used in Python's own test suite -- see the comment to - # said method) - if reboot_forkserver: - try: - from multiprocessing import forkserver - except ImportError: # Incompatible platform # nocover - pass - else: - server_instance: forkserver.ForkServer = forkserver._forkserver - stop = getattr(server_instance, '_stop', None) - assert callable(stop) # Appease the type checker - stop() - lp_cache.add_cleanup(stop) + if _CAN_USE_FORKSERVER and reboot_forkserver: + _stop_forkserver() + lp_cache.add_cleanup(_stop_forkserver) # Mark `multiprocessing` as having been patched lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) +def _stop_forkserver() -> None: + """ + Note: + This uses `ForkServer._stop()` which is private API, but it's + the same hack used in Python's own test suite -- see the comment + to said method + """ + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + def _no_op(*_, **__) -> None: pass diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 340dd963..38119dd3 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -24,8 +24,10 @@ from tempfile import TemporaryDirectory from textwrap import dedent, indent from time import monotonic -from types import ModuleType, TracebackType -from typing import Any, Generic, Literal, TypeVar, cast, final, overload +from types import MappingProxyType, ModuleType, TracebackType +from typing import ( + TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, final, overload, +) from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -38,7 +40,7 @@ ) from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, + _Poller, _PATCHED_MARKER, ) from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, @@ -938,8 +940,21 @@ def compare_with_current_values( cls, old: Mapping[str, Mapping[str, Any]], comparator: Callable[[Any, Any], bool] = operator.is_, - assert_true: bool = True, + assert_true: bool | Mapping[str, Mapping[str, bool]] = True, ) -> dict[str, dict[str, bool]]: + def get_from_mapping(target: str, attr: str) -> bool: + if TYPE_CHECKING: + assert isinstance(assert_true, Mapping) + return assert_true[target][attr] + + def get_from_boolean(*_, **__) -> bool: + return True + + if isinstance(assert_true, Mapping): + get_expected: Callable[[str, str], bool] = get_from_mapping + else: + get_expected = get_from_boolean + result: dict[str, dict[str, bool]] = {} new = cls.fetch_current_values(old) for target, old_values in old.items(): @@ -948,15 +963,18 @@ def compare_with_current_values( for attr, old_value in old_values.items(): print(f'Checking: {target}.{attr}') new_value = new_values[attr] - if assert_true: - assert cmp_results.setdefault( - attr, comparator(new_value, old_value), + cmp_results[attr] = cmp_result = comparator( + new_value, old_value, + ) + expected_result = get_expected(target, attr) + if assert_true and (cmp_result != expected_result): + assert False, ( + f'Comparing `{target}.{attr}` ' + f'(old: {old_value!r} @ {id(old_value):#x}; ' + f'new: {new_value!r} @ {id(new_value):#x}): ' + f'expected comparison with {comparator!r} to return ' + f'{expected_result}, got {cmp_result}' ) - else: - # There's probably a more concise way to write this, - # but we want more info to be available in the - # traceback should the above assertion fail... - cmp_results[attr] = comparator(new_value, old_value) return result @@ -1292,14 +1310,33 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # tested APIs and behaviors MUST NOT be relied upon by end-users. _GLOBAL_PATCHES = { + f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': frozenset({ + 'called', + }), + 'multiprocessing': frozenset({_PATCHED_MARKER}), 'multiprocessing.process.BaseProcess': frozenset({ '_bootstrap', 'terminate', }), 'multiprocessing.spawn': frozenset({'runpy'}), + 'multiprocessing.util': frozenset({ + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', + }), 'os': frozenset({'fork'}), } +@pytest.fixture(scope='module') +def patched_attributes() -> MappingProxyType[str, frozenset[str]]: + result: dict[str, frozenset[str]] = {} + for target, attrs in _GLOBAL_PATCHES.items(): + try: + obj = _import_target(target) + except ImportError: + continue + result[target] = frozenset(a for a in attrs if hasattr(obj, a)) + return MappingProxyType(result) + + @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @pytest.mark.parametrize(('as_module', 'label2'), @@ -1409,6 +1446,7 @@ def test_cache_dump_load( original = create_cache( profiling_targets=['foo', 'bar', 'baz'], main_pid=123456, ) + cache_instances: list[LineProfilingCache] = [original] envvars: set[str] = set(os.environ) try: original.inject_env_vars() # Needed for `.load()` @@ -1417,7 +1455,9 @@ def test_cache_dump_load( assert set(os.environ) == envvars.union(original.environ) > envvars original.dump() loaded = original.load() + cache_instances.append(loaded) reloaded = original.load() + cache_instances.append(reloaded) assert original is not loaded is reloaded # Compare init fields for field in dataclasses.fields(LineProfilingCache): @@ -1428,7 +1468,8 @@ def test_cache_dump_load( == getattr(loaded, field.name) ) finally: # Explicitly cleanup - original.cleanup() + for cache in cache_instances: + cache.cleanup() finally: # Env vars restored after cleanup assert set(os.environ) == envvars @@ -1439,6 +1480,7 @@ def test_cache_dump_load( [(True, 'with-debug'), (False, 'no-debug')]) def test_cache_setup_main_process( create_cache: Callable[..., LineProfilingCache], + patched_attributes: MappingProxyType[str, frozenset[str]], wrap_os_fork: bool, debug: bool, label1: str, label2: str, @@ -1448,11 +1490,17 @@ def test_cache_setup_main_process( as expected. """ cache = create_cache(debug=debug) + # By default, we don't patch the `multiprocessing.util` logging + # facilities patches: dict[str, dict[str, bool]] = { - target: dict.fromkeys(attrs, True) - for target, attrs in _GLOBAL_PATCHES.items() + target: dict.fromkeys(attrs, target != 'multiprocessing.util') + for target, attrs in patched_attributes.items() } - patches['os']['fork'] = wrap_os_fork and (sys.platform != 'win32') + try: + patches['os']['fork'] = wrap_os_fork + except KeyError: + # `os.fork()` pruned because it doesn't exist on e.g. Windows + assert not hasattr(os, 'fork') with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) compare_patched = partial( @@ -1463,8 +1511,7 @@ def test_cache_setup_main_process( # There should be exactly one extra `.pth` file new_pth_hook, = _preserve_pth_files.get_pth_files() - original_pths # Check whether the patches are applied - patch_summary = compare_patched(operator.is_not, assert_true=False) - assert patch_summary == patches + compare_patched(operator.is_not, assert_true=patches) # Check whether the patches are reversed cache.cleanup() compare_patched() @@ -1592,6 +1639,7 @@ def list_profiled_funcs() -> list[str]: @pytest.mark.parametrize('ppid_should_match', [True, False, None]) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], + patched_attributes: MappingProxyType[str, frozenset[str]], another_pid: int, ppid_should_match: bool | None, ) -> None: @@ -1630,23 +1678,36 @@ def test_load_pth_hook( cache.dump() compare = _preserve_attributes.compare_with_current_values - with _preserve_attributes(_GLOBAL_PATCHES) as patched: - load_pth_hook(call_ppid) - # Check that the patches are applied where appropriate - assert ( - getattr(load_pth_hook, 'called', False) == bool(ppid_should_match) - ) - if ppid_should_match: - compare(patched, operator.is_not) - else: # no-op - compare(patched) - return - # Check that calling `load_pth_hook()` again is a no-op - with _preserve_attributes(_GLOBAL_PATCHES) as re_patched: + patches = patched_attributes.copy() + del patches['multiprocessing.util'] # Not patched by default + with _preserve_attributes(patches) as patched: + try: + # NOTE: this creates a cache instance that isn't + # automatically cleaned up by the `create_cache()` + # fixture!!! Hence the try-finally load_pth_hook(call_ppid) - compare(re_patched) + # Check that the patches are applied where appropriate + assert ( + getattr(load_pth_hook, 'called', False) + == bool(ppid_should_match) + ) + if ppid_should_match: + compare(patched, operator.is_not) + else: # no-op + compare(patched) + return + # Check that calling `load_pth_hook()` again is a no-op + with _preserve_attributes(patches) as re_patched: + load_pth_hook(call_ppid) + compare(re_patched) + finally: + try: + current_cache = LineProfilingCache.load() + except Exception: + pass + else: + current_cache.cleanup() # Check that the patches are reversed - LineProfilingCache.load().cleanup() # 'Current' instance compare(patched) From 5499f7b32dda9b378dedf88ad8d034af42001478 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 00:55:37 +0200 Subject: [PATCH 076/111] Bugfix + SIGTERM handling line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing] New key-value pair `catch_sigterm` for whether to catch SIGTERM in child processes started by `multiprocessing` line_profiler/_child_process_profiling/cache.py::LineProfilingCache _setup_common() Updated call signature _handle_sigterm(), _wrap_sigterm() New helper methods for dealing with `SIGTERM` (esp. in child processes created by `multiprocessing`) line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_terminate() Now writing debug messages to the log if the poller timed out and raised an error wrap_bootstrap() Now setting the SIGTERM handler before running the workload tests/test_child_procs.py::patched_attributes Fixed bug where objects with no attributes to be patched remain in the returned mapping --- .../_child_process_profiling/cache.py | 47 +++++++++++++++++-- .../multiprocessing_patches.py | 10 +++- line_profiler/rc/line_profiler.toml | 4 ++ tests/test_child_procs.py | 5 +- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index d7978f84..8ae702ee 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -7,6 +7,7 @@ import atexit import dataclasses import os +import signal try: import _pickle as pickle except ImportError: @@ -17,6 +18,7 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent +from threading import current_thread, main_thread from types import ModuleType from typing import Any, ClassVar, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -292,7 +294,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: self.dump() self.inject_env_vars() _import_sibling('pth_hook').write_pth_hook(self) - self._setup_common(wrap_os_fork, reboot_forkserver=True) + self._setup_common(wrap_os_fork, {'reboot_forkserver': True}) self._replace_loaded_instance() def _setup_in_child_process( @@ -359,7 +361,7 @@ def _setup_in_child_process( self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) # Various setups - self._setup_common(wrap_os_fork, reboot_forkserver=False) + self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate @@ -369,22 +371,59 @@ def _setup_in_child_process( return True def _setup_common( - self, wrap_os_fork: bool, reboot_forkserver: bool, + self, + wrap_os_fork: bool, + mp_apply_kwargs: dict[str, Any] | None = None, ) -> None: if wrap_os_fork: self._wrap_os_fork() _import_sibling('multiprocessing_patches').apply( - self, reboot_forkserver, + self, **(mp_apply_kwargs or {}), ) + def _handle_sigterm(self, signum: int, _) -> None: # nocover + """ + See also: + :py:meth:`coverage.control.Converage._on_sigterm` + """ + try: + name = signal.Signals(signum).name + self._debug_output( + f'Got signal {signum} ({name}), ' + 'cleaning up before passing it on...', + ) + # Also restores handler overwritten by `._wrap_sigterm()` + self.cleanup() + finally: + os.kill(os.getpid(), signum) + + def _wrap_sigterm(self) -> None: # nocover + """ + Side effects: + If on the main thread: + + - :py:func:`signal.signal` called to set + :py:meth:`~._handle_sigterm` as the ``SIGTERM`` handler + + - :py:meth:`~.cleanup` callback registered undoing that + """ + if current_thread() != main_thread(): + return + set_handler = signal.signal + self._debug_output('Adding `SIGTERM` handler...') + handler = set_handler(signal.SIGTERM, self._handle_sigterm) + self.add_cleanup_with_priority(set_handler, 1, signal.SIGTERM, handler) + def _wrap_os_fork(self) -> None: """ Create a wrapper around :py:func:`os.fork` which handles profiling. Side effects: + - :py:func:`os.fork` (if available) replaced with the wrapper + - :py:meth:`~.cleanup` callback registered undoing that """ try: diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c9631ad2..bfca6643 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -26,6 +26,7 @@ from typing import ( Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, ) + from typing_extensions import Concatenate, ParamSpec, Self try: @@ -331,6 +332,9 @@ def wait_for_return( try: with wait_for_return(cache.config): pass + except (_Poller.Timeout, _Poller.TimeoutWarning) as e: + cache._debug_output(f'{type(e).__qualname__}: {e}') + raise finally: # Always call `Process.terminate()` to avoid orphans vanilla_impl(self) @@ -355,6 +359,10 @@ def wrap_bootstrap( :py:mod:`line_profiler` to be loaded before it. Hence the ``# nocover``. """ + # Set a signal handler for SIGTERM to help child processes with + # consistently cleaning up + if _get_config(cache.config)['catch_sigterm']: + cache._wrap_sigterm() try: return vanilla_impl(self, *args, **kwargs) finally: @@ -454,7 +462,7 @@ def apply( .. _GH-126631: https://github.com/python/cpython/issues/126631 """ if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, reboot_forkserver) + _apply_mp_patches(lp_cache, reboot_forkserver=reboot_forkserver) def _apply_patches_generic( diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 5c53aadd..529a6783 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -229,6 +229,10 @@ suffix = '' # the internal log messages are captured and writtent to the debug # logs intercept_logs = false +# - `multiprocessing.catch_sigterm` (bool): +# Whether to set a hook for the child-process session caches to +# `.cleanup()` on SIGTERM +catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.polling] diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 38119dd3..e30309fe 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1333,7 +1333,10 @@ def patched_attributes() -> MappingProxyType[str, frozenset[str]]: obj = _import_target(target) except ImportError: continue - result[target] = frozenset(a for a in attrs if hasattr(obj, a)) + present_attrs = frozenset(a for a in attrs if hasattr(obj, a)) + # Drop if none of the attributes is present + if present_attrs: + result[target] = present_attrs return MappingProxyType(result) From 3362333f6853a845bed0f05f35835c8912ab9547 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 03:39:32 +0200 Subject: [PATCH 077/111] Fix signal handling line_profiler/_child_process_profiling/cache.py::LineProfilingCache cleanup() Now thread-locked to ensure that the signal handler doesn't call `Cleanup.cleanup()` in the middle of another call thereto _handle_signal() Refactored from `._handle_sigterm()` _add_signal_handler() - Refactored from `._wrap_sigterm()` - Now a no-op on Windows line_profiler/_child_process_profiling/multiprocessing_patches.py Updated correspondingly --- .../_child_process_profiling/cache.py | 89 ++++++++++++++----- .../multiprocessing_patches.py | 22 +++-- 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 8ae702ee..7174f3af 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -8,6 +8,7 @@ import dataclasses import os import signal +import sys try: import _pickle as pickle except ImportError: @@ -18,8 +19,8 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from threading import current_thread, main_thread -from types import ModuleType +from threading import current_thread, main_thread, RLock, Thread +from types import FrameType, ModuleType from typing import Any, ClassVar, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -42,6 +43,7 @@ PS = ParamSpec('PS') # Note: `typing.AnyStr` deprecated since 3.13 AnyStr = TypeVar('AnyStr', str, bytes) +_SignalHandler = Callable[[int, FrameType | None], Any] _THIS_SUBPACKAGE, *_ = (lambda: None).__module__.rpartition('.') INHERITED_CACHE_ENV_VARNAME_PREFIX = ( @@ -55,6 +57,9 @@ def _import_sibling(submodule: str) -> ModuleType: return import_module(f'{_THIS_SUBPACKAGE}.{submodule}') +_private_field = partial(dataclasses.field, init=False, repr=False) + + @final @dataclasses.dataclass class LineProfilingCache(Cleanup): @@ -76,12 +81,15 @@ class LineProfilingCache(Cleanup): insert_builtin: bool = True debug: bool = diagnostics.DEBUG - profiler: LineProfiler | None = dataclasses.field( - default=None, init=False, repr=False, + profiler: LineProfiler | None = _private_field(default=None) + _cleanup_stacks: dict[float, list[Callable[[], Any]]] = _private_field( + default_factory=dict, ) - _cleanup_stacks: dict[float, list[Callable[[], Any]]] = dataclasses.field( - default_factory=dict, init=False, repr=False, + _sighandlers: dict[int, _SignalHandler | int | None] = ( + _private_field(default_factory=dict) ) + _rlock: RLock = _private_field(default_factory=RLock) + _loaded_instance: ClassVar[LineProfilingCache | None] = None def __post_init__(self) -> None: @@ -166,6 +174,30 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) + def cleanup(self, *args, **kwargs) -> None: + """ + Perform cleanup. + + Args: + *args, **kwargs + Passed to :py:meth:`Cleanup.cleanup` + + Note: + In child processes we set a ``SIGTERM`` handler to always + call :py:meth:`~.cleanup`. However, this may happen when + we're in the middle of a cleanup call, which results in + undefined behavior. To prevent this, each + :py:meth:`~.cleanup` call is handled by a separate thread + which acquires an instance-specific lock. + """ + thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) + thread.start() + thread.join() + + def _cleanup_worker(self, *args, **kwargs) -> None: + with self._rlock: + super().cleanup(*args, **kwargs) + def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: """ Gather the profiling output files matching ``glob_pattern`` from @@ -381,38 +413,51 @@ def _setup_common( self, **(mp_apply_kwargs or {}), ) - def _handle_sigterm(self, signum: int, _) -> None: # nocover + def _handle_signal(self, signum: int, *_) -> None: # nocover """ See also: :py:meth:`coverage.control.Converage._on_sigterm` """ + name = self._get_signal_name(signum) + msg = f'Cleaning up before passing `{name}` ({signum}) on...' + self._debug_output(msg) try: - name = signal.Signals(signum).name - self._debug_output( - f'Got signal {signum} ({name}), ' - 'cleaning up before passing it on...', - ) - # Also restores handler overwritten by `._wrap_sigterm()` self.cleanup() finally: - os.kill(os.getpid(), signum) + handler = self._sighandlers.pop(signum, None) + if handler is not None: + signal.signal(signum, handler) + signal.raise_signal(signum) - def _wrap_sigterm(self) -> None: # nocover + def _add_signal_handler( + self, signum: int = signal.SIGTERM, + ) -> None: # nocover """ Side effects: - If on the main thread: + If on the main thread and not on Windows: - :py:func:`signal.signal` called to set - :py:meth:`~._handle_sigterm` as the ``SIGTERM`` handler + :py:meth:`~._handle_signal` as the ``SIGTERM`` handler - :py:meth:`~.cleanup` callback registered undoing that + + Note: + ``SIGTERM`` handling is known to be faulty on Windows; see + previous discussions at (examples `1`_, `2`_). + + .. _1: https://github.com/coveragepy/coveragepy/blob/main/\ +coverage/control.py + .. _2: https://stackoverflow.com/questions/35772001/ """ - if current_thread() != main_thread(): + if current_thread() != main_thread() or sys.platform == 'win32': return - set_handler = signal.signal - self._debug_output('Adding `SIGTERM` handler...') - handler = set_handler(signal.SIGTERM, self._handle_sigterm) - self.add_cleanup_with_priority(set_handler, 1, signal.SIGTERM, handler) + name = self._get_signal_name(signum) + self._debug_output(f'Adding `{name}` handler...') + self._sighandlers[signum] = signal.signal(signum, self._handle_signal) + + @staticmethod + def _get_signal_name(signum: int) -> str: + return signal.Signals(signum).name def _wrap_os_fork(self) -> None: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index bfca6643..d156581e 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -332,7 +332,7 @@ def wait_for_return( try: with wait_for_return(cache.config): pass - except (_Poller.Timeout, _Poller.TimeoutWarning) as e: + except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') raise finally: # Always call `Process.terminate()` to avoid orphans @@ -352,17 +352,23 @@ def wrap_bootstrap( ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. - Note: - This is only invoked in child processes, and :py:mod:`coverage` - seem to be having trouble with them in the current setup, - probably due to issues with .pth file precendence causing - :py:mod:`line_profiler` to be loaded before it. Hence the - ``# nocover``. + Notes: + + - This is only invoked in child processes, and + :py:mod:`coverage` seems to be having trouble with them in the + current setup, probably due to issues with .pth file + precendence causing :py:mod:`line_profiler` to be loaded + before it. Hence the ``# nocover``. + + - ``SIGTERM`` handling is not consistent on Windows, so we made + :py:meth:`LineProfilingCache._add_signal_handler` a no-op + there. Hence :py:func:`wrap_terminate` remains necessary in + mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up if _get_config(cache.config)['catch_sigterm']: - cache._wrap_sigterm() + cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) finally: From bfda35dfcca535d62cd8c2edcdfe017f0f51dfd1 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 06:25:44 +0200 Subject: [PATCH 078/111] Fix for profiler cleanup line_profiler/curated_profiling.py::CuratedProfilerContext install() Now handling profiler disabling as a regular cleanup callback uninstall() - Simplifed implementation - Now always calling `.cleanup()` _disable_profiler() New helper method for disabling the profiler (code previosuly lived in `.uninstall()`) line_profiler/_child_process_profiling/cache.py::LineProfilingCache _cleanup_stacks Removed unused attribute (superseded by `._contexts`) cleanup() Added argument `new_thread` for controlling whether to spin up a new thread for do cleanup (note that the profiler has a different `.enable_count` on each thread, so cleaning up on a separate thread may impact proper profiler cleanup) copy() - Simplified implementation - Removed unused arguments `inherit_cleanups` and `inherit_profiler` _setup_in_child_process() Now intercepting the debug-log messages of the created `CuratedProfilerContext` _handle_signal() Now calling `.cleanup()` with `new_thread=True` tests/test_child_procs.py create_cache() Fixed typo in doc test_cache_setup_main_process() - Removed reference to unused/indirectly-used `curated_profiler` fixture - Now explicitly checking for the disabling of the profiler --- .../_child_process_profiling/cache.py | 74 ++++++++++--------- line_profiler/curated_profiling.py | 13 ++-- tests/test_child_procs.py | 5 +- 3 files changed, 50 insertions(+), 42 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 7174f3af..82bc1ef2 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -76,15 +76,12 @@ class LineProfilingCache(Cleanup): profile_imports: bool = False preimports_module: os.PathLike[str] | str | None = None main_pid: int = dataclasses.field(default_factory=os.getpid) - # Note: if we're using the line profiler, `kernprof` always set + # Note: if we're using the line profiler, `kernprof` always sets # `builtin` to true insert_builtin: bool = True debug: bool = diagnostics.DEBUG profiler: LineProfiler | None = _private_field(default=None) - _cleanup_stacks: dict[float, list[Callable[[], Any]]] = _private_field( - default_factory=dict, - ) _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) @@ -95,22 +92,11 @@ class LineProfilingCache(Cleanup): def __post_init__(self) -> None: super().__init__() - def copy( - self, *, - inherit_cleanups: bool = False, - inherit_profiler: bool = False, - **replacements - ) -> Self: + def copy(self, /, **replacements) -> Self: """ Make a copy with optionally replaced fields. Args: - inherit_cleanups (bool): - If true, the copy also makes a (shallow) copy of the - cleanup-callback stack. - inherit_profiler (bool): - If true, the copy also gets a reference to - :py:attr:`~.profiler` **replacements (Any): Optional fields to replace @@ -121,15 +107,7 @@ def copy( init_args: dict[str, Any] = {} for field, value in self._get_init_args().items(): init_args[field] = replacements.get(field, value) - copy = type(self)(**init_args) - if inherit_cleanups: - copy._cleanup_stacks = { - priority: list(callbacks) - for priority, callbacks in self._cleanup_stacks.items() - } - if inherit_profiler: - copy.profiler = self.profiler - return copy + return type(self)(**init_args) @classmethod def load(cls) -> Self: @@ -174,22 +152,35 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) - def cleanup(self, *args, **kwargs) -> None: + def cleanup(self, *args, new_thread: bool = False, **kwargs) -> None: """ Perform cleanup. Args: - *args, **kwargs + new_thread (bool): + Whether to relegate the call to + :py:meth:`Cleanup,cleanup` to a new thread (see Notes) + *args, **kwargs: Passed to :py:meth:`Cleanup.cleanup` Note: - In child processes we set a ``SIGTERM`` handler to always - call :py:meth:`~.cleanup`. However, this may happen when - we're in the middle of a cleanup call, which results in - undefined behavior. To prevent this, each - :py:meth:`~.cleanup` call is handled by a separate thread - which acquires an instance-specific lock. - """ + - In child processes we set a ``SIGTERM`` handler to always + call :py:meth:`~.cleanup`. However, this may happen when + we're already in the middle of a cleanup call, which + results in undefined behavior. To prevent this, we can + supply ``new_thread=True`` so that the + :py:meth:`Cleanup.cleanup` call is handled by a separate + thread which acquires an instance-specific lock. + + - However, this method is supposed to clean up the session + profiler by completely disabling it, and that part must + happen on the main thread or deallocation will be botched. + Hence ``new_thread=True`` is made an option and not the + default. + """ + if not new_thread: + self._cleanup_worker(*args, **kwargs) + return thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) thread.start() thread.join() @@ -356,6 +347,11 @@ def _setup_in_child_process( False the instance has already been set up prior to calling this function, true otherwise """ + def wrap_ctx_debug( + ctx: CuratedProfilerContext, msg: str, + ) -> None: + self._debug_output(f' Context {id(ctx):#x}: {msg}') + if not context: context = '...' self._debug_output(f'Setting up ({context})...') @@ -369,6 +365,8 @@ def _setup_in_child_process( prof = LineProfiler() self.profiler = prof ctx = CuratedProfilerContext(prof, insert_builtin=self.insert_builtin) + if self.debug: + self.patch(ctx, '_debug_output', wrap_ctx_debug.__get__(ctx)) ctx.install() self.add_cleanup(ctx.uninstall) self._debug_output(f'Set up `.profiler` at {id(prof):#x}') @@ -422,7 +420,13 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover msg = f'Cleaning up before passing `{name}` ({signum}) on...' self._debug_output(msg) try: - self.cleanup() + # We don't care about profiler state and such if we're + # already being SIGTERM-ed, so just handle cleanup on a + # separate thread; + # this prevents a duplicate call to `.cleanup()` when we're + # already inside one (e.g. when `Process._bootstrap()` is + # exiting, or the `atexit` hook is triggered) + self.cleanup(new_thread=True) finally: handler = self._sighandlers.pop(signum, None) if handler is not None: diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 3b4c8ac4..80513daa 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -193,6 +193,11 @@ def _global_install(self, prof: ByCountProfilerMixin | None) -> None: # objects to it should be the exception, not the norm. self._kpo(cast(LineProfiler, prof)) + @staticmethod + def _disable_profiler(prof: ByCountProfilerMixin) -> None: + for _ in range(getattr(prof, 'enable_count', 0)): + prof.disable_by_count() + def install(self) -> None: if self._installed: return @@ -208,14 +213,12 @@ def install(self) -> None: # Set up hooks to deal with inserting `.prof` as a builtin name if self.insert_builtin: self.patch(builtins, self.builtin_loc, self.prof) + # Disable the profiler + self.add_cleanup(self._disable_profiler, self.prof) + self.patch(self, '_installed', True) def uninstall(self) -> None: - if not self._installed: - return - # Fully disable the profiler - for _i in range(getattr(self.prof, 'enable_count', 0)): - self.prof.disable_by_count() self.cleanup() def __enter__(self) -> Self: diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index e30309fe..486b6720 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -360,7 +360,7 @@ def create_cache( - ``False``: equivalent to ``None``. - Unless the argument ``_use_curated_profiler: bool = True`` is set - to :py:const`True`, automatically creates an instance of + to :py:const:`False`, automatically creates an instance of :py:class:`LineProfiler` that is curated by a :py:class:`CuratedProfilerContext` and provides it as the :py:attr:`LineProfilingCache.profiler`, and @@ -1550,7 +1550,6 @@ def test_cache_setup_main_process( @_preserve_attributes(_GLOBAL_PATCHES) def test_cache_setup_child( create_cache: Callable[..., LineProfilingCache], - curated_profiler: LineProfiler, ext_module_object: ModuleType, another_pid: int, wrap_os_fork: bool, @@ -1617,6 +1616,8 @@ def list_profiled_funcs() -> list[str]: assert (os.fork is not old_fork) == fork_patched else: # E.g. Windows assert old_fork == _NotSupplied.NOT_SUPPLIED + # Check that after cleaning up the profiler has been disabled + assert not getattr(cache.profiler, 'enable_count', 0) # Check that profiling results have been written to the cache # directory From 96b8975f2969951a2f9ac87bce926a8b3167709f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 29 Apr 2026 07:44:17 +0200 Subject: [PATCH 079/111] Fix test failing corner case --- tests/test_retry_tests.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index b18df4e0..873c7681 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -678,18 +678,21 @@ def parse_line(cls, line: str) -> Self: # that created them went obsolete, before the next rerun; # we can verify that by checking that the ids of the `makefile()` # fixtures appear in contiguous blocks + + # Note: there seems to be a weird corner case where neighboring tests + # may reuse the same fixture id (see failing job 73520441960 in + # pipeline 25091142386); probably has to do with object lifetime. + # So instead of just checking the `fixture_id`, also consult + # `test`; it suffices to see that WITHIN THE SAME TEST we don't have + # fixture values stepping over one another with log.open() as fobj: entries = [LogEntry.parse_line(line.rstrip('\n')) for line in fobj] pprint.pprint(entries) - field: tuple[str, ...] | str - for field in ('test', 'stage'), 'fixture_id': - if isinstance(field, str): - getter: Callable[[LogEntry], Any] = attrgetter(field) - else: - getter = attrgetter(*field) + for fields in ('test', 'stage'), ('test', 'fixture_id'): + getter = attrgetter(*fields) values = [getter(entry) for entry in entries] assert _identical_items_are_adjacent(values), ( - f'Inconsistency in {field} order: {values!r}' + f'Inconsistency in {fields} order: {values!r}' ) From ddbec4e1c8ea00d6623930c6db0e3c6a62e0b648 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 2 May 2026 01:23:12 +0200 Subject: [PATCH 080/111] Streamlined tests line_profiler/_child_process_profiling/multiprocessing_patches.py ::_Poller.__enter__() If `on_timeout='warn'`, the log message is now written BEFORE issuing the warning (so as to avoid having the latter preventing the latter via raising the warning) tests/test_child_procs.py _Param.sorted() New method for sorting the parametrizations of instances test_apply_mp_patches() - Renamed parameter `intercept_logs` from `debug` - Removed several redundant subtests (no need to test `intercept_logs` with all `multiprocessing` start methods) test_multiproc_script_sanity_check() Removed several redundant subtests (tests with `preimports=True` overlapping with `test_apply_mp_patches`) --- .../multiprocessing_patches.py | 4 +- tests/test_child_procs.py | 153 ++++++++++++------ 2 files changed, 106 insertions(+), 51 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index d156581e..26d01c49 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -201,8 +201,10 @@ def error(msg: str) -> NoReturn: raise type(self).Timeout(msg) def warn(msg: str) -> None: - warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) + # Write log before issuing the warning because that may be + # promoted to an exception diagnostics.log.warning(msg) + warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) def ignore(_): pass diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 486b6720..dcd55e92 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -19,6 +19,7 @@ from functools import partial, wraps from io import StringIO from importlib import import_module +from numbers import Real from pathlib import Path from runpy import run_path from tempfile import TemporaryDirectory @@ -692,12 +693,57 @@ def __add__(self, other: Self) -> Self: ) return type(self)(new_params, new_values, new_defaults_tuple) + def sorted( + self, + *, + sort_by: Sequence[str] | None = None, + sortable_types: type[Any] | tuple[type[Any], ...] = (Real, str, bytes), + ) -> Self: + """ + Sort by parametrization values. + + Args: + sort_by (Sequence[str] | None): + Column names to sort by; default is to sort by all + sortable params. + sortable_types (type[Any] | tuple[type[Any], ...]): + Type(s) where if a param has all its values being + instances thereof (excl. :py:const:`None`s), said param + is considered sortable. + + Returns: + New instance + """ + def sort_key(obj: Any) -> tuple[bool, str, Any]: + type_name = '{0.__module__}.{0.__qualname__}'.format(type(obj)) + return (obj is None), type_name, obj + + if sort_by is None: + sort_by = self.params + sortable_columns: set[str] = { + param for param, *values in zip(self.params, *self.values) + if all(isinstance(v, sortable_types) or v is None for v in values) + } + sorted_column_indices: tuple[int, ...] = tuple( + i for i, param in enumerate(sort_by) if param in sortable_columns + ) + + if sorted_column_indices: + new_values = sorted( + self.values, key=lambda vtuple: tuple( + sort_key(vtuple[i]) for i in sorted_column_indices + ), + ) + else: # Fallback + new_values = self.values.copy() + return type(self)(self.params, new_values, self.defaults) + def __call__(self, func: C) -> C: """ Mark a callable as with :py:func:`pytest.mark.parametrize`. """ # Note: `pytest` automatically assumes single-param values to - # be unpackes, so comply here + # be unpacked, so comply here if len(self.params) == 1: marker = pytest.mark.parametrize( self.params[0], [v[0] for v in self.values], @@ -1716,12 +1762,17 @@ def test_load_pth_hook( @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) -@pytest.mark.parametrize('start_method', - ['fork', 'forkserver', 'spawn', 'dummy']) -@pytest.mark.parametrize(('debug', 'label1'), - [(True, 'with-debug'), (False, 'no-debug')]) -@pytest.mark.parametrize(('fail', 'label2'), - [(True, 'failure'), (False, 'success')]) +@(_Params.new('start_method', + ['fork', 'forkserver', 'spawn', 'dummy'], + defaults='dummy') + * _Params.new(('fail', 'label2'), + [(True, 'failure'), (False, 'success')], + defaults=(False, 'success')) + # We only need to check if `intercept_logs` work, the other + # parametrizations don't matter + + _Params.new(('intercept_logs', 'label1'), + [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], + defaults=(False, 'no-intercept-logs'))).sorted() @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) @@ -1731,7 +1782,7 @@ def test_apply_mp_patches( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - debug: bool, + intercept_logs: bool, fail: bool, n: int, nprocs: int, @@ -1760,17 +1811,21 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' cfg_chunks: list[str] = [] - if debug: + if intercept_logs: cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing]\n' 'intercept_logs = true' ) + # This is easier to debug than `ResultMismatch` cfg_chunks.append( '[tool.line_profiler.child_processes.multiprocessing.polling]\n' 'on_timeout = "error"' ) config.write_text('\n\n'.join(cfg_chunks)) + # Note: no need to test the case for `my_local_sum()` separately, + # with `preimports_module=True`, both are just imported and added + # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum called_func = partial( test_module_object.sum_in_child_procs, @@ -1840,7 +1895,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: for path in Path(cache.cache_dir).glob('*.lprof') if is_valid_stats_file(path) } - patterns[re.escape('`multiprocessing` logging (debug)')] = debug + patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs _search_cache_logs(cache, True, patterns) @@ -1864,24 +1919,19 @@ def _get_mp_start_method_fuzzer(label_name: str) -> _Params: return fuzz_fail * fuzz_start -_fuzz_sanity = ( - _Params.new(('run_func', 'label1'), - [(run_module, 'module'), (run_script, 'script')]) - * _Params.new(('use_local_func', 'label2'), - [(True, 'local'), (False, 'ext')]) - # Python can't pickle things unless they resided in a retrievable - # location (so not the script supplied by `python -c`) - + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), - [(run_literal_code, 'literal-code', False, 'ext')]) - # Also fuzz the parallelization-related stuff, esp. check what - # happens if an exception is raised inside the parallelly-run func - + _get_mp_start_method_fuzzer('label3') - + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], - defaults=(None, None)) -) - - -@_fuzz_sanity +@(_Params.new(('run_func', 'label1'), + [(run_module, 'module'), (run_script, 'script')]) + * _Params.new(('use_local_func', 'label2'), + [(True, 'local'), (False, 'ext')]) + # Python can't pickle things unless they resided in a retrievable + # location (so not the script supplied by `python -c`) + + _Params.new(('run_func', 'label1', 'use_local_func', 'label2'), + [(run_literal_code, 'literal-code', False, 'ext')]) + # Also fuzz the parallelization-related stuff, esp. check what + # happens if an exception is raised inside the parallelly-run func + + _get_mp_start_method_fuzzer('label3') + + _Params.new(('nnums', 'nprocs'), [(200, None), (None, 3)], + defaults=(None, None))).sorted() def test_multiproc_script_sanity_check( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, @@ -1951,28 +2001,31 @@ def test_running_multiproc_script( run_func(test_module, tmp_path_factory, runner, outfile, profile) -_fuzz_prof_mp_1 = ( - _Params.new(('run_func', 'label1'), - [(run_module, 'module'), - (run_script, 'script'), - (run_literal_code, 'literal-code')], - defaults=(run_script, 'script')) - + _Params.new(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _get_mp_start_method_fuzzer('label3') -) -_fuzz_prof_mp_2 = ( - _Params.new(('preimports', 'label4'), - [(True, 'with-preimports'), (False, 'no-preimports')], - defaults=(False, 'no-preimports')) - + _Params.new(('use_local_func', 'label5'), - [(True, 'local'), (False, 'external')], - defaults=(False, 'external')) -) - - -@_fuzz_prof_mp_1 -@_fuzz_prof_mp_2 +_fuzz_prof_mp_run_func = _Params.new(('run_func', 'label1'), + [(run_module, 'module'), + (run_script, 'script'), + (run_literal_code, 'literal-code')], + defaults=(run_script, 'script')) + + +@( + (_fuzz_prof_mp_run_func + + _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + + _get_mp_start_method_fuzzer('label3')) + # Test all `multiproc` start methods with both locally- and + # externally-defined profiling targets + * (_Params.new(('preimports', 'label4'), [(False, 'no-preimports')]) + + _Params.new(('use_local_func', 'label5'), + [(True, 'local'), (False, 'external')], + defaults=(False, 'external'))) + # The 'with-preimports' case is already tested rather thoroughly in + # `test_apply_mp_patches()`, so exclude these from the above "main" + # param matrix and just test the different `kernprof` modes via the + # `run_func()`s + + (_fuzz_prof_mp_run_func + + _Params.new(('preimports', 'label4'), [(True, 'with-preimports')])) +).sorted() @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @pytest.mark.parametrize( From a2968137cebc815312cc4222b99c19052a07fa7b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 05:39:32 +0200 Subject: [PATCH 081/111] Update `line_profiler.line_profiler.LineStats` line_profiler/line_profiler.py Remapped some `typing` imports and annotations because we're now on 3.10+ LineStats.__eq__(), .__[i]add__(), from_stats_objects() Fixed misrepresentation of the data structure in docstrings (filenames and function names swapped in `.timings` keys) LineStats.from_files() Added `on_empty` argument to handle empty files (that we know `pickle.load()` will fail on) and special handling therefor, so that error/warning messages are more informative --- line_profiler/line_profiler.py | 114 +++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 48 deletions(-) diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 50ef0d88..98845f3a 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -20,20 +20,10 @@ import tokenize import warnings from argparse import ArgumentParser +from collections.abc import Callable, Collection, Mapping, Sequence from datetime import datetime from os import PathLike -from typing import ( - TYPE_CHECKING, - IO, - Callable, - Literal, - Mapping, - Protocol, - Sequence, - TypeVar, - cast, - Tuple, -) +from typing import TYPE_CHECKING, IO, Any, Literal, Protocol, TypeVar, cast try: from ._line_profiler import ( @@ -63,7 +53,7 @@ class _IPythonLike(Protocol): def register_magics(self, magics: type) -> None: ... PS = ParamSpec('PS') - _TimingsMap = Mapping[Tuple[str, int, str], list[Tuple[int, int, int]]] + _TimingsMap = Mapping[tuple[str, int, str], list[tuple[int, int, int]]] T = TypeVar('T') T_co = TypeVar('T_co', covariant=True) @@ -227,6 +217,15 @@ def tokeneater( return super().tokeneater(type, token, srowcol, erowcol, line) +class _EmptyFileError(OSError): + """ + Error raised when trying to read profiling data from an empty file. + """ + def __init__(self, file: PathLike[str] | str) -> None: + super().__init__(str(file)) + self.file = file + + class _WrapperInfo: """ Helper object for holding the state of a wrapper function. @@ -265,8 +264,8 @@ def __eq__(self, other: object) -> bool: Example: >>> from copy import deepcopy >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = deepcopy(stats1) @@ -275,7 +274,7 @@ def __eq__(self, other: object) -> bool: >>> assert stats2 != stats1 >>> stats3 = deepcopy(stats1) >>> assert stats1 == stats3 is not stats1 - >>> stats3.timings['foo', 1, 'spam.py'][:] = [(2, 11, 330)] + >>> stats3.timings['spam.py', 1, 'foo'][:] = [(2, 11, 330)] >>> assert stats3 != stats1 """ for attr in 'timings', 'unit': @@ -291,20 +290,20 @@ def __add__(self, other: _StatsLike) -> Self: """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_sum = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, ... 1E-6) >>> assert stats1 + stats2 == stats2 + stats1 == stats_sum """ @@ -315,20 +314,20 @@ def __iadd__(self, other: _StatsLike) -> Self: """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_sum = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]}, ... 1E-6) >>> address = id(stats2) >>> stats2 += stats1 @@ -386,6 +385,7 @@ def from_files( file: PathLike[str] | str, /, *files: PathLike[str] | str, + on_empty: Literal['ignore', 'warn', 'error'] = 'warn', on_defective: Literal['ignore', 'warn', 'error'] = 'error', ) -> Self: """ @@ -396,10 +396,11 @@ def from_files( File to load profiling data from *files (PathLike[str] | str): Ditto above - on_defective (Literal['ignore', 'warn', 'error']): - What to do if some files fail to load: ``'ignore'`` - those files, skip them but with a ``'warn'``-ing, or - raise the ``'error'`` as soon as one is encountered + on_empty, on_defective (Literal['ignore', 'warn', 'error']): + What to do if some files are empty (resp. otherwise fail + to load): ``'ignore'`` those files, skip them but with a + ``'warn'``-ing, or raise the ``'error'`` as soon as one + is encountered Returns: instance (LineStats): @@ -407,25 +408,42 @@ def from_files( """ stats_objs = [] failures: dict[str, str] = {} + empty_files: set[str] = set() all_files = [file, *files] + for file in all_files: with open(file, 'rb') as f: try: + if not os.stat(file).st_size: + raise _EmptyFileError(file) stats_objs.append(pickle.load(f)) + except _EmptyFileError as e: + if on_empty == 'error': + raise + empty_files.add(str(e.file)) except Exception as e: if on_defective == 'error': raise failures[str(file)] = f'{type(e).__name__}: {e}' - if failures: - msg = ( - '{} file(s) out of {} failed to load and are skipped: {!r}' - .format(len(failures), len(all_files), failures) + + problems: Collection[Any] + for problems, description, behavior in [ + (list(empty_files), 'is/are empty and thus skipped:', on_empty), + (failures, 'failed to load and is/are skipped', on_defective), + ]: + if not problems: + continue + msg = '{} file(s) out of {} {}: {!r}'.format( + len(problems), len(all_files), description, problems, ) - if on_defective == 'warn': - warnings.warn(msg) + if behavior == 'warn': + # Log before warning because warnings may be promoted to + # errors diagnostics.log.warning(msg) + warnings.warn(msg) else: # 'ignore' diagnostics.log.debug(msg) + if not stats_objs: return cls.get_empty_instance() return cls.from_stats_objects(*stats_objs) @@ -437,23 +455,23 @@ def from_stats_objects( """ Example: >>> stats1 = LineStats( - ... {('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... {('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 2, 1000), (12, 1, 500)]}, ... 1E-6) >>> stats2 = LineStats( - ... {('bar', 10, 'spam.py'): + ... {('spam.py', 10, 'bar'): ... [(11, 10, 20000), (12, 5, 1000)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 5000)]}, + ... ('eggs.py', 5, 'baz'): [(5, 2, 5000)]}, ... 1E-7) >>> stats_combined = LineStats.from_stats_objects( ... stats1, stats2) >>> assert stats_combined.unit == 1E-6 >>> assert stats_combined.timings == { - ... ('foo', 1, 'spam.py'): [(2, 10, 300)], - ... ('bar', 10, 'spam.py'): + ... ('spam.py', 1, 'foo'): [(2, 10, 300)], + ... ('spam.py', 10, 'bar'): ... [(11, 12, 3000), (12, 6, 600)], - ... ('baz', 5, 'eggs.py'): [(5, 2, 500)]} + ... ('eggs.py', 5, 'baz'): [(5, 2, 500)]} """ timings, unit = cls._get_aggregated_timings([stats, *more_stats]) return cls(timings, unit) @@ -890,7 +908,7 @@ def show_func( func_name (str): name of profiled function - timings (List[Tuple[int, int, float]]): + timings (list[tuple[int, int, float]]): Measurements for each line (lineno, nhits, time). unit (float): From 6bdb17f7bd2a6c2683a8b91f24f277e112cad45f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 05:51:20 +0200 Subject: [PATCH 082/111] Various cleanup line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.gather_stats() Now emitting warnings for all anomalous cases (empty and defective files) line_profiler/cleanup.py::Cleanup.patch() Fixed bug where the `priority` argument is unused, instead of passed to `.add_cleanup_with_priority()` as advertised --- line_profiler/_child_process_profiling/cache.py | 4 +++- line_profiler/cleanup.py | 15 +++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 82bc1ef2..39f72297 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -202,7 +202,9 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: ) if not fnames: return LineStats.get_empty_instance() - return LineStats.from_files(*fnames, on_defective='ignore') + return LineStats.from_files( + *fnames, on_empty='warn', on_defective='warn', + ) def _dump_debug_logs(self) -> None: """ diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index 97b6b05a..ac866128 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -15,7 +15,7 @@ from pathlib import Path from types import MethodType from typing import Any, TypeVar, TypedDict, cast -from typing_extensions import ParamSpec, Self, Unpack +from typing_extensions import Concatenate, ParamSpec, Self, Unpack from .line_profiler_utils import block_indent, make_tempfile from . import _diagnostics as diagnostics @@ -517,13 +517,20 @@ def patch( >>> assert obj.foo == 1 >>> assert not hasattr(obj, 'bar') """ - add_cleanup = self.add_cleanup if cleanup else (lambda *_, **__: None) + if cleanup: + add_cleanup: Callable[ + Concatenate[Callable[..., Any], float, ...], Any + ] = self.add_cleanup_with_priority + else: + # ... yeah gotta disagree with flake8, a lambda makes + # perfect sense here + add_cleanup = lambda *_, **__: None # noqa: E731 try: old = getattr(obj, attr) except AttributeError: - add_cleanup(delattr, obj, attr) + add_cleanup(delattr, priority, obj, attr) else: - add_cleanup(setattr, obj, attr, old) + add_cleanup(setattr, priority, obj, attr, old) setattr(obj, attr, value) if name is None: name = self._get_name(obj) From f5faa4814654c4578b61ff51a71c9de38340e15f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 18:58:21 +0200 Subject: [PATCH 083/111] Curb stray warnings for profiling outfiles line_profiler/_child_process_profiling/cache.py::LineProfilingCache gather_stats() - Updated call signature - Now excluding the profiling-stats file from the `multiprocessing` resource-tracker server process, which is often empty because of process longevity and would have resulted in a stray warning _get_profiling_outfiles() New helper method for finding specific profiling-stats files line_profiler/_child_process_profiling/pth_hook.py::write_pth_hook() The .pth file written is now cleaned up with increased priority so as to be less disruptive to other Python processes; most importantly, it is removed before the `multiprocessing` fork-server process is rebooted, so that the new server process will not be polluted by profiling stuff kernprof.py _manage_profiler.__enter__(), .__exit__() Reworked cleanup-task priorities: - Writing of debug logs now managed by the `CuratedProfilerContext` instead of the `LineProfilingCache` - `LineProfilingCache.cleanup()` now called before the gathering of child-process profiling results, to ensure that results from the `multiprocessing` fork-server process is properly handled _prepare_child_profiling_cache() - Updated call signature, taking also a `CuratedProfilerContext` - Said context (instead of the cache) now manages the lifetime of the tempdir --- kernprof.py | 27 ++++--- .../_child_process_profiling/cache.py | 78 +++++++++++++++++-- .../_child_process_profiling/pth_hook.py | 2 + 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/kernprof.py b/kernprof.py index 0e1a4b77..e9612fad 100755 --- a/kernprof.py +++ b/kernprof.py @@ -1207,19 +1207,19 @@ def __enter__(self): ) if self.set_up_child_profiling: self.cache = cache = _prepare_child_profiling_cache( - self.options, self.prof, preimports_file, script_file, + self.options, self._ctx, self.prof, + preimports_file, script_file, ) - self._ctx.add_cleanup(cache.cleanup) # Add deferred callbacks for gathering debug logfiles # (should run right before `.cache.cache_dir` is wiped): # - Write the debug logs to the `._diagnostics` logger if cache.debug: - cache.add_cleanup_with_priority( + self._ctx.add_cleanup_with_priority( cache._dump_debug_logs, CLEANUP_PRIORITIES['gather_logs'], ) # - Write the debug logs to a specific file if self.options.debug_log: - cache.add_cleanup_with_priority( + self._ctx.add_cleanup_with_priority( self._gather_debug_log, CLEANUP_PRIORITIES['gather_logs'], self.options.debug_log, @@ -1230,11 +1230,13 @@ def __exit__(self, *_, **__): try: extra_stats = None if self.set_up_child_profiling: + # Cleaning up here ensures the `multiprocessing` + # fork-server process is rebooted, thus any profiling + # data on it will be properly collected + self.cache.cleanup() extra_stats = self.cache.gather_stats() _post_profile(self.options, self.prof, extra_stats) finally: - # This also calls its `.cleanup()`, and in turn that of - # `.cache`'s self._ctx.uninstall() def _gather_debug_log(self, logfile): @@ -1344,7 +1346,9 @@ def _prepare_exec_script( return script_file, preimports_file -def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): +def _prepare_child_profiling_cache( + options, ctx, prof, preimports_file, script_file +): """ Handle the (line-)profiling of spawned/forked child Python processes. @@ -1364,8 +1368,10 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): ) clean_up = functools.partial(cache.add_cleanup, _remove, missing_ok=True) if not diagnostics.KEEP_TEMPDIRS: - # Defer the scrubbing of the cache dir - cache.add_cleanup_with_priority( + # Defer the scrubbing of the cache dir and let the context + # handle it, ideally speaking the cache dir should survive the + # cache object + ctx.add_cleanup_with_priority( _remove, CLEANUP_PRIORITIES['rm_cache_dir'], cache.cache_dir, recursive=True, ) @@ -1383,6 +1389,9 @@ def _prepare_child_profiling_cache(options, prof, preimports_file, script_file): cache._setup_in_main_process() cache.profiler = prof + # Have the context clean up the cache as a failsafe + ctx.add_cleanup(cache.cleanup) + return cache diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 39f72297..2ff902a7 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -16,12 +16,13 @@ from collections.abc import Collection, Callable, MutableMapping, Iterable from functools import partial, cached_property, wraps from importlib import import_module +from operator import attrgetter from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent from threading import current_thread, main_thread, RLock, Thread from types import FrameType, ModuleType -from typing import Any, ClassVar, TypeVar, cast, final, overload +from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self from .. import _diagnostics as diagnostics @@ -51,6 +52,21 @@ ) CACHE_FILENAME = 'line_profiler_cache.pkl' _DEBUG_LOG_FILENAME_PATTERN = 'debug_log_{main_pid}_{current_pid}.log' +_PROFILING_OUTPUT_PREFIX_PATTERN = ( + 'child-prof-output-{main_pid}-{current_pid}-{prof}-' +) +_DEFAULT_GATHER_STATS_EXCLUDES: set[tuple[str, str]] = { + # Note: the `ResourceTracker` server process is spawned when the + # first `multiprocessing` child process is created via the `spawn` + # or `forkserver` start method. While this server process does not + # meaningfully contribute to the profiling result either way, since + # it can be created with profiling set up, its longevity means that + # `LineProfilingCache.gather_stats()` often catches an empty .lprof + # file which it has occupied but not written to. To reduce noise + # while keeping the warning for other zero-length files, just + # explictly exclude said process + ('multiprocessing.resource_tracker', '_resource_tracker._pid'), +} def _import_sibling(submodule: str) -> ModuleType: @@ -189,13 +205,48 @@ def _cleanup_worker(self, *args, **kwargs) -> None: with self._rlock: super().cleanup(*args, **kwargs) - def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: + def gather_stats( + self, + exclude_pids: Collection[int] | None = None, + *, + on_empty: Literal['error', 'warn', 'ignore'] = 'warn', + on_defective: Literal['error', 'warn', 'ignore'] = 'warn', + ) -> LineStats: """ Gather the profiling output files matching ``glob_pattern`` from :py:attr:`~.cache_dir`, consolidating them into a single :py:class:`LineStats` object. + + Args: + exclude_pids (Collection[int] | None): + Exclude output from child processes with these PIDs; + the default value :py:const:`None` fetches relevant + PIDs dynamically. + on_empty, on_defective (Literal['error', 'warn', 'ignore']): + Passed to :py:meth:`LineStats.from_files`. + + Returns: + :py:class:`LineStats` instance """ - fnames = list(Path(self.cache_dir).glob(glob_pattern)) + if exclude_pids is None: + exclude_pids = set() + for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: + try: + module = import_module(import_target) + except ImportError: + continue + try: + maybe_pid = attrgetter(attr)(module) + except AttributeError: + maybe_pid = None + if maybe_pid is None: + continue + exclude_pids.add(cast(int, maybe_pid)) + + fnames_ = set(self._get_profiling_outfiles()) + for pid in exclude_pids: + fnames_.difference_update(self._get_profiling_outfiles(pid)) + fnames = sorted(fnames_) self._debug_output( 'Loading results from {} child profiling file(s): {!r}' .format(len(fnames), fnames) @@ -203,7 +254,7 @@ def gather_stats(self, glob_pattern: str = '*.lprof') -> LineStats: if not fnames: return LineStats.get_empty_instance() return LineStats.from_files( - *fnames, on_empty='warn', on_defective='warn', + *fnames, on_empty=on_empty, on_defective=on_defective, ) def _dump_debug_logs(self) -> None: @@ -252,10 +303,20 @@ def _gather_debug_log_entries( def _get_debug_logfiles(self) -> Iterable[Path]: pattern = _DEBUG_LOG_FILENAME_PATTERN.format( - main_pid=self.main_pid, current_pid='*', + main_pid=self.main_pid, current_pid='?*', ) return Path(self.cache_dir).glob(pattern) + def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: + prefix = _PROFILING_OUTPUT_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=pid, + # We always format the profiler ID with `hex()`, see + # `._setup_in_child_process()` + prof='0x?*', + ) + return Path(self.cache_dir).glob(prefix + '?*.lprof') + def inject_env_vars( self, env: MutableMapping[str, str] | None = None, ) -> None: @@ -385,8 +446,11 @@ def wrap_ctx_debug( # up to write thereto when the process terminates (with high # priority) prof_outfile = self.make_tempfile( - prefix='child-prof-output-{}-{}-{:#x}-' - .format(self.main_pid, os.getpid(), id(prof)), + prefix=_PROFILING_OUTPUT_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=os.getpid(), + prof=hex(id(prof)), + ), suffix='.lprof', delete=False, ) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/line_profiler/_child_process_profiling/pth_hook.py index 4fa6b9ef..416c3d76 100644 --- a/line_profiler/_child_process_profiling/pth_hook.py +++ b/line_profiler/_child_process_profiling/pth_hook.py @@ -66,6 +66,8 @@ def write_pth_hook(cache): # type: (LineProfilingCache) -> Path prefix=pth_config['prefix'], suffix=pth_config['suffix'] + '.pth', dir=get_path('purelib'), + # Get rid of the .pth file ASAP so as to be the least disruptive + priority=1, ) try: pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( From 952ce8d8700e8edbf67817284030060a71160a65 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 21:02:02 +0200 Subject: [PATCH 084/111] WIP: migrate .pth-related code line_profiler/_child_process_profiling/cache.py::LineProfilingCache write_pth_hook() Refactored from `~~.pth_hook.write_pth_hook` ._setup_in_main_process() Now using `.write_pth_hook()` ._config_source Convenience property for accessing the `ConfigSource` object loaded from `.config` _line_profiler_hooks.py Moved from `line_profiler/_child_process_profiling/pth_hook.py` setup.py Now installing `_line_profiler_hooks` --- .../pth_hook.py => _line_profiler_hooks.py | 0 .../_child_process_profiling/cache.py | 86 +++++++++++++++++-- setup.py | 4 +- 3 files changed, 80 insertions(+), 10 deletions(-) rename line_profiler/_child_process_profiling/pth_hook.py => _line_profiler_hooks.py (100%) diff --git a/line_profiler/_child_process_profiling/pth_hook.py b/_line_profiler_hooks.py similarity index 100% rename from line_profiler/_child_process_profiling/pth_hook.py rename to _line_profiler_hooks.py diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 2ff902a7..f04ea971 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -9,11 +9,14 @@ import os import signal import sys +import sysconfig try: import _pickle as pickle except ImportError: import pickle # type: ignore[assignment,no-redef] -from collections.abc import Collection, Callable, MutableMapping, Iterable +from collections.abc import ( + Collection, Callable, Mapping, MutableMapping, Iterable, +) from functools import partial, cached_property, wraps from importlib import import_module from operator import attrgetter @@ -25,16 +28,13 @@ from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self +from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook from .. import _diagnostics as diagnostics from ..cleanup import Cleanup from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats +from ..toml_config import ConfigSource from ._cache_logging import CacheLoggingEntry -# Note: this should have been defined here in this file, but we moved it -# over to `~._child_process_hook` because that module contains the .pth -# hook, which must run with minimal overhead when a Python process isn't -# associated with a profiled process -from .pth_hook import INHERITED_PID_ENV_VARNAME __all__ = ('LineProfilingCache',) @@ -335,6 +335,67 @@ def inject_env_vars( _format_debug_msg='Injecting env var ${{{1}}}: {2}'.format, ) + def write_pth_hook( + self, *, + prefix: str | None = None, + suffix: str | None = None, + dir: os.PathLike[str] | str | None = None, + # Get rid of the .pth file ASAP so as to be the least disruptive + priority: float = 1, + **kwargs + ) -> Path: + """ + Write a .pth file which allows for setting up profiling in child + Python processes. + + Args: + prefix, suffix (str | None): + Optional filename-stem affixes of the .pth file; default + is to use default values loaded from :py:attr:`.config` + dir (os.PathLike[str] | str | None): + Optional directory to create the .pth file in; default + is to use ``sysconfig.get_path('purelib')`` + priority, **kwargs: + Passed to :py:meth:`.make_tempfile`. + + Returns: + fpath (Path): + Path to the written .pth file + """ + def get_pth_config() -> Mapping[str, Any]: + # Note: the only keys in it should be `prefix` and `suffix` + return ( + self._config_source # Cached + .get_subconfig('child_processes', 'pth_files') + .conf_dict + ) + + if not os.path.exists(self.filename): + self.dump() + assert os.path.exists(self.filename) + + # The string casts are failsafes in case inappropriate values + # (e.g. numbers and booleans) are supplied + if prefix is None: + prefix = str(get_pth_config()['prefix']) + if suffix is None: + suffix = str(get_pth_config()['suffix']) + if dir is None: + dir = sysconfig.get_path('purelib') + + template = 'import {0.__module__}; {0.__module__}.{0.__name__}({1})' + fpath = self.make_tempfile( + prefix=prefix, suffix=suffix + '.pth', dir=dir, priority=priority, + **kwargs, + ) + try: + fpath.write_text(template.format(load_pth_hook, self.main_pid)) + except Exception: + fpath.unlink(missing_ok=True) + raise + + return fpath + def _debug_output(self, msg: str) -> None: """ Beside writing to the logger, also write to the @@ -364,8 +425,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: - A ``.pth`` file written so that child processes automaticaly runs setup code (see - :py:func:`line_profiler._child_process_hook.pth_hook.\ -write_pth_hook`) + :py:meth:`.write_pth_hook`) - :py:func:`os.fork` wrapped so that profiling set up in forked processes is properly handled (if @@ -379,7 +439,7 @@ def _setup_in_main_process(self, wrap_os_fork: bool = True) -> None: """ self.dump() self.inject_env_vars() - _import_sibling('pth_hook').write_pth_hook(self) + self.write_pth_hook() self._setup_common(wrap_os_fork, {'reboot_forkserver': True}) self._replace_loaded_instance() @@ -747,3 +807,11 @@ def _make_debug_entry(self) -> Callable[[str], CacheLoggingEntry]: @cached_property def _consistent_with_loaded_instance(self) -> bool: return type(self).load()._get_init_args() == self._get_init_args() + + @cached_property + def _config_source(self) -> ConfigSource: + if self.config is None: + config: str | None = None + else: + config = str(self.config) + return ConfigSource.from_config(config) diff --git a/setup.py b/setup.py index 739d5d96..a0810cc6 100755 --- a/setup.py +++ b/setup.py @@ -314,7 +314,9 @@ def run_cythonize(force=False): setupkw['long_description_content_type'] = 'text/x-rst' setupkw['license'] = 'BSD' setupkw['packages'] = list(setuptools.find_packages()) - setupkw['py_modules'] = ['kernprof', 'line_profiler'] + setupkw['py_modules'] = [ + 'kernprof', 'line_profiler', '_line_profiler_hooks', + ] setupkw['python_requires'] = '>=3.10' setupkw['license_files'] = ['LICENSE.txt', 'LICENSE_Python.txt'] setupkw['package_data'] = {'line_profiler': ['py.typed', '*.pyi', '*.toml']} From 3a7ce8714aa64d230fdee28c15f15cde8eb641ed Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 4 May 2026 21:29:15 +0200 Subject: [PATCH 085/111] Finish .pth code migration _line_profiler_hooks.py write_pth_hook() Removed (superseded by `LineProfilingCache.write_pth_hook()`) load_pth_hook() Fixed broken relative imports from `line_profiler` tests/test_child_procs.py Fixed import location of `load_pth_hook()` --- _line_profiler_hooks.py | 94 +++++++++------------------------------ tests/test_child_procs.py | 20 ++++----- 2 files changed, 30 insertions(+), 84 deletions(-) diff --git a/_line_profiler_hooks.py b/_line_profiler_hooks.py index 416c3d76..5b5c888a 100644 --- a/_line_profiler_hooks.py +++ b/_line_profiler_hooks.py @@ -1,96 +1,44 @@ """ -Hooks to set up shop in a child Python process and extend profiling -to therein. +Additional hooks installed by :py:mod:`line_profiler`. -Note: - - The current implementation writes temporary .pth files to the +Notes: + - This file and its content should be considered an implmentation + detail of :py:mod:`line_profiler`; currently we just use this to + set up shop in a child Python process, and extend profiling to + therein. + + - This current implementation writes temporary .pth files to the site-packages directory, which are executed for all Python processes referring to the same :path:`lib/`. However, only processes originating from a parent which set the requisite environment variables will execute to the profiling code. + - Said .pth file always import this module; hence, this file is kept - intentionally lean to reduce overhead: - - Imports in this file are deferred to being as late as possible. - - Type annotations are replaced with type comments. - - Non-essential functionalities are split into small separate - submodules (e.g. :py:mod:`~.cache`). + intentionally lean and separate from the main + :py:mod:`line_profiler` package to reduce overhead; e.g. + imports in this file are deferred to being as late as possible. + - Inspired by similar code in :py:mod:`coverage.control` and :py:mod:`pytest_autoprofile.startup_hook`. """ -from typing import TYPE_CHECKING +import os -if TYPE_CHECKING: - from pathlib import Path # noqa: F401 - from .cache import LineProfilingCache # noqa: F401 - -__all__ = ('write_pth_hook', 'load_pth_hook') +__all__ = ('load_pth_hook',) INHERITED_PID_ENV_VARNAME = ( 'LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID' ) -def write_pth_hook(cache): # type: (LineProfilingCache) -> Path - """ - Write a .pth file which allows for setting up profiling in child - Python processes. - - Args: - cache (:py:class:`~.LineProfilingCache`): - Cache object - - Returns: - fpath (Path): - Path to the written .pth file - - Note: - - To be called in the main process. - - The ``cache`` is responsible for deleting the written .pth - file via the registered cleanup callback. - """ - import os - from sysconfig import get_path - from ..toml_config import ConfigSource - - if not os.path.exists(cache.filename): - cache.dump() - assert os.path.exists(cache.filename) - - pth_config = ( - ConfigSource.from_config(cache.config) - .get_subconfig('child_processes', 'pth_files') - .conf_dict - ) - fpath = cache.make_tempfile( - prefix=pth_config['prefix'], - suffix=pth_config['suffix'] + '.pth', - dir=get_path('purelib'), - # Get rid of the .pth file ASAP so as to be the least disruptive - priority=1, - ) - try: - pth_content = 'import {0}; {0}.load_pth_hook({1})'.format( - (lambda: None).__module__, cache.main_pid, - ) - fpath.write_text(pth_content) - except Exception: - fpath.unlink(missing_ok=True) - raise - - return fpath - - -def load_pth_hook(ppid): # type: (int) -> None +def load_pth_hook(ppid: int) -> None: """ Function imported and called by the written .pth file; to reduce overhead, we immediately return if ``ppid`` doesn't match :env:`LINE_PROFILER_PROFILE_CHILD_PROCESSES_CACHE_PID`. """ - from os import environ - try: - env_ppid = int(environ[INHERITED_PID_ENV_VARNAME]) + env_ppid = int(os.environ[INHERITED_PID_ENV_VARNAME]) except (KeyError, ValueError): return if env_ppid != ppid: @@ -100,8 +48,8 @@ def load_pth_hook(ppid): # type: (int) -> None # profiled Python process, so we can be more liberal with the # imports without worrying about overhead import warnings - from .._diagnostics import DEBUG, log - from .cache import LineProfilingCache # noqa: F811 + from line_profiler._diagnostics import DEBUG, log + from line_profiler._child_process_profiling.cache import LineProfilingCache # Note: .pth files may be double-loaded in a virtual environment # (see https://stackoverflow.com/questions/58807569), so work around @@ -115,8 +63,10 @@ def load_pth_hook(ppid): # type: (int) -> None except Exception as e: # nocover if DEBUG: msg = f'{type(e)}: {e}' - warnings.warn(msg) + # Write log befor issuing warning, in case the warning is + # promoted to an exception log.warning(msg) + warnings.warn(msg) load_pth_hook.called = True # type: ignore else: cache.patch(load_pth_hook, 'called', True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index dcd55e92..ec182b12 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -35,11 +35,11 @@ import pytest import ubelt as ub +from _line_profiler_hooks import load_pth_hook from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) -from line_profiler._child_process_profiling.pth_hook import load_pth_hook from line_profiler._child_process_profiling.multiprocessing_patches import ( _Poller, _PATCHED_MARKER, ) @@ -1298,8 +1298,8 @@ def _run_test_module( expected = nnums * (nnums + 1) // 2 output_lines = proc.stdout.splitlines() ResultMismatch.compare(str(expected), output_lines[0]) - # - Temporary `.pth` file(s) created by `~~.pth_hook` has - # been cleaned up + # - Temporary `.pth` file(s) created by + # `LineProfilingCache.write_pth_hook()` has been cleaned up assert _preserve_pth_files.get_pth_files() == old_pth_files # - Profiling results are written to the specified file prof_result: LineStats | None = None @@ -1681,12 +1681,8 @@ def list_profiled_funcs() -> list[str]: _search_cache_logs(cache, debug, patterns) -@_preserve_attributes({ - **_GLOBAL_PATCHES, - 'line_profiler._child_process_profiling.pth_hook.load_pth_hook': - frozenset({'called'}), -}) @pytest.mark.parametrize('ppid_should_match', [True, False, None]) +@_preserve_attributes(_GLOBAL_PATCHES) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], patched_attributes: MappingProxyType[str, frozenset[str]], @@ -1694,16 +1690,16 @@ def test_load_pth_hook( ppid_should_match: bool | None, ) -> None: """ - Simulate calling :py:func:`line_profiler._child_process_profiling\ -.pth_hook.load_pth_hook()` in a child process. + Simulate calling :py:func:`_line_profiler_hooks.load_pth_hook()` in + a child process. Notes: - The function is CALLED in the .pth file, but we don't actually NEED a .pth file to call and test it. - - The counterpart :py:func:`line_profiler\ -._child_process_profiling.pth_hook.write_pth_hook()` + - The counterpart :py:meth:`line_profiler\ +._child_process_profiling.cache.LineProfilingCache.write_pth_hook()` is implicitly tested in :py:func:`test_cache_setup_main_process()`. """ From e8068cc473d26ce478f1cd349783b780454c97be Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 01:47:33 +0200 Subject: [PATCH 086/111] Fixes for cache management line_profiler/_child_process_profiling/cache.py ::LineProfilingCache.gather_stats() Now only excluding profiling-stats files from the "auto-PIDs" if they are empty, to catch the corner case where a properly-profiled child process happens to share a PID with a non-contemporary process that we exclude (currently only the `multiprocessing.resoure_tracker.ResourceTracker` process) tests/test_child_procs.py::test_apply_mp_patches() Added call to `LineProfilingCache.cleanup()` before `.gather_stats()` and the inspection of the cache-dir contents to better replicate the `kernprof` workflow; in particular this makes sure the fork-server process is properly terminated and its profiling-stats file is written to --- line_profiler/_child_process_profiling/cache.py | 16 ++++++++++++++-- tests/test_child_procs.py | 1 + 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index f04ea971..0e1ced85 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -15,7 +15,7 @@ except ImportError: import pickle # type: ignore[assignment,no-redef] from collections.abc import ( - Collection, Callable, Mapping, MutableMapping, Iterable, + Collection, Callable, Iterable, Mapping, MutableMapping, ) from functools import partial, cached_property, wraps from importlib import import_module @@ -228,6 +228,10 @@ def gather_stats( Returns: :py:class:`LineStats` instance """ + def is_empty(path: Path) -> bool: + return not path.stat().st_size + + filter_excludes: Callable[[Iterable[Path]], Iterable[Path]] if exclude_pids is None: exclude_pids = set() for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: @@ -242,10 +246,18 @@ def gather_stats( if maybe_pid is None: continue exclude_pids.add(cast(int, maybe_pid)) + # NOTE: there is no guarantee that the PID hasn't previously + # been used for another child process that we DID properly + # profile and SHOULD include, so we only filter out empty + # files + filter_excludes = partial(filter, is_empty) + else: # User-provided values, who are we to object? + filter_excludes = iter fnames_ = set(self._get_profiling_outfiles()) for pid in exclude_pids: - fnames_.difference_update(self._get_profiling_outfiles(pid)) + excludes = filter_excludes(self._get_profiling_outfiles(pid)) + fnames_.difference_update(excludes) fnames = sorted(fnames_) self._debug_output( 'Loading results from {} child profiling file(s): {!r}' diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index ec182b12..b1aa90be 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -1878,6 +1878,7 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert result == n * (n + 1) // 2 # Check that calls in children are traced + cache.cleanup() stats = cache.profiler.get_stats() stats += cache.gather_stats() entries = stats.timings[timing_key] From 8766714140c8048dfef57c0aa3e204bad9de50ce Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 01:50:37 +0200 Subject: [PATCH 087/111] Refactor `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py _PollerArgs New helper class for getting `_Poller` args from `LineProfilingCache.config` _get_config() Instead of taking an argument from which a `ConfigSource` can be loaded, now taking the `ConfigSource` itself wrap_terminate() Refactored internals into separate functions wrap_bootstrap(), _apply_mp_patches() Updated calls to `_get_config()` --- .../multiprocessing_patches.py | 148 +++++++++--------- 1 file changed, 72 insertions(+), 76 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 26d01c49..80b94de8 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -2,14 +2,14 @@ Patch :py:mod:`multiprocessing` so that profiling extends into processes it creates. -Notes ------ -- Based on the implementations in :py:mod:`coverage.multiproc` and - :py:mod:`pytest_autoprofile._multiprocessing`. -- Results may vary if the process pool is not properly - :py:meth:`multiprocessing.pool.Pool.close`-d and - :py:meth:`multiprocessing.pool.Pool.join`-ed; - see `this caveat `__. """ from __future__ import annotations @@ -17,14 +17,14 @@ import multiprocessing import warnings from collections.abc import Callable, Mapping -from functools import lru_cache, partial +from functools import partial from importlib import import_module from multiprocessing.process import BaseProcess -from os import PathLike from time import sleep, monotonic from types import MappingProxyType from typing import ( - Any, Generic, Literal, Protocol, TypeVar, Union, NoReturn, cast, + Any, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, + cast, final, ) from typing_extensions import Concatenate, ParamSpec, Self @@ -206,14 +206,11 @@ def warn(msg: str) -> None: diagnostics.log.warning(msg) warnings.warn(msg, type(self).TimeoutWarning, stacklevel=3) - def ignore(_): - pass - timeout = self._timeout callback = self._func handle_timeout: Callable[[str], Any] = { - 'error': error, 'warn': warn, 'ignore': ignore, + 'error': error, 'warn': warn, 'ignore': _no_op, }[self._on_timeout] fmt = '.3g' timeout_msg_header = f'{type(self).__name__} at {id(self):#x}' @@ -247,27 +244,64 @@ class TimeoutWarning(Timeout, UserWarning): pass -def _get_config( - config: PathLike[str] | str | bool | None = None, -) -> Mapping[str, Any]: - if config not in (True, False, None): - config = str(config) - return _get_config_cached(cast(Union[str, bool, None], config)) +@final +class _PollerArgs(NamedTuple): + cooldown: float + timeout: float + on_timeout: str | None + @classmethod + def from_config(cls, config: ConfigSource) -> Self: + values = _get_config(config)['polling'] + try: + cooldown = max(float(values['cooldown']), 0) + except (TypeError, ValueError): + cooldown = 0 + try: + timeout = max(float(values['timeout']), 0) + except (TypeError, ValueError): + timeout = 0 + try: + on_timeout: str | None = values['on_timeout'].lower() + except Exception: # Fallback (use `_Poller`'s default) + on_timeout = None + return cls(cooldown, timeout, on_timeout) -@lru_cache() -def _get_config_cached( - config: PathLike[str] | str | bool | None = None, -) -> Mapping[str, Any]: + @classmethod + def get_defaults(cls) -> Self: + namespace = globals() + try: + return namespace['_DEFAULT_POLLER_ARGS'] + except KeyError: + defaults = cls.from_config(ConfigSource.from_default(copy=False)) + return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) + + +def _get_config(config: ConfigSource) -> Mapping[str, Any]: cd = dict( - ConfigSource.from_config(config) - .get_subconfig('child_processes', 'multiprocessing', copy=True) + config.get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) assert isinstance(cd.get('polling'), Mapping) return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) +def _process_has_returned( + proc: BaseProcess, cache: LineProfilingCache, timeout: float, +) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result + + @LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -285,54 +319,15 @@ def wrap_terminate( on the bad path (e.g. the parallel workload errored out), and after the performance-critical part of the code (said workload). """ - # XXX: why can `coverage` get away with not doing all these - # lock-file hijinks and just patching `BaseProcess._bootstrap()`? - def get_poller_args( - config: PathLike[str] | str | bool | None = None, - ) -> tuple[float, float, str | None]: - values = _get_config(config)['polling'] - try: - cooldown = max(float(values['cooldown']), 0) - except (TypeError, ValueError): - cooldown = 0 - try: - timeout = max(float(values['timeout']), 0) - except (TypeError, ValueError): - timeout = 0 - try: - on_timeout: str | None = values['on_timeout'].lower() - except Exception: # Fallback (use `_Poller`'s default) - on_timeout = None - return cooldown, timeout, on_timeout - - def process_has_returned(proc: BaseProcess, timeout: float) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result - - def wait_for_return( - config: PathLike[str] | str | None = None, - ) -> _Poller: - cooldown, timeout, on_timeout = get_poller_args(config) - # `False` -> no resolution, force loading the vanilla file - *_, default_on_timeout = get_poller_args(False) - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = default_on_timeout - return ( - _Poller.poll_until(process_has_returned, self, cooldown) - .with_timeout(timeout, cast(_OnTimeout, on_timeout)) - ) - try: - with wait_for_return(cache.config): + cd, timeout, on_timeout = _PollerArgs.from_config(cache._config_source) + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = _PollerArgs.get_defaults().on_timeout + # `_process_has_returned()` takes a `timeout` which it passes to + # `popen.wait()`; said timeout is essentially a limit as to how + # often the function is called, hence our cooldown + poller = _Poller.poll_until(_process_has_returned, self, cache, cd) + with poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)): pass except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') @@ -369,7 +364,7 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if _get_config(cache.config)['catch_sigterm']: + if _get_config(cache._config_source)['catch_sigterm']: cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) @@ -523,7 +518,8 @@ def _apply_mp_patches( lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) # Intercept `multiprocessing` debug messages if intercept_mp_logs is None: - intercept_mp_logs = _get_config(lp_cache.config)['intercept_logs'] + config = lp_cache._config_source + intercept_mp_logs = _get_config(config)['intercept_logs'] if intercept_mp_logs: lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} From 62a7e5c0f82acf1063eda727296ab2cb2a9e2222 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 5 May 2026 05:29:14 +0200 Subject: [PATCH 088/111] WIP: refactor flaky tests tests/test_child_procs.py _Params.drop_params(), .split_on_params() New helper methods for constructing instances based on existing ones _check_warnings New context manager for helping with checking the absence and presence of warnings at exit time _run_test_module() (This function underlies these tests: - `test_multiproc_script_sanity_check()` - `test_running_multiproc_script` - `test_profiling_multiproc_script_{success,failure}()`) Added protection against stray .pth file created when running `kernprof` test_cache_setup_child() Added check that warnings for empty profiling files are emitted iff expected test_apply_mp_patches_{success,failure}() - Split parametrized test into separate tests, and only allow retries on the known flaky cases - Added check that no stray warnings are emitted test_profiling_multiproc_script_{success,failure}() test_profiling_bare_python_{success,failure}() Split parametrized tests into separate tests, and only allow retries on the known flaky cases --- tests/test_child_procs.py | 791 +++++++++++++++++++++++++++++++++----- 1 file changed, 690 insertions(+), 101 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index b1aa90be..5092b220 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -11,9 +11,10 @@ import subprocess import sys import sysconfig +import warnings from abc import ABC, abstractmethod from collections.abc import ( - Callable, Collection, Generator, Iterable, Mapping, Sequence, + Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, ) from contextlib import AbstractContextManager, ExitStack, nullcontext from functools import partial, wraps @@ -27,7 +28,8 @@ from time import monotonic from types import MappingProxyType, ModuleType, TracebackType from typing import ( - TYPE_CHECKING, Any, Generic, Literal, TypeVar, cast, final, overload, + TYPE_CHECKING, Any, Generic, Literal, Protocol, TypeVar, + cast, final, overload, ) from typing_extensions import Self, ParamSpec from uuid import uuid4 @@ -36,6 +38,7 @@ import ubelt as ub from _line_profiler_hooks import load_pth_hook +from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, @@ -525,9 +528,10 @@ def __init__( actual: Any | _NotSupplied = _NotSupplied.NOT_SUPPLIED, _trunc_tb: int = 0, ) -> None: - msg = f'expected: {expected}' - if actual != _NotSupplied.NOT_SUPPLIED: - msg = f'{msg}, got {actual}' + if actual == _NotSupplied.NOT_SUPPLIED: + msg = f'expected: {expected}' + else: + msg = f'expected {expected}, got {actual}' super().__init__(msg) self.expected = expected self.actual = actual @@ -738,6 +742,119 @@ def sort_key(obj: Any) -> tuple[bool, str, Any]: new_values = self.values.copy() return type(self)(self.params, new_values, self.defaults) + def drop_params(self, params: Collection[str] | str) -> Self: + """ + Return a new instance with the named ``params`` dropped; params + that don't match :py:attr:`.params` are ignored. + + Example: + >>> p = _Params.new(('a', 'b'), [(1, 2), (3, 4)]) + >>> p.drop_params('a') + _Params(params=('b',), values=[(2,), (4,)], defaults=(2,)) + >>> assert p.drop_params(['c', 'd']) == p + """ + def drop(t: tuple[T, ...]) -> tuple[T, ...]: + return tuple(item for i, item in enumerate(t) if i not in dropped) + + if isinstance(params, str): + params = params, + dropped = {i for i, p in enumerate(self.params) if p in params} + return type(self)( + drop(self.params), + [drop(pvalues) for pvalues in self.values], + drop(self.defaults), + ) + + @overload + def split_on_params( + self, params: tuple[str, ...], *, drop_split_params: bool = True, + ) -> dict[tuple[Any, ...], Self]: + ... + + @overload + def split_on_params( + self, params: str, *, drop_split_params: bool = True, + ) -> dict[Any, Self]: + ... + + def split_on_params( + self, params: tuple[str, ...] | str, *, drop_split_params: bool = True, + ) -> dict[tuple[Any, ...], Self] | dict[Any, Self]: + """ + Return new instances splitting on the values of the named + ``params``; params that don't match :py:attr:`.params` results + in an error. + + Example: + >>> p = _Params.new(('a', 'b', 'c'), + ... [(1, 2, True), + ... (1, 2, False), + ... (3, 4, True)]) + + >>> p.split_on_params('a') # doctest: +NORMALIZE_WHITESPACE + {1: _Params(params=('b', 'c'), + values=[(2, True), (2, False)], + defaults=(2, True)), + 3: _Params(params=('b', 'c'), + values=[(4, True)], + defaults=(2, True))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... ('a', 'b'), + ... ) + {(1, 2): _Params(params=('c',), + values=[(True,), (False,)], + defaults=(True,)), + (3, 4): _Params(params=('c',), + values=[(True,)], + defaults=(True,))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... 'a', drop_split_params=False, + ... ) + {1: _Params(params=('a', 'b', 'c'), + values=[(1, 2, True), (1, 2, False)], + defaults=(1, 2, True)), + 3: _Params(params=('a', 'b', 'c'), + values=[(3, 4, True)], + defaults=(1, 2, True))} + + >>> p.split_on_params( # doctest: +NORMALIZE_WHITESPACE + ... ('c', 'd'), + ... ) + Traceback (most recent call last): + ... + ValueError: params = ('c', 'd'): + these params not found: ['d'] + """ + if isinstance(params, str): + params = params, + unpack = True + else: + unpack = False + nonexistent = sorted(set(params) - set(self.params)) + if nonexistent: + raise ValueError( + f'params = {params!r}: these params not found: {nonexistent!r}' + ) + split_params: dict[tuple[Any, ...], list[tuple[Any, ...]]] = {} + indices = tuple(self.params.index(p) for i, p in enumerate(params)) + for pvalues in self.values: + key = tuple(pvalues[i] for i in indices) + split_params.setdefault(key, []).append(pvalues) + new = partial(type(self), params=self.params, defaults=self.defaults) + instances: dict[tuple[Any, ...], Self] = { + key: new(values=values) for key, values in split_params.items() + } + if drop_split_params: + instances = { + key: instance.drop_params(params) + for key, instance in instances.items() + } + if not unpack: + return instances + return {key[0]: instance for key, instance in instances.items()} + def __call__(self, func: C) -> C: """ Mark a callable as with :py:func:`pytest.mark.parametrize`. @@ -1050,6 +1167,278 @@ def _get_path() -> Path: return Path(sysconfig.get_path('purelib')) +class _WarningInfo(Protocol): + @property + def message(self) -> str | Warning: + ... + + @property + def category(self) -> type[Warning]: + ... + + @property + def filename(self) -> str: + ... + + @property + def lineno(self) -> int: + ... + + @property + def line(self) -> str | None: + ... + + +@dataclasses.dataclass +class _WarningMatcher: + message: str | None = None + category: type[Warning] | None = None + module: str | None = None + lineno: int | None = None + _filters: dict[str, Callable[[Any], Any]] = dataclasses.field( + repr=False, init=False, default_factory=dict, + ) + + def __post_init__(self) -> None: + if self.message is not None: + self._filters['message'] = partial( + self._check_message, re.compile(self.message), + ) + if self.category is not None: + self._filters['category'] = partial( + self._check_category, self.category, + ) + if self.module is not None: + self._filters['filename'] = partial( + self._check_module, re.compile(self.module), + ) + if self.lineno is not None: + self._filters['lineno'] = partial(operator.eq, self.lineno) + + def __repr__(self) -> str: + fields: dict[str, Any] = { + field.name: getattr(self, field.name, None) + for field in dataclasses.fields(self) + if field.repr + } + return '{}({})'.format( + type(self).__name__, + ', '.join( + f'{k}={v!r}' for k, v in fields.items() if v is not None + ), + ) + + def match(self, info: _WarningInfo) -> bool: + for field, check in self._filters.items(): + if not check(getattr(info, field)): + return False + return True + + @staticmethod + def _check_message( + msg_regex: re.Pattern, msg: str | Warning, + ) -> re.Match | None: + if not isinstance(msg, str): + msg = str(msg) + return msg_regex.match(msg) + + @staticmethod + def _check_category(parent: type[Any], maybe_child: type[Any]) -> bool: + try: + return issubclass(maybe_child, parent) + except Exception: + return False + + @staticmethod + def _check_module( + module_regex: re.Pattern, filename: str, + ) -> re.Match | None: + module = modpath_to_modname(filename, hide_main=False, hide_init=False) + return module_regex.match(module) + + +@dataclasses.dataclass +class _WarningContext: + catch_warnings: warnings.catch_warnings = dataclasses.field( + default_factory=partial(warnings.catch_warnings, record=True) + ) + checks: list[tuple[_WarningMatcher, bool]] = dataclasses.field( + default_factory=list, + ) + + def forbid_warnings( + self, + message: str | None = None, + category: type[Warning] | None = Warning, + module: str | None = None, + lineno: int | None = None, + ) -> None: + matcher = _WarningMatcher( + message=message, category=category, module=module, lineno=lineno, + ) + self.checks.append((matcher, False)) + + def expect_warnings( + self, + message: str | None = None, + category: type[Warning] | None = Warning, + module: str | None = None, + lineno: int | None = None, + ) -> None: + matcher = _WarningMatcher( + message=message, category=category, module=module, lineno=lineno, + ) + self.checks.append((matcher, True)) + + def check(self, warnings: Sequence[_WarningInfo]) -> None: + for matcher, allowed_or_required in self.checks: + matches = [info for info in warnings if matcher.match(info)] + if matches and not allowed_or_required: + raise ResultMismatch( + expected=f'no warnings matching {matcher!r}', + actual=f'{len(matches)} ({matches!r})', + ) + if not matches and allowed_or_required: + raise ResultMismatch( + expected=f'warnings matching {matcher!r}', + actual=f'none out of {len(warnings)} ({warnings!r})', + ) + + @classmethod + def new(cls, **kwargs) -> Self: + kwargs['record'] = True + return cls(warnings.catch_warnings(**kwargs)) + + +class _check_warnings(Sequence[_WarningInfo]): + """ + Helper context for deferring the checking of warnings to until + context exit. + + Example: + >>> import warnings + + >>> cw = _check_warnings() + + >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... cw.forbid_warnings('foo', UserWarning) + ... warnings.warn('foobar') + ... print('This is printed before the error') + ... + This is printed before the error + Traceback (most recent call last): + ... + test_child_procs.ResultMismatch: expected no warnings matching + _WarningMatcher(message='foo', + category=), + got 1 ([...]) + + >>> with cw: # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... cw.expect_warnings(category=UserWarning) + ... warnings.warn('foobar', Warning) + ... print('This is printed before the error') + ... + This is printed before the error + Traceback (most recent call last): + ... + test_child_procs.ResultMismatch: expected warnings matching + _WarningMatcher(category=), + got none out of 1 ([...]) + >>> assert len(cw) == 1 + >>> assert str(cw[0].message) == 'foobar' + """ + def __init__(self, **kwargs) -> None: + self._new_context: Callable[[], _WarningContext] = partial( + _WarningContext.new, **kwargs, + ) + self._contexts: list[ + tuple[_WarningContext, Sequence[_WarningInfo]] + ] = [] + self._last_captured: Sequence[_WarningInfo] = [] + + def forbid_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('error', *args, **kwargs)``; + at context exit, if ANY matching warning has been issued, an + error will be raised. + """ + ctx, _ = self._current_context + ctx.forbid_warnings(*args, **kwargs) + + def expect_warnings(self, *args, **kwargs) -> None: + """ + Equivalent to calling + ``filterwarnings('always', *args, **kwargs)``; + at context exit, if NO matching warnings have been issued, an + error will be raised. + """ + ctx, _ = self._current_context + ctx.expect_warnings(*args, **kwargs) + + def __enter__(self) -> Self: + ctx = self._new_context() + infos: Sequence[_WarningInfo] | None + infos = ctx.catch_warnings.__enter__() + assert infos is not None + self._contexts.append((ctx, infos)) + return self + + def __exit__(self, *args, **kwargs) -> None: + ctx, infos = self._contexts.pop() + try: + ctx.check(infos) + finally: + self._last_captured = infos + ctx.catch_warnings.__exit__(*args, **kwargs) + + @overload + def __getitem__(self, i: int, /) -> _WarningInfo: + ... + + @overload + def __getitem__(self, i: slice, /) -> list[_WarningInfo]: + ... + + def __getitem__( + self, i: int | slice, /, + ) -> _WarningInfo | Sequence[_WarningInfo]: + return self._current_warnings[i] + + def __len__(self) -> int: + return len(self._current_warnings) + + def __iter__(self) -> Iterator[_WarningInfo]: + return iter(self._current_warnings) + + def __reversed__(self) -> Iterator[_WarningInfo]: + return iter(reversed(self._current_warnings)) + + def __contains__(self, item: Any, /) -> bool: + return item in self._current_warnings + + def index(self, *args, **kwargs) -> int: + return self._current_warnings.index(*args, **kwargs) + + def count(self, *args, **kwargs) -> int: + return self._current_warnings.count(*args, **kwargs) + + @property + def _current_context( + self, + ) -> tuple[_WarningContext, Sequence[_WarningInfo]]: + return self._contexts[-1] + + @property + def _current_warnings(self) -> Sequence[_WarningInfo]: + try: + return self._current_context[1] + except IndexError: + # Outside of contexts, just provide the last captured values + # for convenience + return self._last_captured + + def _import_target(target: str) -> Any: try: return import_module(target) @@ -1215,6 +1604,7 @@ def _run_subproc( ) +@_preserve_pth_files() def _run_test_module( run_helper: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, @@ -1361,7 +1751,7 @@ def _check_output(output: str, tag: str, nhits: int) -> None: }), 'multiprocessing': frozenset({_PATCHED_MARKER}), 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', 'terminate', + '_bootstrap', # 'terminate', }), 'multiprocessing.spawn': frozenset({'runpy'}), 'multiprocessing.util': frozenset({ @@ -1649,15 +2039,23 @@ def list_profiled_funcs() -> list[str]: # Check that on cache cleanup: # - Profiling data is collected # - `os.fork()` is restored + # - The warning for empty profiling files is only issued when + # expected assert func(range(1, n + 1)) == n * (n + 1) // 2 stats = cache.profiler.get_stats() - for callback, has_prof_data, fork_patched in [ - (lambda: None, False, wrap_os_fork), - (cache.cleanup, preimports, False), + for callback, has_nonempty_file, has_stats, fork_patched in [ + (lambda: None, False, False, wrap_os_fork), + (cache.cleanup, True, preimports, False), ]: callback() - gathered = cache.gather_stats() - assert any(gathered.timings.values()) == has_prof_data, gathered + with _check_warnings() as cw: + if has_nonempty_file: + check_warning = cw.forbid_warnings + else: + check_warning = cw.expect_warnings + check_warning(r'.* file\(s\) .* empty', module='line_profiler') + gathered = cache.gather_stats() + assert any(gathered.timings.values()) == has_stats, gathered if hasattr(os, 'fork'): assert (os.fork is not old_fork) == fork_patched else: # E.g. Windows @@ -1757,22 +2155,9 @@ def test_load_pth_hook( compare(patched) -@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) -@(_Params.new('start_method', - ['fork', 'forkserver', 'spawn', 'dummy'], - defaults='dummy') - * _Params.new(('fail', 'label2'), - [(True, 'failure'), (False, 'success')], - defaults=(False, 'success')) - # We only need to check if `intercept_logs` work, the other - # parametrizations don't matter - + _Params.new(('intercept_logs', 'label1'), - [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], - defaults=(False, 'no-intercept-logs'))).sorted() -@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) -def test_apply_mp_patches( +def _test_apply_mp_patches( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -1782,13 +2167,7 @@ def test_apply_mp_patches( fail: bool, n: int, nprocs: int, - label1: str, label2: str, ) -> None: - """ - Test that :py:func:`line_profiler._child_process_profiling\ -.multiprocessing_patches.apply` - works as expected. - """ def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: LineStats.from_files(path, on_defective='error') @@ -1896,21 +2275,108 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: _search_cache_logs(cache, True, patterns) +@(_Params.new('start_method', + ['fork', 'forkserver', 'spawn', 'dummy'], + defaults='dummy') + # We only need to check if `intercept_logs` work, the other + # parametrizations don't matter + + _Params.new(('intercept_logs', 'label'), + [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], + defaults=(False, 'no-intercept-logs'))).sorted() +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_success( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + test_module_object: ModuleType, + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + intercept_logs: bool, + n: int, + nprocs: int, + label: str, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload does not error out. + + See also: + :py:func:`test_apply_mp_patches_failure` + """ + with _check_warnings() as cw: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches( + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + intercept_logs=intercept_logs, + fail=False, + n=n, + nprocs=nprocs, + ) + + +@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) +@pytest.mark.parametrize('start_method', + ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) +def test_apply_mp_patches_failure( + tmp_path_factory: pytest.TempPathFactory, + create_cache: Callable[..., LineProfilingCache], + ext_module_object: ModuleType, + test_module_object: ModuleType, + start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + n: int, + nprocs: int, +) -> None: + """ + Test that :py:func:`line_profiler._child_process_profiling\ +.multiprocessing_patches.apply` + works as expected when the parallel workload errors out. + + See also: + :py:func:`test_apply_mp_patches_success` + """ + with _check_warnings() as cw: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches( + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + intercept_logs=False, + fail=True, + n=n, + nprocs=nprocs, + ) + + # XXX: End of tests for implementation details # ========================= Integration tests ========================== -def _get_mp_start_method_fuzzer(label_name: str) -> _Params: +def _get_mp_start_method_fuzzer(label_name: str | None) -> _Params: """ Returns: :py:class:`_Params` object which does a full Cartesian-product fuzz between ``fail`` (true or false) and ``start_method`` ('fork', 'forkserver', and 'spawn'; default :py:const:`None`) """ + if label_name is None: + label_name, drop_label = '_', True + else: + drop_label = False fuzz_fail = _Params.new(('fail', label_name), [(True, 'failure'), (False, 'success')], defaults=(False, 'success')) + if drop_label: + fuzz_fail = fuzz_fail.drop_params(label_name) fuzz_start = _Params.new('start_method', ['fork', 'forkserver', 'spawn'], defaults=None) return fuzz_fail * fuzz_start @@ -2003,17 +2469,15 @@ def test_running_multiproc_script( (run_script, 'script'), (run_literal_code, 'literal-code')], defaults=(run_script, 'script')) - - -@( +_fuzz_prof_mp_markers = ( (_fuzz_prof_mp_run_func + _Params.new(('prof_child_procs', 'label2'), [(True, 'with-child-prof'), (False, 'no-child-prof')]) - + _get_mp_start_method_fuzzer('label3')) + + _get_mp_start_method_fuzzer(None)) # Test all `multiproc` start methods with both locally- and # externally-defined profiling targets - * (_Params.new(('preimports', 'label4'), [(False, 'no-preimports')]) - + _Params.new(('use_local_func', 'label5'), + * (_Params.new(('preimports', 'label3'), [(False, 'no-preimports')]) + + _Params.new(('use_local_func', 'label4'), [(True, 'local'), (False, 'external')], defaults=(False, 'external'))) # The 'with-preimports' case is already tested rather thoroughly in @@ -2021,16 +2485,77 @@ def test_running_multiproc_script( # param matrix and just test the different `kernprof` modes via the # `run_func()`s + (_fuzz_prof_mp_run_func - + _Params.new(('preimports', 'label4'), [(True, 'with-preimports')])) -).sorted() -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) + + _Params.new(('preimports', 'label3'), [(True, 'with-preimports')])) +).sorted().split_on_params('fail') + + +def _test_profiling_multiproc_script( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + fail: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int, + nprocs: int, +) -> None: + # How many calls do we expect? + nhits = dict.fromkeys( + ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, + ) + # Make sure we're profiling the right function + tag = 'LOCAL' if use_local_func else 'EXT' + tag_call = tag + '-INVOCATION' + tag_loop = tag + '-LOOP' + if not fail: + # The final sum in the parent process should always be profiled + # unless the child processes failed and we never returned from + # `Pool.starmap()` + nhits[tag_call] += 1 + nhits[tag_loop] += nprocs + if prof_child_procs: + # When profiling extends into child processes, each of them + # invokes the sum function once and when combined they loop thru + # all the items + nhits[tag_call] += nprocs + nhits[tag_loop] += nnums + + runner = ['kernprof', '-l'] + runner.extend([ + '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), + '--{}preimports'.format('' if preimports else 'no-'), + ]) + if not use_local_func: + # Also make sure to include the external module in `--prof-mod` + runner.append(f'--prof-mod={ext_module.name}') + run_func( + test_module, tmp_path_factory, + runner=runner, + outfile='out.lprof', + profile=True, + use_local_func=use_local_func, + fail=fail, + start_method=start_method, + nhits=nhits, + nnums=nnums, + nprocs=nprocs, + timeout=_SUBPROC_TIMEOUT, + debug_log=( + 'debug.log' if prof_child_procs and _DEBUG else None + ), + ) + + +@(_fuzz_prof_mp_markers[False]) @pytest.mark.parametrize( # XXX: should we explicitly test the single-proc case? We already # have quite a lot of subtests tho... ('nnums', 'nprocs'), [(2000, 3)], ) -def test_profiling_multiproc_script( +def test_profiling_multiproc_script_success( run_func: Callable[..., subprocess.CompletedProcess], test_module: _ModuleFixture, ext_module: _ModuleFixture, @@ -2038,16 +2563,16 @@ def test_profiling_multiproc_script( prof_child_procs: bool, preimports: bool, use_local_func: bool, - fail: bool, start_method: Literal['fork', 'forkserver', 'spawn'] | None, nnums: int, nprocs: int, # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, label4: str, label5: str, + label1: str, label2: str, label3: str, label4: str, ) -> None: """ Check that `kernprof` can PROFILE the test module in various - contexts, optionally extending profiling into child processes. + contexts when the parallel workload runs without errors, optionally + extending profiling into child processes. Note: This test function is heavily parametrized. Here is why that is @@ -2085,78 +2610,83 @@ def test_profiling_multiproc_script( - ``prof_child_procs`` of course toggles whether to do the patches to set up profiling in child processes. - """ - # How many calls do we expect? - nhits = dict.fromkeys( - ['EXT-INVOCATION', 'EXT-LOOP', 'LOCAL-INVOCATION', 'LOCAL-LOOP'], 0, - ) - # Make sure we're profiling the right function - tag = 'LOCAL' if use_local_func else 'EXT' - tag_call = tag + '-INVOCATION' - tag_loop = tag + '-LOOP' - if not fail: - # The final sum in the parent process should always be profiled - # unless the child processes failed and we never returned from - # `Pool.starmap()` - nhits[tag_call] += 1 - nhits[tag_loop] += nprocs - if prof_child_procs: - # When profiling extends into child processes, each of them - # invokes the sum function once and when combined they loop thru - # all the items - nhits[tag_call] += nprocs - nhits[tag_loop] += nnums - runner = ['kernprof', '-l'] - runner.extend([ - '--{}prof-child-procs'.format('' if prof_child_procs else 'no-'), - '--{}preimports'.format('' if preimports else 'no-'), - ]) - if not use_local_func: - # Also make sure to include the external module in `--prof-mod` - runner.append(f'--prof-mod={ext_module.name}') - run_func( - test_module, tmp_path_factory, - runner=runner, - outfile='out.lprof', - profile=True, + See also: + :py:func:`test_profiling_multiproc_script_failure` + """ + _test_profiling_multiproc_script( + run_func=run_func, + test_module=test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, use_local_func=use_local_func, - fail=fail, + fail=False, start_method=start_method, - nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=_SUBPROC_TIMEOUT, - debug_log=( - 'debug.log' if prof_child_procs and _DEBUG else None - ), ) @pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, subprocess.TimeoutExpired)) -@pytest.mark.parametrize(('use_subprocess', 'label1'), - [(True, 'subprocess.run'), (False, 'os.system')]) -@pytest.mark.parametrize(('prof_child_procs', 'label2'), - [(True, 'with-child-prof'), (False, 'no-child-prof')]) -@pytest.mark.parametrize(('fail', 'label3'), - [(True, 'failure'), (False, 'success')]) -@pytest.mark.parametrize('n', [200]) -def test_profiling_bare_python( +@(_fuzz_prof_mp_markers[True]) +@pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) +def test_profiling_multiproc_script_failure( + run_func: Callable[..., subprocess.CompletedProcess], + test_module: _ModuleFixture, + ext_module: _ModuleFixture, + tmp_path_factory: pytest.TempPathFactory, + prof_child_procs: bool, + preimports: bool, + use_local_func: bool, + start_method: Literal['fork', 'forkserver', 'spawn'] | None, + nnums: int, + nprocs: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, label3: str, label4: str, +) -> None: + """ + Check that `kernprof` can PROFILE the test module in various + contexts when the parallel workload errors out, optionally + extending profiling into child processes. + + See also: + :py:func:`test_profiling_multiproc_script_success` + """ + _test_profiling_multiproc_script( + run_func=run_func, + test_module=test_module, + ext_module=ext_module, + tmp_path_factory=tmp_path_factory, + prof_child_procs=prof_child_procs, + preimports=preimports, + use_local_func=use_local_func, + fail=True, + start_method=start_method, + nnums=nnums, + nprocs=nprocs, + ) + + +_fuzz_bare = ( + _Params.new(('use_subprocess', 'label1'), + [(True, 'subprocess.run'), (False, 'os.system')]) + * _Params.new(('prof_child_procs', 'label2'), + [(True, 'with-child-prof'), (False, 'no-child-prof')]) + * _Params.new('n', [200]) +) + + +def _test_profiling_bare_python( tmp_path_factory: pytest.TempPathFactory, ext_module: _ModuleFixture, use_subprocess: bool, prof_child_procs: bool, fail: bool, n: int, - # Dummy arguments to make `pytest` output more legible - label1: str, label2: str, label3: str, ) -> None: - """ - Check that `kernprof` can profile the target functions if the code - invokes another bare Python process (via either :py:func:`os.system` - or :py:func:`subprocess.run`) that calls them. - """ ext_module.install(children=True) temp_dir = tmp_path_factory.mktemp('mytemp') @@ -2226,3 +2756,62 @@ def test_profiling_bare_python( end='', file=sys.stderr, ) print('-- End of debug logs --', file=sys.stderr) + + +@_fuzz_bare +def test_profiling_bare_python_success( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + # Dummy arguments to make `pytest` output more legible + label1: str, label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits without + errors. + + See also: + :py:func:`test_profiling_bare_python_failure` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=False, + n=n, + ) + + +@pytest.mark.retry(_NUM_RETRIES, + exceptions=(ResultMismatch, subprocess.TimeoutExpired)) +@_fuzz_bare +def test_profiling_bare_python_failure( + tmp_path_factory: pytest.TempPathFactory, + ext_module: _ModuleFixture, + use_subprocess: bool, + prof_child_procs: bool, + n: int, + label1: str, + label2: str, +) -> None: + """ + Check that `kernprof` can profile the target functions if the code + invokes another bare Python process (via either :py:func:`os.system` + or :py:func:`subprocess.run`) that calls them and exits with errors. + + See also: + :py:func:`test_profiling_bare_python_success` + """ + _test_profiling_bare_python( + tmp_path_factory=tmp_path_factory, + ext_module=ext_module, + use_subprocess=use_subprocess, + prof_child_procs=prof_child_procs, + fail=True, + n=n, + ) From 6a7bd1621a35382cbd38b169e0d16a35068e8394 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 04:26:44 +0200 Subject: [PATCH 089/111] Fixup signal handling and child bootstrapping line_profiler/_child_process_profiling cache.py::LineProfilingCache._handle_signal() Added more output and handling for the outcome of the `.cleanup()` call multiprocessing_patches.py::wrap_bootstrap() Now calling cleanup with the `new_thread=True` argument to prevent deadlocks when SIGTERM arrives at an inopportune time -- fellow child threads seem to play well together with the lock, but not the main thread and a child thread line_profiler curated_profiling.py::ClassifiedPreimportTargets.write_preimport_module() line_profiler.py::LineStats.from_files() Fixed calls to `warnings.warn()` so that the warnings are attributed to more informative frames --- line_profiler/_child_process_profiling/cache.py | 12 +++++++++++- .../multiprocessing_patches.py | 6 +++++- line_profiler/curated_profiling.py | 3 ++- line_profiler/line_profiler.py | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 0e1ced85..e814edb4 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -555,8 +555,9 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover :py:meth:`coverage.control.Converage._on_sigterm` """ name = self._get_signal_name(signum) - msg = f'Cleaning up before passing `{name}` ({signum}) on...' + msg = f'Caught `{name}` ({signum}), cleaning up...' self._debug_output(msg) + state = 'succeeded' try: # We don't care about profiler state and such if we're # already being SIGTERM-ed, so just handle cleanup on a @@ -565,8 +566,17 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # already inside one (e.g. when `Process._bootstrap()` is # exiting, or the `atexit` hook is triggered) self.cleanup(new_thread=True) + except BaseException as e: + xc = f'{type(e).__name__}' + msg = str(e) + if msg: + xc = f'{xc}: {msg}' + state = f'failed ({xc})' + raise e finally: handler = self._sighandlers.pop(signum, None) + msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' + self._debug_output(msg) if handler is not None: signal.signal(signum, handler) signal.raise_signal(signum) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 80b94de8..52ddbd78 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -371,7 +371,11 @@ def wrap_bootstrap( finally: msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' cache._debug_output(msg) - cache.cleanup() + # Execute cleanup in a separate thread so as to avoid deadlocks, + # in case when `LineProfilingCache._handle_signal()` caught a + # signal as we're in the middle of this and initiated another + # `.cleanup()` call + cache.cleanup(new_thread=True) def _cache_hook( diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index 80513daa..a120824f 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -71,8 +71,9 @@ def write_preimport_module( invalid_targets, ) ) - warnings.warn(msg) + # Log before warn in case the warning is raised diagnostics.log.warning(msg) + warnings.warn(msg, stacklevel=2) if not self: return None diff --git a/line_profiler/line_profiler.py b/line_profiler/line_profiler.py index 98845f3a..539f8703 100755 --- a/line_profiler/line_profiler.py +++ b/line_profiler/line_profiler.py @@ -440,7 +440,7 @@ def from_files( # Log before warning because warnings may be promoted to # errors diagnostics.log.warning(msg) - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) else: # 'ignore' diagnostics.log.debug(msg) From a7075d64e6b4bb52f417c4bbe00ca55713931947 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 06:09:11 +0200 Subject: [PATCH 090/111] Fix stats file detection in `test_apply_mp_patches_*` --- tests/test_child_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5092b220..126c83d5 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2170,7 +2170,7 @@ def _test_apply_mp_patches( ) -> None: def is_valid_stats_file(path: os.PathLike[str] | str) -> bool: try: - LineStats.from_files(path, on_defective='error') + LineStats.from_files(path, on_empty='error', on_defective='error') except Exception: return False return True From c255740daf6b2bc34751ba4289545a4e91a63bd3 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 08:58:15 +0200 Subject: [PATCH 091/111] Moved code for formatting calls line_profiler/line_profiler_utils.py::CallbackRepr Migrated from `line_profiler/cleanup.py::_CallbackRepr` --- line_profiler/cleanup.py | 199 +---------------------- line_profiler/line_profiler_utils.py | 232 ++++++++++++++++++++++++++- 2 files changed, 235 insertions(+), 196 deletions(-) diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index ac866128..e9e17359 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -3,21 +3,16 @@ """ from __future__ import annotations -import sys -import os from collections.abc import ( - Callable, Generator, Iterable, Mapping, MutableMapping, Collection, + Callable, Generator, Iterable, Mapping, MutableMapping, ) from functools import partial from operator import setitem -from reprlib import Repr -from textwrap import indent from pathlib import Path -from types import MethodType -from typing import Any, TypeVar, TypedDict, cast -from typing_extensions import Concatenate, ParamSpec, Self, Unpack +from typing import Any, TypeVar, cast +from typing_extensions import Concatenate, ParamSpec, Self -from .line_profiler_utils import block_indent, make_tempfile +from .line_profiler_utils import CallbackRepr, make_tempfile from . import _diagnostics as diagnostics @@ -26,194 +21,12 @@ PS = ParamSpec('PS') K = TypeVar('K') V = TypeVar('V') -# Note: `typing.AnyStr` deprecated since 3.13 -AnyStr = TypeVar('AnyStr', str, bytes) _Stacks = dict[float, list[Callable[[], Any]]] _StackContexts = list[_Stacks] -class _ReprAttributes(TypedDict, total=False): - """ - Note: - We use this typed dict instead of directly supplying them in the - :py:meth:`_CallbackRepr.__init__()` signature, because we don't - want to bother with the default values there. - """ - maxlevel: int - maxtuple: int - maxlist: int - maxarray: int - maxdict: int - maxset: int - maxfrozenset: int - maxdeque: int - maxstring: int - maxlog: int - maxother: int - fillvalue: str - indent: str | int | None - - -class _CallbackRepr(Repr): - """ - :py:class:`reprlib.Repr` subclass to help with representing cleanup - callbacks, special-casing certain relevant object types (see - examples below). - - Example: - >>> from functools import partial - >>> from sys import version_info - - >>> class MyEnviron(dict): - ... def some_method(self) -> None: - ... ... - ... - >>> - >>> class MyRepr(_CallbackRepr): - ... # Since we can't instantiate a new `os._Environ`, test - ... # the relevant method with a mock - ... repr_MyEnviron = _CallbackRepr.repr__Environ - ... - >>> - >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) - - Environ-dict formatting: - - >>> my_env = MyEnviron( - ... foo='1', - ... bar='2', - ... this_varname_is_long_but_isnt_truncated=( - ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" - ... ), - ... baz='4', - ... ) - >>> print(r.repr(my_env)) - environ({'foo': '1', 'bar': '2', \ -'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) - - Partial-object formatting: - - >>> r.maxenv = 0 - >>> print(r.repr(my_env.some_method)) - - - Bound-method formatting: - - >>> r.maxargs = 0 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial(, ...) - - Indentation (Python 3.12+): - - >>> if version_info < (3, 12): - ... from pytest import skip - ... - ... skip( - ... '`Repr.indent` not available on {}.{},{}' - ... .format(*sys.version_info) - ... ) - - >>> r = MyRepr(maxenv=2, maxargs=4) - >>> r.indent = 2 - >>> callback_1 = partial(int, base=8) - >>> print(r.repr(callback_1)) - functools.partial( - , - base=8, - ) - - >>> callback_2 = partial(min, 5, 4, 3, 2, 1) - >>> r.indent = '----' - >>> print(r.repr(callback_2)) - functools.partial( - ----, - ----5, - ----4, - ----3, - ----2, - ----..., - ) - - >>> r.indent = ' ' - >>> r.maxenv = 2 - >>> print(r.repr(my_env.some_method)) - - """ - def __init__( - self, - *, - maxargs: int = 5, - maxenv: int = 3, - **kwargs: Unpack[_ReprAttributes] - ) -> None: - super().__init__() # kwargs are 3.12+ - valid_kwargs = ( - _ReprAttributes.__optional_keys__ - | _ReprAttributes.__required_keys__ - ) - for k, v in kwargs.items(): - if k in valid_kwargs: - setattr(self, k, v) - self.maxargs = maxargs - self.maxenv = maxenv - - def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: - get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) - # Truncate envvar values, but not their names - envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] - return self._format_items(envvars, ('environ({', '})'), self.maxenv) - - def repr_method(self, method: MethodType, level: int) -> str: - instance = self.repr1(method.__self__, level-1) - func = getattr(method.__func__, '__qualname__', '?') - prefix, suffix = f'' - # Take care of possible multi-line reprs - return block_indent(instance, prefix) + suffix - - def repr_partial(self, ptl: partial, level: int) -> str: - get: Callable[[Any], str] = partial(self.repr1, level=level-1) - args = [get(arg) for arg in ptl.args] - args.extend('{}={}'.format(k, get(v)) for k, v in ptl.keywords.items()) - args.insert(0, get(ptl.func)) - name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) - # The +1 is to account for `ptl.func` - return self._format_items(args, (name + '(', ')'), self.maxargs + 1) - - def _format_items( - self, - items: Collection[str], - delims: tuple[str, str], - maxlen: int | None = None, - ) -> str: - start, end = delims - if maxlen is not None and len(items) > maxlen: - items = list(items)[:maxlen] + ['...'] - indent_prefix: str | None = self._get_indent() - if indent_prefix is None or not items: - return '{}{}{}'.format(start, ', '.join(items), end) - return '\n'.join([ - start, *(indent(item + ',', indent_prefix) for item in items), end, - ]) - - if sys.version_info >= (3, 12): - # Note: `.indent` only available since 3.12 - def _get_indent(self) -> str | None: - indent = self.indent - if indent is None or isinstance(indent, str): - return indent - return ' ' * indent - else: - @staticmethod - def _get_indent() -> None: - return None - - -_CALLBACK_REPR = _CallbackRepr(maxother=cast(int, float('inf'))).repr +_CALLBACK_REPR_HELPER = CallbackRepr(maxother=cast(int, float('inf'))) +_CALLBACK_REPR = _CALLBACK_REPR_HELPER.repr class Cleanup: diff --git a/line_profiler/line_profiler_utils.py b/line_profiler/line_profiler_utils.py index ea86695e..eae5d27f 100644 --- a/line_profiler/line_profiler_utils.py +++ b/line_profiler/line_profiler_utils.py @@ -6,14 +6,22 @@ import enum import os +import sys +from collections.abc import Callable, Collection, Mapping, Sequence +from functools import partial from pathlib import Path +from reprlib import Repr from tempfile import mkstemp from textwrap import indent -from typing import TYPE_CHECKING -from typing_extensions import Self +from types import MethodType +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar +from typing_extensions import Self, Unpack -__all__ = ('StringEnum', 'block_indent', 'make_tempfile') +__all__ = ('StringEnum', 'CallbackRepr', 'block_indent', 'make_tempfile') + +# Note: `typing.AnyStr` deprecated since 3.13 +AnyStr = TypeVar('AnyStr', str, bytes) class _StrEnumBase(str, enum.Enum): @@ -96,6 +104,224 @@ def _missing_(cls, value: object) -> Self | None: return members.get(value.casefold()) +class _ReprAttributes(TypedDict, total=False): + """ + Note: + We use this typed dict instead of directly supplying them in the + :py:meth:`CallbackRepr.__init__()` signature, because we don't + want to bother with the default values there. + """ + maxlevel: int + maxtuple: int + maxlist: int + maxarray: int + maxdict: int + maxset: int + maxfrozenset: int + maxdeque: int + maxstring: int + maxlog: int + maxother: int + fillvalue: str + indent: str | int | None + + +class CallbackRepr(Repr): + """ + :py:class:`reprlib.Repr` subclass to help with representing cleanup + callbacks, special-casing certain relevant object types (see + examples below). + + Example: + >>> from functools import partial + >>> from sys import version_info + + >>> class MyEnviron(dict): + ... def some_method(self) -> None: + ... ... + ... + >>> + >>> class MyRepr(CallbackRepr): + ... # Since we can't instantiate a new `os._Environ`, test + ... # the relevant method with a mock + ... repr_MyEnviron = CallbackRepr.repr__Environ + ... + >>> + >>> r = MyRepr(maxenv=3, maxargs=4, maxstring=15) + + Environ-dict formatting: + + >>> my_env = MyEnviron( + ... foo='1', + ... bar='2', + ... this_varname_is_long_but_isnt_truncated=( + ... "THIS VALUE IS TRUNCATED BECAUSE IT'S TOO LONG" + ... ), + ... baz='4', + ... ) + >>> print(r.repr(my_env)) + environ({'foo': '1', 'bar': '2', \ +'this_varname_is_long_but_isnt_truncated': 'THIS ... LONG', ...}) + + Partial-object formatting: + + >>> r.maxenv = 0 + >>> print(r.repr(my_env.some_method)) + + + Bound-method formatting: + + >>> r.maxargs = 0 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial(, ...) + + Indentation (Python 3.12+): + + >>> if version_info < (3, 12): + ... from pytest import skip + ... + ... skip( + ... '`Repr.indent` not available on {}.{},{}' + ... .format(*sys.version_info) + ... ) + + >>> r = MyRepr(maxenv=2, maxargs=4) + >>> r.indent = 2 + >>> callback_1 = partial(int, base=8) + >>> print(r.repr(callback_1)) + functools.partial( + , + base=8, + ) + + >>> callback_2 = partial(min, 5, 4, 3, 2, 1) + >>> r.indent = '----' + >>> print(r.repr(callback_2)) + functools.partial( + ----, + ----5, + ----4, + ----3, + ----2, + ----..., + ) + + >>> r.indent = ' ' + >>> r.maxenv = 2 + >>> print(r.repr(my_env.some_method)) + + """ + def __init__( + self, + *, + maxargs: int = 5, + maxenv: int = 3, + **kwargs: Unpack[_ReprAttributes] + ) -> None: + super().__init__() # kwargs are 3.12+ + valid_kwargs = ( + _ReprAttributes.__optional_keys__ + | _ReprAttributes.__required_keys__ + ) + for k, v in kwargs.items(): + if k in valid_kwargs: + setattr(self, k, v) + self.maxargs = maxargs + self.maxenv = maxenv + + def repr__Environ(self, env: os._Environ[AnyStr], level: int) -> str: + """ + Format :py:data:`os.environ` or :py:data:`os.environb`. + """ + get: Callable[[AnyStr], str] = partial(self.repr1, level=level-1) + # Truncate envvar values, but not their names + envvars = ['{!r}: {}'.format(k, get(v)) for k, v in env.items()] + return self._format_items(envvars, ('environ({', '})'), self.maxenv) + + def repr_method(self, method: MethodType, level: int) -> str: + """ + Format a :py:class:`types.MethodType`. + """ + instance = self.repr1(method.__self__, level-1) + func = getattr(method.__func__, '__qualname__', '?') + prefix, suffix = f'' + # Take care of possible multi-line reprs + return block_indent(instance, prefix) + suffix + + def repr_partial(self, ptl: partial, level: int) -> str: + """ + Format a :py:func:`functools.partial`. + """ + name = '{0.__module__}.{0.__qualname__}'.format(type(ptl)) + # The +1 is to account for `ptl.func` + return self._format_call( + level, (name + '(', ')'), self.maxargs + 1, + [ptl.func, *ptl.args], ptl.keywords, + ) + + def format_call(self, /, *args, **kwargs) -> str: + """ + Convenience method for Formating a call a la + :py:meth:`inspect.BoundArguments.__str__`. + + Example: + >>> r = CallbackRepr(maxargs=3, maxlist=3) + >>> print(r.format_call( + ... [1, 2, 3, 4, 5], 'foo', spam=1, ham=2, + ... )) + ([1, 2, 3, ...], 'foo', spam=1, ...) + """ + return self._format_call( + self.maxlevel, ('(', ')'), self.maxargs, args, kwargs, + ) + + def _format_call( + self, + level: int, + delims: tuple[str, str], + maxargs: int, + args: Sequence[Any], + kwargs: Mapping[str, Any], + ) -> str: + get: Callable[[Any], str] = partial(self.repr1, level=level-1) + args = [get(arg) for arg in args] + args.extend('{}={}'.format(k, get(v)) for k, v in kwargs.items()) + return self._format_items(args, delims, maxargs) + + def _format_items( + self, + items: Collection[str], + delims: tuple[str, str], + maxlen: int | None = None, + ) -> str: + start, end = delims + if maxlen is not None and len(items) > maxlen: + items = list(items)[:maxlen] + ['...'] + indent_prefix: str | None = self._get_indent() + if indent_prefix is None or not items: + return '{}{}{}'.format(start, ', '.join(items), end) + return '\n'.join([ + start, *(indent(item + ',', indent_prefix) for item in items), end, + ]) + + if sys.version_info >= (3, 12): + # Note: `.indent` only available since 3.12 + def _get_indent(self) -> str | None: + indent = self.indent + if indent is None or isinstance(indent, str): + return indent + return ' ' * indent + else: + @staticmethod + def _get_indent() -> None: + return None + + def block_indent(string: str, prefix: str, fill_char: str = ' ') -> str: r""" Example: From 4d80ab36389404f86fb60b666bad43764531bf55 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 09:44:45 +0200 Subject: [PATCH 092/111] Refactored call formatting line_profiler/_child_process_profiling/ cache.py::LineProfilingCache _dump_stats() New pseudo-method set by `._setup_in_child_process()` for writing profiling output @_method_wrapper Refactored implementation to use `._format_call()` for formatting calls in debug-log messages _format_call() New convenience method for formatting a call runpy_patches.py::_run() Refactored implementation to use: - `cache._format_call()` for formatting calls in debug-log messages - `Cleanup.patch()` for monkeypatching `exec()` --- .../_child_process_profiling/cache.py | 67 ++++++++++--------- .../_child_process_profiling/runpy_patches.py | 30 +++------ 2 files changed, 45 insertions(+), 52 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index e814edb4..f6a79edb 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -30,7 +30,7 @@ from _line_profiler_hooks import INHERITED_PID_ENV_VARNAME, load_pth_hook from .. import _diagnostics as diagnostics -from ..cleanup import Cleanup +from ..cleanup import Cleanup, _CALLBACK_REPR_HELPER from ..curated_profiling import CuratedProfilerContext from ..line_profiler import LineProfiler, LineStats from ..toml_config import ConfigSource @@ -102,6 +102,7 @@ class LineProfilingCache(Cleanup): _private_field(default_factory=dict) ) _rlock: RLock = _private_field(default_factory=RLock) + _dump_stats: Callable[..., None] | None = _private_field(default=None) _loaded_instance: ClassVar[LineProfilingCache | None] = None @@ -526,7 +527,8 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - self.add_cleanup_with_priority(prof.dump_stats, 1, prof_outfile) + dump_stats = self._dump_stats = partial(prof.dump_stats, prof_outfile) + self.add_cleanup_with_priority(dump_stats, 1) # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -755,39 +757,31 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: cache = cls.load() write = cache._debug_output debug_: bool | None = debug + call = partial(wrapper, cache, vanilla_impl, *args, **kwargs) + if debug_ is None: debug_ = cache.debug - if debug_: - arg_reprs: list[str] = [repr(arg) for arg in args] - arg_reprs.extend(f'{k}={v!r}' for k, v in kwargs.items()) - formatted_call = f'{name}({", ".join(arg_reprs)})' - write(f'Wrapped call made: {formatted_call}...') - try: - result = wrapper(cache, vanilla_impl, *args, **kwargs) - except Exception as e: - if debug_: - write( - 'Wrapped call failed: ' - f'{formatted_call} -> {type(e).__name__}: {e}', - ) - raise + call_fmt = cache._format_call(name, *args, **kwargs) + write(f'Wrapped call made: {call_fmt}...') + state = 'succeeded' + try: + result = call() + except Exception as e: + state = 'failed' + outcome = f'{type(e).__name__}' + if str(e): + outcome = f'{outcome}: {e}' + raise e + else: + outcome = _CALLBACK_REPR_HELPER.repr(result) + return result + finally: + write(f'Wrapped call {state}: {call_fmt} -> {outcome}') else: - if debug_: - write( - 'Wrapped call succeeded: ' - f'{formatted_call} -> {result!r}', - ) - return result - - if ( - hasattr(vanilla_impl, '__module__') - and hasattr(vanilla_impl, '__qualname__') - ): - name = '{0.__module__}.{0.__qualname__}'.format(vanilla_impl) - else: - name = f'' + return call() + name = cls._get_name(vanilla_impl) return wrapped_impl for field in 'name', 'qualname', 'doc': @@ -797,6 +791,19 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: setattr(inner_wrapper, dunder, value) return inner_wrapper + @classmethod + def _format_call( + cls, func: Callable[..., Any] | str, /, *args, **kwargs, + ) -> str: + if isinstance(func, partial): + return cls._format_call( + func.func, [*func.args, *args], {**func.keywords, **kwargs}, + ) + call = _CALLBACK_REPR_HELPER.format_call(*args, **kwargs) + if not isinstance(func, str): + func = cls._get_name(func) + return func + call + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/runpy_patches.py b/line_profiler/_child_process_profiling/runpy_patches.py index 659f4de8..22a600ea 100644 --- a/line_profiler/_child_process_profiling/runpy_patches.py +++ b/line_profiler/_child_process_profiling/runpy_patches.py @@ -16,6 +16,7 @@ from ..autoprofile.ast_tree_profiler import AstTreeProfiler from ..autoprofile.run_module import AstTreeModuleProfiler from ..autoprofile.util_static import modname_to_modpath +from ..cleanup import Cleanup from .cache import LineProfilingCache @@ -67,13 +68,8 @@ def _exec( is always executed. """ assert cache.rewrite_module - cache._debug_output('Calling via {}: `exec({})`'.format( - THIS_MODULE, - ', '.join( - [repr(a) for a in (_code, *args)] - + [f'{k}={v!r}' for k, v in kwargs.items()] - ), - )) + call = cache._format_call('exec', _code, *args, **kwargs) + cache._debug_output(f'Calling via {THIS_MODULE}: `{call}`') fname = str(cache.rewrite_module) code_writer = CodeWriter( fname, @@ -95,14 +91,8 @@ def _run( /, *args: PS.args, **kwargs: PS.kwargs ) -> T: - cache._debug_output('Calling via {}: `runpy.{}({})`'.format( - THIS_MODULE, - name, - ', '.join( - [repr(a) for a in (target, *args)] - + [f'{k}={v!r}' for k, v in kwargs.items()] - ), - )) + call = cache._format_call('runpy.' + name, target, *args, **kwargs) + cache._debug_output(f'Calling via {THIS_MODULE}: `{call}`') if cache.rewrite_module: try: filename = resolve_target_to_path(target) @@ -119,14 +109,10 @@ def _run( # If we are about to run the code to be autoprofiled, monkey-patch # `exec()` into the `runpy` namespace which just rewrites # `cache.rewrite_module` and executes it - if profile: - # Dodge attr-defined errors and their ilk - vars(runpy)['exec'] = partial(_exec, cache, CodeWriter) - try: + with Cleanup() as cleanup: + if profile: + cleanup.patch(runpy, 'exec', partial(_exec, cache, CodeWriter)) return func(target, *args, **kwargs) - finally: - if hasattr(runpy, 'exec'): - del runpy.exec def create_runpy_wrapper(cache: LineProfilingCache) -> ModuleType: From 380ee3bc342174fedc43d0b832cf2b31ea0b695e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 6 May 2026 23:56:05 +0200 Subject: [PATCH 093/111] Extended `@pytest.mark.retry` tests/conftest.py _RetryFailure New exception class for when a retry condition fails evaluation _RetryHelper condition New attribute for text condition manage_call() Added handling for evaluating `.condition` before each retry using the test function's globals locals get_helper() Added handling for the `condition=...` parameter of the marker: - If `False`, returns `None` - If a string, returns an instance with `.condition` set pytest_configure() Updated help text for the `@pytest.mark.retry` marker tests/test_retry_tests.py _TestModule run() New params `additional_std{out,err}_lines` for additional patterns to match the output streams to check_verbose_summary(), check_concise_summary() New params `std{out,err}` for additional patterns to match the output streams to conditions_module New `_TestModule` fixture for a test module using `@pytest.mark.retry(condition=...)` test_retry_conditions() New test using `conditions_module` to check the behavior of `@pytest.mark.retry(condition=...)` --- tests/conftest.py | 83 ++++++++++++++++++- tests/test_retry_tests.py | 166 +++++++++++++++++++++++++++++++++++--- 2 files changed, 236 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a11bb3aa..cfbf0c33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,6 +41,38 @@ def __call__(self, *, pyfuncitem: pytest.Function) -> Any: ... +class _RetryFailure(RuntimeError): + def __init__( + self, + previous_error: Exception, + condition_error: Exception, + condition: str | None = None + ) -> None: + self.previous_error = previous_error + self.condition_error = condition_error + self.condition = condition + super().__init__(self._format_message()) + + condition_error.__cause__ = previous_error + self.__cause__ = condition_error + + def _format_message(self) -> str: + prev = self._format_exception(self.previous_error) + if len(prev.split()) > 1: + prev = f'({prev})' + condition = self._format_exception(self.condition_error) + if self.condition: + condition = f'(condition: {self.condition!r} -> {condition})' + return '{} -> {}'.format(prev, condition) + + @staticmethod + def _format_exception(xc: Exception) -> str: + msg = type(xc).__name__ + if str(xc): + return f'{msg}: {xc}' + return msg + + @final @dataclasses.dataclass class _RetryEntry: @@ -118,6 +150,7 @@ class _RetryHelper: retries: int = 1 exceptions: type[Exception] | tuple[type[Exception], ...] = () reset_fixtures: bool | Collection[str] = True + condition: str | None = None name: ClassVar[str] = 'retry' def __post_init__(self) -> None: @@ -171,6 +204,21 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: for i in range(1 + self.retries): if i: reset_fixtures(func) + if self.condition: + cond, error = self._check_condition(self.condition, func) + if error: # Bail + # XXX: would be nice if we can directly force an + # internal error, but that doesn't seem to be + # possible from within `pytest_pyfunc_call()`; + # directly calling `pytest_internalerror()` + # results in botched teardown and weird + # tracebacks, and leaves the test session in a + # bad state... + assert xc is not None + raise _RetryFailure(xc, cond, self.condition) + if not cond: + i -= 1 + break try: result = impl(pyfuncitem=func) except self.exceptions as e: @@ -195,6 +243,21 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: else: raise xc + @staticmethod + def _check_condition( + condition: str, func: pytest.Function, + ) -> tuple[Any, Literal[False]] | tuple[Exception, Literal[True]]: + global_ns: dict[str, Any] | None = None + try: + global_ns = func.obj.__globals__ + except AttributeError: # Not a `types.FunctionType` + pass + local_ns = func.funcargs + try: + return (eval(condition, global_ns, local_ns), False) + except Exception as e: + return (e, True) + @staticmethod def _reset_between_retries( func: pytest.Function, @@ -297,11 +360,13 @@ def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: retries: int = 0 xc: set[type[Exception]] = set() reset_fixtures: bool | set[str] = True + condition: bool | str | None = None for mark in pyfuncitem.iter_markers(): if mark.name != cls.name: continue instance = cls(*mark.args, **mark.kwargs) retries += instance.retries + condition = instance.condition if isinstance(instance.exceptions, tuple): xc.update(instance.exceptions) else: @@ -322,9 +387,14 @@ def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: reset_fixtures = bool(instance.reset_fixtures) if not retries: return None + if not (condition is None or isinstance(condition, str)): + if condition: + condition = None + else: + return None if not xc: xc = {Exception} - return cls(retries, tuple(xc), reset_fixtures) + return cls(retries, tuple(xc), reset_fixtures, condition) def _pluralize(noun: str, count: int, plural: str | None = None) -> str: @@ -347,7 +417,8 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry(retries=1, exceptions=Exception, reset_fixtures=True): + retry(retries=1, exceptions=Exception, \ +reset_fixtures=True, condition=None): mark the test for retrying upon failure. @@ -359,7 +430,13 @@ def pytest_configure(config: pytest.Config) -> None: reset_fixtures (bool | Collection[str]): Names of function-scoped fixtures to reset between retries, `True` reset all such fixtures, - `False` none thereof + `False` none thereof; + condition (bool | str | None): + Optional condition for retry: + if a boolean, only retry if true; + if a string, only retry if it `eval()`s true + (w/globals of the test module and locals from the fixtures + and parametrizations) """.split()) config.addinivalue_line('markers', help_text) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 873c7681..52338e78 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -7,7 +7,7 @@ import re import pprint import textwrap -from collections.abc import Callable, Generator, Iterable +from collections.abc import Callable, Collection, Generator, Iterable from dataclasses import dataclass from functools import partial from operator import attrgetter @@ -280,6 +280,67 @@ def test_three_xc_types(items: Iterable[Any]) -> None: ''' assert int(next(items)) < 0 """ +TEST_CONDITIONS = """ +from __future__ import annotations + +from sys import version_info + +import pytest + + +@pytest.mark.retry(2, condition=(11 % 2)) +def test_concrete_positive_condition() -> None: + ''' + This should fail after 2 retries because its condition is true. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition=('a' in 'foo')) +def test_concrete_negative_condition() -> None: + ''' + This should fail without retries because its condition is false. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='version_info.major >= 3') +def test_dynamic_positive_condition_test_module_globals() -> None: + ''' + This should fail after 1 retry because the condition evaluates to + true on the test module's ``globals()``. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='version_info.major < 3') +def test_dynamic_negative_condition_test_module_globals() -> None: + ''' + This should fail without retries because the condition evaluates to + false on the test module's ``globals()``. + ''' + raise RuntimeError + + +@pytest.mark.retry(condition='foo == 1') +def test_bad_dynamic_condition() -> None: + ''' + This should fail without retries because the condition cannot be + evaluated (``NameError: name 'foo' is not defined``). + ''' + raise RuntimeError('bar') + + +@pytest.mark.retry(condition='n % 2') +@pytest.mark.parametrize('n', [0, 1, 2]) +def test_dynamic_condition_test_params(n: int) -> None: + ''' + Subtests `[0]` and `[2]` (resp. subtest `[1]`) should fail without + retries (resp. with 1 retry) because the condition evaluates to + false (resp. true) on the test's parametrization. + ''' + raise RuntimeError +""" @dataclass @@ -325,6 +386,8 @@ def run( check_summary: Literal['verbose', 'concise'] | None = None, check_warnings: int | None = None, runner: _RunPytest_Method = 'runpytest', + additional_stdout_lines: Collection[str] = (), + additional_stderr_lines: Collection[str] = (), ) -> pytest.RunResult: """ Args: @@ -344,6 +407,11 @@ def run( 'runpytest_subprocess']): The :py:class:`pytest.Pytester` method used to run the test module + additional_stdout_lines, additional_stderr_lines \ +(Collection[str]): + Additional regex patterns (other than the + automatically-generated ones) to match against the + output streams Returns: :py:class:`pytest.RunResult` object returned by the @@ -374,9 +442,14 @@ def run( self.check_results(result, check_warnings) if check_summary is not None: if check_summary == 'verbose': - self.check_verbose_summary(result) + checker = self.check_verbose_summary else: - self.check_concise_summary(result) + checker = self.check_concise_summary + checker( + result, + stdout=additional_stdout_lines, + stderr=additional_stderr_lines, + ) return result finally: for path in tempfiles: @@ -403,7 +476,12 @@ def check_results( counts[outcome.status] = counts.get(outcome.status, 0) + 1 result.assert_outcomes(warnings=warnings, **counts) - def check_verbose_summary(self, result: pytest.RunResult) -> None: + def check_verbose_summary( + self, + result: pytest.RunResult, + stdout: Collection[str] = (), + stderr: Collection[str] = (), + ) -> None: lines: list[str] = [] counts: dict[_Status, int] = {} for outcomes in self.expected_outcomes.values(): @@ -422,11 +500,14 @@ def check_verbose_summary(self, result: pytest.RunResult) -> None: lines.extend( self._format_header(status, n) for status, n in counts.items() ) + self._check_lines(result, [*lines, *stdout], stderr) - print(f'Expecting these lines in the output: {lines!r}...') - result.stdout.re_match_lines_random(lines) - - def check_concise_summary(self, result: pytest.RunResult) -> None: + def check_concise_summary( + self, + result: pytest.RunResult, + stdout: Collection[str] = (), + stderr: Collection[str] = (), + ) -> None: lines: list[str] = [] counts: dict[_Status, int] = {} test_names: dict[_Status, dict[str, set[str]]] = {} @@ -459,8 +540,7 @@ def check_concise_summary(self, result: pytest.RunResult) -> None: parent_test, n, '' if n == 1 else 's', )) - print(f'Expecting these lines in the output: {lines!r}...') - result.stdout.re_match_lines_random(lines) + self._check_lines(result, [*lines, *stdout], stderr) for status, n in counts.items(): header = self._format_header(status, n) @@ -471,6 +551,20 @@ def check_concise_summary(self, result: pytest.RunResult) -> None: for test_name in names: assert test_name in line + @staticmethod + def _check_lines( + result: pytest.RunResult, + stdout: Collection[str], + stderr: Collection[str], + ) -> None: + for stream, lines in { + 'stdout': list(stdout), 'stderr': list(stderr), + }.items(): + if not lines: + continue + print(f'Expecting these lines in the {stream}: {lines!r}...') + getattr(result, stream).re_match_lines_random(lines) + @staticmethod def _find_line(pattern: str, text: str) -> str: pattern = f'^.*{pattern}.*' @@ -616,6 +710,37 @@ def exceptions_module( ) +@pytest.fixture +def conditions_module( + pytester: pytest.Pytester, +) -> Generator[_TestModule, None, None]: + test = partial(_TestOutcome, status='failed') + param_test_name = 'test_dynamic_condition_test_params' + param_test = partial(test(param_test_name).subtest, status='failed') + yield _TestModule( + 'test_conditions', + TEST_CONDITIONS, + { + 'test_concrete_positive_condition': + [test('test_concrete_positive_condition', retries=2)], + 'test_concrete_negative_condition': + [test('test_concrete_negative_condition')], + 'test_dynamic_positive_condition_test_module_globals': + [test( + 'test_dynamic_positive_condition_test_module_globals', + retries=1, + )], + 'test_dynamic_negative_condition_test_module_globals': + [test('test_dynamic_negative_condition_test_module_globals')], + 'test_bad_dynamic_condition': + [test('test_bad_dynamic_condition')], + param_test_name: + [param_test('0'), param_test('1', retries=1), param_test('2')], + }, + pytester, + ) + + @pytest.mark.parametrize('verbose', [True, False]) def test_fixture_scoping(counters_module: _TestModule, verbose: bool) -> None: """ @@ -705,3 +830,24 @@ def test_exception_restrictions(exceptions_module: _TestModule) -> None: '--verbose', check_results=True, check_summary='verbose', check_warnings=0, ) + + +def test_retry_conditions(conditions_module: _TestModule) -> None: + """ + Test that the decorator correctly handles retry conditions. + """ + conditions_module.run( + '--verbose', + check_results=True, + check_summary='verbose', + check_warnings=0, + # `test_bad_dynamic_condition()` should have failed with a + # `_RetryFailure`, listing the error encountered in the last + # trial and the error encountered when `eval()`-ing the + # condition + additional_stdout_lines=[ + 'FAILED +.*::test_bad_dynamic_condition - ' + r".*_RetryFailure: +\(RuntimeError: bar\) +" + r"-> +\(condition: +'foo == 1' +-> +NameError: .*'foo'.*\)" + ], + ) From 8c2ec71b1eaa688c210afe65862dcc01972a7e82 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 03:37:47 +0200 Subject: [PATCH 094/111] Reworked `multiprocessing_patches` line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper New pickleable wrapper object around a callable to be sent as tasks to child processes, pking sure that rofiling output is written before the call result is pushed back to the parent (and thus leaving the child prone to termination) Patch New helper object for declaratively constructing patches for subcomponents, reworked from `_apply_patches_generic()`; the patches are: - `pool`: Patch `multiprocessing.pool.Pool`, so that each tasks write profiling output before pushing result back to the parent process - `process`: Patch `multiprocessing.process.BaseProcess`, so that each child process write profiling output before exiting - `logging`: Patch logging functions in `multiprocessing.util`, so that the internal logs of `multiprocessing` are teed to the session cache's debug logs wrap_get_tasks(), wrap_guarded_task_generation() New wrappers for the eponymous methods in `multiprocessing.pool.Pool` which wraps the input callables in a `TaskWrapper` apply() - Merged with `_apply_patches_generic()` which it was a thin wrapper for - Added param `patches` for explicitly supplying which patches to apply - Refactored internals to work with `Patch` objects line_profiler/rc/line_profiler.toml [tool.line_profiler.child_processes.multiprocessing] Removed key `intercept_logs` (superseded by `patches.logging`) [tool.line_profiler.child_processes.multiprocessing.patches] New subtable of boolean switches for whether to apply each of the aforementioned patches [toll.line_profiler.child_processes.multiprocessing.polling] Updated obsolete comments --- .../multiprocessing_patches.py | 402 ++++++++++++++---- line_profiler/rc/line_profiler.toml | 36 +- 2 files changed, 341 insertions(+), 97 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 52ddbd78..efecb2f7 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,16 +14,21 @@ """ from __future__ import annotations +import dataclasses import multiprocessing import warnings -from collections.abc import Callable, Mapping +from collections.abc import Callable, Collection, Mapping, Sequence from functools import partial from importlib import import_module +from inspect import getattr_static, signature from multiprocessing.process import BaseProcess +from multiprocessing.pool import Pool +from operator import attrgetter from time import sleep, monotonic -from types import MappingProxyType +from types import MappingProxyType, ModuleType from typing import ( - Any, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, + TYPE_CHECKING, + Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, cast, final, ) @@ -54,10 +59,18 @@ T = TypeVar('T') +T1 = TypeVar('T1') +T2 = TypeVar('T2') PS = ParamSpec('PS') +PS1 = ParamSpec('PS1') +PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] +_PatchName = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' +_LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] + +# ------------------------------ Helpers ------------------------------- class _Wrapper(Protocol, Generic[PS, T]): @@ -277,13 +290,185 @@ def get_defaults(cls) -> Self: return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) +class TaskWrapper(Generic[PS, T]): + """ + Pickle-able wrapper around the supplied task callable, which writes + to the session's profiling-stats file on exit. + """ + def __init__(self, func: Callable[PS, T]) -> None: + self.func = func + try: + self.__signature__ = signature(func) + except Exception: # nocover + # Can happen with e.g. certain builin/c-based callables + pass + + def __call__(self, *args, **kwargs) -> T: + dump_stats = LineProfilingCache.load()._dump_stats + try: + return self.func(*args, **kwargs) + finally: + if dump_stats is not None: + dump_stats() + + +@dataclasses.dataclass +class Patch: + """ + Patch to apply to a component in :py:mod:`multiprocessing`. + + Attributes: + submodule (str): + Name of the :py:mod:`multiprocessing` submodule. + targets (dict[str,\ +dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]]]): + Dictionary mapping (dot-chained) names in said submodule to + a dictionary of patches; said patches dictionary should have + the format of + ``dict[simple_attribute, wrapper | [wrapper1, ...]]``. See + Example for details. + + Example: + Consider + ``Patch('foo', {'bar.baz': {'foobar': foofoo},\ +'': {'spam': [ham, eggs]}})``. + This instance would perform the following patches on the module + ``multiprocessing.foo``: + + - Replace ``multiprocessing.foo.bar.baz.foobar`` with + ``foofoo(multiprocessing.foo.bar.baz.foobar)`` + + - Replace ``multiprocessing.foo.spam`` with + ``eggs(ham(multiprocessing.foo.spam))``; + note that the two wrappers are applied in order to the + original attribute. + """ + submodule: str + targets: dict[ + str, dict[str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]]] + ] = dataclasses.field(default_factory=dict) + package: ClassVar[str] = 'multiprocessing' + + def add_target( + self, + target: str, + patches: Mapping[ + str, Callable[[Any], Any] | Sequence[Callable[[Any], Any]] + ], + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Returns: + This instance + """ + self.targets.setdefault(target, {}).update(patches) + return self + + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + static: bool = False, + ) -> list[str]: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patches + static (bool): + Whether to use :py:func:`inspect.getattr_static` to + retrieve to the attributes to be patched on the patch + targets + + Returns: + replacements (list[str]): + Names of entities replaced + """ + submod_name = f'{self.package}.{self.submodule}' + get_attribute = getattr_static if static else getattr + result: list[str] = [] + try: + mod = self.load_module() + except ImportError: # nocover + return [] + + for target in sorted(self.targets, key=len, reverse=True): + if TYPE_CHECKING: + # See `ty` issue #2572 + assert isinstance(target, str) + if target: + try: + obj: Any = attrgetter(target)(mod) + except AttributeError: # nocover + continue + name = f'{submod_name}.{target}' + else: + obj, name = mod, submod_name + replace = partial(cache.patch, obj, cleanup=cleanup, name=name) + for method, method_wrappers in self.targets[target].items(): + if callable(method_wrappers): + method_wrappers = cast( + Sequence[Callable[[Any], Any]], (method_wrappers,), + ) + try: + impl = get_attribute(obj, method) + except AttributeError: + continue + for wrapper in method_wrappers: + impl = wrapper(impl) + replace(method, impl) + result.append(f'{name}.{method}') + return result + + def load_module(self) -> ModuleType: + """ + Returns: + Module object :py:attr:`.module` points to + """ + return import_module(self.module) + + @staticmethod + def _join(s: str, *strs: str, sep: str = '.') -> str: + return sep.join(string for string in (s, *strs) if string) + + @property + def module(self) -> str: + """ + Module where the patches are applied + """ + return self._join(self.package, self.submodule) + + @property + def summary(self) -> MappingProxyType[str, frozenset[str]]: + """ + Summary of the dotted paths to the patched objects and their + patched attributes + """ + add_prefix = partial(self._join, self.module) + return MappingProxyType({ + add_prefix(target): frozenset(patches) + for target, patches in self.targets.items() + }) + + def _get_config(config: ConfigSource) -> Mapping[str, Any]: cd = dict( config.get_subconfig('child_processes', 'multiprocessing', copy=True) .conf_dict ) + assert isinstance(cd.get('patches'), Mapping) assert isinstance(cd.get('polling'), Mapping) - return MappingProxyType({**cd, 'polling': MappingProxyType(cd['polling'])}) + return MappingProxyType({ + **cd, + 'patches': MappingProxyType(cd['patches']), + 'polling': MappingProxyType(cd['polling']), + }) def _process_has_returned( @@ -302,6 +487,53 @@ def _process_has_returned( return result +def _no_op(*_, **__) -> None: + pass + + +# ---------------- `multiprocessing.pool.Pool` patches ----------------- + + +@LineProfilingCache._method_wrapper +def wrap_get_tasks( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._get_tasks` so that the writing of + profiling stats is handled within the callables sent to the child + processes before the parent process assumes control. + + Note: + :py:meth:`Pool._get_tasks` is a static method. + """ + return vanilla_impl(TaskWrapper(func), *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_guarded_task_generation( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], + self: Pool, + result_job: int, + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._guarded_task_generation` so that the + writing of profiling stats is handled within the callables sent to + the child processes before the parent process assumes control. + """ + return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) + + +# ----------- `multiprocessing.process.BaseProcess` patches ------------ + + @LineProfilingCache._method_wrapper def wrap_terminate( cache: LineProfilingCache, @@ -378,6 +610,9 @@ def wrap_bootstrap( cache.cleanup(new_thread=True) +# --------------- `multiprocessing.util` logging patches --------------- + + def _cache_hook( vanilla_impl: Callable[PS, T], get_logging_message: Callable[PS, str], @@ -412,8 +647,47 @@ def get_msg(msg: str, *_, **__) -> str: ) +# -------------------------- Applying patches -------------------------- + + +_PATCHES: dict[_PatchName, Patch] = { + 'process': Patch('process').add_target( + 'BaseProcess', + {'terminate': wrap_terminate, '_bootstrap': wrap_bootstrap}, + ), + 'pool': Patch('pool').add_target( + 'Pool', { + # `._get_task()` is a static method, so the wrapper function + # needs additional wrapping + '_get_tasks': [wrap_get_tasks, staticmethod], + '_guarded_task_generation': wrap_guarded_task_generation, + }, + ), + 'logging': Patch('util').add_target( + # The logging functions exists directly in the module namespace + # so no further attribute access is needed + '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, + ), +} + + +def _stop_forkserver() -> None: + """ + Note: + This uses `ForkServer._stop()` which is private API, but it's + the same hack used in Python's own test suite -- see the comment + to said method + """ + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + def apply( - lp_cache: LineProfilingCache, reboot_forkserver: bool = True, + lp_cache: LineProfilingCache, + reboot_forkserver: bool = True, + patches: Collection[_PatchName] | None = None, ) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by @@ -421,21 +695,35 @@ def apply( Args: lp_cache (LineProfilingCache): - Cache instance governing the profiling run + Cache instance governing the profiling run. reboot_forkserver (bool): Whether to reboot the global :py:class`multiprocessing.forkserver.ForkServer` instance so as to ensure that profiling happens on processes forked - therefrom (see Note) + therefrom (see Note). + patches \ +(Collection[Literal['pool', 'process', 'logging'] | None]): + Patches to apply to :py:mod:`multiprocessing`; see the + following section for a description of each; + the default is taken from the TOML config file. + + Patches: + ``'pool'``: + Patch :py:class:`multiprocessing.pool.Pool`'s + ``._get_tasks()`` and ``._guarded_task_generation()`` + methods so that parallel tasks write profiling output. + ``'process'``: + Patch :py:class:`multiprocessing.process.BaseProcess`'s + ``.terminate()`` and ``._bootstrap()`` methods so that child + processes write profiling output on exit and are given + enough time for that. + ``'logging'``: + Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. + ``debug()`` and ``info()``) so that their messages are teed + to the cache's debug log. Side effects: - - :py:mod:`multiprocessing` marked as having been set up - - - The following methods and functions patched: - - - :py:meth:`multiprocessing.process.BaseProcess.terminate` - - - :py:meth:`multiprocessing.process.BaseProcess._bootstrap` + - The aforementioned patches applied - If ``reboot_forkserver=True``, fork-server process rebooted: @@ -468,66 +756,23 @@ def apply( .. _GH-126631: https://github.com/python/cpython/issues/126631 """ - if not getattr(multiprocessing, _PATCHED_MARKER, False): - _apply_mp_patches(lp_cache, reboot_forkserver=reboot_forkserver) - - -def _apply_patches_generic( - lp_cache: LineProfilingCache, - submodule: str, - targets: Mapping[str, Mapping[str, Callable[[Any], Any]]], - cleanup: bool = True, -) -> None: - submod_name = 'multiprocessing.' + submodule - try: - mod = import_module(submod_name) - except ImportError: # nocover + if getattr(multiprocessing, _PATCHED_MARKER, False): return - for target, patches in targets.items(): - if target: - try: - obj: Any = getattr(mod, target) - except AttributeError: # nocover - continue - name = f'{submod_name}.{target}' - else: - obj, name = mod, submod_name - replace = partial(lp_cache.patch, obj, cleanup=cleanup, name=name) - for method, method_wrapper in patches.items(): - try: - vanilla = getattr(obj, method) - except AttributeError: - continue - replace(method, method_wrapper(vanilla)) - - -def _apply_mp_patches( - lp_cache: LineProfilingCache, - reboot_forkserver: bool = True, - intercept_mp_logs: bool | None = None, -) -> None: - # In a child process, we don't care about polluting the - # `multiprocessing` namespace, so don't bother with cleanup - apply_patches = partial(_apply_patches_generic, lp_cache) - # Patch `multiprocessing.process.BaseProcess` methods - # Note: the type checkers seem to need some help figuring the - # `patches` out... so do explicit `cast()`s - apply_patches( - 'process', - {'BaseProcess': {'terminate': wrap_terminate, - '_bootstrap': wrap_bootstrap}}, - ) + if patches is None: + config = _get_config(lp_cache._config_source)['patches'] + patches_ = {patch for patch, applied in config.items() if applied} + else: + patches_ = {p.lower() for p in patches} # Patch `multiprocessing.spawn` if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) - # Intercept `multiprocessing` debug messages - if intercept_mp_logs is None: - config = lp_cache._config_source - intercept_mp_logs = _get_config(config)['intercept_logs'] - if intercept_mp_logs: - lfuncs = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] - lpatches = {func: partial(partial, tee_log, func) for func in lfuncs} - apply_patches('util', {'': lpatches}) + # Patch methods/functions in these entities: + # - `multiprocessing.pool.Pool` + # - `multiprocessing.process.BaseProcess` + # - `multiprocessing.util` + for name, patch in _PATCHES.items(): + if name in patches_: + patch.apply(lp_cache) # Stop the current `ForkServer` server process: # - Now, so that the (rebooted) fork-server process has profiling # set up; and @@ -537,20 +782,3 @@ def _apply_mp_patches( lp_cache.add_cleanup(_stop_forkserver) # Mark `multiprocessing` as having been patched lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) - - -def _stop_forkserver() -> None: - """ - Note: - This uses `ForkServer._stop()` which is private API, but it's - the same hack used in Python's own test suite -- see the comment - to said method - """ - # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) - stop() - - -def _no_op(*_, **__) -> None: - pass diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 529a6783..20eb2dd7 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -224,26 +224,42 @@ suffix = '' [tool.line_profiler.child_processes.multiprocessing] -# - `multiprocessing.intercept_logs` (bool): -# Whether to patch the `multiprocessing.util` logging funcions so that -# the internal log messages are captured and writtent to the debug -# logs -intercept_logs = false # - `multiprocessing.catch_sigterm` (bool): # Whether to set a hook for the child-process session caches to # `.cleanup()` on SIGTERM catch_sigterm = true +# List of individual patches to apply to `multiprocessing` + +[tool.line_profiler.child_processes.multiprocessing.patches] + +# - `multiprocessing.patches.pool` (bool): +# Whether to patch `multiprocessing.pool.Pool`, so that each tasks +# write profiling output before pushing result back to the parent +# process +pool = true +# - `multiprocessing.patches.process` (bool): +# Whether to patch `multiprocessing.process.BaseProcess`, so that each +# child process write profiling output before exiting +process = true +# - `multiprocessing.patches.logging` (bool): +# Whether to patch logging functions in `multiprocessing.util`, so +# that the internal logs of `multiprocessing` are teed to the session +# cache's debug logs +logging = false + +# Polling controls, used for the `pool` patch + [tool.line_profiler.child_processes.multiprocessing.polling] # - `multiprocessing.polling.cooldown` (float): -# Cooldown time (seconds) before successive polls on lock files (set -# to <= 0 to disable cooldowns) +# Cooldown time (seconds) before successive polls on child processes +# (set to <= 0 to disable cooldowns) cooldown = 0.03125 # 1/32-nd of a second # - `multiprocessing.polling.timeout` (float): -# Time (seconds) before the main process disregards the existence of -# lock file and unblocks calls to `.terminate()` a (most probably -# errored-out) child process anyway (set to <= 0 to disable timeouts) +# Time (seconds) before the main process disregards the alive-ness of +# child processes, and unblocks calls to `.terminate()` a (most +# probably errored-out) child anyway (set to <= 0 to disable timeouts) timeout = 0.25 # - `multiprocessing.polling.on_timeout` # (Literal['error', 'warn', 'ignore']): From f1e1ad69cd5e0703e979d3dc8d1c4445466358f9 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 07:00:13 +0200 Subject: [PATCH 095/111] Fix: `getattr_static` line_profiler/_child_process_profiling/multiprocessing_patches.py Patch.apply(static=...) Changed default to true _PATCHES['pool'].targets['Pool']['_get_tasks'] Updated to work with the new default line_profiler/cleanup.py::Cleanup.patch() Added new parameter `static` to toggle whether to use `inspect.getattr_static()` to get the pre-patch value; now defaulting to true, while the old behavior is equivalent to false --- .../_child_process_profiling/multiprocessing_patches.py | 6 ++++-- line_profiler/cleanup.py | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index efecb2f7..0e3cc190 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -371,7 +371,7 @@ def apply( cache: LineProfilingCache, *, cleanup: bool = True, - static: bool = False, + static: bool = True, ) -> list[str]: """ Apply the patch. @@ -659,7 +659,9 @@ def get_msg(msg: str, *_, **__) -> str: 'Pool', { # `._get_task()` is a static method, so the wrapper function # needs additional wrapping - '_get_tasks': [wrap_get_tasks, staticmethod], + '_get_tasks': [ + attrgetter('__func__'), wrap_get_tasks, staticmethod, + ], '_guarded_task_generation': wrap_guarded_task_generation, }, ), diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index e9e17359..d4bd55cf 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -7,6 +7,7 @@ Callable, Generator, Iterable, Mapping, MutableMapping, ) from functools import partial +from inspect import getattr_static from operator import setitem from pathlib import Path from typing import Any, TypeVar, cast @@ -292,6 +293,7 @@ def make_tempfile( def patch( self, obj: Any, attr: str, value: Any, *, name: str | None = None, + static: bool = True, cleanup: bool = True, priority: float = 0, ) -> None: @@ -307,6 +309,9 @@ def patch( Value to be assigned to said attribute of ``obj`` name (str | None): Optional name for ``obj`` to be used in debug messages + static (bool): + Whether to use :py:func:`inspect.getattr_static` to + get the current value of the attribute cleanup (bool): Whether to reverse the patch (by resetting or deleting the attribute) on cleanup @@ -338,8 +343,10 @@ def patch( # ... yeah gotta disagree with flake8, a lambda makes # perfect sense here add_cleanup = lambda *_, **__: None # noqa: E731 + get_attribute = getattr_static if static else getattr + try: - old = getattr(obj, attr) + old = get_attribute(obj, attr) except AttributeError: add_cleanup(delattr, priority, obj, attr) else: From ee04e004deaed0831ebcdc97553df6505178a0e0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Thu, 7 May 2026 07:16:15 +0200 Subject: [PATCH 096/111] WIP: test update tests/test_child_procs.py _preserve_obj_attributes _preserve_attributes[.fetch_current_values()] New param `static` for toggling whether to use `inspect.getattr_static()` _preserve_attributes.compare_with_current_values() - New param `static` for toggling whether to use `inspect.getattr_static()` - Refactored to give more useful output and to only raise the `AssertionError` after all comparisons have been made get_patched_attributes() New convenience function superseding the `patched_attributes` fixture test_cache_setup_main_process() Refactored to test the various combinations of applying `multiprocessing` patches test_apply_mp_patches_{success,failure}() - Refactored to test the various combinations of applying `multiprocessing` patches - Further narrowed the scope of the `@pytest.mark.retry` on the failing case: retries are only attempted when the fully-deterministic `pool` patch isn't used test_profiling_{multiproc_script,bare_python}_failure() No longer using the `@pytest.mark.retry` decorator because the 'pool' patch is used by default test_load_pth_hook() Updated to check (against) the correct patched attributes --- tests/test_child_procs.py | 348 +++++++++++++++++++++++++++++--------- 1 file changed, 272 insertions(+), 76 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 126c83d5..6216d3be 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -14,10 +14,11 @@ import warnings from abc import ABC, abstractmethod from collections.abc import ( - Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, + Callable, + Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, ) from contextlib import AbstractContextManager, ExitStack, nullcontext -from functools import partial, wraps +from functools import lru_cache, partial, wraps from io import StringIO from importlib import import_module from numbers import Real @@ -38,18 +39,19 @@ import ubelt as ub from _line_profiler_hooks import load_pth_hook -from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler._child_process_profiling.cache import LineProfilingCache from line_profiler._child_process_profiling.runpy_patches import ( create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, _PATCHED_MARKER, + _Poller, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) +from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.curated_profiling import ( CuratedProfilerContext, ClassifiedPreimportTargets, ) from line_profiler.line_profiler import LineProfiler, LineStats +from line_profiler.toml_config import ConfigSource T = TypeVar('T') @@ -520,6 +522,23 @@ class _NotSupplied(enum.Enum): NOT_SUPPLIED = enum.auto() +class _GetAttr(Protocol): + """ + Function signature for functions that behave like + :py:func:`getattr``. + """ + @overload + def __call__(self, obj: Any, attr: str, /) -> Any: + ... + + @overload + def __call__(self, obj: Any, attr: str, default: Any, /) -> Any: + ... + + def __call__(self, *args): + ... + + @final class ResultMismatch(ValueError): def __init__( @@ -969,17 +988,19 @@ def _debug(self, msg: str, **kwargs) -> None: class _preserve_obj_attributes(_CallableContextManager[dict[str, Any]]): def __init__( - self, obj: Any, attrs: Collection[str], debug: bool = _DEBUG, + self, obj: Any, attrs: Collection[str], *, + static: bool = True, debug: bool = _DEBUG, ) -> None: self.obj = obj self.attrs = set(attrs) self._callbacks: list[Callable[[], None]] = [] self.debug = debug + self.static = static def __enter__(self) -> dict[str, Any]: def get_repr(attr: str) -> str: try: - value = getattr(self.obj, attr) + value = get_attribute(self.obj, attr) except ValueError: return '' else: @@ -1000,9 +1021,14 @@ def reset(attr: str, value: Any) -> None: )) setattr(self.obj, attr, value) + if self.static: + get_attribute: _GetAttr = inspect.getattr_static + else: + get_attribute = getattr + result: dict[str, Any] = {} for attr in self.attrs: - old = getattr(self.obj, attr, _NotSupplied.NOT_SUPPLIED) + old = get_attribute(self.obj, attr, _NotSupplied.NOT_SUPPLIED) if old is _NotSupplied.NOT_SUPPLIED: callback = partial(delete, attr) else: @@ -1066,12 +1092,14 @@ class _preserve_attributes(_CallableContextManager[dict[str, dict[str, Any]]]): >>> assert main is not line_profiler.main """ def __init__( - self, targets: Mapping[str, Collection[str]], debug: bool = _DEBUG, + self, targets: Mapping[str, Collection[str]], *, + static: bool = True, debug: bool = _DEBUG, ) -> None: self.targets = { target: set(attrs) for target, attrs in targets.items() } self._stacks: list[ExitStack] = [] + self.static = static self.debug = debug def __enter__(self) -> dict[str, dict[str, Any]]: @@ -1080,7 +1108,8 @@ def __enter__(self) -> dict[str, dict[str, Any]]: result: dict[str, Any] = {} for target, attrs in self.targets.items(): result[target] = stack.enter_context(_preserve_obj_attributes( - _import_target(target), attrs, debug=self.debug, + _import_target(target), attrs, + debug=self.debug, static=self.static, )) return result @@ -1089,13 +1118,17 @@ def __exit__(self, *_, **__) -> None: @staticmethod def fetch_current_values( - targets: Mapping[str, Collection[str]], + targets: Mapping[str, Collection[str]], static: bool = True, ) -> dict[str, dict[str, Any]]: result: dict[str, dict[str, Any]] = {} na = _NotSupplied.NOT_SUPPLIED + if static: + get: _GetAttr = inspect.getattr_static + else: + get = getattr for target, attrs in targets.items(): obj = _import_target(target) - result[target] = {attr: getattr(obj, attr, na) for attr in attrs} + result[target] = {attr: get(obj, attr, na) for attr in attrs} return result @classmethod @@ -1104,6 +1137,7 @@ def compare_with_current_values( old: Mapping[str, Mapping[str, Any]], comparator: Callable[[Any, Any], bool] = operator.is_, assert_true: bool | Mapping[str, Mapping[str, bool]] = True, + static: bool = True, ) -> dict[str, dict[str, bool]]: def get_from_mapping(target: str, attr: str) -> bool: if TYPE_CHECKING: @@ -1119,7 +1153,8 @@ def get_from_boolean(*_, **__) -> bool: get_expected = get_from_boolean result: dict[str, dict[str, bool]] = {} - new = cls.fetch_current_values(old) + new = cls.fetch_current_values(old, static) + failures: list[str] = [] for target, old_values in old.items(): new_values = new[target] cmp_results = result[target] = {} @@ -1129,15 +1164,31 @@ def get_from_boolean(*_, **__) -> bool: cmp_results[attr] = cmp_result = comparator( new_value, old_value, ) + format_msg = partial( + '{}: {}'.format, + f'Compared `{target}.{attr}` ' + f'(old: {old_value!r} @ {id(old_value):#x}; ' + f'new: {new_value!r} @ {id(new_value):#x})', + ) expected_result = get_expected(target, attr) - if assert_true and (cmp_result != expected_result): - assert False, ( - f'Comparing `{target}.{attr}` ' - f'(old: {old_value!r} @ {id(old_value):#x}; ' - f'new: {new_value!r} @ {id(new_value):#x}): ' - f'expected comparison with {comparator!r} to return ' - f'{expected_result}, got {cmp_result}' + if assert_true: + if cmp_result == expected_result: + message = format_msg( + f'comparison result with {comparator!r} is ' + f'{cmp_result} (as expected)' + ) + else: + message = format_msg( + f'expected comparison with {comparator!r} to ' + f'return {expected_result}, got {cmp_result}' + ) + failures.append(message) + else: + message = format_msg( + f'comparison result with {comparator!r}: {cmp_result}' ) + print(message) + assert (not failures), '\n'.join(failures) return result @@ -1745,35 +1796,125 @@ def _check_output(output: str, tag: str, nhits: int) -> None: # XXX: Tests in this section concerns implementation details, and the # tested APIs and behaviors MUST NOT be relied upon by end-users. -_GLOBAL_PATCHES = { - f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': frozenset({ - 'called', - }), +_PatchSummary = Mapping[str, Set[str]] + +_GLOBAL_MINIMAL_PATCHES = { 'multiprocessing': frozenset({_PATCHED_MARKER}), - 'multiprocessing.process.BaseProcess': frozenset({ - '_bootstrap', # 'terminate', - }), 'multiprocessing.spawn': frozenset({'runpy'}), - 'multiprocessing.util': frozenset({ - 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', - }), 'os': frozenset({'fork'}), } +if not hasattr(os, 'fork'): # E.g. Windows + _GLOBAL_MINIMAL_PATCHES.pop('os') + + +def get_patched_attributes( + applied_mp_patches: Collection[str] | None = None, +) -> MappingProxyType[str, frozenset[str]]: + if applied_mp_patches is None: + applied_mp_patches = { + patch for patch, applied in ( + ConfigSource.from_config() + .get_subconfig('child_processes', 'multiprocessing', 'patches') + .conf_dict.items() + ) if applied + } + return _get_patched_attributes(frozenset(applied_mp_patches)) + + +@lru_cache() +def _get_patched_attributes( + applied_mp_patches: frozenset[str], +) -> MappingProxyType[str, frozenset[str]]: + # Get the contents of the individual patches + patches = _GLOBAL_MINIMAL_PATCHES.copy() + for patch in applied_mp_patches: + maybe_patch = MP_PATCHES.get(patch) # type: ignore + if maybe_patch: + for target, attrs in maybe_patch.summary.items(): + patches[target] = patches.get(target, frozenset()) | attrs + return MappingProxyType({ + target: frozenset(attrs) + for target, attrs in _filter_patches(patches).items() + }) + + +def _get_toml_patches_section(mp_patches: Collection[str]) -> str: + mp_patches_as_dict = {name: name in mp_patches for name in MP_PATCHES} + return ( + '[tool.line_profiler.child_processes.multiprocessing.patches]\n' + + '\n'.join( + f'{patch} = {str(applied).lower()}' + for patch, applied in mp_patches_as_dict.items() + ) + ) + + +def _summarize_patches( + summaries: Collection[tuple[bool, _PatchSummary]] +) -> dict[str, dict[str, bool]]: + """ + Example: + >>> _summarize_patches([(False, {'foo': {'bar'}})]) + {'foo': {'bar': False}} + >>> _summarize_patches([ # doctest: +NORMALIZE_WHITESPACE + ... (False, {'foo': {'bar', 'baz'}}), + ... (True, {'foo': {'baz', 'foobar'}, 'spam': {'ham'}}), + ... (False, {'foo': {'baz'}, 'spam': {'eggs'}}) + ... ]) + {'foo': {'bar': False, 'baz': True, 'foobar': True}, + 'spam': {'eggs': False, 'ham': True}} + """ + def get_all_mentioned(s: Iterable[_PatchSummary]) -> dict[str, set[str]]: + all_items: dict[str, set[str]] = {} + for summary in s: + for target, attrs in summary.items(): + all_items.setdefault(target, set()).update(attrs) + return all_items + + all_items = get_all_mentioned(s for _, s in summaries) + all_patched = get_all_mentioned(s for applied, s in summaries if applied) + result: dict[str, dict[str, bool]] = { + target: + {attr: attr in all_patched.get(target, set()) for attr in attrs} + for target, attrs in all_items.items() + } + # Normalize the order for convenience + return { + target: dict(sorted(attrs.items())) + for target, attrs in sorted(result.items()) + } -@pytest.fixture(scope='module') -def patched_attributes() -> MappingProxyType[str, frozenset[str]]: - result: dict[str, frozenset[str]] = {} - for target, attrs in _GLOBAL_PATCHES.items(): +def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: + result: dict[str, set[str]] = {} + for target, attrs in summary.items(): try: obj = _import_target(target) except ImportError: continue - present_attrs = frozenset(a for a in attrs if hasattr(obj, a)) + present_attrs = {a for a in attrs if hasattr(obj, a)} # Drop if none of the attributes is present if present_attrs: result[target] = present_attrs - return MappingProxyType(result) + return result + + +# This is only patched if we called +# `_line_profiler_hooks.load_pth_hook()` +_HOOK_PATCHES = { + f'{load_pth_hook.__module__}.{load_pth_hook.__qualname__}': + frozenset({'called'}), +} +# Upper limit of what we could've patched +_GLOBAL_PATCHES = { + **_GLOBAL_MINIMAL_PATCHES, + **_HOOK_PATCHES, + **get_patched_attributes(MP_PATCHES), +} +# Actual patches using the default config +DEFAULT_GLOBAL_PATCHES = { + **_GLOBAL_MINIMAL_PATCHES, **get_patched_attributes(), +} @pytest.mark.parametrize(('run_profiled_code', 'label1'), @@ -1913,33 +2054,59 @@ def test_cache_dump_load( assert set(os.environ) == envvars -@pytest.mark.parametrize(('wrap_os_fork', 'label1'), - [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) -@pytest.mark.parametrize(('debug', 'label2'), - [(True, 'with-debug'), (False, 'no-debug')]) +@(_Params.new(('wrap_os_fork', 'label1'), + [(True, 'with-wrap-fork'), (False, 'no-wrap-fork')]) + + _Params.new(('debug', 'label2'), + [(True, 'with-debug'), (False, 'no-debug')]) + + _Params.new(('patch_pool', 'patch_process', 'intercept_logs', 'label3'), + [(True, True, True, 'all-patches'), + (True, True, False, 'pool-and-process'), + (True, False, True, 'pool-and-logging'), + (True, False, False, 'pool-only'), + (False, True, True, 'process-and-logging'), + (False, True, False, 'process-only'), + (False, False, True, 'logging-only'), + (False, False, False, 'no-patches')])).sorted() def test_cache_setup_main_process( + tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], - patched_attributes: MappingProxyType[str, frozenset[str]], wrap_os_fork: bool, debug: bool, - label1: str, label2: str, + patch_pool: bool, + patch_process: bool, + intercept_logs: bool, + label1: str, label2: str, label3: str, ) -> None: """ Test that :py:meth:`LineProfilingCache._setup_in_main_process` works as expected. """ - cache = create_cache(debug=debug) - # By default, we don't patch the `multiprocessing.util` logging - # facilities - patches: dict[str, dict[str, bool]] = { - target: dict.fromkeys(attrs, target != 'multiprocessing.util') - for target, attrs in patched_attributes.items() - } + mp_patches: set[str] = set() + if patch_pool: + mp_patches.add('pool') + if patch_process: + mp_patches.add('process') + if intercept_logs: + mp_patches.add('logging') + + config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' + config.write_text(_get_toml_patches_section(mp_patches)) + cache = create_cache(debug=debug, config=config) + + # Check that only the requested patches are applied + patches = _summarize_patches([ + (True, _GLOBAL_MINIMAL_PATCHES), + *( + (name in mp_patches, _filter_patches(patch.summary)) + for name, patch in MP_PATCHES.items() + ), + ]) try: patches['os']['fork'] = wrap_os_fork except KeyError: # `os.fork()` pruned because it doesn't exist on e.g. Windows assert not hasattr(os, 'fork') + with ExitStack() as stack: patched = stack.enter_context(_preserve_attributes(patches)) compare_patched = partial( @@ -2083,7 +2250,6 @@ def list_profiled_funcs() -> list[str]: @_preserve_attributes(_GLOBAL_PATCHES) def test_load_pth_hook( create_cache: Callable[..., LineProfilingCache], - patched_attributes: MappingProxyType[str, frozenset[str]], another_pid: int, ppid_should_match: bool | None, ) -> None: @@ -2122,8 +2288,7 @@ def test_load_pth_hook( cache.dump() compare = _preserve_attributes.compare_with_current_values - patches = patched_attributes.copy() - del patches['multiprocessing.util'] # Not patched by default + patches = {**DEFAULT_GLOBAL_PATCHES, **_HOOK_PATCHES} with _preserve_attributes(patches) as patched: try: # NOTE: this creates a cache instance that isn't @@ -2163,7 +2328,7 @@ def _test_apply_mp_patches( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], - intercept_logs: bool, + mp_patches: Collection[str], fail: bool, n: int, nprocs: int, @@ -2185,17 +2350,14 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: ) config = tmp_path_factory.mktemp('myconfig') / 'mytoml.toml' - cfg_chunks: list[str] = [] - if intercept_logs: - cfg_chunks.append( - '[tool.line_profiler.child_processes.multiprocessing]\n' - 'intercept_logs = true' - ) - # This is easier to debug than `ResultMismatch` - cfg_chunks.append( + intercept_logs = 'logging' in mp_patches + patch_process = 'process' in mp_patches + cfg_chunks: list[str] = [ + _get_toml_patches_section(mp_patches), + # This is easier to debug than `ResultMismatch` '[tool.line_profiler.child_processes.multiprocessing.polling]\n' - 'on_timeout = "error"' - ) + 'on_timeout = "error"', + ] config.write_text('\n\n'.join(cfg_chunks)) # Note: no need to test the case for `my_local_sum()` separately, @@ -2266,11 +2428,16 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Check the debug logs to see if we have done everything right, esp. # the logging interception part not covered by other tests - patterns: dict[str, bool] = { - 'Cleanup succeeded.*: .*dump_stats.*' + re.escape(path.name): True - for path in Path(cache.cache_dir).glob('*.lprof') - if is_valid_stats_file(path) - } + patterns: dict[str, bool] = {} + if patch_process: + # Note: if we're not using `Process`-based patch, there is no + # guaratee that the profiling result is written via cleanup + iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') + iter_stats = filter(is_valid_stats_file, iter_stats) + pat = 'Cleanup succeeded.*: .*dump_stats.*{}' + patterns.update({ + pat.format(re.escape(path.name)): True for path in iter_stats + }) patterns[re.escape('`multiprocessing` logging (debug)')] = intercept_logs _search_cache_logs(cache, True, patterns) @@ -2280,9 +2447,13 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: defaults='dummy') # We only need to check if `intercept_logs` work, the other # parametrizations don't matter - + _Params.new(('intercept_logs', 'label'), + + _Params.new(('intercept_logs', 'label1'), [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], defaults=(False, 'no-intercept-logs'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) def test_apply_mp_patches_success( tmp_path_factory: pytest.TempPathFactory, @@ -2290,10 +2461,13 @@ def test_apply_mp_patches_success( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + patch_pool: bool, + patch_process: bool, intercept_logs: bool, n: int, nprocs: int, - label: str, + label1: str, + label2: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2303,6 +2477,13 @@ def test_apply_mp_patches_success( See also: :py:func:`test_apply_mp_patches_failure` """ + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') + if intercept_logs: + mp_patches.append('logging') with _check_warnings() as cw: cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') @@ -2312,16 +2493,27 @@ def test_apply_mp_patches_success( ext_module_object=ext_module_object, test_module_object=test_module_object, start_method=start_method, - intercept_logs=intercept_logs, + mp_patches=mp_patches, fail=False, n=n, nprocs=nprocs, ) -@pytest.mark.retry(_NUM_RETRIES, exceptions=(ResultMismatch, _Poller.Timeout)) +@pytest.mark.retry( + _NUM_RETRIES, + exceptions=(ResultMismatch, _Poller.Timeout), + # Patching `Pool` should be foolproof no matter the platform + # (as long as we use `Pool` for our parallelism); + # `Process` though... + condition='not patch_pool', +) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label'), + [(True, True, 'pool-and-process'), + (True, False, 'pool-only'), + (False, True, 'process-only')]) @pytest.mark.parametrize(('n', 'nprocs'), [(100, 2)]) def test_apply_mp_patches_failure( tmp_path_factory: pytest.TempPathFactory, @@ -2329,8 +2521,11 @@ def test_apply_mp_patches_failure( ext_module_object: ModuleType, test_module_object: ModuleType, start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], + patch_pool: bool, + patch_process: bool, n: int, nprocs: int, + label: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2340,6 +2535,11 @@ def test_apply_mp_patches_failure( See also: :py:func:`test_apply_mp_patches_success` """ + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') with _check_warnings() as cw: cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') @@ -2349,7 +2549,7 @@ def test_apply_mp_patches_failure( ext_module_object=ext_module_object, test_module_object=test_module_object, start_method=start_method, - intercept_logs=False, + mp_patches=mp_patches, fail=True, n=n, nprocs=nprocs, @@ -2629,8 +2829,6 @@ def test_profiling_multiproc_script_success( ) -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @(_fuzz_prof_mp_markers[True]) @pytest.mark.parametrize(('nnums', 'nprocs'), [(2000, 3)]) def test_profiling_multiproc_script_failure( @@ -2787,8 +2985,6 @@ def test_profiling_bare_python_success( ) -@pytest.mark.retry(_NUM_RETRIES, - exceptions=(ResultMismatch, subprocess.TimeoutExpired)) @_fuzz_bare def test_profiling_bare_python_failure( tmp_path_factory: pytest.TempPathFactory, From 1d9a972c802768b5080e27782ddfc92a39fb738f Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 8 May 2026 03:23:15 +0200 Subject: [PATCH 097/111] Reworked `multiprocessing_patches`... again line_profiler/_child_process_profiling/cache.py::LineProfilingCache _additional_data New free-for-all field allowing for unstructured data to be `.dump()`-ed and `.load()`-ed .dump(), .load(), ._from_path() Updated implementations to handle `._additional_data` _warn_possible_lack_of_stats() New method for arbitrary (child or main) process to communicate that some PIDs are expected to not have written to their profiling-stats file ("maybe-bad PIDs"); this fixes the stray empty-file warning sometimes emitted when the 'process' patch is not applied _get_pids_possibly_lacking_stats() New method for retrieving said PIDs line_profiler/_child_process_profiling/multiprocessing_patches.py Massively refactored. _PollerArgs.new() Refactored from `.from_config()` MPConfig New object for holding the submodule-specific config options extracted from `LineProfilingCache.load().config` _Patch New protocol for objects (e/g. `Patch`) capable of `.apply()`-ing themselves via a `LineProfilingCache` and providing a `.summary` of the patched entities Patch.add_method() New convenience method for adding a single (instance, class, or static) method to be patched wrap_repopulate_pool_static() New wrapper for `Pool._repopulate_pool_static()` which alerts the session cache to "maybe-bad PIDs"; this is needed because when wrapping only `Pool` and not `Process`, processes which exited without receiving a single task would leave their profiling-stats file empty (see `LineProfilingCache._warn_possible_lack_of_stats()` above) wrap_terminate(), wrap_bootstrap() Simplified implementations using `MPConfig` RebootForkserverPatch, RunpyPatch New helper `_Patch` classes consisting of functionalities previously embedded in `apply()` ResourceTrackerPatch New helper `_Patch` class patching `multiprocessing.resource_tracker.ResourceTracker` to set up the reporting of "maybe-bad PIDs"; this is needed because the resource-tracker server, while not directly handling the profiled parallel workload, is long-lived and may not have exited (and thus written profiling stats) at stats-gathering time; supersedes `cache.py::_DEFAULT_GATHER_STATS_EXCLUDES` apply() - Renamed parameter `lp_cache` -> `cache` - Simplified implementation because the components are all outsourced to `_Patch` objects tests/test_child_procs.py test_cache_dump_load() Added check for the round-tripping of `._additional_data` _test_apply_mp_patches() Fixed minor typing issue _GLOBAL_MINIMAL_PATCHES, DEFAULT_GLOBAL_PATCHES, etc. Updated definitions so that we check for the patching and restoration of the correct attributes --- .../_child_process_profiling/cache.py | 102 +++- .../multiprocessing_patches.py | 565 ++++++++++++++---- tests/test_child_procs.py | 79 ++- 3 files changed, 572 insertions(+), 174 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index f6a79edb..ac70eb8e 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -19,7 +19,6 @@ ) from functools import partial, cached_property, wraps from importlib import import_module -from operator import attrgetter from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent @@ -55,18 +54,9 @@ _PROFILING_OUTPUT_PREFIX_PATTERN = ( 'child-prof-output-{main_pid}-{current_pid}-{prof}-' ) -_DEFAULT_GATHER_STATS_EXCLUDES: set[tuple[str, str]] = { - # Note: the `ResourceTracker` server process is spawned when the - # first `multiprocessing` child process is created via the `spawn` - # or `forkserver` start method. While this server process does not - # meaningfully contribute to the profiling result either way, since - # it can be created with profiling set up, its longevity means that - # `LineProfilingCache.gather_stats()` often catches an empty .lprof - # file which it has occupied but not written to. To reduce noise - # while keeping the warning for other zero-length files, just - # explictly exclude said process - ('multiprocessing.resource_tracker', '_resource_tracker._pid'), -} +_POSSIBLE_EMPTY_STATS_PREFIX_PATTERN = ( + 'ignore-empty-stats-file-{main_pid}-{current_pid}-' +) def _import_sibling(submodule: str) -> ModuleType: @@ -103,6 +93,10 @@ class LineProfilingCache(Cleanup): ) _rlock: RLock = _private_field(default_factory=RLock) _dump_stats: Callable[..., None] | None = _private_field(default=None) + # These are unstructured fields; other components can decide on what + # to put in them. They are also pickled by `.dump()`, and are thus + # retrievable in `.load()`-ed instances. + _additional_data: dict[str, Any] = _private_field(default_factory=dict) _loaded_instance: ClassVar[LineProfilingCache | None] = None @@ -163,7 +157,10 @@ def dump(self) -> None: Note: Cleanup callbacks are not serialized. """ - content = self._get_init_args() + content = { + 'init_args': self._get_init_args(), + 'additional_data': self._additional_data, + } msg = f'Dumping instance data to {self.filename}: {content!r}' self._debug_output(msg) with open(self.filename, mode='wb') as fobj: @@ -234,23 +231,11 @@ def is_empty(path: Path) -> bool: filter_excludes: Callable[[Iterable[Path]], Iterable[Path]] if exclude_pids is None: - exclude_pids = set() - for import_target, attr in _DEFAULT_GATHER_STATS_EXCLUDES: - try: - module = import_module(import_target) - except ImportError: - continue - try: - maybe_pid = attrgetter(attr)(module) - except AttributeError: - maybe_pid = None - if maybe_pid is None: - continue - exclude_pids.add(cast(int, maybe_pid)) # NOTE: there is no guarantee that the PID hasn't previously # been used for another child process that we DID properly # profile and SHOULD include, so we only filter out empty # files + exclude_pids = self._get_pids_possibly_lacking_stats() filter_excludes = partial(filter, is_empty) else: # User-provided values, who are we to object? filter_excludes = iter @@ -314,11 +299,13 @@ def _gather_debug_log_entries( for entry in CacheLoggingEntry.from_file(log) ) + def _glob(self, *args, **kwargs) -> Iterable[Path]: + return Path(self.cache_dir).glob(*args, **kwargs) + def _get_debug_logfiles(self) -> Iterable[Path]: - pattern = _DEBUG_LOG_FILENAME_PATTERN.format( + return self._glob(_DEBUG_LOG_FILENAME_PATTERN.format( main_pid=self.main_pid, current_pid='?*', - ) - return Path(self.cache_dir).glob(pattern) + )) def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: prefix = _PROFILING_OUTPUT_PREFIX_PATTERN.format( @@ -328,7 +315,7 @@ def _get_profiling_outfiles(self, pid: Any = '?*') -> Iterable[Path]: # `._setup_in_child_process()` prof='0x?*', ) - return Path(self.cache_dir).glob(prefix + '?*.lprof') + return self._glob(prefix + '?*.lprof') def inject_env_vars( self, env: MutableMapping[str, str] | None = None, @@ -652,6 +639,46 @@ def wrapper() -> int: self.patch(os, 'fork', wrapper, name='os') + def _warn_possible_lack_of_stats( + self, pids: int | Collection[int], + ) -> None: + """ + Register PID(s) which may have created a profiling stats file + without writing to it; when calling :py:meth:`.gather_stats`, + empty stats files associated with those PIDs are ignored instead + of warned against or treated as an error. + """ + if not isinstance(pids, Collection): + pids = pids, + with self._empty_stats_pid_registry.open(mode='a') as fobj: + print(*pids, sep='\n', file=fobj) + + def _get_pids_possibly_lacking_stats(self) -> set[int]: + """ + See also + :py:meth:`._warn_possible_lack_of_stats` + """ + prefix = _POSSIBLE_EMPTY_STATS_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid='?*', # Gather from all child processes + ) + result: set[int] = set() + for registry in self._glob(prefix + '?*.dat'): + from_reg: set[int] = set() + with registry.open() as fobj: + for line in fobj: + try: + from_reg.add(int(line)) + except ValueError: + pass + if from_reg: + self._debug_output( + f'Loaded {len(from_reg)} PID(s) possibly lacking ' + f'profiling output from {registry.name!r}: {from_reg!r}' + ) + result.update(from_reg) + return result + def make_tempfile(self, **kwargs) -> Path: """ Create a fresh tempfile under :py:attr:`~.cache_dir`. The other @@ -677,7 +704,10 @@ def _replace_loaded_instance(self, force: bool = False) -> bool: @classmethod def _from_path(cls, fname: os.PathLike[str] | str) -> Self: with open(fname, mode='rb') as fobj: - return cls(**pickle.load(fobj)) + content = pickle.load(fobj) + instance = cls(**content['init_args']) + instance._additional_data.update(content.get('additional_data', {})) + return instance def _get_init_args(self) -> dict[str, Any]: init_fields = [ @@ -844,3 +874,11 @@ def _config_source(self) -> ConfigSource: else: config = str(self.config) return ConfigSource.from_config(config) + + @cached_property + def _empty_stats_pid_registry(self) -> Path: + prefix = _POSSIBLE_EMPTY_STATS_PREFIX_PATTERN.format( + main_pid=self.main_pid, + current_pid=os.getpid(), + ) + return self.make_tempfile(prefix=prefix, suffix='.dat', delete=False) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0e3cc190..67299fdb 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,20 +16,22 @@ import dataclasses import multiprocessing +import sys import warnings -from collections.abc import Callable, Collection, Mapping, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module from inspect import getattr_static, signature +from multiprocessing.context import BaseContext from multiprocessing.process import BaseProcess from multiprocessing.pool import Pool from operator import attrgetter from time import sleep, monotonic -from types import MappingProxyType, ModuleType +from types import MappingProxyType as mappingproxy, MethodType, ModuleType from typing import ( TYPE_CHECKING, Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, - cast, final, + cast, final, overload, ) from typing_extensions import Concatenate, ParamSpec, Self @@ -48,6 +50,12 @@ _CAN_USE_FORKSERVER = ( 'forkserver' in multiprocessing.get_all_start_methods() ) +try: + from multiprocessing import resource_tracker +except ImportError: + _CAN_USE_RESOURCE_TRACKER = False +else: + _CAN_USE_RESOURCE_TRACKER = True from .. import _diagnostics as diagnostics from ..toml_config import ConfigSource @@ -61,14 +69,18 @@ T = TypeVar('T') T1 = TypeVar('T1') T2 = TypeVar('T2') +P = TypeVar('P', bound=BaseProcess) +Pt = TypeVar('Pt', bound='_Patch') PS = ParamSpec('PS') PS1 = ParamSpec('PS1') PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] -_PatchName = Literal['pool', 'process', 'logging'] +PublicPatch = Literal['pool', 'process', 'logging'] _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] +_PATCHES: dict[str, '_Patch'] = {} + # ------------------------------ Helpers ------------------------------- @@ -264,30 +276,94 @@ class _PollerArgs(NamedTuple): on_timeout: str | None @classmethod - def from_config(cls, config: ConfigSource) -> Self: - values = _get_config(config)['polling'] + def new(cls, cooldown: Any, timeout: Any, on_timeout: Any) -> Self: try: - cooldown = max(float(values['cooldown']), 0) + cd = max(float(cooldown), 0) except (TypeError, ValueError): - cooldown = 0 + cd = 0 try: - timeout = max(float(values['timeout']), 0) + to = max(float(timeout), 0) except (TypeError, ValueError): - timeout = 0 + to = 0 try: - on_timeout: str | None = values['on_timeout'].lower() + ot: str | None = on_timeout.lower() except Exception: # Fallback (use `_Poller`'s default) - on_timeout = None - return cls(cooldown, timeout, on_timeout) + ot = None + return cls(cd, to, ot) + + +@final +@dataclasses.dataclass +class MPConfig: + """ + Consolidate the config options into a structured object. + """ + catch_sigterm: bool + patches: dict[PublicPatch, bool] + polling: _PollerArgs + + def _get_terminate_poller( + self, cache: LineProfilingCache, process: BaseProcess, + ) -> _Poller: + cd, timeout, on_timeout = self.polling + if on_timeout not in ('ignore', 'warn', 'error'): + on_timeout = self.get_defaults().polling.on_timeout + # `_process_has_returned()` takes a `timeout` which it passes to + # `popen.wait()`; said timeout is essentially a limit as to how + # often the function is called, hence our cooldown + poller = _Poller.poll_until( + self._process_has_returned, process, cache, cd, + ) + return poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)) + + @classmethod + def from_config(cls, config: ConfigSource) -> Self: + loaded = ( + config + .get_subconfig('child_processes', 'multiprocessing') + .conf_dict + ) + polling = _PollerArgs.new(**loaded['polling']) + return cls( + catch_sigterm=loaded['catch_sigterm'], + patches=dict(loaded['patches']), + polling=polling, + ) + + @classmethod + def from_cache(cls, cache: LineProfilingCache) -> Self: + key = 'mp_config' + try: + return cache._additional_data[key] + except KeyError: + config = cls.from_config(cache._config_source) + return cache._additional_data.setdefault(key, config) @classmethod def get_defaults(cls) -> Self: namespace = globals() + name = '_DEFAULT_CONFIG' try: - return namespace['_DEFAULT_POLLER_ARGS'] + return namespace[name] except KeyError: defaults = cls.from_config(ConfigSource.from_default(copy=False)) - return namespace.setdefault('_DEFAULT_POLLER_ARGS', defaults) + return namespace.setdefault(name, defaults) + + @staticmethod + def _process_has_returned( + proc: BaseProcess, cache: LineProfilingCache, timeout: float, + ) -> bool: + popen = getattr(proc, '_popen', None) + if popen is None: + msg, result = 'No associated process', True + else: + result = popen.wait(timeout) is not None + if result: + msg = f'Process {popen.pid} has returned' + else: + msg = f'Waiting for process {popen.pid} to return...' + cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') + return result class TaskWrapper(Generic[PS, T]): @@ -312,6 +388,47 @@ def __call__(self, *args, **kwargs) -> T: dump_stats() +def _no_op(*_, **__) -> None: + pass + + +# ---------------------- Patching infrastructure ----------------------- + + +class _Patch(Protocol): + """ + Interface for patches. + """ + def apply( + self, + cache: LineProfilingCache, + *, + cleanup: bool = True, + **kwargs + ) -> Any: + """ + Apply the patch. + + Args: + cache (LineProfilingCache): + Session cache + cleanup (bool): + Whether ``cache.cleanup()`` should reverse the patch + **kwargs + Individual implementations should pick the ones they + need and ignore the rest. + """ + ... + + @property + def summary(self) -> Mapping[str, Set[str]]: + """ + A mapping from dotted-path names of objects to the set of + attributes patched thereon. + """ + ... + + @dataclasses.dataclass class Patch: """ @@ -360,18 +477,71 @@ def add_target( Convenience method for gradually constructing the patch with a fluent interface. + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + patches (Mapping[str, Callable[[Any], Any] \ +| Sequence[Callable[[Any], Any]]]): + Mapping from patched attrbute names to the wrappers to + apply thereto; sequences of wrappers are applied in + order + Returns: This instance """ self.targets.setdefault(target, {}).update(patches) return self + def add_method( + self, + target: str, + method: str, + wrapper: Callable[[Any], Any], + methodtype: ( + type[classmethod] | type[staticmethod] + | Literal['class', 'static'] | None + ) = None, + ) -> Self: + """ + Convenience method for gradually constructing the patch with a + fluent interface. + + Args: + target (str): + Dotted path to the object in :py:attr:`.submodule` + method (str): + Name of the (class, static, or instance) method to patch + wrapper (Callable[[Any], Any]): + Wrapping callable which takes the method-implementaion + callable and returns a wrapper thereof + methodtype (type[classmethod] | type[staticmethod] | \ +Literal['class', 'static'] | None): + Optional type of the method if not an instance method; + the strings ``'class'`` and ``'static'`` are respective + shorthands for :py:class:`classmethod` and + :py:class:`staticmethod` + + Returns: + This instance + """ + wrappers: Callable[[Any], Any] | list[Callable[[Any], Any]] + if methodtype is None: + wrappers = wrapper + else: + if methodtype == 'class': + methodtype = classmethod + elif methodtype == 'static': + methodtype = staticmethod + wrappers = [attrgetter('__func__'), wrapper, methodtype] + return self.add_target(target, {method: wrappers}) + def apply( self, cache: LineProfilingCache, *, cleanup: bool = True, static: bool = True, + **_ ) -> list[str]: """ Apply the patch. @@ -380,7 +550,7 @@ def apply( cache (LineProfilingCache): Session cache cleanup (bool): - Whether ``cache.cleanup()`` should reverse the patches + Whether ``cache.cleanup()`` should reverse the patch static (bool): Whether to use :py:func:`inspect.getattr_static` to retrieve to the attributes to be patched on the patch @@ -445,50 +615,46 @@ def module(self) -> str: return self._join(self.package, self.submodule) @property - def summary(self) -> MappingProxyType[str, frozenset[str]]: + def summary(self) -> mappingproxy[str, frozenset[str]]: """ Summary of the dotted paths to the patched objects and their patched attributes """ add_prefix = partial(self._join, self.module) - return MappingProxyType({ + return mappingproxy({ add_prefix(target): frozenset(patches) for target, patches in self.targets.items() }) -def _get_config(config: ConfigSource) -> Mapping[str, Any]: - cd = dict( - config.get_subconfig('child_processes', 'multiprocessing', copy=True) - .conf_dict - ) - assert isinstance(cd.get('patches'), Mapping) - assert isinstance(cd.get('polling'), Mapping) - return MappingProxyType({ - **cd, - 'patches': MappingProxyType(cd['patches']), - 'polling': MappingProxyType(cd['polling']), - }) - - -def _process_has_returned( - proc: BaseProcess, cache: LineProfilingCache, timeout: float, -) -> bool: - popen = getattr(proc, '_popen', None) - if popen is None: - msg, result = 'No associated process', True - else: - result = popen.wait(timeout) is not None - if result: - msg = f'Process {popen.pid} has returned' - else: - msg = f'Waiting for process {popen.pid} to return...' - cache._debug_output(f' {type(proc).__name__} @ {id(proc):#x}: {msg}') - return result +@overload +def _register_patch(name: str, patch: Pt) -> Pt: + ... -def _no_op(*_, **__) -> None: - pass +@overload +def _register_patch(name: str, patch: None = None) -> _Patch: + ... + + +def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: + """ + Register the ``patch`` under ``name`` and return it as-is. If + ``patch`` isn't provided, look for the existing patch registered + under the name. + + Note: + Patches named with leading double underscores are applied no + matter the user input (e.g. via ``apply(..., patches=...)`` or + the config file). + """ + if patch is not None: + if _PATCHES.setdefault(name, patch) is not patch: + raise ValueError( + f'name = {name!r}, patch = {patch!r}: ' + 'name already in use by {_PATCHES[name]}' + ) + return _PATCHES[name] # ---------------- `multiprocessing.pool.Pool` patches ----------------- @@ -508,7 +674,7 @@ def wrap_get_tasks( processes before the parent process assumes control. Note: - :py:meth:`Pool._get_tasks` is a static method. + :py:meth:`.Pool._get_tasks` is a static method. """ return vanilla_impl(TaskWrapper(func), *args, **kwargs) @@ -531,6 +697,58 @@ def wrap_guarded_task_generation( return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) +@LineProfilingCache._method_wrapper +def wrap_repopulate_pool_static( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[BaseContext, type[P], int, list[P], PS], None + ], + ctx: BaseContext, + Process: type[P], + processes: int, + pool: list[P], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`.Pool._repopulate_pool_static` so that we can + keep track of the PIDs of the created child processes. + + Note: + :py:meth:`.Pool._repopulate_pool_static` is a static method. + """ + try: + vanilla_impl(ctx, Process, processes, pool, *args, **kwargs) + finally: + patches = MPConfig.from_cache(cache).patches + if not patches.get('process', False): + # Notes: + # - if the `process` patch is active, child processes should + # always have called `.cache.profiler.dump_stats()` at + # least once before exiting, so there's no need to warn + # the cache against children possibly not having written + # profiling data. + # - Since the vanilla implementation calls `Process.start()` + # on each child, they should all have valid PIDs. However: + # - The process might have terminated for whatever reason. + # and + # - `multiprocessing.dummy.DummyProcess` doesn't have + # `.pid`. + # So we add a fallback to `None` just in case. + pids = cast( + set[int], + {getattr(process, 'pid', None) for process in pool} - {None}, + ) + cache._warn_possible_lack_of_stats(pids) + + +_patch_pool = partial( + _register_patch('pool', Patch('pool')).add_method, 'Pool', +) +_patch_pool('_get_tasks', wrap_get_tasks, 'static') +_patch_pool('_guarded_task_generation', wrap_guarded_task_generation) +_patch_pool('_repopulate_pool_static', wrap_repopulate_pool_static, 'static') + # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -541,7 +759,7 @@ def wrap_terminate( self: BaseProcess, ) -> None: """ - Wrap around :py:meth:`BaseProcess.terminate` to make sure that we + Wrap around :py:meth:`.BaseProcess.terminate` to make sure that we don't actually kill the child (OS-level) process before it has the chance to properly clean up. @@ -552,14 +770,8 @@ def wrap_terminate( after the performance-critical part of the code (said workload). """ try: - cd, timeout, on_timeout = _PollerArgs.from_config(cache._config_source) - if on_timeout not in ('ignore', 'warn', 'error'): - on_timeout = _PollerArgs.get_defaults().on_timeout - # `_process_has_returned()` takes a `timeout` which it passes to - # `popen.wait()`; said timeout is essentially a limit as to how - # often the function is called, hence our cooldown - poller = _Poller.poll_until(_process_has_returned, self, cache, cd) - with poller.with_timeout(timeout, cast(_OnTimeout, on_timeout)): + config = MPConfig.from_cache(cache) + with config._get_terminate_poller(cache, self): pass except _Poller.Timeout as e: # Also handles `~.TimeoutWarning` cache._debug_output(f'{type(e).__qualname__}: {e}') @@ -577,7 +789,7 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`BaseProcess._bootstrap` to run + Wrap around :py:meth:`.BaseProcess._bootstrap` to run ``LineProfilingCache.load().cleanup()`` so that profiling results can be gathered. @@ -590,13 +802,13 @@ def wrap_bootstrap( before it. Hence the ``# nocover``. - ``SIGTERM`` handling is not consistent on Windows, so we made - :py:meth:`LineProfilingCache._add_signal_handler` a no-op + :py:meth:`.LineProfilingCache._add_signal_handler` a no-op there. Hence :py:func:`wrap_terminate` remains necessary in mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if _get_config(cache._config_source)['catch_sigterm']: + if MPConfig.from_cache(cache).catch_sigterm: cache._add_signal_handler() try: return vanilla_impl(self, *args, **kwargs) @@ -610,6 +822,15 @@ def wrap_bootstrap( cache.cleanup(new_thread=True) +_patch_process = partial( + _register_patch('process', Patch('process')).add_method, 'BaseProcess', +) +_patch_process('_bootstrap', wrap_bootstrap) +# We only need to patch `Process.terminate()` if we can't do SIGTERM +# handling, i.e. on Windows +if sys.platform == 'win32': + _patch_process('terminate', wrap_terminate) + # --------------- `multiprocessing.util` logging patches --------------- @@ -634,7 +855,7 @@ def tee_log( **kwargs: PS.kwargs ) -> None: """ - Wrap around logging functions like +Wrap around logging functions like :py:func:`multiprocessing.util.debug` so that we can tee log messages from the package to our own logs. """ @@ -647,56 +868,165 @@ def get_msg(msg: str, *_, **__) -> str: ) -# -------------------------- Applying patches -------------------------- +_register_patch('logging', Patch('util')).add_target( + # The logging functions exists directly in the module namespace so + # no further attribute access is needed + '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, +) + +# --------------------------- Misc. patches ---------------------------- -_PATCHES: dict[_PatchName, Patch] = { - 'process': Patch('process').add_target( - 'BaseProcess', - {'terminate': wrap_terminate, '_bootstrap': wrap_bootstrap}, - ), - 'pool': Patch('pool').add_target( - 'Pool', { - # `._get_task()` is a static method, so the wrapper function - # needs additional wrapping - '_get_tasks': [ - attrgetter('__func__'), wrap_get_tasks, staticmethod, - ], - '_guarded_task_generation': wrap_guarded_task_generation, - }, - ), - 'logging': Patch('util').add_target( - # The logging functions exists directly in the module namespace - # so no further attribute access is needed - '', {func: partial(partial, tee_log, func) for func in _LOGGERS}, - ), -} - - -def _stop_forkserver() -> None: +class RebootForkserverPatch: """ + Reboot the process backing the global + :py:class:`multiprocessing.forkserver.ForkServer` instance: + + - When the patch is applied, so as to ensure that child processes + forked therefrom actually receives the active patches; and + + - When the session cache is cleaned up, so that child processes + forked therefrom is no longer polluted by the patches. + Note: - This uses `ForkServer._stop()` which is private API, but it's - the same hack used in Python's own test suite -- see the comment - to said method + This uses + :py:method:`multiprocessing.forkserver.ForkServer._stop()` which + is private API, but it's the same hack used in Python's own test + suite -- see the comment to said method. """ - # Appease the type-checker since `._stop()` is not public API - stop = getattr(forkserver._forkserver, '_stop', None) - assert callable(stop) - stop() + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({}) + + @classmethod + def apply(cls, cache: LineProfilingCache, **_) -> None: + if not _CAN_USE_FORKSERVER: + return + cls.reboot() + cache.add_cleanup(cls.reboot) + + @staticmethod + def reboot() -> None: + # Appease the type-checker since `._stop()` is not public API + stop = getattr(forkserver._forkserver, '_stop', None) + assert callable(stop) + stop() + + +class ResourceTrackerPatch: + """ + Patch :py:mod:`multiprocessing.resource_tracker` so that + :py:func:`multiprocessing.resource_tracker.ensure_running` and the + eponymous method of + :py:class:`multiprocessing.resource_tracker.ResourceTracker` report + the resource-tracker server PIDs to the session cache. + + Note: + The ``ResourceTracker`` server process is spawned when the first + :py:mod:`multiprocessing` child process is created via the + ``spawn`` or ``forkserver`` start methods. While this server + process does not meaningfully contribute to the profiling result + either way, since it can be created with profiling set up, its + longevity means that :py:meth:`.LineProfilingCache.gather_stats` + often catches empty .lprof files which it has occupied but not + written to. + + To reduce noise while keeping the empty-file warning for other + output files, we report the PIDs used by the server to the + session cache so that they can be ignored if necessary. + """ + if _CAN_USE_RESOURCE_TRACKER: + summary: ClassVar[mappingproxy[str, frozenset[str]]] = mappingproxy({ + 'multiprocessing.resource_tracker': + frozenset({'ensure_running'}), + 'multiprocessing.resource_tracker.ResourceTracker': + frozenset({'ensure_running'}), + }) + else: + summary = mappingproxy({}) + + @staticmethod + @LineProfilingCache._method_wrapper + def wrap_ensure_running( + cache: LineProfilingCache, + vanilla_impl: Callable[['resource_tracker.ResourceTracker'], None], + self: 'resource_tracker.ResourceTracker', + ) -> None: + """ + Wrap around :py:meth:`multiprocessing.resource_tracker\ +.ResourceTracker.ensure_running` + so that the session cache can keep track of the PIDs used by the + resource-tracer server. + """ + maybe_pids: set[int | None] = {getattr(self, '_pid', None)} + try: + vanilla_impl(self) + finally: + maybe_pids.add(getattr(self, '_pid', None)) + pids = cast(set[int], maybe_pids - {None}) + if pids: + cache._warn_possible_lack_of_stats(pids) + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if _CAN_USE_RESOURCE_TRACKER: + patch = partial(cache.patch, cleanup=cleanup) + # Patch the method on the class + method = resource_tracker.ResourceTracker.ensure_running + method = cls.wrap_ensure_running(method) + patch(resource_tracker.ResourceTracker, 'ensure_running', method) + # Patch the preexisting bound method on the module + instance = resource_tracker._resource_tracker + bound_method = MethodType(method, instance) + patch(resource_tracker, 'ensure_running', bound_method) + return list(cls.summary) + + +class RunpyPatch: + """ + Patch the copy of :py:mod:`runpy` in the + :py:mod:`multiprocessing.spawn` namespace so that subprocesses can + perform rewrite-based profiling as with + :py:func:`line_profiler.autoprofile.autoprofile.run`. + + See also: + :py:mod:`line_profiler._child_process_profiling.runpy_patches` + """ + summary: ClassVar[mappingproxy[str, frozenset[str]]] + if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): + summary = mappingproxy({'multiprocessing.spawn': frozenset({'runpy'})}) + else: + summary = mappingproxy({}) + + @classmethod + def apply( + cls, cache: LineProfilingCache, *, cleanup: bool = True, **_, + ) -> list[str]: + if cls.summary: + patch = partial(cache.patch, cleanup=cleanup) + patch(spawn, 'runpy', create_runpy_wrapper(cache)) + return list(cls.summary) + + +# See `ty` issue #3429 for why we need the casts +_register_patch('__reboot_forkserver', cast(_Patch, RebootForkserverPatch)) +_register_patch('__resource_tracker', cast(_Patch, ResourceTrackerPatch)) +_register_patch('__spawn_runpy', cast(_Patch, RunpyPatch)) + +# -------------------------- Applying patches -------------------------- def apply( - lp_cache: LineProfilingCache, + cache: LineProfilingCache, reboot_forkserver: bool = True, - patches: Collection[_PatchName] | None = None, + patches: Collection[PublicPatch] | None = None, ) -> None: """ Set up profiling in :py:mod:`multiprocessing` child processes by applying patches to the module. Args: - lp_cache (LineProfilingCache): + cache (LineProfilingCache): Cache instance governing the profiling run. reboot_forkserver (bool): Whether to reboot the global @@ -716,9 +1046,9 @@ def apply( methods so that parallel tasks write profiling output. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s - ``.terminate()`` and ``._bootstrap()`` methods so that child - processes write profiling output on exit and are given - enough time for that. + ``._bootstrap()`` method (and ``.terminate()`` on Windows) + so that child processes write profiling output on exit and + are given enough time for that. ``'logging'``: Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed @@ -731,9 +1061,9 @@ def apply( - Immediately - - When ``lp_cache.cleanup()`` is run + - When ``cache.cleanup()`` is run - - Cleanup callbacks registered via ``lp_cache.add_cleanup()`` + - Cleanup callbacks registered via ``cache.add_cleanup()`` Note: Rebooting the fork server is necessary because its process @@ -761,26 +1091,21 @@ def apply( if getattr(multiprocessing, _PATCHED_MARKER, False): return if patches is None: - config = _get_config(lp_cache._config_source)['patches'] - patches_ = {patch for patch, applied in config.items() if applied} + patches_dict = MPConfig.from_cache(cache).patches + patches_: set[str] = {p for p, use in patches_dict.items() if use} else: patches_ = {p.lower() for p in patches} - # Patch `multiprocessing.spawn` - if _CAN_USE_SPAWN and hasattr(spawn, 'runpy'): - lp_cache.patch(spawn, 'runpy', create_runpy_wrapper(lp_cache)) - # Patch methods/functions in these entities: - # - `multiprocessing.pool.Pool` - # - `multiprocessing.process.BaseProcess` - # - `multiprocessing.util` for name, patch in _PATCHES.items(): if name in patches_: - patch.apply(lp_cache) - # Stop the current `ForkServer` server process: - # - Now, so that the (rebooted) fork-server process has profiling - # set up; and - # - Also as a part of cache cleanup - if _CAN_USE_FORKSERVER and reboot_forkserver: - _stop_forkserver() - lp_cache.add_cleanup(_stop_forkserver) + should_apply = True + elif name.startswith('__'): + should_apply = (name != '__reboot_forkserver' or reboot_forkserver) + else: + should_apply = False + if should_apply: + msg = f'applying `multiprocessing` patch {name!r}' + cache._debug_output(msg.capitalize() + '...') + patch.apply(cache) + cache._debug_output('Done with ' + msg) # Mark `multiprocessing` as having been patched - lp_cache.patch(multiprocessing, _PATCHED_MARKER, True) + cache.patch(multiprocessing, _PATCHED_MARKER, True) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 6216d3be..19c7ca0f 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -3,6 +3,7 @@ import dataclasses import enum import inspect +import itertools import multiprocessing.pool import operator import os @@ -1798,14 +1799,6 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _PatchSummary = Mapping[str, Set[str]] -_GLOBAL_MINIMAL_PATCHES = { - 'multiprocessing': frozenset({_PATCHED_MARKER}), - 'multiprocessing.spawn': frozenset({'runpy'}), - 'os': frozenset({'fork'}), -} -if not hasattr(os, 'fork'): # E.g. Windows - _GLOBAL_MINIMAL_PATCHES.pop('os') - def get_patched_attributes( applied_mp_patches: Collection[str] | None = None, @@ -1827,11 +1820,11 @@ def _get_patched_attributes( ) -> MappingProxyType[str, frozenset[str]]: # Get the contents of the individual patches patches = _GLOBAL_MINIMAL_PATCHES.copy() - for patch in applied_mp_patches: - maybe_patch = MP_PATCHES.get(patch) # type: ignore - if maybe_patch: - for target, attrs in maybe_patch.summary.items(): - patches[target] = patches.get(target, frozenset()) | attrs + iter_summaries = ( + MP_PATCHES[patch].summary + for patch in applied_mp_patches if patch in MP_PATCHES + ) + patches = _get_patch_summary_union(patches, *iter_summaries) return MappingProxyType({ target: frozenset(attrs) for target, attrs in _filter_patches(patches).items() @@ -1849,6 +1842,16 @@ def _get_toml_patches_section(mp_patches: Collection[str]) -> str: ) +def _get_patch_summary_union( + *summaries: _PatchSummary, +) -> dict[str, frozenset[str]]: + result: dict[str, frozenset[str]] = {} + for summary in summaries: + for target, attrs in summary.items(): + result[target] = result.get(target, frozenset()) | frozenset(attrs) + return result + + def _summarize_patches( summaries: Collection[tuple[bool, _PatchSummary]] ) -> dict[str, dict[str, bool]]: @@ -1899,6 +1902,30 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: return result +_GLOBAL_MINIMAL_PATCHES = { + 'multiprocessing': frozenset({_PATCHED_MARKER}), +} +# Get patches that are dynamically resolved: while these patches are +# always applied, some of the patch targets are +# platform-/Pyhon-version-specific and may not always exist +_dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( + patch.summary for name, patch in MP_PATCHES.items() + # Basic `multiprocessing` patches are always applied + if name.startswith('__') +) +_dynamically_resolved_patch_summaries = itertools.chain( + _dynamically_resolved_patch_summaries, + # some platforms e.g. Windows don't have `fork()` + [{'os': frozenset({'fork'})}], +) +_dynamically_resolved_patch_summaries = cast( # See `ty` issue #3428 + Iterable[_PatchSummary], + map(_filter_patches, _dynamically_resolved_patch_summaries), +) +_GLOBAL_MINIMAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, *_dynamically_resolved_patch_summaries, +) + # This is only patched if we called # `_line_profiler_hooks.load_pth_hook()` _HOOK_PATCHES = { @@ -1906,15 +1933,17 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: frozenset({'called'}), } # Upper limit of what we could've patched -_GLOBAL_PATCHES = { - **_GLOBAL_MINIMAL_PATCHES, - **_HOOK_PATCHES, - **get_patched_attributes(MP_PATCHES), -} +_GLOBAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, + _HOOK_PATCHES, + get_patched_attributes([ + name for name in MP_PATCHES if not name.startswith('__') + ]), +) # Actual patches using the default config -DEFAULT_GLOBAL_PATCHES = { - **_GLOBAL_MINIMAL_PATCHES, **get_patched_attributes(), -} +DEFAULT_GLOBAL_PATCHES = _get_patch_summary_union( + _GLOBAL_MINIMAL_PATCHES, get_patched_attributes(), +) @pytest.mark.parametrize(('run_profiled_code', 'label1'), @@ -2030,6 +2059,8 @@ def test_cache_dump_load( envvars: set[str] = set(os.environ) try: original.inject_env_vars() # Needed for `.load()` + # Also test slipping stuff into the `._additional_data` + original._additional_data['foo'] = [1, 'string', None] try: # Env vars should be inserted assert set(os.environ) == envvars.union(original.environ) > envvars @@ -2047,6 +2078,8 @@ def test_cache_dump_load( getattr(original, field.name) == getattr(loaded, field.name) ) + # Compare `._additional_data` + assert original._additional_data == loaded._additional_data finally: # Explicitly cleanup for cache in cache_instances: cache.cleanup() @@ -2433,7 +2466,9 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # Note: if we're not using `Process`-based patch, there is no # guaratee that the profiling result is written via cleanup iter_stats: Iterable[Path] = Path(cache.cache_dir).glob('*.lprof') - iter_stats = filter(is_valid_stats_file, iter_stats) + iter_stats = cast( # See `ty` issue #3428 + Iterable[Path], filter(is_valid_stats_file, iter_stats), + ) pat = 'Cleanup succeeded.*: .*dump_stats.*{}' patterns.update({ pat.format(re.escape(path.name)): True for path in iter_stats From 22c56082dabc5c29b08ed19de57b4237a61b887b Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 11 May 2026 20:07:15 +0200 Subject: [PATCH 098/111] Retry tests for consistency instead of leniency tests/conftest.py RetryMarker Refactored from `_RetryHelper`: - `pytest_pyfunc_call()` now issues a warning for malformed `@pytest.mark.retry(...)` instead of erroring out - New attribute `require` for controlling whether to require the test to pass on 'all' or 'any' retries; old and default behavior equivalent to `'any'` - New attribute `is_active` for representing `@pytest.mark.retry(...)` setups where retries shouldn't be attempted - Constructor method `from_test_func()` refactored from `get_helper()`, and now always return an instance instead of `None` - New constructor method `from_arguments()` with doctest RetryMarkerWarning New warning class for malformed `@pytest.mark.retry(...)` markers RetryConditionFailure Refactored from `_RetryFailure` tests/test_retry_tests.py Simplified implementation of `_TestModule` fixtures test_bad_markers() New test that malformed `@pytest.mark.retry(...)` results in the appropriate warnings instead of test failure test_requirement() New test that `@pytest.mark.retry(require=...)` works as expected tests/test_child_procs.py::test_apply_mp_patches_failure() Now retrying with `require='all'` to test AGAINST flakiness --- tests/conftest.py | 476 ++++++++++++++++++++++++++++++-------- tests/test_child_procs.py | 1 + tests/test_retry_tests.py | 451 +++++++++++++++++++++++++----------- 3 files changed, 703 insertions(+), 225 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cfbf0c33..205ecf8d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,20 +1,35 @@ """ A simple :py:deco:`pytest.mark.retry` decorator. -Function-scoped fixtures are re-fetched bewteen retries. +Function-scoped fixtures are re-fetched between retries. + +Note: + - This file is designed to also function as a standalone + ``conftest.py`` file. + + - Adapted from `pytest-mark-retry`_. + +.. _pytest-mark-retry: https://gitlab.com/TTsangSC/pytest-mark-retry """ from __future__ import annotations +import ast import dataclasses +import os +import sys import warnings from collections.abc import ( Callable, Collection, Generator, Hashable, Iterable, Mapping, ) -from functools import cached_property, partial +from functools import cached_property, lru_cache, partial +from importlib.util import find_spec +from inspect import Signature, signature from operator import contains from pathlib import Path from typing import ( - TYPE_CHECKING, Any, ClassVar, Literal, Protocol, TypeVar, cast, final, + TYPE_CHECKING, + Any, ClassVar, Literal, Protocol, TypedDict, TypeVar, cast, overload, ) +from typing_extensions import Self import pytest from _pytest.compat import NOTSET @@ -23,14 +38,24 @@ from _pytest.scope import Scope from _pytest.unittest import TestCaseFunction try: - from pytest import TerminalReporter + from pytest import TerminalReporter # type: ignore except ImportError: # pytest < ~8.4 - from _pytest.terminal import TerminalReporter + from _pytest.terminal import TerminalReporter # type: ignore +__all__ = ( + 'RetryMarker', + 'RetryMarkerWarning', + 'RetryConditionFailure', + 'pytest_addhooks', + 'pytest_configure', + 'pytest_terminal_summary', +) + _Status = Literal['passed', 'failed', 'skipped'] +_Require = Literal['any', 'all'] F = TypeVar('F', bound=TestCaseFunction) -FCls = TypeVar('FCls', bound=type[TestCaseFunction]) +FCls = TypeVar('FCls', bound='type[TestCaseFunction]') T = TypeVar('T') _FUNCTION_SCOPE = Scope.Function @@ -41,12 +66,41 @@ def __call__(self, *, pyfuncitem: pytest.Function) -> Any: ... -class _RetryFailure(RuntimeError): +class _RetryError(RuntimeError): + """ + Base class for errors associated with retrying tests. + """ + + +class RetryMarkerWarning(_RetryError, UserWarning): + """ + Warning issued when the :deco:`pytest.mark.retry` markers on a test + fail to resolve to a valid :py:class:`RetryMarker` instance. + """ + @classmethod + def warn_from_error( + cls, xc: Exception, *args, **kwargs + ) -> None: + msg = 'disregarding invalid `@pytest.mark.retry` marker' + msg = f'{msg}: ({_format_exception(xc)})' + if 'PYTEST_CURRENT_TEST' in os.environ: + msg = f'{os.environ["PYTEST_CURRENT_TEST"]}: {msg}' + if sys.version_info < (3, 12): # Compatibility + kwargs.pop('skip_file_prefixes', None) + return warnings.warn(msg, cls, *args, **kwargs) + + +class RetryConditionFailure(_RetryError): + """ + Error raised when an attempt to retry a test failed because we + can't :py:func:`eval` the condition. + """ def __init__( self, previous_error: Exception, condition_error: Exception, - condition: str | None = None + # Should't be here if the condition isn't a string, but whatever + condition: str | bool | None = None ) -> None: self.previous_error = previous_error self.condition_error = condition_error @@ -57,23 +111,15 @@ def __init__( self.__cause__ = condition_error def _format_message(self) -> str: - prev = self._format_exception(self.previous_error) + prev = _format_exception(self.previous_error) if len(prev.split()) > 1: prev = f'({prev})' - condition = self._format_exception(self.condition_error) + condition = _format_exception(self.condition_error) if self.condition: condition = f'(condition: {self.condition!r} -> {condition})' return '{} -> {}'.format(prev, condition) - @staticmethod - def _format_exception(xc: Exception) -> str: - msg = type(xc).__name__ - if str(xc): - return f'{msg}: {xc}' - return msg - -@final @dataclasses.dataclass class _RetryEntry: func: pytest.Function @@ -93,7 +139,7 @@ def _get_name(self, with_params: bool) -> str: return name @classmethod - def add_entry(cls, func: pytest.Function, *args, **kwargs) -> _RetryEntry: + def add_entry(cls, func: pytest.Function, *args, **kwargs) -> Self: assert func.config is not None entry = cls(func, *args, **kwargs) entry.get_entries(func.config).append(entry) @@ -141,32 +187,105 @@ def _name_prefixes(self) -> tuple[str, str]: _RETRY_ENTRIES_KEY = pytest.StashKey[list[_RetryEntry]]() -_NEXTITEMS_KEY = pytest.StashKey[dict[pytest.Item, pytest.Item | None]]() -@final +def _retry_marker_sig_helper( + retries=1, *, + exceptions=None, reset_fixtures=None, condition=None, require=None, +): + """ + Dummy callable helping with :py:meth:`RetryMarker.from_test_func`, + so that we can handle marker stacking in a sane way withou having to + special-case the defaults of the class constructor. + """ + pass + + +if sys.version_info >= (3, 10): + _keyword = partial(dataclasses.field, kw_only=True) +else: + _keyword = dataclasses.field + + +class _RetryMarkerArgs(TypedDict, total=False): + retries: int + exceptions: type[Exception] | tuple[type[Exception], ...] + reset_fixtures: bool | Collection[str] + condition: str | bool | None + require: _Require + + @dataclasses.dataclass -class _RetryHelper: +class RetryMarker: + """ + Object representing the :deco:`pytest.mark.retry` marks on a test + function, managing test retries. + + Attributes: + retries (int): + Number of retries to attempt; should be positive. + exceptions (type[Exception] | tuple[type[Exception], ...]): + "Allowed" exception type(s) which result in retries; + mismatching exceptions are propagated normally, resuling in + a failure + reset_fixtures (bool | Collection[str]): + Whether to reset the function-scoped fixtures when retrying; + if a collection of names, only reset matching fixtures + condition (str | bool | None): + Only attempt retries if this is true (or :py:const:`None`); + if a string, it is :py:func:`eval`-ed to the condition + before each retry using the globals of the test function, + and the fixtures and parametrizations as the locals + require (Literal['any', 'all']): + Whether 'any' or 'all' attempts to run a test function + should pass for the test to pass + + See also: + :py:meth:`.from_arguments` for examples + """ retries: int = 1 - exceptions: type[Exception] | tuple[type[Exception], ...] = () - reset_fixtures: bool | Collection[str] = True - condition: str | None = None + exceptions: type[Exception] | tuple[type[Exception], ...] = ( + _keyword(default=Exception) + ) + reset_fixtures: bool | Collection[str] = _keyword(default=True) + condition: str | bool | None = _keyword(default=None) + require: _Require = _keyword(default='any') + name: ClassVar[str] = 'retry' + _sig: ClassVar[Signature] = signature(_retry_marker_sig_helper) def __post_init__(self) -> None: - if not (int(self.retries) == self.retries > 0): - raise TypeError( - f'.entries = {self.retries!r}: expected a positive integer' - ) + # Normalize `.retries` + try: + self.retries = max(0, int(self.retries)) + except Exception as e: + msg = f'.retries = {self.retries!r}' + msg = f'{msg}: not a valid number {_format_exception(e)}' + raise TypeError(msg).with_traceback(e.__traceback__) + # Check `.exceptions` if isinstance(self.exceptions, tuple): xc: tuple[type[Exception], ...] = self.exceptions else: xc = self.exceptions, - if not all(issubclass(X, Exception) for X in xc): + if not all( + isinstance(X, type) and issubclass(X, Exception) for X in xc + ): raise TypeError( f'.exceptions = {self.exceptions!r}: ' 'expected an exception type or a tuple thereof' ) + # Check `.condition` + if isinstance(self.condition, str): + try: + ast.parse(self.condition, mode='eval') + except Exception as e: + msg = f'.condition = {self.condition!r}' + msg = f'{msg}: not a valid expression ({_format_exception(e)})' + raise ValueError(msg).with_traceback(e.__traceback__) + # Check `.require` + if self.require not in ('all', 'any'): + msg = f'.require = {self.require!r}: expected \'any\' or \'all\'' + raise TypeError(msg) def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: """ @@ -204,32 +323,34 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: for i in range(1 + self.retries): if i: reset_fixtures(func) - if self.condition: - cond, error = self._check_condition(self.condition, func) - if error: # Bail - # XXX: would be nice if we can directly force an - # internal error, but that doesn't seem to be - # possible from within `pytest_pyfunc_call()`; - # directly calling `pytest_internalerror()` - # results in botched teardown and weird - # tracebacks, and leaves the test session in a - # bad state... - assert xc is not None - raise _RetryFailure(xc, cond, self.condition) - if not cond: - i -= 1 - break + cond, error = self._check_condition(self.condition, func) + if error: # Bail + # XXX: would be nice if we can directly force an + # internal error, but that doesn't seem to be + # possible from within `pytest_pyfunc_call()`; + # directly calling `pytest_internalerror()` + # results in botched teardown and weird + # tracebacks, and leaves the test session in a + # bad state... + assert xc is not None + raise RetryConditionFailure(xc, cond, self.condition) + if not cond: + i -= 1 + break try: result = impl(pyfuncitem=func) except self.exceptions as e: - # `ty` doesn't agree that `e` is an exception... + # `ty` doesn't agree that `e` is an exception (#3432)... xc = cast(Exception, e) + if self.require == 'all': + break except Exception as e: # Uncaught exc. -> break to raise xc = e break else: # Correct execution -> break to return xc = None - break + if self.require == 'any': + break if i: if xc is None: status: _Status = 'passed' @@ -245,8 +366,15 @@ def manage_call(self, impl: _PyfuncCallImpl, func: pytest.Function) -> Any: @staticmethod def _check_condition( - condition: str, func: pytest.Function, + condition: str | bool | None, func: pytest.Function, ) -> tuple[Any, Literal[False]] | tuple[Exception, Literal[True]]: + if condition in (True, None): # Always retry + return (True, False) + if condition in (False,): # Never retry + return (False, False) + + if TYPE_CHECKING: # Help narrowing + assert isinstance(condition, str) global_ns: dict[str, Any] | None = None try: global_ns = func.obj.__globals__ @@ -347,54 +475,192 @@ def pytest_pyfunc_call(cls, pyfuncitem: pytest.Function) -> Any: number of retries if necessary. """ pm = pyfuncitem.config.pluginmanager - impl: _PyfuncCallImpl = pm.subset_hook_caller( + pytest_pyfunc_call: _PyfuncCallImpl = pm.subset_hook_caller( 'pytest_pyfunc_call', [cls], ) - helper = cls.get_helper(pyfuncitem) - if helper: - return helper.manage_call(impl, pyfuncitem) - return impl(pyfuncitem=pyfuncitem) + try: + helper = cls.from_test_func(pyfuncitem) + except Exception as e: + # Level 1 is the `.warn_from_error()` frame, 2 is here, 3 is + # where the error actually happened + warn = RetryMarkerWarning.warn_from_error + skip_ = {_find_module_path('_pytest'), _find_module_path('pluggy')} + skip = cast(tuple[str, ...], tuple(skip_ - {None})) + warn(e, stacklevel=3, skip_file_prefixes=skip) + else: + if helper.is_active: + return helper.manage_call(pytest_pyfunc_call, pyfuncitem) + return pytest_pyfunc_call(pyfuncitem=pyfuncitem) + + @classmethod + def from_test_func(cls, func: pytest.Function, /) -> Self: + """ + Returns: + Instance combining the stack of :deco:`pytest.mark.retry` + decorators on the :py:class:`pytest.Function`. + """ + marks = (m for m in func.iter_markers() if m.name == cls.name) + return cls.from_arguments(cls._get_marker_args(mark) for mark in marks) + + @classmethod + @overload + def from_arguments(cls, args: Iterable[_RetryMarkerArgs] = (), /) -> Self: + ... + + @classmethod + @overload + def from_arguments(cls, *args: _RetryMarkerArgs) -> Self: + ... @classmethod - def get_helper(cls, pyfuncitem: pytest.Function) -> _RetryHelper | None: + def from_arguments(cls, *args) -> Self: + """ + Invocations: + (, ...) -> RetryMarker + ([, ...]) -> RetryMarker + + Examples: + >>> empty = RetryMarker.from_arguments() + >>> assert not empty.retries + >>> assert empty == RetryMarker.from_arguments([]) + + >>> default = RetryMarker.from_arguments({}) + >>> assert default.retries == 1 + >>> assert default.exceptions in ((Exception,), Exception) + >>> assert default.reset_fixtures == True + >>> assert default.condition is None + >>> assert default.require == 'any' + >>> assert default.is_active + >>> assert default == RetryMarker.from_arguments([{}]) + + Some arguments result in inactive instances (i.e. no + retries): + + >>> bad_xc = RetryMarker.from_arguments({'exceptions': ()}) + >>> assert not bad_xc.exceptions + >>> assert not bad_xc.is_active + + >>> bad_retries = RetryMarker.from_arguments( + ... {}, {}, {'retries': -5}, + ... ) + >>> assert not bad_retries.retries + >>> assert not bad_retries.is_active + + >>> bad_cond = RetryMarker.from_arguments( + ... {'condition': False}, + ... ) + >>> assert bad_cond.condition == False + >>> assert not bad_cond.is_active + + Congruent values are unioned: + + >>> stacked_xcs = RetryMarker.from_arguments([ + ... {'exceptions': ()}, + ... {'exceptions': ValueError}, + ... {'retries': 3, 'exceptions': (TypeError, OSError)}, + ... ]) + >>> assert stacked_xcs.retries == 5 + >>> assert set(stacked_xcs.exceptions) == { + ... ValueError, TypeError, OSError, + ... } + + >>> stacked_resets_1 = RetryMarker.from_arguments( + ... {'reset_fixtures': ['foo', 'bar']}, + ... {'reset_fixtures': ['baz']}, + ... ) + >>> sorted(stacked_resets_1.reset_fixtures) + ['bar', 'baz', 'foo'] + + Incongruent values override one another: + + >>> stacked_resets_2 = RetryMarker.from_arguments( + ... {'reset_fixtures': ['foo', 'bar']}, + ... {'reset_fixtures': False}, + ... ) + >>> assert stacked_resets_2.reset_fixtures == False + + >>> stacked_conditions = RetryMarker.from_arguments( + ... {'condition': 'foo==bar'}, + ... {'condition': False}, + ... ) + >>> assert stacked_conditions.condition == False + >>> assert not stacked_conditions.is_active + + >>> stacked_requires = RetryMarker.from_arguments( + ... {'require': 'any'}, + ... {'require': 'all'}, + ... ) + >>> assert stacked_requires.require == 'all' + """ retries: int = 0 - xc: set[type[Exception]] = set() + xc: set[type[Exception]] | None = None reset_fixtures: bool | set[str] = True condition: bool | str | None = None - for mark in pyfuncitem.iter_markers(): - if mark.name != cls.name: - continue - instance = cls(*mark.args, **mark.kwargs) - retries += instance.retries - condition = instance.condition - if isinstance(instance.exceptions, tuple): - xc.update(instance.exceptions) - else: - xc.add(instance.exceptions) - if ( - reset_fixtures not in (True, False) - and instance.reset_fixtures not in (True, False) - ): # Both collections of fixture names - if TYPE_CHECKING: - assert not isinstance(reset_fixtures, bool) - assert not isinstance(instance.reset_fixtures, bool) - reset_fixtures.update(instance.reset_fixtures) - elif instance.reset_fixtures not in (True, False): - if TYPE_CHECKING: - assert not isinstance(instance.reset_fixtures, bool) - reset_fixtures = set(instance.reset_fixtures) - else: - reset_fixtures = bool(instance.reset_fixtures) - if not retries: - return None - if not (condition is None or isinstance(condition, str)): - if condition: - condition = None + require: _Require | None = None + + if args: + if isinstance(args[0], Mapping): + iter_args = args else: - return None - if not xc: - xc = {Exception} - return cls(retries, tuple(xc), reset_fixtures, condition) + assert len(args) == 1 + iter_args = args[0] + else: + iter_args = args + + # `ty` needs some help here... hence the `cast()` + for bound_args in cast(Iterable[_RetryMarkerArgs], iter_args): + retries += bound_args.get('retries', 1) + if 'exceptions' in bound_args: + xc_new = bound_args['exceptions'] + if xc is None: + xc = set() + if isinstance(xc_new, type): + xc.add(xc_new) + else: + xc.update(xc_new) + if 'reset_fixtures' in bound_args: + rf_new = bound_args['reset_fixtures'] + if isinstance(rf_new, Collection): + if isinstance(reset_fixtures, Collection): + reset_fixtures.update(rf_new) + else: + reset_fixtures = set(rf_new) + else: + reset_fixtures = bool(rf_new) + if 'condition' in bound_args: + condition = bound_args['condition'] + if 'require' in bound_args: + require = bound_args['require'] + + kwargs: _RetryMarkerArgs = { + 'retries': retries, 'reset_fixtures': reset_fixtures, + } + if xc is not None: + kwargs['exceptions'] = tuple(xc) + if condition is not None: + kwargs['condition'] = condition + if require is not None: + kwargs['require'] = require + return cls(**kwargs) + + @classmethod + def _get_marker_args(cls, mark: pytest.Mark) -> _RetryMarkerArgs: + args = cls._sig.bind(*mark.args, **mark.kwargs).arguments + return cast(_RetryMarkerArgs, args) + + @property + def is_active(self) -> bool: + """ + Whether the instance should possibly attempt retries in any + condition + """ + if not self.retries: + return False + if self.exceptions == (): + return False + if self.condition in (False,): + return False + return True def _pluralize(noun: str, count: int, plural: str | None = None) -> str: @@ -403,13 +669,33 @@ def _pluralize(noun: str, count: int, plural: str | None = None) -> str: return f'{count} {noun if count == 1 else plural}' +def _format_exception(xc: Exception) -> str: + msg = type(xc).__name__ + if str(xc): + return f'{msg}: {xc}' + return msg + + +@lru_cache() +def _find_module_path(module: str) -> str | None: + spec = find_spec(module) + if spec is None or spec.origin is None: + return None + file = Path(spec.origin) + if not file.exists(): + return None + if file.name == '__init__.py': # Package + file = file.parent + return str(file) + + def pytest_addhooks(pluginmanager: pytest.PytestPluginManager) -> None: """ - Register :py:class:`_RetryHelper` as a plugin so that its + Register :py:class:`RetryMarker` as a plugin so that its :py:meth:`pytest_pyfunc_call` method can safely call other implementations without recursing to itself. """ - pluginmanager.register(_RetryHelper) + pluginmanager.register(RetryMarker) def pytest_configure(config: pytest.Config) -> None: @@ -417,8 +703,8 @@ def pytest_configure(config: pytest.Config) -> None: Register the :py:deco:`pytest.mark.retry` marker. """ help_text = ' '.join(""" - retry(retries=1, exceptions=Exception, \ -reset_fixtures=True, condition=None): + retry(retries=1, *, exceptions=Exception, \ +reset_fixtures=True, condition=None, require='any'): mark the test for retrying upon failure. @@ -436,7 +722,11 @@ def pytest_configure(config: pytest.Config) -> None: if a boolean, only retry if true; if a string, only retry if it `eval()`s true (w/globals of the test module and locals from the fixtures - and parametrizations) + and parametrizations); + require (Literal['any', 'all']): + If 'any', stop retrying and record a test pass if ANY + attempt passes; + if 'all', only record a test pass if ALL attempts pass """.split()) config.addinivalue_line('markers', help_text) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 19c7ca0f..5c2e2669 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -2542,6 +2542,7 @@ def test_apply_mp_patches_success( # (as long as we use `Pool` for our parallelism); # `Process` though... condition='not patch_pool', + require='all', # Check consistency ) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 52338e78..16340003 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -1,15 +1,31 @@ """ Tests to make sure that our :py:deco:`pytest.mark.retry` decorator works. + +Notes: + This test module is written to work both: + + - When :py:mod:`pytest_mark_retry` (`link`_) is installed from + source along with this file and the rest of the test suite, or + + - In a test directory containing (among other things): + + - This file as a standalone test module, and + + - A ``conftest.py`` containing the content of single-file module + ``pytest_mark_retry.py``. + +.. _link: https://gitlab.com/TTsangSC/pytest-mark-retry """ from __future__ import annotations import re import pprint import textwrap -from collections.abc import Callable, Collection, Generator, Iterable +from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass -from functools import partial +from functools import cached_property, partial +from importlib.util import find_spec from operator import attrgetter from pathlib import Path from shutil import rmtree @@ -22,12 +38,11 @@ pytest_plugins = ('pytester',) _Status = Literal['passed', 'failed', 'skipped'] -_RunPytest_Method = Literal[ +_RunPytestMethod = Literal[ 'runpytest', 'runpytest_inprocess', 'runpytest_subprocess', ] -_RunPytest = Callable[..., pytest.RunResult] -_RunnerGetter = Callable[[str, str], _RunPytest] +PROJECT_MODULE = 'pytest_mark_retry' TEST_COUNTERS = """ from __future__ import annotations @@ -335,12 +350,95 @@ def test_bad_dynamic_condition() -> None: @pytest.mark.parametrize('n', [0, 1, 2]) def test_dynamic_condition_test_params(n: int) -> None: ''' - Subtests `[0]` and `[2]` (resp. subtest `[1]`) should fail without - retries (resp. with 1 retry) because the condition evaluates to - false (resp. true) on the test's parametrization. + Subtests ``[0]`` and ``[2]`` (resp. subtest ``[1]``) should fail + without retries (resp. with 1 retry) because the condition evaluates + to false (resp. true) on the test's parametrization. ''' raise RuntimeError """ +TEST_BAD_MARKERS = """ +from __future__ import annotations + +import pytest + + +@pytest.mark.retry(1, 2) # `exceptions` cannot be 2 +def test_passing_bad_exceptions() -> None: + ''' + This test passes with a warning because its retry marker has an + invalid :py:attr:`RetryMarker.exceptions`. + ''' + pass + + +@pytest.mark.retry(foo=1) # No argument named `foo` +def test_passing_stray_arg() -> None: + ''' + This test also passes with a warning because its retry marker has am + stray argument ``foo`` + ''' + pass + + +@pytest.mark.retry(condition='') # Syntax error +def test_failing_bad_condition() -> None: + ''' + This test fails with a warning and without retries, because its + retry marker got a bad :py:attr:`RetryMarker.condition`. + ''' + assert False +""" +TEST_REQUIRE = """ +from __future__ import annotations + +import itertools + +import pytest + + +@pytest.fixture(scope='module') +def counter() -> itertools.count: + return itertools.count() + + +@pytest.fixture +def index(counter: itertools.count) -> int: + return next(counter) + + +@pytest.mark.retry(3) +def test_passing_retry_require_any(index: int) -> None: + ''' + This passes with two retries and leave ``index`` at 2. + ''' + assert index >= 2 + + +@pytest.mark.retry(3, require='any') +def test_failing_retry_require_any(index: int) -> None: + ''' + This fails with three retries and leave ``index`` at 6. + ''' + assert index < 3 + + +@pytest.mark.retry(3, require='all') +def test_failing_retry_require_all(index: int) -> None: + ''' + This fails with zero retries and leave ``index`` at 7. + ''' + # Fails right out the gate, no need to continue retrying + assert index > 7 + + +@pytest.mark.retry(3, require='all') +def test_passing_retry_require_all(index: int) -> None: + ''' + This passes with three retries and leave ``index`` at 11. + ''' + # All attempts pass, but we are instructed to exhaust the retries + assert index > 0 +""" @dataclass @@ -385,7 +483,7 @@ def run( check_results: bool = False, check_summary: Literal['verbose', 'concise'] | None = None, check_warnings: int | None = None, - runner: _RunPytest_Method = 'runpytest', + runner: _RunPytestMethod = 'runpytest', additional_stdout_lines: Collection[str] = (), additional_stderr_lines: Collection[str] = (), ) -> pytest.RunResult: @@ -420,7 +518,11 @@ def run( tempfiles: list[Path] = [] tempdirs: list[Path] = [] try: - conftests: list[str] = [self._get_proj_conftest().read_text()] + conftests: list[str] = [] + if not self.marker_plugin_globally_installed: + # If we don't do this the project will be loaded twice + # as a plugin, leading to a clash + conftests.append(self.marker_plugin_path.read_text()) if self.conftest: conftests.append(self.conftest) # Create separate conftest.py in nested subdirs to avoid @@ -474,7 +576,9 @@ def check_results( for outcomes in self.expected_outcomes.values(): for outcome in outcomes: counts[outcome.status] = counts.get(outcome.status, 0) + 1 - result.assert_outcomes(warnings=warnings, **counts) + result.assert_outcomes( + warnings=warnings, **cast(dict[str, int], counts), + ) def check_verbose_summary( self, @@ -551,6 +655,32 @@ def check_concise_summary( for test_name in names: assert test_name in line + @property + def marker_plugin_path(self) -> Path: + return self._source[0] + + @property + def marker_plugin_globally_installed(self) -> bool: + return self._source[1] + + @cached_property + def _source(self) -> tuple[Path, bool]: + sources = { + f'module `{PROJECT_MODULE}`': (self._get_proj_module_path, True), + repr('conftest.py'): (self._get_proj_conftest, False), + } + for src, (get_path, retry_globally_installed) in sources.items(): + try: + path = get_path() + assert 'class RetryMarker' in path.read_text() + print(f'Loaded project source from {src}: {str(path)!r}') + return path, retry_globally_installed + except Exception: + pass + raise RuntimeError( + f'Failed to load the project source from any of: {sources!r}', + ) + @staticmethod def _check_lines( result: pytest.RunResult, @@ -580,9 +710,15 @@ def _format_header(status: _Status, n: int) -> str: ) @staticmethod - def _get_proj_conftest() -> Path: + def _get_proj_conftest() -> Path: # If installed as the conftest return Path(__file__).parent / 'conftest.py' + @staticmethod + def _get_proj_module_path() -> Path: # If installed as a module + spec = find_spec(PROJECT_MODULE) + assert spec and spec.origin + return Path(spec.origin) + @staticmethod def _strip(text: str) -> str: return textwrap.dedent(text).strip('\n') @@ -616,129 +752,131 @@ def _identical_items_are_adjacent(items: Iterable[Any]) -> bool: return True +def _outcomes_to_outcome_dict( + outcomes: Iterable[_TestOutcome], +) -> dict[str, list[_TestOutcome]]: + """ + Example: + >>> o0 = _TestOutcome('foo', 'passed', 0) + >>> o1 = _TestOutcome('bar[1-2-3]', 'failed', 1) + >>> o2 = _TestOutcome('bar[4-5-6]', 'passed', 2) + >>> outcomes = {'foo': [o0], 'bar': [o1, o2]} + >>> assert _outcomes_to_outcome_dict([o1, o0, o2]) == outcomes + """ + result: dict[str, list[_TestOutcome]] = {} + for outcome in outcomes: + name = outcome.name + if name.endswith(']') and '[' in name: # Subtest + base_name, *_ = name.partition('[') + else: + base_name = name + result.setdefault(base_name, []).append(outcome) + return result + + @pytest.fixture -def counters_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: +def counters_module(pytester: pytest.Pytester) -> _TestModule: dynamic_p = _TestOutcome('test_dynamic_fixtures_persisted').subtest static_p = _TestOutcome('test_static_fixtures_persisted').subtest dynamic_r = _TestOutcome('test_dynamic_fixtures_reset').subtest static_r = _TestOutcome('test_static_fixtures_reset').subtest - outcomes = { - 'test_dynamic_fixtures_persisted': [ - dynamic_p('func-0'), - dynamic_p('func-2', retries=2), - dynamic_p('func-6', status='failed', retries=3), - dynamic_p('module-4', status='failed', retries=3), - dynamic_p('module-5', retries=1), - ], - 'test_static_fixtures_persisted': [ - static_p('func-3', retries=3), - static_p('func-4', status='failed', retries=3), - static_p('module-4'), - static_p('module-9', retries=2), - ], - 'test_dynamic_fixtures_reset': [ - dynamic_r('func-0'), - dynamic_r('func-1', status='failed', retries=1), - dynamic_r('module-11', retries=1), - ], - 'test_static_fixtures_reset': [ - static_r('func-0'), - static_r('func-1', status='failed', retries=2), - static_r('module-14', retries=2), - ], - } - yield _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) + outcomes = _outcomes_to_outcome_dict([ + dynamic_p('func-0'), + dynamic_p('func-2', retries=2), + dynamic_p('func-6', status='failed', retries=3), + dynamic_p('module-4', status='failed', retries=3), + dynamic_p('module-5', retries=1), + static_p('func-3', retries=3), + static_p('func-4', status='failed', retries=3), + static_p('module-4'), + static_p('module-9', retries=2), + dynamic_r('func-0'), + dynamic_r('func-1', status='failed', retries=1), + dynamic_r('module-11', retries=1), + static_r('func-0'), + static_r('func-1', status='failed', retries=2), + static_r('module-14', retries=2), + ]) + return _TestModule('test_counters', TEST_COUNTERS, outcomes, pytester) @pytest.fixture -def teardown_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: - yield _TestModule( - 'test_teardown', - TEST_TEARDOWN, - { - 'test_no_fixture_reset': - [_TestOutcome('test_no_fixture_reset', 'failed', 2)], - 'test_with_fixture_reset': - [_TestOutcome('test_with_fixture_reset', 'failed', 1)], - }, - pytester, - conftest=""" - from __future__ import annotations - - from pathlib import Path - - import pytest - - - def pytest_addoption(parser: pytest.Parser) -> None: - parser.addoption( - '--my-temp-dir', - type=Path, - help=f'persisted tempdir location for {__file__!r}', - ) - parser.addoption( - '--my-log', - type=Path, - help=f'log file location for tempfile creation/deletion', - ) - """, - ) +def teardown_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_no_fixture_reset', 'failed', 2), + _TestOutcome('test_with_fixture_reset', 'failed', 1), + ]) + cf = """ + from __future__ import annotations + + from pathlib import Path + + import pytest + + + def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + '--my-temp-dir', + type=Path, + help=f'persisted tempdir location for {__file__!r}', + ) + parser.addoption( + '--my-log', + type=Path, + help=f'log file location for tempfile creation/deletion', + ) + """ + return _TestModule('test_teardown', TEST_TEARDOWN, outcomes, pytester, cf) @pytest.fixture -def exceptions_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: - yield _TestModule( - 'test_exceptions', - TEST_EXCEPTIONS, - { - 'test_all_xc_types': - [_TestOutcome('test_all_xc_types', retries=3)], - 'test_one_xc_type': - [_TestOutcome('test_one_xc_type', 'failed', 1)], - 'test_two_xc_types': - [_TestOutcome('test_two_xc_types', 'failed', 2)], - 'test_three_xc_types': - [_TestOutcome('test_three_xc_types', retries=3)], - }, - pytester, - ) +def exceptions_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_all_xc_types', retries=3), + _TestOutcome('test_one_xc_type', 'failed', 1), + _TestOutcome('test_two_xc_types', 'failed', 2), + _TestOutcome('test_three_xc_types', retries=3), + ]) + return _TestModule('test_exceptions', TEST_EXCEPTIONS, outcomes, pytester) @pytest.fixture -def conditions_module( - pytester: pytest.Pytester, -) -> Generator[_TestModule, None, None]: +def conditions_module(pytester: pytest.Pytester) -> _TestModule: test = partial(_TestOutcome, status='failed') param_test_name = 'test_dynamic_condition_test_params' param_test = partial(test(param_test_name).subtest, status='failed') - yield _TestModule( - 'test_conditions', - TEST_CONDITIONS, - { - 'test_concrete_positive_condition': - [test('test_concrete_positive_condition', retries=2)], - 'test_concrete_negative_condition': - [test('test_concrete_negative_condition')], - 'test_dynamic_positive_condition_test_module_globals': - [test( - 'test_dynamic_positive_condition_test_module_globals', - retries=1, - )], - 'test_dynamic_negative_condition_test_module_globals': - [test('test_dynamic_negative_condition_test_module_globals')], - 'test_bad_dynamic_condition': - [test('test_bad_dynamic_condition')], - param_test_name: - [param_test('0'), param_test('1', retries=1), param_test('2')], - }, - pytester, - ) + outcomes = _outcomes_to_outcome_dict([ + test('test_concrete_positive_condition', retries=2), + test('test_concrete_negative_condition'), + test('test_dynamic_positive_condition_test_module_globals', retries=1), + test('test_dynamic_negative_condition_test_module_globals'), + test('test_bad_dynamic_condition'), + param_test('0'), + param_test('1', retries=1), + param_test('2'), + ]) + return _TestModule('test_conditions', TEST_CONDITIONS, outcomes, pytester) + + +@pytest.fixture +def bad_markers_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_passing_bad_exceptions'), + _TestOutcome('test_passing_stray_arg'), + _TestOutcome('test_failing_bad_condition', 'failed'), + ]) + return _TestModule('test_bad', TEST_BAD_MARKERS, outcomes, pytester) + + +@pytest.fixture +def require_module(pytester: pytest.Pytester) -> _TestModule: + outcomes = _outcomes_to_outcome_dict([ + _TestOutcome('test_passing_retry_require_any', retries=2), + _TestOutcome('test_failing_retry_require_any', 'failed', 3), + _TestOutcome('test_failing_retry_require_all', 'failed'), + _TestOutcome('test_passing_retry_require_all', retries=3), + ]) + return _TestModule('test_require', TEST_REQUIRE, outcomes, pytester) @pytest.mark.parametrize('verbose', [True, False]) @@ -804,9 +942,10 @@ def parse_line(cls, line: str) -> Self: # we can verify that by checking that the ids of the `makefile()` # fixtures appear in contiguous blocks - # Note: there seems to be a weird corner case where neighboring tests - # may reuse the same fixture id (see failing job 73520441960 in - # pipeline 25091142386); probably has to do with object lifetime. + # Note: there seems to be a weird corner case where neighboring + # tests may reuse the same fixture id (see `line_profiler` failing + # job 73520441960 in pipeline 25091142386); probably has to do with + # object lifetime. # So instead of just checking the `fixture_id`, also consult # `test`; it suffices to see that WITHIN THE SAME TEST we don't have # fixture values stepping over one another @@ -836,18 +975,66 @@ def test_retry_conditions(conditions_module: _TestModule) -> None: """ Test that the decorator correctly handles retry conditions. """ + # `test_bad_dynamic_condition()` should have failed with a + # `RetryConditionFailure`, listing the error encountered in the last + # trial and the error encountered when `eval()`-ing the condition + # (Note: grepping for the entire error message in the short test + # summary is fragile since it may be elided; so we just use a + # separate pattern to grep it from the tracebacks) + lines = [ + 'FAILED .*::test_bad_dynamic_condition - .*RetryConditionFailure', + r'.*RetryConditionFailure: \(RuntimeError: bar\) ' + r"-> \(condition: 'foo == 1' -> NameError: .*'foo'.*\)", + ] conditions_module.run( '--verbose', - check_results=True, - check_summary='verbose', - check_warnings=0, - # `test_bad_dynamic_condition()` should have failed with a - # `_RetryFailure`, listing the error encountered in the last - # trial and the error encountered when `eval()`-ing the - # condition - additional_stdout_lines=[ - 'FAILED +.*::test_bad_dynamic_condition - ' - r".*_RetryFailure: +\(RuntimeError: bar\) +" - r"-> +\(condition: +'foo == 1' +-> +NameError: .*'foo'.*\)" + check_results=True, check_summary='verbose', check_warnings=0, + additional_stdout_lines=lines, + ) + + +def test_bad_markers(bad_markers_module: _TestModule) -> None: + """ + Test that the decorator gracefully handles incorrect constructions. + """ + stdout = bad_markers_module.run( + '--verbose', + check_results=True, check_summary='verbose', check_warnings=3, + ).stdout + # Check the warnings emitted + # (Since we want to match across multiple lines we can't use + # `additional_stdout_lines`) + errors: str | Sequence[str] + pattern = ( + '{0}\n' + r'.*RetryMarkerWarning: .*{0}.*: disregarding .* marker: \(.*{1}.*\)' + ) + for test, errors in { + 'test_passing_bad_exceptions': [ + r'TypeError: \.exceptions = .*2.*: expected .*exception', + r'TypeError: .*not iterable', + r'TypeError: too many positional arguments', + r'TypeError: .*takes 1 positional argument but' ], + 'test_passing_stray_arg': + r'TypeError: .*unexpected keyword argument \'foo\'', + 'test_failing_bad_condition': + r'ValueError: \.condition = \'\': not a valid expression ' + r'\(SyntaxError.*\)', + }.items(): + if isinstance(errors, str): + errors = [errors] + messages = [pattern.format(re.escape(test), error) for error in errors] + if not any(re.search(msg, str(stdout)) for msg in messages): + msg = f'none of the patterns {messages!r} matched {stdout!r}' + raise AssertionError(msg) + + +def test_requirement(require_module: _TestModule) -> None: + """ + Test that the decorator correctly handles requirements that all + trials should pass (via ``require='all'``). + """ + require_module.run( + '--verbose', check_results=True, check_summary='verbose', ) From 859b77b4f096e163ccc3656006a5be633e4f46eb Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Tue, 12 May 2026 23:18:15 +0200 Subject: [PATCH 099/111] Better debugging output line_profiler/cleanup.py::Cleanup.cleanup() Now taking an optional `reason` argument which is reflected in log messages line_profiler/_child_process_profiling/cache.py ::LineProfilingCache._handle_signal() - Updated the log message written to (hopefully) make it easier to catch corner cases where the `.cleanup()` call somehow didn't happen - Added a `reason` to the `.cleanup()` call line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_bootstrap() line_profiler/curated_profiling.py::CuratedProfilerContext.uninstall() Added `reason`s to the `.cleanup()` calls --- .../_child_process_profiling/cache.py | 12 ++++++---- .../multiprocessing_patches.py | 7 +++--- line_profiler/cleanup.py | 23 +++++++++++++------ line_profiler/curated_profiling.py | 2 +- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ac70eb8e..ece8edc2 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -522,7 +522,7 @@ def wrap_ctx_debug( # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate - atexit.register(self.cleanup) + atexit.register(partial(self.cleanup, reason='`atexit` callback')) self._debug_output(f'Setup successful ({context})') return True @@ -544,9 +544,9 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover :py:meth:`coverage.control.Converage._on_sigterm` """ name = self._get_signal_name(signum) - msg = f'Caught `{name}` ({signum}), cleaning up...' - self._debug_output(msg) - state = 'succeeded' + # Shouldn't happen, but all kinds of weird things happen at the + # interpreter's EoL... + state = 'not initiated?!' try: # We don't care about profiler state and such if we're # already being SIGTERM-ed, so just handle cleanup on a @@ -554,7 +554,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # this prevents a duplicate call to `.cleanup()` when we're # already inside one (e.g. when `Process._bootstrap()` is # exiting, or the `atexit` hook is triggered) - self.cleanup(new_thread=True) + self.cleanup(new_thread=True, reason=f'caught `{name}` ({signum})') except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -562,6 +562,8 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover xc = f'{xc}: {msg}' state = f'failed ({xc})' raise e + else: + state = 'succeeded' finally: handler = self._sighandlers.pop(signum, None) msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 67299fdb..61834172 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -813,13 +813,14 @@ def wrap_bootstrap( try: return vanilla_impl(self, *args, **kwargs) finally: - msg = 'Calling cleanup hook via `BaseProcess._bootstrap`' - cache._debug_output(msg) # Execute cleanup in a separate thread so as to avoid deadlocks, # in case when `LineProfilingCache._handle_signal()` caught a # signal as we're in the middle of this and initiated another # `.cleanup()` call - cache.cleanup(new_thread=True) + cache.cleanup( + new_thread=True, + reason='exiting `multiprocessing.Process._bootstrap`', + ) _patch_process = partial( diff --git a/line_profiler/cleanup.py b/line_profiler/cleanup.py index d4bd55cf..11d21890 100644 --- a/line_profiler/cleanup.py +++ b/line_profiler/cleanup.py @@ -80,11 +80,13 @@ def __exit__(self, *_, **__) -> Any: Call ``~.cleanup(1)``, clearing the level of cleanup stacks we previously :py:meth:`~.__enter__`-ed into. """ - self.cleanup(1) + self.cleanup(1, reason='context exit') # Cleanup methods - def cleanup(self, levels: int | None = None) -> None: + def cleanup( + self, levels: int | None = None, *, reason: str | None = None, + ) -> None: """ Pop cleanup callbacks from the internal stacks added via :py:meth:`~.add_cleanup` etc. and call them in order. @@ -93,6 +95,8 @@ def cleanup(self, levels: int | None = None) -> None: levels (int | None): Number of stack levels to clear; passing :py:const`None` clears the entire stack of callback stacks + reason (str | None): + Optional description of the reason for cleaning up """ def pop_all_contexts( contexts: _StackContexts, @@ -114,19 +118,24 @@ def pop_n_levels_of_contexts( pop_contexts = pop_all_contexts(self._contexts) else: pop_contexts = pop_n_levels_of_contexts(self._contexts, levels) - cleanup = partial(self._cleanup, self._debug_output) + cleanup = partial(self._cleanup, self._debug_output, reason=reason) for stacks in pop_contexts: cleanup(stacks) @staticmethod - def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: + def _cleanup( + log: Callable[[str], Any], stacks: _Stacks, reason: str | None, + ) -> None: ncallbacks_total = sum(len(stack) for stack in stacks.values()) + note = f'{ncallbacks_total} callback(s)' + if reason: + note = f'{reason}; {note}' if not ncallbacks_total: - log('Cleanup aborted (no registered callbacks)') + log(f'Cleanup aborted ({note})') return # Bookend the cleanup loop with log messages to help detect if # child processes are prematurely terminated - log(f'Starting cleanup ({ncallbacks_total} callback(s))...') + log(f'Starting cleanup ({note})...') ncallbacks_run = 0 for priority in sorted(stacks, reverse=True): callbacks = stacks.pop(priority) @@ -145,7 +154,7 @@ def _cleanup(log: Callable[[str], Any], stacks: _Stacks) -> None: f'- Cleanup {state} ' f'({ncallbacks_run}/{ncallbacks_total}): {msg}', ) - log(f'... cleanup completed ({ncallbacks_total} callback(s))') + log(f'... cleanup completed ({note})') def add_cleanup( self, callback: Callable[PS, Any], *args: PS.args, **kwargs: PS.kwargs, diff --git a/line_profiler/curated_profiling.py b/line_profiler/curated_profiling.py index a120824f..e87dec60 100644 --- a/line_profiler/curated_profiling.py +++ b/line_profiler/curated_profiling.py @@ -220,7 +220,7 @@ def install(self) -> None: self.patch(self, '_installed', True) def uninstall(self) -> None: - self.cleanup() + self.cleanup(reason='uninstalling profiling context') def __enter__(self) -> Self: self.install() From 4a22dbaa9ea9495ef0f0233a239ea84c57de0c2e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 01:00:33 +0200 Subject: [PATCH 100/111] Reworked `multiprocessing.pool` patch line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper, wrap_get_tasks(), wrap_guarded_task_generation() Removed wrap_worker() New wrapper function for wrapping `multiprocessing.pool.worker()` and writing profiling output ASAP as it catches the sentinel value --- .../multiprocessing_patches.py | 179 +++++++----------- line_profiler/rc/line_profiler.toml | 7 +- 2 files changed, 75 insertions(+), 111 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 61834172..a1c40c1c 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -16,15 +16,15 @@ import dataclasses import multiprocessing +import os import sys import warnings from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module -from inspect import getattr_static, signature -from multiprocessing.context import BaseContext +from inspect import getattr_static from multiprocessing.process import BaseProcess -from multiprocessing.pool import Pool +from multiprocessing.queues import SimpleQueue from operator import attrgetter from time import sleep, monotonic from types import MappingProxyType as mappingproxy, MethodType, ModuleType @@ -58,6 +58,7 @@ _CAN_USE_RESOURCE_TRACKER = True from .. import _diagnostics as diagnostics +from ..cleanup import Cleanup from ..toml_config import ConfigSource from .cache import LineProfilingCache from .runpy_patches import create_runpy_wrapper @@ -366,32 +367,22 @@ def _process_has_returned( return result -class TaskWrapper(Generic[PS, T]): - """ - Pickle-able wrapper around the supplied task callable, which writes - to the session's profiling-stats file on exit. - """ - def __init__(self, func: Callable[PS, T]) -> None: - self.func = func - try: - self.__signature__ = signature(func) - except Exception: # nocover - # Can happen with e.g. certain builin/c-based callables - pass - - def __call__(self, *args, **kwargs) -> T: - dump_stats = LineProfilingCache.load()._dump_stats - try: - return self.func(*args, **kwargs) - finally: - if dump_stats is not None: - dump_stats() - - def _no_op(*_, **__) -> None: pass +def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: + key = 'mp_added_sigterm_handler' + if not MPConfig.from_cache(cache).catch_sigterm: + return + if cache.main_pid == os.getpid(): # Not in a child process + return + if cache._additional_data.get(key, False): + # Already added (e.g. by another plugin) + return + cache._add_signal_handler() + cache._additional_data[key] = True + # ---------------------- Patching infrastructure ----------------------- @@ -660,94 +651,67 @@ def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: # ---------------- `multiprocessing.pool.Pool` patches ----------------- -@LineProfilingCache._method_wrapper -def wrap_get_tasks( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: - """ - Wrap around :py:meth:`.Pool._get_tasks` so that the writing of - profiling stats is handled within the callables sent to the child - processes before the parent process assumes control. - - Note: - :py:meth:`.Pool._get_tasks` is a static method. - """ - return vanilla_impl(TaskWrapper(func), *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_guarded_task_generation( - _, # No need to use the cache, but `_method_wrapper` expects it - vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], - self: Pool, - result_job: int, - func: Callable[PS1, T1], - *args: PS2.args, - **kwargs: PS2.kwargs -) -> T2: - """ - Wrap around :py:meth:`.Pool._guarded_task_generation` so that the - writing of profiling stats is handled within the callables sent to - the child processes before the parent process assumes control. - """ - return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) - - -@LineProfilingCache._method_wrapper -def wrap_repopulate_pool_static( +@LineProfilingCache._method_wrapper # nocover +def wrap_worker( cache: LineProfilingCache, - vanilla_impl: Callable[ - Concatenate[BaseContext, type[P], int, list[P], PS], None - ], - ctx: BaseContext, - Process: type[P], - processes: int, - pool: list[P], + vanilla_impl: Callable[Concatenate[SimpleQueue, PS], None], + inqueue: SimpleQueue, *args: PS.args, **kwargs: PS.kwargs ) -> None: """ - Wrap around :py:meth:`.Pool._repopulate_pool_static` so that we can - keep track of the PIDs of the created child processes. + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes can write profiling output as soon as the pool runs out of + tasks. Note: - :py:meth:`.Pool._repopulate_pool_static` is a static method. + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. """ - try: - vanilla_impl(ctx, Process, processes, pool, *args, **kwargs) - finally: - patches = MPConfig.from_cache(cache).patches - if not patches.get('process', False): - # Notes: - # - if the `process` patch is active, child processes should - # always have called `.cache.profiler.dump_stats()` at - # least once before exiting, so there's no need to warn - # the cache against children possibly not having written - # profiling data. - # - Since the vanilla implementation calls `Process.start()` - # on each child, they should all have valid PIDs. However: - # - The process might have terminated for whatever reason. - # and - # - `multiprocessing.dummy.DummyProcess` doesn't have - # `.pid`. - # So we add a fallback to `None` just in case. - pids = cast( - set[int], - {getattr(process, 'pid', None) for process in pool} - {None}, + # Set a signal handler for SIGTERM to help child processes with + # consistently cleaning up + _add_sigterm_handler_in_child(cache) + # Note: using the `cache` itself as the context manager is prone to + # deadlock + with Cleanup() as cleanup: + if isinstance(inqueue.get, MethodType): + get = partial(_wrap_queue_get, cache, inqueue.get.__func__) + cleanup.patch( + inqueue, 'get', MethodType(get, inqueue), + name='._inqueue', ) - cache._warn_possible_lack_of_stats(pids) + return vanilla_impl(inqueue, *args, **kwargs) -_patch_pool = partial( - _register_patch('pool', Patch('pool')).add_method, 'Pool', -) -_patch_pool('_get_tasks', wrap_get_tasks, 'static') -_patch_pool('_guarded_task_generation', wrap_guarded_task_generation) -_patch_pool('_repopulate_pool_static', wrap_repopulate_pool_static, 'static') +def _wrap_queue_get( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[SimpleQueue, PS], T], + self: SimpleQueue, + /, + *args: PS.args, + **kwargs: PS.kwargs +) -> T: + result = vanilla_impl(self, *args, **kwargs) + ntasks: dict[int, int] + ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) + queue_id = id(self) + if result is None: + n = ntasks.pop(queue_id, 0) + msg = f'`multiprocessing.pool.worker`: recieved {n} task(s) in total' + cache._debug_output(msg) + # Got sentinel value, process is about to exit + cache.cleanup( + new_thread=True, + reason='ran out of tasks in `multiprocessing.process.worker()`', + ) + else: + ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 + return result + + +_patch_pool = _register_patch('pool', Patch('pool')).add_method +_patch_pool('', 'worker', wrap_worker) # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -808,8 +772,7 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - if MPConfig.from_cache(cache).catch_sigterm: - cache._add_signal_handler() + _add_sigterm_handler_in_child(cache) try: return vanilla_impl(self, *args, **kwargs) finally: @@ -819,7 +782,7 @@ def wrap_bootstrap( # `.cleanup()` call cache.cleanup( new_thread=True, - reason='exiting `multiprocessing.Process._bootstrap`', + reason='exiting `multiprocessing.Process._bootstrap()`', ) @@ -1042,9 +1005,9 @@ def apply( Patches: ``'pool'``: - Patch :py:class:`multiprocessing.pool.Pool`'s - ``._get_tasks()`` and ``._guarded_task_generation()`` - methods so that parallel tasks write profiling output. + Patch :py:func:`multiprocessing.pool.worker` so that + profiling output is written as each child process runs out + of task. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s ``._bootstrap()`` method (and ``.terminate()`` on Windows) diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 20eb2dd7..493d9ab8 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -234,14 +234,15 @@ catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool.Pool`, so that each tasks -# write profiling output before pushing result back to the parent -# process +# Whether to patch `multiprocessing.pool.worker()`, so that each child +# process write profiling output as it runs out of tasks pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each # child process write profiling output before exiting process = true +# NOTE: for the best result, stick to the default and have both applied + # - `multiprocessing.patches.logging` (bool): # Whether to patch logging functions in `multiprocessing.util`, so # that the internal logs of `multiprocessing` are teed to the session From 4b2ff5e65a66a84d25eb73c2b8cf84f1f6cb5fe2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 05:28:17 +0200 Subject: [PATCH 101/111] Fix signal-handling edge cases line_profiler/_child_process_profiling/cache.py::LineProfilingCache _stats_dumper New private attribute superseding `._dump_stats` cleanup() No longer overriding the base-class method _setup_in_child_process() Now setting up the `._stats_dumper` attribute instead of `._dump_stats` _handle_signal() - No longer calling `.cleanup()` to avoid deadlocks - Added handling for when the signal handler was originally set from outside of Python line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_runner(), wrap_bootstrap() No longer calling `LineProfilingCache.load().cleanup()` with the `new_thread` argument --- .../_child_process_profiling/cache.py | 74 ++++++------------- .../multiprocessing_patches.py | 12 +-- 2 files changed, 27 insertions(+), 59 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index ece8edc2..060e3cd6 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -22,7 +22,7 @@ from pathlib import Path from pickle import HIGHEST_PROTOCOL from textwrap import indent -from threading import current_thread, main_thread, RLock, Thread +from threading import current_thread, main_thread from types import FrameType, ModuleType from typing import Any, ClassVar, Literal, TypeVar, cast, final, overload from typing_extensions import Concatenate, ParamSpec, Self @@ -91,8 +91,7 @@ class LineProfilingCache(Cleanup): _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) - _rlock: RLock = _private_field(default_factory=RLock) - _dump_stats: Callable[..., None] | None = _private_field(default=None) + _stats_dumper: Cleanup | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus # retrievable in `.load()`-ed instances. @@ -166,43 +165,6 @@ def dump(self) -> None: with open(self.filename, mode='wb') as fobj: pickle.dump(content, fobj, protocol=HIGHEST_PROTOCOL) - def cleanup(self, *args, new_thread: bool = False, **kwargs) -> None: - """ - Perform cleanup. - - Args: - new_thread (bool): - Whether to relegate the call to - :py:meth:`Cleanup,cleanup` to a new thread (see Notes) - *args, **kwargs: - Passed to :py:meth:`Cleanup.cleanup` - - Note: - - In child processes we set a ``SIGTERM`` handler to always - call :py:meth:`~.cleanup`. However, this may happen when - we're already in the middle of a cleanup call, which - results in undefined behavior. To prevent this, we can - supply ``new_thread=True`` so that the - :py:meth:`Cleanup.cleanup` call is handled by a separate - thread which acquires an instance-specific lock. - - - However, this method is supposed to clean up the session - profiler by completely disabling it, and that part must - happen on the main thread or deallocation will be botched. - Hence ``new_thread=True`` is made an option and not the - default. - """ - if not new_thread: - self._cleanup_worker(*args, **kwargs) - return - thread = Thread(target=self._cleanup_worker, args=args, kwargs=kwargs) - thread.start() - thread.join() - - def _cleanup_worker(self, *args, **kwargs) -> None: - with self._rlock: - super().cleanup(*args, **kwargs) - def gather_stats( self, exclude_pids: Collection[int] | None = None, @@ -514,9 +476,18 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - dump_stats = self._dump_stats = partial(prof.dump_stats, prof_outfile) + dump_stats = partial(prof.dump_stats, prof_outfile) self.add_cleanup_with_priority(dump_stats, 1) + # Create a cleanup object for the express purpose of dumping + # stats in an emergency (e.g. when a signal is caught) + self._stats_dumper = dumper = Cleanup() + self.patch( + dumper, '_debug_output', self._debug_output, + cleanup=False, name=f'{self!r}._stats_dumper', + ) + dumper.add_cleanup(dump_stats) + # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -548,13 +519,11 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # interpreter's EoL... state = 'not initiated?!' try: - # We don't care about profiler state and such if we're - # already being SIGTERM-ed, so just handle cleanup on a - # separate thread; - # this prevents a duplicate call to `.cleanup()` when we're - # already inside one (e.g. when `Process._bootstrap()` is - # exiting, or the `atexit` hook is triggered) - self.cleanup(new_thread=True, reason=f'caught `{name}` ({signum})') + # Just dump the stats ASAP without running `.cleanup()` to + # avoid deadlocks + self._debug_output(f'Caught `{name}` ({signum}), dumping stats...') + if self._stats_dumper is not None: + self._stats_dumper.cleanup() except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -566,11 +535,14 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover state = 'succeeded' finally: handler = self._sighandlers.pop(signum, None) - msg = f'Cleanup {state}, passing `{name}` onto {handler!r}...' + msg = f'Stat-dumping {state}, passing `{name}` onto {handler!r}...' self._debug_output(msg) - if handler is not None: + if handler is None: + msg = 'original handler set from outside of Python' + raise RuntimeError(msg) + else: signal.signal(signum, handler) - signal.raise_signal(signum) + signal.raise_signal(signum) def _add_signal_handler( self, signum: int = signal.SIGTERM, diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index a1c40c1c..c493def0 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -701,10 +701,8 @@ def _wrap_queue_get( msg = f'`multiprocessing.pool.worker`: recieved {n} task(s) in total' cache._debug_output(msg) # Got sentinel value, process is about to exit - cache.cleanup( - new_thread=True, - reason='ran out of tasks in `multiprocessing.process.worker()`', - ) + reason = 'ran out of tasks in `multiprocessing.process.worker()`' + cache.cleanup(reason=reason) else: ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 return result @@ -780,10 +778,8 @@ def wrap_bootstrap( # in case when `LineProfilingCache._handle_signal()` caught a # signal as we're in the middle of this and initiated another # `.cleanup()` call - cache.cleanup( - new_thread=True, - reason='exiting `multiprocessing.Process._bootstrap()`', - ) + reason = 'exiting `multiprocessing.Process._bootstrap()`' + cache.cleanup(reason=reason) _patch_process = partial( From 519f2b5532b562493bba2c42b57285e5c9e9c550 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 08:02:05 +0200 Subject: [PATCH 102/111] Partial rollback of 4a22dba line_profiler/_child_process_profiling/cache.py::LineProfilingCache _stats_dumper Now a convenience subtype of `Cleanup` which can be directly called _setup_in_child_process(), _handle_signal() Updated setup and handling for `._stats_dumper` line_profiler/_child_process_profiling/multiprocessing_patches.py TaskWrapper, wrap_get_tasks(), wrap_guarded_task_generation() Partially reverted deletion because we need these on Windows wrap_worker() Patch now only applied when on POSIX (Windows uses the above instead) --- .../_child_process_profiling/cache.py | 54 +++++--- .../multiprocessing_patches.py | 115 ++++++++++++++++-- line_profiler/rc/line_profiler.toml | 10 +- 3 files changed, 150 insertions(+), 29 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 060e3cd6..c4653388 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -66,6 +66,23 @@ def _import_sibling(submodule: str) -> ModuleType: _private_field = partial(dataclasses.field, init=False, repr=False) +class _DumpStatsHelper(Cleanup): + def __init__( + self, prof: LineProfiler, outfile: os.PathLike[str] | str, + ) -> None: + super().__init__() + callback = self._callback = partial(prof.dump_stats, outfile) + self.add_cleanup(callback) + + def __repr__(self) -> str: + name = type(self).__name__ + get_repr = _CALLBACK_REPR_HELPER.repr + return f'<{name} @ {hex(id(self))}: {get_repr(self._callback)}>' + + def __call__(self) -> None: + self._callback() + + @final @dataclasses.dataclass class LineProfilingCache(Cleanup): @@ -91,7 +108,7 @@ class LineProfilingCache(Cleanup): _sighandlers: dict[int, _SignalHandler | int | None] = ( _private_field(default_factory=dict) ) - _stats_dumper: Cleanup | None = _private_field(default=None) + _stats_dumper: _DumpStatsHelper | None = _private_field(default=None) # These are unstructured fields; other components can decide on what # to put in them. They are also pickled by `.dump()`, and are thus # retrievable in `.load()`-ed instances. @@ -464,9 +481,11 @@ def wrap_ctx_debug( code = compile(fobj.read(), self.preimports_module, 'exec') exec(code, {}) # Use a fresh, empty namespace - # Occupy a tempfile slot in `.cache_dir` and set the profiler - # up to write thereto when the process terminates (with high - # priority) + # - Occupy a tempfile slot in `.cache_dir` + # - Set the profiler up to write thereto when the process + # terminates (with high priority) + # (Also keep a separate reference to the callback for e.g. + # dumping stats ASAP when a signal is caught) prof_outfile = self.make_tempfile( prefix=_PROFILING_OUTPUT_PREFIX_PATTERN.format( main_pid=self.main_pid, @@ -476,17 +495,15 @@ def wrap_ctx_debug( suffix='.lprof', delete=False, ) - dump_stats = partial(prof.dump_stats, prof_outfile) - self.add_cleanup_with_priority(dump_stats, 1) - - # Create a cleanup object for the express purpose of dumping - # stats in an emergency (e.g. when a signal is caught) - self._stats_dumper = dumper = Cleanup() + self._stats_dumper = dumper = _DumpStatsHelper(prof, prof_outfile) self.patch( + # If we call `dumper.cleanup()` instead of `dumper` (e.g. + # in `._handle_signal()`), the subsequent debug-log messages + # are attributed to and handled by this cache instance dumper, '_debug_output', self._debug_output, - cleanup=False, name=f'{self!r}._stats_dumper', + cleanup=False, name='._stats_dumper', ) - dumper.add_cleanup(dump_stats) + self.add_cleanup_with_priority(self._stats_dumper, 1) # Various setups self._setup_common(wrap_os_fork, {'reboot_forkserver': False}) @@ -519,11 +536,14 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover # interpreter's EoL... state = 'not initiated?!' try: - # Just dump the stats ASAP without running `.cleanup()` to - # avoid deadlocks - self._debug_output(f'Caught `{name}` ({signum}), dumping stats...') - if self._stats_dumper is not None: - self._stats_dumper.cleanup() + # Just use the `._stats_dumper` to dump the stats ASAP + # without running this cache's `.cleanup()` to avoid + # deadlocks + if self._stats_dumper is None: + state = 'unavailable' + else: + reason = f'caught `{name}` ({signum})' + self._stats_dumper.cleanup(reason=reason) except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index c493def0..9a3c775d 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -22,7 +22,8 @@ from collections.abc import Callable, Collection, Mapping, Sequence, Set from functools import partial from importlib import import_module -from inspect import getattr_static +from inspect import getattr_static, signature +from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess from multiprocessing.queues import SimpleQueue from operator import attrgetter @@ -78,6 +79,7 @@ _OnTimeout = Literal['ignore', 'warn', 'error'] PublicPatch = Literal['pool', 'process', 'logging'] +_CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' _LOGGERS = ['sub_debug', 'debug', 'info', 'sub_warning', 'warn'] _PATCHES: dict[str, '_Patch'] = {} @@ -367,6 +369,42 @@ def _process_has_returned( return result +class TaskWrapper(Generic[PS, T]): + """ + Pickle-able wrapper around the supplied task callable, which writes + to the session's profiling-stats file on exit. + + Note: + Since this produces extra overhead for each invocation of the + callable, it is only used when we can't reliably do + end-of-process cleanup. This mainly happens on Windows, where + we can't catch and handle ``SIGTERM``. + """ + def __init__(self, func: Callable[PS, T]) -> None: + self.func = func + try: + self.__signature__ = signature(func) + except Exception: # nocover + # Can happen with e.g. certain builin/c-based callables + pass + + def __call__(self, *args, **kwargs) -> T: + stats_dumper: Callable[[], None] | None = None + try: + stats_dumper = LineProfilingCache.load()._stats_dumper + except Exception: + pass + + try: + return self.func(*args, **kwargs) + finally: + if stats_dumper is not None: + # Calling the `_DumpStatsHelper` instead of using its + # `.cleanup()` prevents excessive per-task debugging + # output (and extra *extra* overhead) + stats_dumper() + + def _no_op(*_, **__) -> None: pass @@ -651,6 +689,43 @@ def _register_patch(name: str, patch: _Patch | None = None) -> _Patch: # ---------------- `multiprocessing.pool.Pool` patches ----------------- +@LineProfilingCache._method_wrapper +def wrap_get_tasks( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Callable[PS1, T1], PS2], T2], + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._get_tasks` so that the writing of + profiling stats is handled within the callables sent to the child + processes before the parent process assumes control. + + Note: + :py:meth:`.Pool._get_tasks` is a static method. + """ + return vanilla_impl(TaskWrapper(func), *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_guarded_task_generation( + _, # No need to use the cache, but `_method_wrapper` expects it + vanilla_impl: Callable[Concatenate[Pool, int, Callable[PS1, T1], PS2], T2], + self: Pool, + result_job: int, + func: Callable[PS1, T1], + *args: PS2.args, + **kwargs: PS2.kwargs +) -> T2: + """ + Wrap around :py:meth:`.Pool._guarded_task_generation` so that the + writing of profiling stats is handled within the callables sent to + the child processes before the parent process assumes control. + """ + return vanilla_impl(self, result_job, TaskWrapper(func), *args, **kwargs) + + @LineProfilingCache._method_wrapper # nocover def wrap_worker( cache: LineProfilingCache, @@ -664,10 +739,14 @@ def wrap_worker( processes can write profiling output as soon as the pool runs out of tasks. - Note: - This is only called in child processes and thus we can't - reliably measure coverage thereon; see also - :py:func:`wrap_bootstrap`. + Notes: + - This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + + - This only works reliably for POSIX because we can handle + ``SIGTERM`` on child processes and ensure that they aren't + prematurely terminated. """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up @@ -709,7 +788,18 @@ def _wrap_queue_get( _patch_pool = _register_patch('pool', Patch('pool')).add_method -_patch_pool('', 'worker', wrap_worker) +if _CAN_CATCH_SIGTERM: + # Only write profiling output once per process if it can be helped + _patch_pool('', 'worker', wrap_worker) +else: + # Don't have a choice on platform like Windows, the only reliable + # way to ensure that the child survives until profiling output is + # written is to write it for every task before control is returned + # to the parent + _patch_pool('Pool', '_get_tasks', wrap_get_tasks, 'static') + _patch_pool( + 'Pool', '_guarded_task_generation', wrap_guarded_task_generation, + ) # ----------- `multiprocessing.process.BaseProcess` patches ------------ @@ -788,7 +878,7 @@ def wrap_bootstrap( _patch_process('_bootstrap', wrap_bootstrap) # We only need to patch `Process.terminate()` if we can't do SIGTERM # handling, i.e. on Windows -if sys.platform == 'win32': +if not _CAN_CATCH_SIGTERM: _patch_process('terminate', wrap_terminate) # --------------- `multiprocessing.util` logging patches --------------- @@ -1001,9 +1091,14 @@ def apply( Patches: ``'pool'``: - Patch :py:func:`multiprocessing.pool.worker` so that - profiling output is written as each child process runs out - of task. + On Windows + Patch :py:class:`multiprocessing.pool.Pool`'s + ``._get_tasks()`` and ``._guarded_task_generation()`` + methods so that parallel tasks write profiling output. + Else + Patch :py:func:`multiprocessing.pool.worker` so that + profiling output is written as each child process runs + out of task. ``'process'``: Patch :py:class:`multiprocessing.process.BaseProcess`'s ``._bootstrap()`` method (and ``.terminate()`` on Windows) diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 493d9ab8..592beba9 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -234,8 +234,14 @@ catch_sigterm = true [tool.line_profiler.child_processes.multiprocessing.patches] # - `multiprocessing.patches.pool` (bool): -# Whether to patch `multiprocessing.pool.worker()`, so that each child -# process write profiling output as it runs out of tasks +# Whether to patch `multiprocessing.pool`: +# - POSIX: +# Patch `multiprocessing.pool.worker()` so that each child process +# write profiling output as it runs out of tasks +# - Windows: +# Patch `multiprocessing.pool.Pool` so that each task writes +# profiling output before pushing the result back to the parent +# process pool = true # - `multiprocessing.patches.process` (bool): # Whether to patch `multiprocessing.process.BaseProcess`, so that each From bb4d94101472043fca0d4e455355301f102a40c5 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 09:18:47 +0200 Subject: [PATCH 103/111] Updated tests tests/test_child_procs.py::test_apply_mp_patches_{success,failure}() - Updated implementations - Loosened checks against empty-file warnings - Updated retrying rules for `test_apply_mp_patches_failure()`: - Only retrying when on Windows and `patch_pool` is false - Reverted back to `require='any'` --- tests/test_child_procs.py | 117 +++++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 53 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 5c2e2669..8c7e495c 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -66,13 +66,9 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -# XXX: owing to the shenanigans in -# `line_profiler._child_process_profiling.multiprocessing_patches`, -# there is a risk that failing child processes are not properly -# `.terminate()`-ed. So just put in a timeout... -_NUM_RETRIES = 2 _SUBPROC_TIMEOUT = 5 # Seconds _DEBUG = True +_WINDOWS = sys.platform == 'win32' def strip(s: str) -> str: @@ -1540,7 +1536,7 @@ def _search_cache_logs( # though an "intentionally" undocumented API (cpython issue #10308), # it's been around since 2.4, seems stable enough, and does exactly what # is needed -if sys.platform == 'win32': +if _WINDOWS: concat_command_line: Callable[ [Sequence[str]], str ] = subprocess.list2cmdline @@ -2355,7 +2351,7 @@ def test_load_pth_hook( @_preserve_pth_files() @_preserve_attributes(_GLOBAL_PATCHES) -def _test_apply_mp_patches( +def _test_apply_mp_patches_inner( tmp_path_factory: pytest.TempPathFactory, create_cache: Callable[..., LineProfilingCache], ext_module_object: ModuleType, @@ -2477,6 +2473,27 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: _search_cache_logs(cache, True, patterns) +def _test_apply_mp_patches( + patch_pool: bool, patch_process: bool, intercept_logs: bool, **kwargs +) -> None: + mp_patches: list[str] = [] + if patch_pool: + mp_patches.append('pool') + if patch_process: + mp_patches.append('process') + if intercept_logs: + mp_patches.append('logging') + with _check_warnings() as cw: + # Note: we can't guarantee that everything is set up early + # enough for cleanup to occur properly in child processes which + # don't receive any tasks, unless both the `process` and `pool` + # patches are applied + if patch_pool and patch_process: + cw.forbid_warnings(category=UserWarning, module='line_profiler') + cw.forbid_warnings(module='multiprocessing') + _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) + + @(_Params.new('start_method', ['fork', 'forkserver', 'spawn', 'dummy'], defaults='dummy') @@ -2512,37 +2529,37 @@ def test_apply_mp_patches_success( See also: :py:func:`test_apply_mp_patches_failure` """ - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - if intercept_logs: - mp_patches.append('logging') - with _check_warnings() as cw: - cw.forbid_warnings(category=UserWarning, module='line_profiler') - cw.forbid_warnings(module='multiprocessing') - _test_apply_mp_patches( - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - mp_patches=mp_patches, - fail=False, - n=n, - nprocs=nprocs, - ) + _test_apply_mp_patches( + patch_pool, + patch_process, + intercept_logs, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + fail=False, + n=n, + nprocs=nprocs, + ) +# XXX: on POSIX child processes can hang around for long enough for +# profiling-stats collection to occur somewhat robustly, thanks to +# signal handling. But unfortunately on Windows: +# - When `patch_pool` is true, we wrap the task callables so that they +# always write profiling stats before returning/erroring out. This +# incurs extra overhead, but effectively prevents the reliquishing of +# control back to the parent process before the stats are ready. +# - However, when `patch_pool` is false, we can only try to block/delay +# child-process termination. A timeout is used to prevent indefinite +# waits for them to finish, and there's always the off chance that the +# end-of-process cleanup still haven't finished at the end. +# Hence the conditional need for retries... @pytest.mark.retry( - _NUM_RETRIES, + retries=2, + condition='_WINDOWS and not patch_pool', exceptions=(ResultMismatch, _Poller.Timeout), - # Patching `Pool` should be foolproof no matter the platform - # (as long as we use `Pool` for our parallelism); - # `Process` though... - condition='not patch_pool', - require='all', # Check consistency ) @pytest.mark.parametrize('start_method', ['fork', 'forkserver', 'spawn', 'dummy']) @@ -2571,25 +2588,19 @@ def test_apply_mp_patches_failure( See also: :py:func:`test_apply_mp_patches_success` """ - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - with _check_warnings() as cw: - cw.forbid_warnings(category=UserWarning, module='line_profiler') - cw.forbid_warnings(module='multiprocessing') - _test_apply_mp_patches( - tmp_path_factory=tmp_path_factory, - create_cache=create_cache, - ext_module_object=ext_module_object, - test_module_object=test_module_object, - start_method=start_method, - mp_patches=mp_patches, - fail=True, - n=n, - nprocs=nprocs, - ) + _test_apply_mp_patches( + patch_pool, + patch_process, + intercept_logs=False, + tmp_path_factory=tmp_path_factory, + create_cache=create_cache, + ext_module_object=ext_module_object, + test_module_object=test_module_object, + start_method=start_method, + fail=True, + n=n, + nprocs=nprocs, + ) # XXX: End of tests for implementation details From b3475237881d518582fac33093c300bc381d2592 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Wed, 13 May 2026 10:13:15 +0200 Subject: [PATCH 104/111] Fix `tests/test_retry_tests.py` tests/test_retry_tests.py::test_retry_conditions() Fix failure when running in a narrow terminal; see `pytest-mark-retry` commit d53a515 --- tests/test_retry_tests.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/test_retry_tests.py b/tests/test_retry_tests.py index 16340003..a3251671 100644 --- a/tests/test_retry_tests.py +++ b/tests/test_retry_tests.py @@ -978,11 +978,10 @@ def test_retry_conditions(conditions_module: _TestModule) -> None: # `test_bad_dynamic_condition()` should have failed with a # `RetryConditionFailure`, listing the error encountered in the last # trial and the error encountered when `eval()`-ing the condition - # (Note: grepping for the entire error message in the short test - # summary is fragile since it may be elided; so we just use a - # separate pattern to grep it from the tracebacks) + # (Note: grepping for the error details in the short test summary is + # fragile since it may be elided or even entirely omitted; so we + # just use a separate pattern to grep it from the tracebacks) lines = [ - 'FAILED .*::test_bad_dynamic_condition - .*RetryConditionFailure', r'.*RetryConditionFailure: \(RuntimeError: bar\) ' r"-> \(condition: 'foo == 1' -> NameError: .*'foo'.*\)", ] From 8269898f7e3d4389326ecb0d10b67929f75e24b2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 15 May 2026 21:21:50 +0200 Subject: [PATCH 105/111] More robust PID management line_profiler/_child_process_profiling/multiprocessing_patches.py wrap_worker_pool() Renamed from `wrap_worker()` because another function now also patches `multiprocessing.pool.worker()` wrap_handle_results(), wrap_worker_pid(), wrap_process() Parts of the new patch `child_pids` which patchs resp. - `multiprocessing.pool.Pool._handle_results()` - `multiprocessing.pool.worker()` - `multiprocessing.pool.Pool.Process()` To help the parent process keep track of which child process having completed which task, and hence be able to warn the session cache against profiling-stats file expected to be empty apply() Added documentation on the `child_pids` patch line_profiler/rc/line_profiler.toml ::[tool.line_profiler.child_processes.multiprocessing.patches] Added boolean key `child_pids` for whether to apply the new patch --- .../multiprocessing_patches.py | 256 +++++++++++++++++- line_profiler/rc/line_profiler.toml | 6 + 2 files changed, 248 insertions(+), 14 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 9a3c775d..0eae8cd8 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -20,13 +20,13 @@ import sys import warnings from collections.abc import Callable, Collection, Mapping, Sequence, Set -from functools import partial +from functools import partial, wraps from importlib import import_module from inspect import getattr_static, signature from multiprocessing.pool import Pool from multiprocessing.process import BaseProcess -from multiprocessing.queues import SimpleQueue from operator import attrgetter +from queue import SimpleQueue from time import sleep, monotonic from types import MappingProxyType as mappingproxy, MethodType, ModuleType from typing import ( @@ -34,7 +34,6 @@ Any, ClassVar, Generic, Literal, NamedTuple, Protocol, TypeVar, NoReturn, cast, final, overload, ) - from typing_extensions import Concatenate, ParamSpec, Self try: @@ -77,7 +76,7 @@ PS1 = ParamSpec('PS1') PS2 = ParamSpec('PS2') _OnTimeout = Literal['ignore', 'warn', 'error'] -PublicPatch = Literal['pool', 'process', 'logging'] +PublicPatch = Literal['pool', 'process', 'logging', 'child_pids'] _CAN_CATCH_SIGTERM = sys.platform != 'win32' _PATCHED_MARKER = '__line_profiler_patched_multiprocessing__' @@ -93,6 +92,18 @@ def __call__(self, func: Callable[PS, T], /) -> Callable[PS, T]: ... +class _Queue(Protocol): + """ + Protocol for methods common to e.g. :py:class:`queue.SimpleQueue` + and :py:class:`multiprocessing.queues.SimpleQueue`. + """ + def put(self, obj: Any) -> None: + ... + + def get(self) -> Any: + ... + + class _Poller: """ Poll a callable until it returns true-y. @@ -405,6 +416,34 @@ def __call__(self, *args, **kwargs) -> T: stats_dumper() +class _QueuePIDWrapper: + """ + Wrap around a :py:class:`queue.SimpleQueue` (used by + :py:mod:`multiprocessing.dummy`) so that the PID info is attached to + the :py:meth:`.SimpleQueue.put` tuple. + + Notes: + - Used by the ``child_pids`` patch. + + - While the PID info is useless when using + :py:mod:`multiprocessing.dummy`, it is nonetheless necessary + because :py:meth:`multiprocessing.pool.Pool._handle_results` + has also been patched to expect the queue-item-getter function + to return the tuple ``(pid, original_put_value)``. + """ + def __init__(self, queue: _Queue) -> None: + self._queue = queue + + def __getattr__(self, attr: str) -> Any: + return getattr(self._queue, attr) + + def put(self, obj: Any) -> None: + self._queue.put((os.getpid(), obj)) + + def get(self) -> Any: + return self._queue.get() + + def _no_op(*_, **__) -> None: pass @@ -727,10 +766,10 @@ def wrap_guarded_task_generation( @LineProfilingCache._method_wrapper # nocover -def wrap_worker( +def wrap_worker_pool( cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[SimpleQueue, PS], None], - inqueue: SimpleQueue, + vanilla_impl: Callable[Concatenate[_Queue, PS], None], + inqueue: _Queue, *args: PS.args, **kwargs: PS.kwargs ) -> None: @@ -755,7 +794,7 @@ def wrap_worker( # deadlock with Cleanup() as cleanup: if isinstance(inqueue.get, MethodType): - get = partial(_wrap_queue_get, cache, inqueue.get.__func__) + get = partial(_wrap_inqueue_get, cache, inqueue.get.__func__) cleanup.patch( inqueue, 'get', MethodType(get, inqueue), name='._inqueue', @@ -763,14 +802,18 @@ def wrap_worker( return vanilla_impl(inqueue, *args, **kwargs) -def _wrap_queue_get( +def _wrap_inqueue_get( cache: LineProfilingCache, - vanilla_impl: Callable[Concatenate[SimpleQueue, PS], T], - self: SimpleQueue, + vanilla_impl: Callable[Concatenate[_Queue, PS], T], + self: _Queue, /, *args: PS.args, **kwargs: PS.kwargs ) -> T: + """ + Intecept the sentinel value (:py:const:`None`) signifiying the end + of the queue and perform cleanup. + """ result = vanilla_impl(self, *args, **kwargs) ntasks: dict[int, int] ntasks = cache._additional_data.setdefault('mp_queue_ntasks', {}) @@ -790,7 +833,7 @@ def _wrap_queue_get( _patch_pool = _register_patch('pool', Patch('pool')).add_method if _CAN_CATCH_SIGTERM: # Only write profiling output once per process if it can be helped - _patch_pool('', 'worker', wrap_worker) + _patch_pool('', 'worker', wrap_worker_pool) else: # Don't have a choice on platform like Windows, the only reliable # way to ensure that the child survives until profiling output is @@ -855,7 +898,7 @@ def wrap_bootstrap( - ``SIGTERM`` handling is not consistent on Windows, so we made :py:meth:`.LineProfilingCache._add_signal_handler` a no-op - there. Hence :py:func:`wrap_terminate` remains necessary in + there. Hence :py:func:`wrap_terminate` remains necessary for mitigating unclean exits. """ # Set a signal handler for SIGTERM to help child processes with @@ -881,6 +924,178 @@ def wrap_bootstrap( if not _CAN_CATCH_SIGTERM: _patch_process('terminate', wrap_terminate) +# ---------------------- PID bookkeeping patches ----------------------- + + +@LineProfilingCache._method_wrapper +def wrap_handle_results( + cache: LineProfilingCache, + vanilla_impl: Callable[ + Concatenate[_Queue, Callable[[], tuple[Any, ...] | None], PS], + None + ], + outqueue: _Queue, + # Since we patched `outqueue.put()` in the child process, the result + # tuple pushed to the parent has an extra item (the child PID) + get: Callable[[], tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:meth:`multiprocessing.pool.Pool._handle_results` so + that it handles the extra info (PID of child process handling the + task) included by :py:func:`.wrap_worker_pid`. + + Note: + :py:meth:`.Pool._handle_results` is a static method. + """ + # Somehow this doesn't type-check with either `mypy` or `ty` when + # we use a `TypeVar` instead of `Any` with the tuple items... + # (see `ty` issue #3467) + wrapped_get = partial(_wrap_outqueue_quick_get, cache, get) + vanilla_impl(outqueue, wrapped_get, *args, **kwargs) + + +@LineProfilingCache._method_wrapper # nocover +def wrap_worker_pid( + _, # We don't need the cache instance, but `@_method_wrapper` does + vanilla_impl: Callable[Concatenate[_Queue, _Queue, PS], None], + inqueue: _Queue, + outqueue: _Queue, + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Wrap around :py:func:`multiprocessing.pool.worker` so that child + processes report their PIDs as they pass the task results back to + the parent. + + Note: + This is only called in child processes and thus we can't + reliably measure coverage thereon; see also + :py:func:`wrap_bootstrap`. + """ + # Note: using the `cache` itself as the context manager is prone to + # deadlock + with Cleanup() as cleanup: + if isinstance(outqueue, SimpleQueue): + # `multiprocessing.dummy` instantiates C-based queue + # objects, which doesn't permit assigning to the instance + # (because it doesn't have an instance dict)... + # so just wrap the queue with a helper class + outqueue = _QueuePIDWrapper(outqueue) + elif isinstance(outqueue.put, MethodType): + put = partial(_wrap_outqueue_put, outqueue.put.__func__) + cleanup.patch( + outqueue, 'put', MethodType(put, outqueue), + name='._outqueue', + ) + return vanilla_impl(inqueue, outqueue, *args, **kwargs) + + +@LineProfilingCache._method_wrapper +def wrap_process( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, P], + *args: PS.args, + **kwargs: PS.kwargs +) -> P: + """ + Wrap around :py:func:`multiprocessing.pool.Pool.Process` so that the + processes created can report on usage when + :py:meth:`.BaseProcess.join`-ed or + :py:meth:`.BaseProcess.terminate`-ed. + + Note: + :py:meth:`.Pool.Process` is a static method. + """ + proc = vanilla_impl(*args, **kwargs) + # Note: since we don't clean up here, there's no need to instantiate + # another `Cleanup` helper + name = f'<{type(proc).__name__} @ {hex(id(proc))}>' + patch = partial(cache.patch, cleanup=False, name=name) + for method, action in ('join', 'joining'), ('terminate', 'terminating'): + bound = getattr(proc, method) + if isinstance(bound, MethodType): + finalize = _wrap_process_finalize(cache, bound.__func__, action) + patch(proc, method, MethodType(finalize, proc)) + return proc + + +def _wrap_process_finalize( + cache: LineProfilingCache, + vanilla_impl: Callable[Concatenate[P, PS], None], + action: str, +) -> Callable[Concatenate[P, PS], None]: + """ + Check if the process has run any tasks; + if not, report to the cache. + + Note: + Since the process object is pickled, this method has to directly + return a function object instead of merely being + :py:func:`partial`-ed and wrapped in a + :py:class:`types.MethodType`. + """ + @wraps(vanilla_impl) + def finaize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + pid: int | None = getattr(self, 'pid', None) + if pid is not None: + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + msg = f'{action} process {pid} which ran {ntasks} task(s)' + cache._debug_output(msg) + vanilla_impl(self, *args, **kwargs) + + action = action.capitalize() + return finaize + + +def _wrap_outqueue_put( + vanilla_impl: Callable[ + Concatenate[_Queue, tuple[Any, ...], PS], None + ], + self: _Queue, + obj: tuple[Any, ...], + *args: PS.args, + **kwargs: PS.kwargs +) -> None: + """ + Smuggle in the PID of the child process so that the parent can keep + track of which child completed what task. + """ + vanilla_impl(self, (os.getpid(), obj), *args, **kwargs) + + +def _wrap_outqueue_quick_get( + cache: LineProfilingCache, + vanilla_impl: Callable[PS, tuple[int, tuple[Any, ...]] | None], + *args: PS.args, + **kwargs: PS.kwargs +) -> tuple[Any, ...] | None: + """ + Take and process the PID of the child process completing the task. + """ + result = vanilla_impl(*args, **kwargs) + if result is None: + return None + pid, orig_result = result + ntasks = _get_ntasks(cache) + ntasks[pid] = ntasks.get(pid, 0) + 1 + return orig_result + + +def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: + key = 'mp_proc_ntasks' + return cache._additional_data.setdefault(key, cast(dict[int, int], {})) + + +_patch_pid = _register_patch('child_pids', Patch('pool')).add_method +_patch_pid('', 'worker', wrap_worker_pid) +_patch_pid('Pool', '_handle_results', wrap_handle_results, 'static') +_patch_pid('Pool', 'Process', wrap_process, 'static') + # --------------- `multiprocessing.util` logging patches --------------- @@ -1084,7 +1299,8 @@ def apply( so as to ensure that profiling happens on processes forked therefrom (see Note). patches \ -(Collection[Literal['pool', 'process', 'logging'] | None]): +(Collection[Literal['pool', 'process', 'logging', 'child_pids'] \ +| None]): Patches to apply to :py:mod:`multiprocessing`; see the following section for a description of each; the default is taken from the TOML config file. @@ -1108,6 +1324,18 @@ def apply( Patch :py:mod:`multiprocessing.util`'s logging methods (e.g. ``debug()`` and ``info()``) so that their messages are teed to the cache's debug log. + ``'child_pids'``: + Patch the following components of + :py:mod:`multiprocess.pool` so that the parent process keeps + track of the workload executed by each child process, + reducing stray warnings about the lack of profiling stats + reported thereby: + + - :py:func:`multiprocessing.pool.worker` + + - :py:meth:`multiprocessing.pool.Pool._handle_results` + + - :py:meth:`multiprocessing.pool.Pool.Process` Side effects: - The aforementioned patches applied diff --git a/line_profiler/rc/line_profiler.toml b/line_profiler/rc/line_profiler.toml index 592beba9..bdd8549b 100644 --- a/line_profiler/rc/line_profiler.toml +++ b/line_profiler/rc/line_profiler.toml @@ -254,6 +254,12 @@ process = true # that the internal logs of `multiprocessing` are teed to the session # cache's debug logs logging = false +# - `multiprocessing.patches.child_pids` (bool): +# Whether to patch `multiprocessing.pool.worker()` and +# `multiprocessing.pool.Pool` so that the the parent process keeps +# track of the workload sent to each child, suppressing stray warnings +# about empty output files where appropriate +child_pids = true # Polling controls, used for the `pool` patch From e4bafdd57f45fdd33e884fab34971e17de30a7f4 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Fri, 15 May 2026 21:51:35 +0200 Subject: [PATCH 106/111] Updated tests tests/test_child_procs.py _test_apply_mp_patches() - Parameters `patch_pool`, `patch_process`, and `intercept_logs` now permit the value `None` for using the default-config values - New parameter `trace_pids` for toggling the `child_pids` patch - Updated the `line_profiler` warning check to only happen when the `child_pids` patch is applied (since that is when we can reliably weed out the empty-file warnings) test_apply_mp_patches_success() - Reworked most subtests to just use the default value for whether to apply the `logging` patch - Added subtest for whether to apply the `child_pids` patch test_apply_mp_patches_failure() Now using the default for whether to apply the `logging` patch --- tests/test_child_procs.py | 64 +++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 8c7e495c..26172b81 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -45,7 +45,7 @@ create_runpy_wrapper, ) from line_profiler._child_process_profiling.multiprocessing_patches import ( - _Poller, _PATCHED_MARKER, _PATCHES as MP_PATCHES, + _Poller, MPConfig, _PATCHED_MARKER, _PATCHES as MP_PATCHES, ) from line_profiler.autoprofile.util_static import modpath_to_modname from line_profiler.curated_profiling import ( @@ -1795,6 +1795,9 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _PatchSummary = Mapping[str, Set[str]] +_mp_patch_is_internal: Callable[[str], bool] +_mp_patch_is_internal = operator.methodcaller('startswith', '__') + def get_patched_attributes( applied_mp_patches: Collection[str] | None = None, @@ -1828,7 +1831,10 @@ def _get_patched_attributes( def _get_toml_patches_section(mp_patches: Collection[str]) -> str: - mp_patches_as_dict = {name: name in mp_patches for name in MP_PATCHES} + mp_patches_as_dict = { + name: name in mp_patches for name in MP_PATCHES + if not _mp_patch_is_internal(name) + } return ( '[tool.line_profiler.child_processes.multiprocessing.patches]\n' + '\n'.join( @@ -1907,7 +1913,7 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _dynamically_resolved_patch_summaries: Iterable[_PatchSummary] = ( patch.summary for name, patch in MP_PATCHES.items() # Basic `multiprocessing` patches are always applied - if name.startswith('__') + if _mp_patch_is_internal(name) ) _dynamically_resolved_patch_summaries = itertools.chain( _dynamically_resolved_patch_summaries, @@ -1933,7 +1939,7 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _GLOBAL_MINIMAL_PATCHES, _HOOK_PATCHES, get_patched_attributes([ - name for name in MP_PATCHES if not name.startswith('__') + name for name in MP_PATCHES if not _mp_patch_is_internal(name) ]), ) # Actual patches using the default config @@ -1941,6 +1947,8 @@ def _filter_patches(summary: _PatchSummary) -> dict[str, set[str]]: _GLOBAL_MINIMAL_PATCHES, get_patched_attributes(), ) +_DEFAULT_MP_CONFIG = MPConfig.from_config(ConfigSource.from_default()) + @pytest.mark.parametrize(('run_profiled_code', 'label1'), [(True, 'run-profiled'), (False, 'run-unrelated')]) @@ -2128,6 +2136,7 @@ def test_cache_setup_main_process( *( (name in mp_patches, _filter_patches(patch.summary)) for name, patch in MP_PATCHES.items() + if not _mp_patch_is_internal(name) ), ]) try: @@ -2474,21 +2483,24 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: def _test_apply_mp_patches( - patch_pool: bool, patch_process: bool, intercept_logs: bool, **kwargs + patch_pool: bool | None = None, + patch_process: bool | None = None, + intercept_logs: bool | None = None, + trace_pids: bool | None = None, + **kwargs ) -> None: - mp_patches: list[str] = [] - if patch_pool: - mp_patches.append('pool') - if patch_process: - mp_patches.append('process') - if intercept_logs: - mp_patches.append('logging') + patches = cast(dict[str, bool], _DEFAULT_MP_CONFIG.patches.copy()) + for name, applied in { + 'pool': patch_pool, 'process': patch_process, + 'logging': intercept_logs, 'child_pids': trace_pids, + }.items(): + if applied is not None: + patches[name] = applied + mp_patches = [name for name, applied in patches.items() if applied] with _check_warnings() as cw: - # Note: we can't guarantee that everything is set up early - # enough for cleanup to occur properly in child processes which - # don't receive any tasks, unless both the `process` and `pool` - # patches are applied - if patch_pool and patch_process: + if 'child_pids' in mp_patches: + # With PID bookkeeping we should be able to weed out all + # the child processes which didn't perform any work cw.forbid_warnings(category=UserWarning, module='line_profiler') cw.forbid_warnings(module='multiprocessing') _test_apply_mp_patches_inner(mp_patches=mp_patches, **kwargs) @@ -2497,12 +2509,16 @@ def _test_apply_mp_patches( @(_Params.new('start_method', ['fork', 'forkserver', 'spawn', 'dummy'], defaults='dummy') - # We only need to check if `intercept_logs` work, the other + # We only need to check if `intercept_logs = logging` work, the other # parametrizations don't matter + _Params.new(('intercept_logs', 'label1'), - [(True, 'with-intercept-logs'), (False, 'no-intercept-logs')], - defaults=(False, 'no-intercept-logs'))).sorted() -@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label2'), + [(True, 'with-logging'), (False, 'no-logging')], + defaults=(None, 'default-logging')) + # Same deal with `trace_pids = child_pids` + + _Params.new(('trace_pids', 'label2'), + [(True, 'with-child_pids'), (False, 'no-child-pids')], + defaults=(None, 'default-child-pids'))).sorted() +@pytest.mark.parametrize(('patch_pool', 'patch_process', 'label3'), [(True, True, 'pool-and-process'), (True, False, 'pool-only'), (False, True, 'process-only')]) @@ -2515,11 +2531,13 @@ def test_apply_mp_patches_success( start_method: Literal['fork', 'forkserver', 'spawn', 'dummy'], patch_pool: bool, patch_process: bool, - intercept_logs: bool, + intercept_logs: bool | None, + trace_pids: bool | None, n: int, nprocs: int, label1: str, label2: str, + label3: str, ) -> None: """ Test that :py:func:`line_profiler._child_process_profiling\ @@ -2533,6 +2551,7 @@ def test_apply_mp_patches_success( patch_pool, patch_process, intercept_logs, + trace_pids, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, @@ -2591,7 +2610,6 @@ def test_apply_mp_patches_failure( _test_apply_mp_patches( patch_pool, patch_process, - intercept_logs=False, tmp_path_factory=tmp_path_factory, create_cache=create_cache, ext_module_object=ext_module_object, From 08c489c51e71defd9282f5c439aefcb4d8e4f0a5 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sat, 16 May 2026 23:13:24 +0200 Subject: [PATCH 107/111] Timeout for `test_apply_mp_patches_*()` tests/test_child_procs.py @_timeout, _TestTimeout New decorator and error class for timing out function execution using threads _test_apply_mp_patches_inner() Now timing out the tested function (`test_module_object.sum_in_child_procs()`) which uses `multiprocessing` --- tests/test_child_procs.py | 128 +++++++++++++++++++++++++++++++++++--- 1 file changed, 121 insertions(+), 7 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 26172b81..19fce375 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -7,11 +7,13 @@ import multiprocessing.pool import operator import os +import pickle import re import shlex import subprocess import sys import sysconfig +import threading import warnings from abc import ABC, abstractmethod from collections.abc import ( @@ -20,8 +22,11 @@ ) from contextlib import AbstractContextManager, ExitStack, nullcontext from functools import lru_cache, partial, wraps -from io import StringIO +from io import BytesIO, StringIO from importlib import import_module +from multiprocessing.pool import ( # type: ignore + ExceptionWithTraceback as ExceptionHelper, +) from numbers import Real from pathlib import Path from runpy import run_path @@ -30,7 +35,7 @@ from time import monotonic from types import MappingProxyType, ModuleType, TracebackType from typing import ( - TYPE_CHECKING, Any, Generic, Literal, Protocol, TypeVar, + TYPE_CHECKING, Any, Generic, IO, Literal, Protocol, TypeVar, cast, final, overload, ) from typing_extensions import Self, ParamSpec @@ -66,7 +71,7 @@ NUM_PROCS = 4 START_METHODS = set(multiprocessing.get_all_start_methods()) -_SUBPROC_TIMEOUT = 5 # Seconds +_TEST_TIMEOUT = 5 # Seconds _DEBUG = True _WINDOWS = sys.platform == 'win32' @@ -611,6 +616,13 @@ def rich_message(self) -> str: return msg +class _TestTimeout(RuntimeError): + """ + Error raised by the :py:func:`_timeout` decorator. + """ + pass + + @final @dataclasses.dataclass class _Params: @@ -1788,6 +1800,101 @@ def _check_output(output: str, tag: str, nhits: int) -> None: _run_test_module, _run_as_literal_code, profiled_code_is_tempfile=True, ) + +@overload +def _timeout( + func: Callable[PS, T], *, timeout: float = _TEST_TIMEOUT, +) -> Callable[PS, T]: + ... + + +@overload +def _timeout( + func: None = None, *, timeout: float = _TEST_TIMEOUT, +) -> Callable[[Callable[PS, T]], Callable[PS, T]]: + ... + + +def _timeout( + func: Callable[PS, T] | None = None, *, + timeout: float = _TEST_TIMEOUT, +) -> Callable[PS, T] | Callable[[Callable[PS, T]], Callable[PS, T]]: + """ + Decorate the test function so that it is run in another thread and + can be timed out. + + Example: + >>> from time import sleep + + >>> @_timeout(timeout=.5) + ... def my_func( + ... n: int, delay: float = 1, error: bool = False, + ... ) -> list[int]: + ... sleep(delay) + ... if error: + ... raise RuntimeError('my error message') + ... return list(range(n)) + + Normal execution: + + >>> my_func(3, 0) + [3] + [0, 1, 2, 3] + + Erroring out: + + >>> my_func(3, 0, error=True) + Traceback (most recent call last): + ... + RuntimeError: my error message + + Timing out: + + >>> my_func(4, delay=5) # doctest: +NORMALIZE_WHITESPACE + Traceback (most recent call last): + ... + test_child_procs._TestTimeout: + my_func(4, delay=5): timed out after 0.5 s + """ + if func is None: + return cast( + Callable[[Callable[PS, T]], Callable[PS, T]], + partial(_timeout, timeout=timeout), + ) + + @wraps(func) + def inner_wrapper( + fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs + ) -> None: + try: + result = True, func(*args, **kwargs) + except Exception as e: + result = False, ExceptionHelper(e, e.__traceback__) + pickle.dump(result, fobj) + + @wraps(func) + def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: + with BytesIO() as bio: + thread = new_thread(args=(bio, *args), kwargs=kwargs) + thread.start() + thread.join(timeout) + if not thread.is_alive(): + successful, result = pickle.loads(bio.getvalue()) + if successful: + return result + assert isinstance(result, Exception) + raise result + args_repr = [repr(a) for a in args] + args_repr.extend(f'{k}={v!r}' for k, v in kwargs.items()) + name = getattr(func, '__name__', repr(func)) + call_repr = f'{name}({", ".join(args_repr)})' + msg = f'{call_repr}: timed out after {timeout:.2g} s' + raise _TestTimeout(msg) + + new_thread = partial(threading.Thread, target=inner_wrapper, daemon=True) + + return wrapper + + # ============================= Unit tests ============================= # XXX: Tests in this section concerns implementation details, and the @@ -2402,8 +2509,14 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: # with `preimports_module=True`, both are just imported and added # to the profiler, so the code paths are the same profiled_func = ext_module_object.my_external_sum + # Note: it would have been more intuitive to just apply `@_timeout` + # to this whole function and run it all in a new thread, but that + # seem to interact adversely with patch application and prof-data + # collection... so just timeout the function invoking + # `multiprocessing` + sum_with_timeout = _timeout(test_module_object.sum_in_child_procs) called_func = partial( - test_module_object.sum_in_child_procs, + sum_with_timeout, n=nprocs, my_sum=profiled_func, start_method=start_method, @@ -2438,7 +2551,8 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') if fail: - xc_context: AbstractContextManager[Any] = pytest.raises(RuntimeError) + xc_context: AbstractContextManager[Any] + xc_context = pytest.raises(RuntimeError, match='^forced failure$') nloops_expected = n else: xc_context = nullcontext() @@ -2807,7 +2921,7 @@ def _test_profiling_multiproc_script( nhits=nhits, nnums=nnums, nprocs=nprocs, - timeout=_SUBPROC_TIMEOUT, + timeout=_TEST_TIMEOUT, debug_log=( 'debug.log' if prof_child_procs and _DEBUG else None ), @@ -2997,7 +3111,7 @@ def _test_profiling_bare_python( """.format(concat_command_line(sub_cmd))) cmd.extend(['-c', code]) proc = _run_subproc( - cmd, text=True, capture_output=True, timeout=_SUBPROC_TIMEOUT, + cmd, text=True, capture_output=True, timeout=_TEST_TIMEOUT, ) nhits = {'EXT-INVOCATION': 1, 'EXT-LOOP': n} From b48aaa225aff0c8c48d678d6fc1f55b25b8965b2 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Sun, 17 May 2026 00:27:19 +0200 Subject: [PATCH 108/111] PID-tracking fix line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_process() - Fixed bug where the if `Process.join()` and `Process.terminate()` are called more than once, the second call onwards will always report the process' having completed 0 tasks - Enclosed the bookkeeping in the above methods in a try-except block so that the original implementation is always run --- .../multiprocessing_patches.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0eae8cd8..0004be1b 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -1038,18 +1038,23 @@ def _wrap_process_finalize( :py:class:`types.MethodType`. """ @wraps(vanilla_impl) - def finaize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: - pid: int | None = getattr(self, 'pid', None) - if pid is not None: - ntasks = _get_ntasks(cache).pop(pid, 0) - if not ntasks: - cache._warn_possible_lack_of_stats(pid) - msg = f'{action} process {pid} which ran {ntasks} task(s)' - cache._debug_output(msg) - vanilla_impl(self, *args, **kwargs) + def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + try: + pid: int | None = getattr(self, 'pid', None) + checked_procs = _get_checked_processes(cache) + identifier = id(self), pid + if not (pid is None or identifier in checked_procs): + ntasks = _get_ntasks(cache).pop(pid, 0) + if not ntasks: + cache._warn_possible_lack_of_stats(pid) + msg = f'{action} process {pid} which ran {ntasks} task(s)' + cache._debug_output(msg) + checked_procs.add(cast(tuple[int, int], identifier)) + finally: + vanilla_impl(self, *args, **kwargs) action = action.capitalize() - return finaize + return finalize def _wrap_outqueue_put( @@ -1091,6 +1096,15 @@ def _get_ntasks(cache: LineProfilingCache) -> dict[int, int]: return cache._additional_data.setdefault(key, cast(dict[int, int], {})) +def _get_checked_processes( + cache: LineProfilingCache, +) -> set[tuple[int, int]]: + key = 'mp_proc_checked_workload' + return cache._additional_data.setdefault( + key, cast(set[tuple[int, int]], set()), + ) + + _patch_pid = _register_patch('child_pids', Patch('pool')).add_method _patch_pid('', 'worker', wrap_worker_pid) _patch_pid('Pool', '_handle_results', wrap_handle_results, 'static') From ea30899153f3e89499c788cc971b13aa4d3f7f5e Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 02:05:35 +0200 Subject: [PATCH 109/111] Minor test fixes tests/test_child_procs.py @_timeout Now no longer raising an error from the new thread when the main thread has timed out, causing the file handle to be destroyed _run_test_module(), run_{module,script,literal_code}() Fixed bug where if the subprocess timed out the debug logs won't be written _test_apply_mp_patches_inner() No longer using `pytest.raises()` to check for the error when `fail=True` to allow for other `RuntimeError`s (e.g. timeouts) to be propagated --- tests/test_child_procs.py | 58 +++++++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/tests/test_child_procs.py b/tests/test_child_procs.py index 19fce375..86033b3d 100644 --- a/tests/test_child_procs.py +++ b/tests/test_child_procs.py @@ -20,7 +20,7 @@ Callable, Collection, Generator, Iterable, Iterator, Mapping, Sequence, Set, ) -from contextlib import AbstractContextManager, ExitStack, nullcontext +from contextlib import ExitStack from functools import lru_cache, partial, wraps from io import BytesIO, StringIO from importlib import import_module @@ -1732,12 +1732,12 @@ def _run_test_module( if debug_log: runner_args.extend(['--debug-log', debug_log]) old_pth_files = _preserve_pth_files.get_pth_files() - proc = run_helper( - runner_args, test_args, test_module, - text=True, capture_output=True, check=(check and not fail), - **kwargs - ) try: + proc = run_helper( + runner_args, test_args, test_module, + text=True, capture_output=True, check=(check and not fail), + **kwargs + ) # Checks: if fail: # - The process has failed as expected @@ -1769,7 +1769,7 @@ def _run_test_module( for tag, num in (nhits or {}).items(): _check_output(proc.stdout, tag, num) finally: - if debug_log is not None: + if debug_log is not None and os.path.exists(debug_log): with open(debug_log) as fobj: print('-- Combined debug logs --', file=sys.stderr) print(indent(fobj.read(), ' '), end='', file=sys.stderr) @@ -1862,14 +1862,25 @@ def _timeout( ) @wraps(func) - def inner_wrapper( + def worker( fobj: IO[bytes], /, *args: PS.args, **kwargs: PS.kwargs ) -> None: try: result = True, func(*args, **kwargs) except Exception as e: result = False, ExceptionHelper(e, e.__traceback__) - pickle.dump(result, fobj) + # Do this instead of directly using `pickle.dump(..., fobj)` so + # that pickling errors and file-handle-related errors are + # handled separately + serialized = pickle.dumps(result, protocol=pickle.HIGHEST_PROTOCOL) + try: + fobj.write(serialized) + fobj.flush() + except Exception: + # Since this is run in a daemon thread, by the time this + # write happens the main thread could've already timed out + # and destroyed `fobj`... in that case just gracefully exit + pass @wraps(func) def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: @@ -1890,7 +1901,7 @@ def wrapper(*args: PS.args, **kwargs: PS.kwargs) -> T: msg = f'{call_repr}: timed out after {timeout:.2g} s' raise _TestTimeout(msg) - new_thread = partial(threading.Thread, target=inner_wrapper, daemon=True) + new_thread = partial(threading.Thread, target=worker, daemon=True) return wrapper @@ -2550,13 +2561,10 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: assert ext_module_object.__file__ loop_line = get_lineno(ext_module_object.__file__, 'EXT-LOOP') - if fail: - xc_context: AbstractContextManager[Any] - xc_context = pytest.raises(RuntimeError, match='^forced failure$') - nloops_expected = n - else: - xc_context = nullcontext() - nloops_expected = n + nprocs + nloops_expected = n + if not fail: + # Counts from the one final sum over the parallel results + nloops_expected += nprocs if start_method not in ('dummy', *START_METHODS): pytest.skip( @@ -2564,10 +2572,20 @@ def get_lineno(path: os.PathLike[str] | str, query: str) -> int: 'not available on the platform' ) - with xc_context: + # Note: manually handle the error here instead of using + # `pytest.raises()` since we want certain `RuntimeError`s to be + # propagated and handled by `@pytest.mark.retry` + fail_msg = 'forced failure' + try: result = called_func(n) - # Check correctness of the results - if not fail: + except RuntimeError as e: + if not (fail and str(e) == fail_msg): + raise + else: + if fail: + msg = f"expected `RuntimeError({fail_msg!r})`, no error raised" + raise ValueError(msg) + else: # Check correctness of the results assert result == n * (n + 1) // 2 # Check that calls in children are traced From 7f6b54a74a00359b7123e4512c24996c92ae7da0 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 08:00:37 +0200 Subject: [PATCH 110/111] Unreg. `atexit` hook in `multiprocessing` procs line_profiler/_child_process_profiling/cache.py _DumpStatsHelper.cleanup() New parameter `force` for forcing the writing of the profiling stats even if the cleanup stacks are empty LineProfilingCache._atexit_hook() New property for a persistent handle on the hook passed to `atexit.register()`, so that it can be unregistered when necessary line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_{worker_pool,bootstrap}() - Now unregistering `LineProfilingCache.load()._atexit_hook()` with `atexit.unregister()` to hopefully take care of end-of-process deadlocks in child processes - Now prioritizing cleanup speed in child processes, using `cache._stats_dumper.cleanup()` instead of `cache.cleanup()` where appropriate to avoid complications --- .../_child_process_profiling/cache.py | 13 ++- .../multiprocessing_patches.py | 94 ++++++++++++++----- 2 files changed, 84 insertions(+), 23 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index c4653388..185b8076 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -82,6 +82,11 @@ def __repr__(self) -> str: def __call__(self) -> None: self._callback() + def cleanup(self, *args, force: bool = False, **kwargs) -> None: + if force and not any(self._current_context.values()): + self.add_cleanup(self._callback) + super().cleanup(*args, **kwargs) + @final @dataclasses.dataclass @@ -510,7 +515,7 @@ def wrap_ctx_debug( # Set `.cleanup()` as an atexit hook to handle everything when # the child process is about to terminate - atexit.register(partial(self.cleanup, reason='`atexit` callback')) + atexit.register(self._atexit_hook) self._debug_output(f'Setup successful ({context})') return True @@ -543,7 +548,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover state = 'unavailable' else: reason = f'caught `{name}` ({signum})' - self._stats_dumper.cleanup(reason=reason) + self._stats_dumper.cleanup(force=True, reason=reason) except BaseException as e: xc = f'{type(e).__name__}' msg = str(e) @@ -876,3 +881,7 @@ def _empty_stats_pid_registry(self) -> Path: current_pid=os.getpid(), ) return self.make_tempfile(prefix=prefix, suffix='.dat', delete=False) + + @cached_property + def _atexit_hook(self) -> Callable[[], None]: + return partial(self.cleanup, reason='`atexit` callback') diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 0004be1b..7c37bfe3 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -14,6 +14,7 @@ """ from __future__ import annotations +import atexit import dataclasses import multiprocessing import os @@ -400,20 +401,20 @@ def __init__(self, func: Callable[PS, T]) -> None: pass def __call__(self, *args, **kwargs) -> T: - stats_dumper: Callable[[], None] | None = None + callback: Callable[[], None] = _no_op try: - stats_dumper = LineProfilingCache.load()._stats_dumper + cache = LineProfilingCache.load() except Exception: pass + else: + # Note: this doesn't write debugging output... doing so for + # every task would be excessive + callback = partial(_dump_stats_quick, cache) try: return self.func(*args, **kwargs) finally: - if stats_dumper is not None: - # Calling the `_DumpStatsHelper` instead of using its - # `.cleanup()` prevents excessive per-task debugging - # output (and extra *extra* overhead) - stats_dumper() + callback() class _QueuePIDWrapper: @@ -448,6 +449,32 @@ def _no_op(*_, **__) -> None: pass +def _setup_in_mp_child(cache: LineProfilingCache) -> None: + """ + Perform :py:mod:`multiprocessing`-specific setup in a child process + curated by the module. Currently it does the following: + + - Set up ``cache`` to handle ``SIGTERM`` on POSIX if not already + set. + + - Unregister the :py:mod:`atexit` hook associated with ``cache`` to + avoid possible clashes with the profiling-file writing managed by + this module. + """ + xc: Exception | None = None + for setup in [_add_sigterm_handler_in_child, _unregister_atexit_hook]: + try: + setup(cache) + except Exception as e: + xc = e + if xc is not None: + xc_str = type(xc).__name__ + if str(xc): + xc_str = f'{xc_str}: {xc}' + cache._debug_output(f'Setup failed in process {os.getpid()}: {xc_str}') + raise xc + + def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: key = 'mp_added_sigterm_handler' if not MPConfig.from_cache(cache).catch_sigterm: @@ -460,6 +487,35 @@ def _add_sigterm_handler_in_child(cache: LineProfilingCache) -> None: cache._add_signal_handler() cache._additional_data[key] = True + +def _unregister_atexit_hook(cache: LineProfilingCache) -> None: + atexit.unregister(cache._atexit_hook) + + +def _dump_stats_quick( + cache: LineProfilingCache, + *, + reason: str | None = None, + debug: bool = False, +) -> None: + """ + Note: + We don't really care about cleanup in the child process, so just + dump the stats and bail to reduce the chance of end-of-process + shenanigans causing a deadlock... + but do use ``._stats_dumper.cleanup()`` instead of + ``.__call__()`` so that we get debugging output (if ``debug`` is + true) + """ + stats_dumper = cache._stats_dumper + if stats_dumper is None: + return + if debug: + stats_dumper.cleanup(force=True, reason=reason) + else: + stats_dumper() + + # ---------------------- Patching infrastructure ----------------------- @@ -789,7 +845,7 @@ def wrap_worker_pool( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - _add_sigterm_handler_in_child(cache) + _setup_in_mp_child(cache) # Note: using the `cache` itself as the context manager is prone to # deadlock with Cleanup() as cleanup: @@ -802,7 +858,7 @@ def wrap_worker_pool( return vanilla_impl(inqueue, *args, **kwargs) -def _wrap_inqueue_get( +def _wrap_inqueue_get( # nocover cache: LineProfilingCache, vanilla_impl: Callable[Concatenate[_Queue, PS], T], self: _Queue, @@ -811,7 +867,7 @@ def _wrap_inqueue_get( **kwargs: PS.kwargs ) -> T: """ - Intecept the sentinel value (:py:const:`None`) signifiying the end + Intercept the sentinel value (:py:const:`None`) signifiying the end of the queue and perform cleanup. """ result = vanilla_impl(self, *args, **kwargs) @@ -824,7 +880,8 @@ def _wrap_inqueue_get( cache._debug_output(msg) # Got sentinel value, process is about to exit reason = 'ran out of tasks in `multiprocessing.process.worker()`' - cache.cleanup(reason=reason) + if cache.main_pid != os.getpid(): + _dump_stats_quick(cache, debug=True, reason=reason) else: ntasks[queue_id] = ntasks.get(queue_id, 0) + 1 return result @@ -884,9 +941,8 @@ def wrap_bootstrap( *args: PS.args, **kwargs: PS.kwargs ) -> T: """ - Wrap around :py:meth:`.BaseProcess._bootstrap` to run - ``LineProfilingCache.load().cleanup()`` so that profiling results - can be gathered. + Wrap around :py:meth:`.BaseProcess._bootstrap` so that profiling + stats are written at the end. Notes: @@ -903,16 +959,12 @@ def wrap_bootstrap( """ # Set a signal handler for SIGTERM to help child processes with # consistently cleaning up - _add_sigterm_handler_in_child(cache) + _setup_in_mp_child(cache) try: return vanilla_impl(self, *args, **kwargs) finally: - # Execute cleanup in a separate thread so as to avoid deadlocks, - # in case when `LineProfilingCache._handle_signal()` caught a - # signal as we're in the middle of this and initiated another - # `.cleanup()` call - reason = 'exiting `multiprocessing.Process._bootstrap()`' - cache.cleanup(reason=reason) + reason = 'exiting `multiprocessing.process.BaseProcess._bootstrap`' + _dump_stats_quick(cache, debug=True, reason=reason) _patch_process = partial( From 3c874304596f7f783345f6142448c13878dd4e64 Mon Sep 17 00:00:00 2001 From: "Terence S.-C. Tsang" Date: Mon, 18 May 2026 12:32:36 +0200 Subject: [PATCH 111/111] More defensive error handling line_profiler/_child_process_profiling/cache.py::LineProfilingCache @_method_wrapper Now also handling `BaseException` _format_exception() New convenience method line_profiler/_child_process_profiling/multiprocessing_patches.py ::wrap_process() Added more debugging output in `.join()` and `.terminate()` of the created `BaseProcess` --- .../_child_process_profiling/cache.py | 27 +++++++++++-------- .../multiprocessing_patches.py | 22 ++++++++++++--- 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/line_profiler/_child_process_profiling/cache.py b/line_profiler/_child_process_profiling/cache.py index 185b8076..b658e8e0 100644 --- a/line_profiler/_child_process_profiling/cache.py +++ b/line_profiler/_child_process_profiling/cache.py @@ -550,11 +550,7 @@ def _handle_signal(self, signum: int, *_) -> None: # nocover reason = f'caught `{name}` ({signum})' self._stats_dumper.cleanup(force=True, reason=reason) except BaseException as e: - xc = f'{type(e).__name__}' - msg = str(e) - if msg: - xc = f'{xc}: {msg}' - state = f'failed ({xc})' + state = f'failed ({self._format_exception(e)})' raise e else: state = 'succeeded' @@ -793,16 +789,18 @@ def wrapped_impl(*args: PS.args, **kwargs: PS.kwargs) -> T: if debug_: call_fmt = cache._format_call(name, *args, **kwargs) write(f'Wrapped call made: {call_fmt}...') - state = 'succeeded' try: result = call() - except Exception as e: - state = 'failed' - outcome = f'{type(e).__name__}' - if str(e): - outcome = f'{outcome}: {e}' + except BaseException as e: + # Note: be more defensive than normal and + # prepared to deal with `BaseException`; this + # decorator is often used for functions invoked + # in child processes which don't cleanly + # terminate + state, outcome = 'failed', cache._format_exception(e) raise e else: + state = 'succeeded' outcome = _CALLBACK_REPR_HELPER.repr(result) return result finally: @@ -833,6 +831,13 @@ def _format_call( func = cls._get_name(func) return func + call + @staticmethod + def _format_exception(xc: BaseException) -> str: + formatted = type(xc).__name__ + if str(xc): + formatted = f'{formatted}: {xc}' + return formatted + @property def environ(self) -> dict[str, str]: """ diff --git a/line_profiler/_child_process_profiling/multiprocessing_patches.py b/line_profiler/_child_process_profiling/multiprocessing_patches.py index 7c37bfe3..b30b1bca 100644 --- a/line_profiler/_child_process_profiling/multiprocessing_patches.py +++ b/line_profiler/_child_process_profiling/multiprocessing_patches.py @@ -1091,7 +1091,10 @@ def _wrap_process_finalize( """ @wraps(vanilla_impl) def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: + log = cache._debug_output + call = cache._format_call(vanilla_impl, self, *args, **kwargs) try: + log(f'Wrapped call made: {call}') pid: int | None = getattr(self, 'pid', None) checked_procs = _get_checked_processes(cache) identifier = id(self), pid @@ -1099,11 +1102,24 @@ def finalize(self: P, *args: PS.args, **kwargs: PS.kwargs) -> None: ntasks = _get_ntasks(cache).pop(pid, 0) if not ntasks: cache._warn_possible_lack_of_stats(pid) - msg = f'{action} process {pid} which ran {ntasks} task(s)' - cache._debug_output(msg) + log(f'{action} process {pid} which ran {ntasks} task(s)...') checked_procs.add(cast(tuple[int, int], identifier)) + except BaseException as e: + log( + f'Error in bookkeeping ({cache._format_exception(e)}), ' + 'invoking base implementation nonetheless...' + ) + raise e finally: - vanilla_impl(self, *args, **kwargs) + try: + vanilla_impl(self, *args, **kwargs) + except BaseException as e: + state = f'failed ({cache._format_exception(e)})' + raise e + else: + state = 'succeeded' + finally: + log(f'Wrapped call {call} {state}') action = action.capitalize() return finalize