From 70e365c0ca6c95461aa92dc059fef7b64e3290b0 Mon Sep 17 00:00:00 2001 From: Guo Ci Date: Wed, 6 May 2026 08:08:13 -0400 Subject: [PATCH 01/15] Replace use of Python keyword in `issubclass` function documentation (#142357) --- Doc/library/functions.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 06fd5cdc7be2a6..4394dc0690cf7e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1107,13 +1107,13 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: issubclass(class, classinfo, /) +.. function:: issubclass(cls, classinfo, /) - Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual + Return ``True`` if *cls* is a subclass (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a + or a :ref:`types-union`, in which case return ``True`` if *cls* is a subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` exception is raised. From 347f53853faa4777e3e63a8a58a0d5e926bd3e54 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Wed, 6 May 2026 15:58:45 +0300 Subject: [PATCH 02/15] gh-149415: Replace `typing._LazyAnnotationLib` with `lazy import` (#149416) --- Lib/typing.py | 74 +++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index e7563a53878da5..5b1e223d59641e 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -28,6 +28,7 @@ import sys import types from types import GenericAlias +lazy import annotationlib from _typing import ( _idfunc, @@ -163,15 +164,6 @@ 'Unpack', ] -class _LazyAnnotationLib: - def __getattr__(self, attr): - global _lazy_annotationlib - import annotationlib - _lazy_annotationlib = annotationlib - return getattr(annotationlib, attr) - -_lazy_annotationlib = _LazyAnnotationLib() - def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" @@ -255,7 +247,7 @@ def _type_repr(obj): if isinstance(obj, tuple): # Special case for `repr` of types with `ParamSpec`: return '[' + ', '.join(_type_repr(t) for t in obj) + ']' - return _lazy_annotationlib.type_repr(obj) + return annotationlib.type_repr(obj) def _collect_type_parameters( @@ -463,7 +455,7 @@ def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset() recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ - if isinstance(t, _lazy_annotationlib.ForwardRef): + if isinstance(t, annotationlib.ForwardRef): # If the forward_ref has __forward_module__ set, evaluate() infers the globals # from the module, and it will probably pick better than the globals we have here. # We do this only for calls from get_type_hints() (which opts in through the @@ -1004,7 +996,7 @@ def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): kwargs['module'] = parent_fwdref.__forward_module__ if parent_fwdref.__owner__ is not None: kwargs['owner'] = parent_fwdref.__owner__ - forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) + forward_ref = annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ return forward_ref @@ -1039,18 +1031,18 @@ def evaluate_forward_ref( VALUE. """ - if format == _lazy_annotationlib.Format.STRING: + if format == annotationlib.Format.STRING: return forward_ref.__forward_arg__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref if format is None: - format = _lazy_annotationlib.Format.VALUE + format = annotationlib.Format.VALUE value = forward_ref.evaluate(globals=globals, locals=locals, type_params=type_params, owner=owner, format=format) - if (isinstance(value, _lazy_annotationlib.ForwardRef) - and format == _lazy_annotationlib.Format.FORWARDREF): + if (isinstance(value, annotationlib.ForwardRef) + and format == annotationlib.Format.FORWARDREF): return value if isinstance(value, str): @@ -1891,8 +1883,8 @@ def _get_protocol_attrs(cls): annotations = base.__annotations__ except Exception: # Only go through annotationlib to handle deferred annotations if we need to - annotations = _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF + annotations = annotationlib.get_annotations( + base, format=annotationlib.Format.FORWARDREF ) for attr in (*base.__dict__, *annotations): if not attr.startswith('_abc_') and attr not in EXCLUDED_ATTRIBUTES: @@ -2140,8 +2132,8 @@ def _proto_hook(cls, other): try: annos = base.__annotations__ except Exception: - annos = _lazy_annotationlib.get_annotations( - base, format=_lazy_annotationlib.Format.FORWARDREF + annos = annotationlib.get_annotations( + base, format=annotationlib.Format.FORWARDREF ) if attr in annos: break @@ -2428,14 +2420,14 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, """ if getattr(obj, '__no_type_check__', None): return {} - Format = _lazy_annotationlib.Format + Format = annotationlib.Format if format is None: format = Format.VALUE # Classes require a special treatment. if isinstance(obj, type): hints = {} for base in reversed(obj.__mro__): - ann = _lazy_annotationlib.get_annotations(base, format=format) + ann = annotationlib.get_annotations(base, format=format) if format == Format.STRING: hints.update(ann) continue @@ -2468,7 +2460,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: return {k: _strip_annotations(t) for k, t in hints.items()} - hints = _lazy_annotationlib.get_annotations(obj, format=format) + hints = annotationlib.get_annotations(obj, format=format) if ( not hints and not isinstance(obj, types.ModuleType) @@ -3020,10 +3012,10 @@ def _make_eager_annotate(types): for key, val in types.items()} def annotate(format): match format: - case _lazy_annotationlib.Format.VALUE | _lazy_annotationlib.Format.FORWARDREF: + case annotationlib.Format.VALUE | annotationlib.Format.FORWARDREF: return checked_types - case _lazy_annotationlib.Format.STRING: - return _lazy_annotationlib.annotations_to_string(types) + case annotationlib.Format.STRING: + return annotationlib.annotations_to_string(types) case _: raise NotImplementedError(format) return annotate @@ -3053,9 +3045,9 @@ def __new__(cls, typename, bases, ns): types = ns["__annotations__"] field_names = list(types) annotate = _make_eager_annotate(types) - elif (original_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: - types = _lazy_annotationlib.call_annotate_function( - original_annotate, _lazy_annotationlib.Format.FORWARDREF) + elif (original_annotate := annotationlib.get_annotate_from_class_namespace(ns)) is not None: + types = annotationlib.call_annotate_function( + original_annotate, annotationlib.Format.FORWARDREF) field_names = list(types) # For backward compatibility, type-check all the types at creation time @@ -3063,9 +3055,9 @@ def __new__(cls, typename, bases, ns): _type_check(typ, "field annotation must be a type") def annotate(format): - annos = _lazy_annotationlib.call_annotate_function( + annos = annotationlib.call_annotate_function( original_annotate, format) - if format != _lazy_annotationlib.Format.STRING: + if format != annotationlib.Format.STRING: return {key: _type_check(val, f"field {key} annotation must be a type") for key, val in annos.items()} return annos @@ -3207,9 +3199,9 @@ def __new__(cls, name, bases, ns, total=True, closed=None, if ns_annotations is not None: own_annotate = None own_annotations = ns_annotations - elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: - own_annotations = _lazy_annotationlib.call_annotate_function( - own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict + elif (own_annotate := annotationlib.get_annotate_from_class_namespace(ns)) is not None: + own_annotations = annotationlib.call_annotate_function( + own_annotate, annotationlib.Format.FORWARDREF, owner=tp_dict ) else: own_annotate = None @@ -3276,20 +3268,20 @@ def __annotate__(format): base_annotate = base.__annotate__ if base_annotate is None: continue - base_annos = _lazy_annotationlib.call_annotate_function( + base_annos = annotationlib.call_annotate_function( base_annotate, format, owner=base) annos.update(base_annos) if own_annotate is not None: - own = _lazy_annotationlib.call_annotate_function( + own = annotationlib.call_annotate_function( own_annotate, format, owner=tp_dict) - if format != _lazy_annotationlib.Format.STRING: + if format != annotationlib.Format.STRING: own = { n: _type_check(tp, msg, module=tp_dict.__module__) for n, tp in own.items() } - elif format == _lazy_annotationlib.Format.STRING: - own = _lazy_annotationlib.annotations_to_string(own_annotations) - elif format in (_lazy_annotationlib.Format.FORWARDREF, _lazy_annotationlib.Format.VALUE): + elif format == annotationlib.Format.STRING: + own = annotationlib.annotations_to_string(own_annotations) + elif format in (annotationlib.Format.FORWARDREF, annotationlib.Format.VALUE): own = own_checked_annotations else: raise NotImplementedError(format) @@ -3886,7 +3878,7 @@ def __getattr__(attr): are only created on-demand here. """ if attr == "ForwardRef": - obj = _lazy_annotationlib.ForwardRef + obj = annotationlib.ForwardRef elif attr in {"Pattern", "Match"}: import re obj = _alias(getattr(re, attr), 1) From 0b75b7338d3a85199396404106aa011559c9c250 Mon Sep 17 00:00:00 2001 From: Wulian233 <1055917385@qq.com> Date: Wed, 6 May 2026 20:59:08 +0800 Subject: [PATCH 03/15] gh-100239: specialize mixed int/float inplace binary ops (GH-149413) --- Lib/test/test_opcache.py | 24 ++++++++++++++++++++++++ Python/specialize.c | 8 ++++++++ 2 files changed, 32 insertions(+) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index bd97b0b4d61474..7946550ec0db63 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1419,6 +1419,30 @@ def binary_op_add_extend(): self.assertEqual(c, 2.0) c = b / a self.assertEqual(c, 0.5) + c = a + c += b + self.assertEqual(c, 9.0) + c = b + c += a + self.assertEqual(c, 9.0) + c = a + c -= b + self.assertEqual(c, 3.0) + c = b + c -= a + self.assertEqual(c, -3.0) + c = a + c *= b + self.assertEqual(c, 18.0) + c = b + c *= a + self.assertEqual(c, 18.0) + c = a + c /= b + self.assertEqual(c, 2.0) + c = b + c /= a + self.assertEqual(c, 0.5) binary_op_add_extend() self.assert_specialized(binary_op_add_extend, "BINARY_OP_EXTEND") diff --git a/Python/specialize.c b/Python/specialize.c index c54807931f2326..459e69de5709b8 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2278,12 +2278,20 @@ static _PyBinaryOpSpecializationDescr binaryop_extend_descrs[] = { {NB_SUBTRACT, float_compactlong_guard, float_compactlong_subtract, &PyFloat_Type, 1, NULL, NULL}, {NB_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div, &PyFloat_Type, 1, NULL, NULL}, {NB_MULTIPLY, float_compactlong_guard, float_compactlong_multiply, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_ADD, float_compactlong_guard, float_compactlong_add, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_SUBTRACT, float_compactlong_guard, float_compactlong_subtract, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_TRUE_DIVIDE, nonzero_float_compactlong_guard, float_compactlong_true_div, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_MULTIPLY, float_compactlong_guard, float_compactlong_multiply, &PyFloat_Type, 1, NULL, NULL}, /* long-float arithmetic: guards also check NaN and compactness. */ {NB_ADD, compactlong_float_guard, compactlong_float_add, &PyFloat_Type, 1, NULL, NULL}, {NB_SUBTRACT, compactlong_float_guard, compactlong_float_subtract, &PyFloat_Type, 1, NULL, NULL}, {NB_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div, &PyFloat_Type, 1, NULL, NULL}, {NB_MULTIPLY, compactlong_float_guard, compactlong_float_multiply, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_ADD, compactlong_float_guard, compactlong_float_add, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_SUBTRACT, compactlong_float_guard, compactlong_float_subtract, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_TRUE_DIVIDE, nonzero_compactlong_float_guard, compactlong_float_true_div, &PyFloat_Type, 1, NULL, NULL}, + {NB_INPLACE_MULTIPLY, compactlong_float_guard, compactlong_float_multiply, &PyFloat_Type, 1, NULL, NULL}, /* list-list concatenation: _PyList_Concat always allocates a new list */ {NB_ADD, NULL, _PyList_Concat, &PyList_Type, 1, &PyList_Type, &PyList_Type}, From ffa4d47abdbf38bb5d022d2e37949226cdff7198 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Wed, 6 May 2026 16:14:49 +0300 Subject: [PATCH 04/15] gh-146238: Add missing tests for 'e', 'Zf' and 'Zd' array type codes in test_buffer (#149345) This amends e79fd60. I'll not fix this for 'F'/'D' complex types as they might be removed. --- Lib/test/test_buffer.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index f08faa14b24c64..7454c8a15391e9 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -143,7 +143,7 @@ def native_type_range(fmt): # Format codes supported by array.array ARRAY = NATIVE.copy() for k in NATIVE: - if not k in "bBhHiIlLfd": + if k not in list("bBhHiIlLefd") + ['Zf', 'Zd']: del ARRAY[k] BYTEFMT = NATIVE.copy() @@ -4495,8 +4495,10 @@ def test_bytearray_alignment(self): def test_array_alignment(self): # gh-140557: pointer alignment of buffers including empty allocation # should match the maximum array alignment. - align = max(struct.calcsize(fmt) for fmt in ARRAY) - cases = [array.array(fmt) for fmt in ARRAY] + formats = [fmt for fmt in ARRAY + if struct.calcsize(fmt) <= struct.calcsize('P')] + align = max(struct.calcsize(fmt) for fmt in formats) + cases = [array.array(fmt) for fmt in formats] # Empty arrays self.assertEqual( [_testcapi.buffer_pointer_as_int(case) % align for case in cases], From e7613f2735d5c45d86351729e0483db25af50a21 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 6 May 2026 16:48:04 +0300 Subject: [PATCH 05/15] gh-148766: Add colour to Python help (#148767) --- Doc/whatsnew/3.15.rst | 5 + Lib/test/test_cmd_line.py | 24 + ...-04-19-22-35-39.gh-issue-148766.coLWln.rst | 2 + Python/initconfig.c | 482 +++++++++++------- 4 files changed, 332 insertions(+), 181 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-19-22-35-39.gh-issue-148766.coLWln.rst diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1a5e5fe2fc0be6..1043fe08d5b075 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -652,6 +652,11 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) +* The interpreter help (such as ``python --help``) is now in color. + This can be controlled by :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148766`.) + * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 8740f65b7b0d1d..7f9e44d70001b7 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -57,6 +57,7 @@ def verify_valid_flag(self, cmd_line): return out @support.cpython_only + @support.force_not_colorized def test_help(self): self.verify_valid_flag('-h') self.verify_valid_flag('-?') @@ -68,6 +69,7 @@ def test_help(self): self.assertLess(len(lines), 50) @support.cpython_only + @support.force_not_colorized def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) @@ -81,6 +83,7 @@ def test_help_env(self): "env vars should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) @@ -89,6 +92,7 @@ def test_help_xoptions(self): "options should be sorted alphabetically") @support.cpython_only + @support.force_not_colorized def test_help_all(self): out = self.verify_valid_flag('--help-all') lines = out.splitlines() @@ -100,6 +104,25 @@ def test_help_all(self): # but the rest should be ASCII-only b''.join(lines[1:]).decode('ascii') + @support.cpython_only + @support.force_colorized + def test_help_colorized(self): + rc, out, err = assert_python_ok("--help", FORCE_COLOR="1") + # Check ANSI color codes are present + self.assertIn(b"\x1b[", out) + # Check that key text elements are still present + self.assertIn(b"usage:", out) + self.assertIn(b"-h", out) + self.assertIn(b"--help-all", out) + self.assertIn(b"cmd", out) + self.assertIn(b"Arguments:", out) + + @support.cpython_only + @support.force_not_colorized + def test_help_not_colorized(self): + rc, out, err = assert_python_ok("--help") + self.assertNotIn(b"\x1b[", out) + def test_optimize(self): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') @@ -1063,6 +1086,7 @@ def test_argv0_normalization(self): self.assertEqual(proc.stdout.strip(), b'0') @support.cpython_only + @support.force_not_colorized def test_parsing_error(self): args = [sys.executable, '-I', '--unknown-option'] proc = subprocess.run(args, diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-19-22-35-39.gh-issue-148766.coLWln.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-19-22-35-39.gh-issue-148766.coLWln.rst new file mode 100644 index 00000000000000..946473d700f13a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-19-22-35-39.gh-issue-148766.coLWln.rst @@ -0,0 +1,2 @@ +The interpreter help (such as ``python --help``) is now in color. Patch by +Hugo van Kemenade. diff --git a/Python/initconfig.c b/Python/initconfig.c index 8dc9602ff13df7..a996fb117aab9d 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -250,224 +250,340 @@ static void initconfig_free_config(const PyConfig *config); /* --- Command line options --------------------------------------- */ -/* Short usage message (with %s for argv0) */ +/* + * Help text markup (matching Lib/_colorize.py Argparse theme). + * + * Color spans, #X{...} where "}" resets to default color: + * #b{...} label bold yellow + * #B{...} summary label yellow + * #E{...} env var (primary) bold cyan + * #e{...} env var reference cyan + * #h{...} heading bold blue + * #L{...} long option bold cyan + * #s{...} short option bold green + * #S{...} summary short opt green + * + * Runtime substitutions (no "{" follows): + * #P program name (bold magenta) + * #D path separator (DELIM) + * #H PYTHONHOMEHELP default search path + * + * fprint_help() walks the string, expanding color codes only when colorize=1 + * and substituting runtime values regardless. + */ + +#if defined(MS_WINDOWS) +# define PYTHONHOMEHELP "\\python{major}{minor}" +#else +# define PYTHONHOMEHELP "/lib/pythonX.X" +#endif + +/* Determine if we can emit ANSI color codes on the given stream. + * Logic mirrors Lib/_colorize.py:can_colorize(). */ +static int +_Py_can_colorize(FILE *f) +{ + const char *env; + + env = Py_GETENV("PYTHON_COLORS"); + if (env) { + if (strcmp(env, "0") == 0) { + return 0; + } + if (strcmp(env, "1") == 0) { + return 1; + } + } + if (getenv("NO_COLOR")) { + return 0; + } + if (getenv("FORCE_COLOR")) { + return 1; + } + env = getenv("TERM"); + if (env && strcmp(env, "dumb") == 0) { + return 0; + } +#if defined(MS_WINDOWS) && defined(HAVE_WINDOWS_CONSOLE_IO) + { + DWORD mode = 0; + DWORD nStdHandle = (f == stderr) ? STD_ERROR_HANDLE + : STD_OUTPUT_HANDLE; + HANDLE handle = GetStdHandle(nStdHandle); + if (!GetConsoleMode(handle, &mode) + || !(mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING)) + { + return 0; + } + } +#endif + return isatty(fileno(f)); +} + +/* Walk help text, expanding markup: + * #X{...} color span (only emitted when colorize=1; '}' resets). + * #X runtime substitution (program name, DELIM, PYTHONHOMEHELP). + * See the markup table above the macro/comment block. */ +static void +fprint_help(FILE *f, const char *text, int colorize, const wchar_t *program) +{ + for (const char *p = text; *p; ) { + if (*p == '#' && p[1]) { + char code = p[1]; + if (p[2] == '{') { + /* Color span open */ + const char *seq = NULL; + switch (code) { + case 'h': seq = "\x1b[1;34m"; break; // heading + case 'E': seq = "\x1b[1;36m"; break; // env var primary + case 'e': seq = "\x1b[36m"; break; // env var reference + case 'L': seq = "\x1b[1;36m"; break; // long option + case 'b': seq = "\x1b[1;33m"; break; // label + case 'B': seq = "\x1b[33m"; break; // summary label + case 's': seq = "\x1b[1;32m"; break; // short option + case 'S': seq = "\x1b[32m"; break; // summary short option + } + if (colorize && seq) fputs(seq, f); + p += 3; // skip "#X{" + continue; + } + /* Runtime substitution */ + switch (code) { + case 'P': // program name with bold magenta + if (colorize) fputs("\x1b[1;35m", f); + if (program) fprintf(f, "%ls", program); + if (colorize) fputs("\x1b[0m", f); + break; + case 'D': + fputc((char)DELIM, f); + break; + case 'H': + fputs(PYTHONHOMEHELP, f); + break; + default: // unknown: emit literally + fputc('#', f); + fputc(code, f); + break; + } + p += 2; // skip "#X" + continue; + } + if (*p == '}') { + if (colorize) fputs("\x1b[0m", f); + p++; + continue; + } + fputc(*p++, f); + } +} + +/* Short usage message */ static const char usage_line[] = -"usage: %ls [option] ... [-c cmd | -m mod | file | -] [arg] ...\n"; +"#h{usage:} #P [#S{option}] #S{...} " +"[#S{-c} #B{cmd} | #S{-m} #B{mod} | #S{file} | #S{-}] " +"[#S{arg}] #S{...}\n" +; /* Long help message */ /* Lines sorted by option name; keep in sync with usage_envvars* below */ -static const char usage_help[] = "\ -Options (and corresponding environment variables):\n\ --b : issue warnings about converting bytes/bytearray to str and comparing\n\ - bytes/bytearray with str or bytes with int. (-bb: issue errors)\n\ - deprecated since 3.15 and will become no-op in 3.17.\n\ --B : don't write .pyc files on import; also PYTHONDONTWRITEBYTECODE=x\n\ --c cmd : program passed in as string (terminates option list)\n\ --d : turn on parser debugging output (for experts only, only works on\n\ - debug builds); also PYTHONDEBUG=x\n\ --E : ignore PYTHON* environment variables (such as PYTHONPATH)\n\ --h : print this help message and exit (also -? or --help)\n\ --i : inspect interactively after running script; forces a prompt even\n\ - if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ --I : isolate Python from the user's environment (implies -E, -P and -s)\n\ --m mod : run library module as a script (terminates option list)\n\ --O : remove assert and __debug__-dependent statements; add .opt-1 before\n\ - .pyc extension; also PYTHONOPTIMIZE=x\n\ --OO : do -O changes and also discard docstrings; add .opt-2 before\n\ - .pyc extension\n\ --P : don't prepend a potentially unsafe path to sys.path; also\n\ - PYTHONSAFEPATH\n\ --q : don't print version and copyright messages on interactive startup\n\ --s : don't add user site directory to sys.path; also PYTHONNOUSERSITE=x\n\ --S : don't imply 'import site' on initialization\n\ --u : force the stdout and stderr streams to be unbuffered;\n\ - this option has no effect on stdin; also PYTHONUNBUFFERED=x\n\ --v : verbose (trace import statements); also PYTHONVERBOSE=x\n\ - can be supplied multiple times to increase verbosity\n\ --V : print the Python version number and exit (also --version)\n\ - when given twice, print more information about the build\n\ --W arg : warning control; arg is action:message:category:module:lineno\n\ - also PYTHONWARNINGS=arg\n\ --x : skip first line of source, allowing use of non-Unix forms of #!cmd\n\ --X opt : set implementation-specific option\n\ ---check-hash-based-pycs always|default|never:\n\ - control how Python invalidates hash-based .pyc files\n\ ---help-env: print help about Python environment variables and exit\n\ ---help-xoptions: print help about implementation-specific -X options and exit\n\ ---help-all: print complete help information and exit\n\ -\n\ -Arguments:\n\ -file : program read from script file\n\ -- : program read from stdin (default; interactive mode if a tty)\n\ -arg ...: arguments passed to program in sys.argv[1:]\n\ -"; - -static const char usage_xoptions[] = "\ -The following implementation-specific options are available:\n\ --X context_aware_warnings=[0|1]: if true (1) then the warnings module will\n\ - use a context variables; if false (0) then the warnings module will\n\ - use module globals, which is not concurrent-safe; set to true for\n\ - free-threaded builds and false otherwise; also\n\ - PYTHON_CONTEXT_AWARE_WARNINGS\n\ --X cpu_count=N: override the return value of os.cpu_count();\n\ - -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ --X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ --X disable-remote-debug: disable remote debugging; also PYTHON_DISABLE_REMOTE_DEBUG\n\ --X faulthandler: dump the Python traceback on fatal errors;\n\ - also PYTHONFAULTHANDLER\n\ --X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ - for installed Python and \"off\" for a local build;\n\ - also PYTHON_FROZEN_MODULES\n\ -" +static const char usage_help[] = +"#h{Options (and corresponding environment variables):}\n" +"#s{-b} : issue warnings about converting bytes/bytearray to str and comparing\n" +" bytes/bytearray with str or bytes with int. (#S{-bb}: issue errors)\n" +" deprecated since 3.15 and will become no-op in 3.17.\n" +"#s{-B} : don't write .pyc files on import; also #e{PYTHONDONTWRITEBYTECODE}#B{=x}\n" +"#s{-c} #b{cmd} : program passed in as string (terminates option list)\n" +"#s{-d} : turn on parser debugging output (for experts only, only works on\n" +" debug builds); also #e{PYTHONDEBUG}#B{=x}\n" +"#s{-E} : ignore #e{PYTHON*} environment variables (such as #e{PYTHONPATH})\n" +"#s{-h} : print this help message and exit (also #S{-?} or #e{--help})\n" +"#s{-i} : inspect interactively after running script; forces a prompt even\n" +" if stdin does not appear to be a terminal; also #e{PYTHONINSPECT}#B{=x}\n" +"#s{-I} : isolate Python from the user's environment (implies #S{-E}, #S{-P} and #S{-s})\n" +"#s{-m} #b{mod} : run library module as a script (terminates option list)\n" +"#s{-O} : remove assert and __debug__-dependent statements; add .opt-1 before\n" +" .pyc extension; also #e{PYTHONOPTIMIZE}#B{=x}\n" +"#s{-OO} : do #S{-O} changes and also discard docstrings; add .opt-2 before\n" +" .pyc extension\n" +"#s{-P} : don't prepend a potentially unsafe path to sys.path; also\n" +" #e{PYTHONSAFEPATH}\n" +"#s{-q} : don't print version and copyright messages on interactive startup\n" +"#s{-s} : don't add user site directory to sys.path; also #e{PYTHONNOUSERSITE}#B{=x}\n" +"#s{-S} : don't imply 'import site' on initialization\n" +"#s{-u} : force the stdout and stderr streams to be unbuffered;\n" +" this option has no effect on stdin; also #e{PYTHONUNBUFFERED}#B{=x}\n" +"#s{-v} : verbose (trace import statements); also #e{PYTHONVERBOSE}#B{=x}\n" +" can be supplied multiple times to increase verbosity\n" +"#s{-V} : print the Python version number and exit (also #e{--version})\n" +" when given twice, print more information about the build\n" +"#s{-W} #b{arg} : warning control; #B{arg} is action:message:category:module:lineno\n" +" also #e{PYTHONWARNINGS}#B{=arg}\n" +"#s{-x} : skip first line of source, allowing use of non-Unix forms of #!cmd\n" +"#s{-X} #b{opt} : set implementation-specific option\n" +"#L{--check-hash-based-pycs} #b{always|default|never}:\n" +" control how Python invalidates hash-based .pyc files\n" +"#L{--help-env}: print help about Python environment variables and exit\n" +"#L{--help-xoptions}: print help about implementation-specific #S{-X} options and exit\n" +"#L{--help-all}: print complete help information and exit\n" +"\n" +"#h{Arguments:}\n" +"#s{file} : program read from script file\n" +"#s{-} : program read from stdin (default; interactive mode if a tty)\n" +"#s{arg} #b{...}: arguments passed to program in sys.argv[1:]\n" +; + +static const char usage_xoptions[] = +"#h{The following implementation-specific options are available:}\n" +"#s{-X} #L{context_aware_warnings}#b{=[0|1]}: if true (#B{1}) then the warnings module will\n" +" use a context variables; if false (#B{0}) then the warnings module will\n" +" use module globals, which is not concurrent-safe; set to true for\n" +" free-threaded builds and false otherwise; also\n" +" #e{PYTHON_CONTEXT_AWARE_WARNINGS}\n" +"#s{-X} #L{cpu_count}#b{=N}: override the return value of os.cpu_count();\n" +" #S{-X} #e{cpu_count}#B{=default} cancels overriding; also #e{PYTHON_CPU_COUNT}\n" +"#s{-X} #L{dev} : enable Python Development Mode; also #e{PYTHONDEVMODE}\n" +"#s{-X} #L{disable-remote-debug}: disable remote debugging; also #e{PYTHON_DISABLE_REMOTE_DEBUG}\n" +"#s{-X} #L{faulthandler}: dump the Python traceback on fatal errors;\n" +" also #e{PYTHONFAULTHANDLER}\n" +"#s{-X} #L{frozen_modules}#b{=[on|off]}: whether to use frozen modules; the default is \"#B{on}\"\n" +" for installed Python and \"#B{off}\" for a local build;\n" +" also #e{PYTHON_FROZEN_MODULES}\n" #ifdef Py_GIL_DISABLED -"-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" +"#s{-X} #L{gil}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) the GIL; also #e{PYTHON_GIL}\n" #endif -"\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ - log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ --X int_max_str_digits=N: limit the size of int<->str conversions;\n\ - 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ --X lazy_imports=[all|none|normal]: control global lazy imports;\n\ - default is normal; also PYTHON_LAZY_IMPORTS\n\ --X no_debug_ranges: don't include extra location information in code objects;\n\ - also PYTHONNODEBUGRANGES\n\ --X pathconfig_warnings=[0|1]: if true (1) then path configuration is allowed\n\ - to log warnings into stderr; if false (0) suppress these warnings;\n\ - set to true by default; also PYTHON_PATHCONFIG_WARNINGS\n\ --X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ --X perf_jit: support the Linux \"perf\" profiler with DWARF support;\n\ - also PYTHON_PERF_JIT_SUPPORT=1\n\ -" +"#s{-X} #L{importtime}#b{[=2]}: show how long each import takes; use #S{-X} #e{importtime}#B{=2} to\n" +" log imports of already-loaded modules; also #e{PYTHONPROFILEIMPORTTIME}\n" +"#s{-X} #L{int_max_str_digits}#b{=N}: limit the size of int<->str conversions;\n" +" 0 disables the limit; also #e{PYTHONINTMAXSTRDIGITS}\n" +"#s{-X} #L{lazy_imports}#b{=[all|none|normal]}: control global lazy imports;\n" +" default is #B{normal}; also #e{PYTHON_LAZY_IMPORTS}\n" +"#s{-X} #L{no_debug_ranges}: don't include extra location information in code objects;\n" +" also #e{PYTHONNODEBUGRANGES}\n" +"#s{-X} #L{pathconfig_warnings}#b{=[0|1]}: if true (#B{1}) then path configuration is allowed\n" +" to log warnings into stderr; if false (#B{0}) suppress these warnings;\n" +" set to true by default; also #e{PYTHON_PATHCONFIG_WARNINGS}\n" +"#s{-X} #L{perf}: support the Linux \"perf\" profiler; also #e{PYTHONPERFSUPPORT}#B{=1}\n" +"#s{-X} #L{perf_jit}: support the Linux \"perf\" profiler with DWARF support;\n" +" also #e{PYTHON_PERF_JIT_SUPPORT}#B{=1}\n" #ifdef Py_DEBUG -"-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" +"#s{-X} #L{presite}#b{=MOD}: import this module before site; also #e{PYTHON_PRESITE}\n" #endif -"\ --X pycache_prefix=PATH: write .pyc files to a parallel tree instead of to the\n\ - code tree; also PYTHONPYCACHEPREFIX\n\ -" +"#s{-X} #L{pycache_prefix}#b{=PATH}: write .pyc files to a parallel tree instead of to the\n" +" code tree; also #e{PYTHONPYCACHEPREFIX}\n" #ifdef Py_STATS -"-X pystats: enable pystats collection at startup; also PYTHONSTATS\n" +"#s{-X} #L{pystats}: enable pystats collection at startup; also #e{PYTHONSTATS}\n" #endif -"\ --X showrefcount: output the total reference count and number of used\n\ - memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter; only works on debug builds\n\ --X thread_inherit_context=[0|1]: enable (1) or disable (0) threads inheriting\n\ - context vars by default; enabled by default in the free-threaded\n\ - build and disabled otherwise; also PYTHON_THREAD_INHERIT_CONTEXT\n\ -" +"#s{-X} #L{showrefcount}: output the total reference count and number of used\n" +" memory blocks when the program finishes or after each statement in\n" +" the interactive interpreter; only works on debug builds\n" +"#s{-X} #L{thread_inherit_context}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) threads inheriting\n" +" context vars by default; enabled by default in the free-threaded\n" +" build and disabled otherwise; also #e{PYTHON_THREAD_INHERIT_CONTEXT}\n" #ifdef Py_GIL_DISABLED -"-X tlbc=[0|1]: enable (1) or disable (0) thread-local bytecode. Also\n\ - PYTHON_TLBC\n" +"#s{-X} #L{tlbc}#b{=[0|1]}: enable (#B{1}) or disable (#B{0}) thread-local bytecode. Also\n" +" #e{PYTHON_TLBC}\n" #endif -"\ --X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ - of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ --X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ --X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ - also PYTHONWARNDEFAULTENCODING\ -"; +"#s{-X} #L{tracemalloc}#b{[=N]}: trace Python memory allocations; N sets a traceback limit\n" +" of #B{N} frames (default: #B{1}); also #e{PYTHONTRACEMALLOC}#B{=N}\n" +"#s{-X} #L{utf8}#b{[=0|1]}: enable (#B{1}) or disable (#B{0}) UTF-8 mode; also #e{PYTHONUTF8}\n" +"#s{-X} #L{warn_default_encoding}: enable opt-in EncodingWarning for 'encoding=None';\n" +" also #e{PYTHONWARNDEFAULTENCODING}\n" +; /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = -"Environment variables that change behavior:\n" -"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" -"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" -"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" +"#h{Environment variables that change behavior:}\n" +"#E{PYTHONASYNCIODEBUG}: enable asyncio debug mode\n" +"#E{PYTHON_BASIC_REPL}: use the traditional parser-based REPL\n" +"#E{PYTHONBREAKPOINT}: if this variable is set to #B{0}, it disables the default\n" " debugger. It can be set to the callable of your debugger of\n" " choice.\n" -"PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" -"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" -" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" +"#E{PYTHONCASEOK} : ignore case in 'import' statements (Windows)\n" +"#E{PYTHONCOERCECLOCALE}: if this variable is set to #B{0}, it disables the locale\n" +" coercion behavior. Use #e{PYTHONCOERCECLOCALE}#B{=warn} to request\n" " display of locale coercion and locale compatibility warnings\n" " on stderr.\n" -"PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" -" various kinds of output. Setting it to 0 deactivates\n" +"#E{PYTHON_COLORS} : if this variable is set to #B{1}, the interpreter will colorize\n" +" various kinds of output. Setting it to #B{0} deactivates\n" " this behavior.\n" #ifdef Py_TRACE_REFS -"PYTHONDUMPREFS : dump objects and reference counts still alive after shutdown\n" -"PYTHONDUMPREFSFILE: dump objects and reference counts to the specified file\n" +"#E{PYTHONDUMPREFS} : dump objects and reference counts still alive after shutdown\n" +"#E{PYTHONDUMPREFSFILE}: dump objects and reference counts to the specified file\n" #endif #ifdef __APPLE__ -"PYTHONEXECUTABLE: set sys.argv[0] to this value (macOS only)\n" +"#E{PYTHONEXECUTABLE}: set sys.argv[0] to this value (macOS only)\n" #endif -"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +"#E{PYTHONHASHSEED} : if this variable is set to 'random', a random value is used\n" " to seed the hashes of str and bytes objects. It can also be\n" " set to an integer in the range [0,4294967295] to get hash\n" " values with a predictable seed.\n" -"PYTHON_HISTORY : the location of a .python_history file.\n" -"PYTHONHOME : alternate directory (or %lc).\n" -" The default module search path uses %s.\n" -"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" +"#E{PYTHON_HISTORY} : the location of a .python_history file.\n" +"#E{PYTHONHOME} : alternate directory (or #D).\n" +" The default module search path uses #H.\n" +"#E{PYTHONIOENCODING}: encoding[:errors] used for stdin/stdout/stderr\n" #ifdef MS_WINDOWS -"PYTHONLEGACYWINDOWSFSENCODING: use legacy \"mbcs\" encoding for file system\n" -"PYTHONLEGACYWINDOWSSTDIO: use legacy Windows stdio\n" +"#E{PYTHONLEGACYWINDOWSFSENCODING}: use legacy \"mbcs\" encoding for file system\n" +"#E{PYTHONLEGACYWINDOWSSTDIO}: use legacy Windows stdio\n" #endif -"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +"#E{PYTHONMALLOC} : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use #e{PYTHONMALLOC}#B{=debug} to\n" " install debug hooks.\n" -"PYTHONMALLOCSTATS: print memory allocator statistics\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +"#E{PYTHONMALLOCSTATS}: print memory allocator statistics\n" +"#E{PYTHONPATH} : '#D'-separated list of directories prefixed to the\n" " default module search path. The result is sys.path.\n" -"PYTHONPLATLIBDIR: override sys.platlibdir\n" -"PYTHONSTARTUP : file executed on interactive startup (no default)\n" -"PYTHONUSERBASE : defines the user base directory (site.USER_BASE)\n" +"#E{PYTHONPLATLIBDIR}: override sys.platlibdir\n" +"#E{PYTHONSTARTUP} : file executed on interactive startup (no default)\n" +"#E{PYTHONUSERBASE} : defines the user base directory (site.USER_BASE)\n" "\n" -"These variables have equivalent command-line options (see --help for details):\n" -"PYTHON_CONTEXT_AWARE_WARNINGS: if true (1), enable thread-safe warnings\n" -" module behaviour (-X context_aware_warnings)\n" -"PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" -"PYTHONDEBUG : enable parser debug mode (-d)\n" -"PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" -"PYTHONDONTWRITEBYTECODE: don't write .pyc files (-B)\n" -"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors (-X faulthandler)\n" -"PYTHON_FROZEN_MODULES: whether to use frozen modules; the default is \"on\"\n" -" for installed Python and \"off\" for a local build\n" -" (-X frozen_modules)\n" +"#h{These variables have equivalent command-line options (see }#e{--help} for details):\n" +"#E{PYTHON_CONTEXT_AWARE_WARNINGS}: if true (#B{1}), enable thread-safe warnings\n" +" module behaviour (#S{-X} #e{context_aware_warnings})\n" +"#E{PYTHON_CPU_COUNT}: override the return value of os.cpu_count() (#S{-X} #e{cpu_count})\n" +"#E{PYTHONDEBUG} : enable parser debug mode (#S{-d})\n" +"#E{PYTHONDEVMODE} : enable Python Development Mode (#S{-X} #e{dev})\n" +"#E{PYTHONDONTWRITEBYTECODE}: don't write .pyc files (#S{-B})\n" +"#E{PYTHONFAULTHANDLER}: dump the Python traceback on fatal errors (#S{-X} #e{faulthandler})\n" +"#E{PYTHON_FROZEN_MODULES}: whether to use frozen modules; the default is \"#B{on}\"\n" +" for installed Python and \"#B{off}\" for a local build\n" +" (#S{-X} #e{frozen_modules})\n" #ifdef Py_GIL_DISABLED -"PYTHON_GIL : when set to 0, disables the GIL (-X gil)\n" +"#E{PYTHON_GIL} : when set to #B{0}, disables the GIL (#S{-X} #e{gil})\n" #endif -"PYTHONINSPECT : inspect interactively after running script (-i)\n" -"PYTHONINTMAXSTRDIGITS: limit the size of int<->str conversions;\n" -" 0 disables the limit (-X int_max_str_digits=N)\n" -"PYTHON_LAZY_IMPORTS: control global lazy imports (-X lazy_imports)\n" -"PYTHONNODEBUGRANGES: don't include extra location information in code objects\n" -" (-X no_debug_ranges)\n" -"PYTHONNOUSERSITE: disable user site directory (-s)\n" -"PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHON_PERF_JIT_SUPPORT: enable Linux \"perf\" profiler support with JIT\n" -" (-X perf_jit)\n" -"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" +"#E{PYTHONINSPECT} : inspect interactively after running script (#S{-i})\n" +"#E{PYTHONINTMAXSTRDIGITS}: limit the size of int<->str conversions;\n" +" 0 disables the limit (#S{-X} #e{int_max_str_digits}#B{=N})\n" +"#E{PYTHON_LAZY_IMPORTS}: control global lazy imports (#S{-X} #e{lazy_imports})\n" +"#E{PYTHONNODEBUGRANGES}: don't include extra location information in code objects\n" +" (#S{-X} #e{no_debug_ranges})\n" +"#E{PYTHONNOUSERSITE}: disable user site directory (#S{-s})\n" +"#E{PYTHONOPTIMIZE} : enable level 1 optimizations (#S{-O})\n" +"#E{PYTHON_PERF_JIT_SUPPORT}: enable Linux \"perf\" profiler support with JIT\n" +" (#S{-X} #e{perf_jit})\n" +"#E{PYTHONPERFSUPPORT}: support the Linux \"perf\" profiler (#S{-X} #e{perf})\n" #ifdef Py_DEBUG -"PYTHON_PRESITE: import this module before site (-X presite)\n" +"#E{PYTHON_PRESITE}: import this module before site (#S{-X} #e{presite})\n" #endif -"PYTHONPROFILEIMPORTTIME: show how long each import takes (-X importtime)\n" -"PYTHONPYCACHEPREFIX: root directory for bytecode cache (pyc) files\n" -" (-X pycache_prefix)\n" -"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path.\n" +"#E{PYTHONPROFILEIMPORTTIME}: show how long each import takes (#S{-X} #e{importtime})\n" +"#E{PYTHONPYCACHEPREFIX}: root directory for bytecode cache (pyc) files\n" +" (#S{-X} #e{pycache_prefix})\n" +"#E{PYTHONSAFEPATH} : don't prepend a potentially unsafe path to sys.path.\n" #ifdef Py_STATS -"PYTHONSTATS : turns on statistics gathering (-X pystats)\n" +"#E{PYTHONSTATS} : turns on statistics gathering (#S{-X} #e{pystats})\n" #endif -"PYTHON_THREAD_INHERIT_CONTEXT: if true (1), threads inherit context vars\n" -" (-X thread_inherit_context)\n" +"#E{PYTHON_THREAD_INHERIT_CONTEXT}: if true (#B{1}), threads inherit context vars\n" +" (#S{-X} #e{thread_inherit_context})\n" #ifdef Py_GIL_DISABLED -"PYTHON_TLBC : when set to 0, disables thread-local bytecode (-X tlbc)\n" +"#E{PYTHON_TLBC} : when set to #B{0}, disables thread-local bytecode (#S{-X} #e{tlbc})\n" #endif -"PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" -"PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" -"PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" -"PYTHONVERBOSE : trace import statements (-v)\n" -"PYTHONWARNDEFAULTENCODING: enable opt-in EncodingWarning for 'encoding=None'\n" -" (-X warn_default_encoding)\n" -"PYTHONWARNINGS : warning control (-W)\n" +"#E{PYTHONTRACEMALLOC}: trace Python memory allocations (#S{-X} #e{tracemalloc})\n" +"#E{PYTHONUNBUFFERED}: disable stdout/stderr buffering (#S{-u})\n" +"#E{PYTHONUTF8} : control the UTF-8 mode (#S{-X} #e{utf8})\n" +"#E{PYTHONVERBOSE} : trace import statements (#S{-v})\n" +"#E{PYTHONWARNDEFAULTENCODING}: enable opt-in EncodingWarning for 'encoding=None'\n" +" (#S{-X} #e{warn_default_encoding})\n" +"#E{PYTHONWARNINGS} : warning control (#S{-W})\n" ; -#if defined(MS_WINDOWS) -# define PYTHONHOMEHELP "\\python{major}{minor}" -#else -# define PYTHONHOMEHELP "/lib/pythonX.X" -#endif - /* --- Global configuration variables ----------------------------- */ @@ -2935,25 +3051,29 @@ static void config_usage(int error, const wchar_t* program) { FILE *f = error ? stderr : stdout; + int colorize = _Py_can_colorize(f); - fprintf(f, usage_line, program); - if (error) + fprint_help(f, usage_line, colorize, program); + if (error) { fprintf(f, "Try `python -h' for more information.\n"); + } else { - fputs(usage_help, f); + fprint_help(f, usage_help, colorize, NULL); } } static void config_envvars_usage(void) { - printf(usage_envvars, (wint_t)DELIM, PYTHONHOMEHELP, (wint_t)DELIM); + int colorize = _Py_can_colorize(stdout); + fprint_help(stdout, usage_envvars, colorize, NULL); } static void config_xoptions_usage(void) { - puts(usage_xoptions); + int colorize = _Py_can_colorize(stdout); + fprint_help(stdout, usage_xoptions, colorize, NULL); } static void From 1bdfc0f253730077ccd3a4b0714388e8227b1b71 Mon Sep 17 00:00:00 2001 From: Daniele Parmeggiani <8658291+dpdani@users.noreply.github.com> Date: Wed, 6 May 2026 14:50:24 +0100 Subject: [PATCH 06/15] gh-146270: Fix `PyMember_SetOne(..., NULL)` not being atomic (gh-148800) Fixes a sequential consistency bug whereby two threads that are deleting a struct member may observe both their deletions to be successful. --- Lib/test/test_free_threading/test_slots.py | 35 ++++++++++++++----- ...-04-20-15-25-55.gh-issue-146270.qZYfyc.rst | 1 + Python/structmember.c | 26 +++++++------- 3 files changed, 41 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-04-20-15-25-55.gh-issue-146270.qZYfyc.rst diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index a3b9f4b0175ae7..a73525e1bebfb4 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -16,18 +16,19 @@ def run_in_threads(targets): thread.join() +class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + @threading_helper.requires_working_threading() class TestSlots(TestCase): def test_object(self): - class Spam: - __slots__ = [ - "eggs", - ] - - def __init__(self, initial_value): - self.eggs = initial_value - spam = Spam(0) iters = 20_000 @@ -43,6 +44,24 @@ def reader(): run_in_threads([writer, reader, reader, reader]) + def test_del_object_is_atomic(self): + # Testing whether the implementation of `del slots_object.attribute` + # removes the attribute atomically, thus avoiding non-sequentially- + # consistent behaviors. + # https://github.com/python/cpython/issues/146270 + def deleter(spam, successes): + try: + del spam.eggs + successes.append(True) + except AttributeError: + successes.append(False) + + for _ in range(10): + spam = Spam(0) + successes = [] + threading_helper.run_concurrently(deleter, nthreads=4, args=(spam, successes)) + self.assertEqual(sum(successes), 1) + def test_T_BOOL(self): spam_old = _testcapi._test_structmembersType_OldAPI() spam_new = _testcapi._test_structmembersType_NewAPI() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-20-15-25-55.gh-issue-146270.qZYfyc.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-20-15-25-55.gh-issue-146270.qZYfyc.rst new file mode 100644 index 00000000000000..46c292e183e0fd --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-20-15-25-55.gh-issue-146270.qZYfyc.rst @@ -0,0 +1 @@ +Fix a sequential consistency bug in ``structmember.c``. diff --git a/Python/structmember.c b/Python/structmember.c index b88e13ac0462b8..adea8216b8796b 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -171,19 +171,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_AttributeError, "readonly attribute"); return -1; } - if (v == NULL) { - if (l->type == Py_T_OBJECT_EX) { - /* Check if the attribute is set. */ - if (*(PyObject **)addr == NULL) { - PyErr_SetString(PyExc_AttributeError, l->name); - return -1; - } - } - else if (l->type != _Py_T_OBJECT) { - PyErr_SetString(PyExc_TypeError, - "can't delete numeric/char attribute"); - return -1; - } + if (v == NULL && l->type != Py_T_OBJECT_EX && l->type != _Py_T_OBJECT) { + PyErr_SetString(PyExc_TypeError, + "can't delete numeric/char attribute"); + return -1; } switch (l->type) { case Py_T_BOOL:{ @@ -334,6 +325,15 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) oldv = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, Py_XNewRef(v)); Py_END_CRITICAL_SECTION(); + if (v == NULL && oldv == NULL && l->type == Py_T_OBJECT_EX) { + // Raise an exception when attempting to delete an already deleted + // attribute. + // Differently from Py_T_OBJECT_EX, _Py_T_OBJECT does not raise an + // exception here (PyMember_GetOne will return Py_None instead of + // NULL). + PyErr_SetString(PyExc_AttributeError, l->name); + return -1; + } Py_XDECREF(oldv); break; case Py_T_CHAR: { From aeb02ac42b113bff8218df890b7102d8abc66a9d Mon Sep 17 00:00:00 2001 From: Jeff Lyon <146767590+secengjeff@users.noreply.github.com> Date: Wed, 6 May 2026 06:56:17 -0700 Subject: [PATCH 07/15] gh-137586: Replace 'osascript' with 'open' on macOS in webbrowser (#146439) Co-authored-by: Claude Sonnet 4.6 Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Gregory P. Smith --- Doc/deprecations/pending-removal-in-3.17.rst | 5 ++ Doc/library/webbrowser.rst | 21 ++++- Doc/whatsnew/3.15.rst | 20 +++++ Lib/test/test_webbrowser.py | 89 +++++++++++++++++-- Lib/webbrowser.py | 88 ++++++++++++++++-- ...-03-26-01-42-20.gh-issue-137586.KmHRwR.rst | 3 + ...-03-26-01-42-15.gh-issue-137586.j3SkOm.rst | 4 + 7 files changed, 215 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-03-26-01-42-20.gh-issue-137586.KmHRwR.rst create mode 100644 Misc/NEWS.d/next/Security/2026-03-26-01-42-15.gh-issue-137586.j3SkOm.rst diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 952ffad64356d9..8ee7f335cc9514 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -37,6 +37,11 @@ Pending removal in Python 3.17 is deprecated and scheduled for removal in Python 3.17. (Contributed by Stan Ulbrych in :gh:`136702`.) +* :mod:`webbrowser`: + + - :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS`. (:gh:`137586`) + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst index 389648d4f393e4..30e4df1688d7a0 100644 --- a/Doc/library/webbrowser.rst +++ b/Doc/library/webbrowser.rst @@ -172,13 +172,15 @@ for the controller classes, all defined in this module. +------------------------+-----------------------------------------+-------+ | ``'windows-default'`` | ``WindowsDefault`` | \(2) | +------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | ``MacOSXOSAScript('default')`` | \(3) | +| ``'macos'`` | ``MacOS('default')`` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'safari'`` | ``MacOSXOSAScript('safari')`` | \(3) | +| ``'safari'`` | ``MacOS('safari')`` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'google-chrome'`` | ``Chrome('google-chrome')`` | | +| ``'chrome'`` | ``MacOS('google chrome')`` | \(3) | ++------------------------+-----------------------------------------+-------+ +| ``'firefox'`` | ``MacOS('firefox')`` | \(3) | +------------------------+-----------------------------------------+-------+ -| ``'chrome'`` | ``Chrome('chrome')`` | | +| ``'google-chrome'`` | ``Chrome('google-chrome')`` | | +------------------------+-----------------------------------------+-------+ | ``'chromium'`` | ``Chromium('chromium')`` | | +------------------------+-----------------------------------------+-------+ @@ -221,6 +223,17 @@ Notes: .. versionchanged:: 3.13 Support for iOS has been added. +.. versionadded:: next + :class:`!MacOS` has been added as a replacement for :class:`!MacOSXOSAScript`, + opening browsers via :program:`/usr/bin/open` instead of :program:`osascript`. + +.. deprecated-removed:: next 3.17 + :class:`!MacOSXOSAScript` is deprecated in favour of :class:`!MacOS`. + Using :program:`/usr/bin/open` instead of :program:`osascript` is a + security and usability improvement: :program:`osascript` may be blocked + on managed systems due to its abuse potential as a general-purpose + scripting interpreter. + Here are some simple examples:: url = 'https://docs.python.org/' diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 1043fe08d5b075..6007d772f8e2d7 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1693,6 +1693,20 @@ wave (Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) +webbrowser +---------- + +* On macOS, the new :class:`!webbrowser.MacOS` class opens URLs via + :program:`/usr/bin/open` instead of constructing and executing AppleScript + via :program:`osascript`. The default browser is detected from the + LaunchServices preferences file using :mod:`plistlib`, with + :class:`!com.apple.Safari` as the fallback on fresh installations. + For non-HTTP(S) URLs, :program:`open -b ` is used to route the + URL through a browser rather than the OS file handler, preventing + file injection attacks. + (Contributed by Jeff Lyon in :gh:`137586`.) + + xml --- @@ -2132,6 +2146,12 @@ New deprecations merely imported or accessed from the :mod:`!typing` module. +* :mod:`webbrowser`: + + * :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS` and scheduled for removal in Python 3.17. + (Contributed by Jeff Lyon in :gh:`137586`.) + * ``__version__`` * The ``__version__``, ``version`` and ``VERSION`` attributes have been diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 2ba3af8d5bf22f..51d627d24c5a8a 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -5,6 +5,7 @@ import subprocess import sys import unittest +import warnings import webbrowser from test import support from test.support import force_not_colorized_test_class @@ -335,6 +336,83 @@ def close(self): return None +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSTest(unittest.TestCase): + + def test_default(self): + browser = webbrowser.get() + self.assertIsInstance(browser, webbrowser.MacOS) + self.assertEqual(browser.name, 'default') + + def test_default_http_open(self): + # http/https URLs use /usr/bin/open directly — no bundle ID needed. + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_default_non_http_uses_bundle_id(self): + # Non-http(s) URLs (e.g. file://) must be routed through the browser + # via -b to prevent OS file handler dispatch. + file_url = 'file:///tmp/test.html' + browser = webbrowser.MacOS('default') + with mock.patch('webbrowser._macos_default_browser_bundle_id', + return_value='com.google.Chrome'), \ + mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(file_url) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.google.Chrome', file_url], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_known_browser_uses_bundle_id(self): + # Named browsers with a known bundle ID use /usr/bin/open -b. + browser = webbrowser.MacOS('safari') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.apple.Safari', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_unknown_browser_falls_back_to_dash_a(self): + # Named browsers not in the bundle ID map fall back to -a. + browser = webbrowser.MacOS('lynx') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-a', 'lynx', URL], + stderr=subprocess.DEVNULL, + ) + + def test_open_failure(self): + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=1) + result = browser.open(URL) + self.assertFalse(result) + + +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSXOSAScriptDeprecationTest(unittest.TestCase): + + def test_deprecation_warning(self): + with self.assertWarns(DeprecationWarning): + webbrowser.MacOSXOSAScript('default') + + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") @requires_subprocess() class MacOSXOSAScriptTest(unittest.TestCase): @@ -345,17 +423,14 @@ def setUp(self): env.unset("BROWSER") support.patch(self, os, "popen", self.mock_popen) + self.enterContext(warnings.catch_warnings()) + warnings.simplefilter("ignore", DeprecationWarning) self.browser = webbrowser.MacOSXOSAScript("default") def mock_popen(self, cmd, mode): self.popen_pipe = MockPopenPipe(cmd, mode) return self.popen_pipe - def test_default(self): - browser = webbrowser.get() - assert isinstance(browser, webbrowser.MacOSXOSAScript) - self.assertEqual(browser.name, "default") - def test_default_open(self): url = "https://python.org" self.browser.open(url) @@ -381,7 +456,9 @@ def test_default_browser_lookup(self): self.assertIn(f'open location "{url}"', script) def test_explicit_browser(self): - browser = webbrowser.MacOSXOSAScript("safari") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + browser = webbrowser.MacOSXOSAScript("safari") browser.open("https://python.org") script = self.popen_pipe.pipe.getvalue() self.assertIn('tell application "safari"', script) diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index c2ee0df0ef8885..ec8b544a6b0523 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,7 +1,8 @@ """Interfaces for launching and remotely controlling web browsers.""" -# Maintained by Georg Brandl. +import builtins # because we override open import os +lazy import plistlib import shlex import shutil import sys @@ -492,10 +493,15 @@ def register_standard_browsers(): _tryorder = [] if sys.platform == 'darwin': - register("MacOSX", None, MacOSXOSAScript('default')) - register("chrome", None, MacOSXOSAScript('google chrome')) - register("firefox", None, MacOSXOSAScript('firefox')) - register("safari", None, MacOSXOSAScript('safari')) + register("MacOS", None, MacOS('default')) + register("MacOSX", None, MacOS('default')) # backward compat alias + register("chrome", None, MacOS('google chrome')) + register("chromium", None, MacOS('chromium')) + register("firefox", None, MacOS('firefox')) + register("safari", None, MacOS('safari')) + register("opera", None, MacOS('opera')) + register("microsoft-edge", None, MacOS('microsoft edge')) + register("brave", None, MacOS('brave browser')) # macOS can use below Unix support (but we prefer using the macOS # specific stuff) @@ -614,8 +620,80 @@ def open(self, url, new=0, autoraise=True): # if sys.platform == 'darwin': + def _macos_default_browser_bundle_id(): + """Return the bundle ID of the default web browser. + + Reads the LaunchServices preferences file that macOS maintains + when the user sets a default browser. Returns 'com.apple.Safari' + if the file is absent or no https handler is recorded, because on + a fresh macOS installation Safari is the default browser and the + LaunchServices plist is not written until the user explicitly + changes their default browser. + """ + plist = os.path.expanduser( + '~/Library/Preferences/com.apple.LaunchServices/' + 'com.apple.launchservices.secure.plist' + ) + try: + with builtins.open(plist, 'rb') as f: + data = plistlib.load(f) + for handler in data.get('LSHandlers', []): + if handler.get('LSHandlerURLScheme') == 'https': + return (handler.get('LSHandlerRoleAll') + or handler.get('LSHandlerRoleViewer')) + except (OSError, KeyError, ValueError): + pass + return 'com.apple.Safari' + + class MacOS(BaseBrowser): + """Launcher class for macOS browsers, using /usr/bin/open. + + For http/https URLs with the default browser, /usr/bin/open is called + directly; macOS routes these to the registered browser. + + For all other URL schemes (e.g. file://) and for named browsers, + /usr/bin/open -b is used so that the URL is always passed + to a browser application rather than dispatched by the OS file handler. + This prevents file injection attacks where a file:// URL pointing to an + executable bundle could otherwise be launched by the OS. + + Named browsers with known bundle IDs use -b; unknown names fall back + to -a. + """ + + _BUNDLE_IDS = { + 'google chrome': 'com.google.Chrome', + 'firefox': 'org.mozilla.firefox', + 'safari': 'com.apple.Safari', + 'chromium': 'org.chromium.Chromium', + 'opera': 'com.operasoftware.Opera', + 'microsoft edge': 'com.microsoft.edgemac', + 'brave browser': 'com.brave.Browser', + } + + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + self._check_url(url) + if self.name == 'default': + proto, sep, _ = url.partition(':') + if sep and proto.lower() in {'http', 'https'}: + cmd = ['/usr/bin/open', url] + else: + bundle_id = _macos_default_browser_bundle_id() + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + bundle_id = self._BUNDLE_IDS.get(self.name.lower()) + if bundle_id: + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + cmd = ['/usr/bin/open', '-a', self.name, url] + proc = subprocess.run(cmd, stderr=subprocess.DEVNULL) + return proc.returncode == 0 + class MacOSXOSAScript(BaseBrowser): def __init__(self, name='default'): + import warnings + warnings._deprecated("webbrowser.MacOSXOSAScript", remove=(3, 17)) super().__init__(name) def open(self, url, new=0, autoraise=True): diff --git a/Misc/NEWS.d/next/Library/2026-03-26-01-42-20.gh-issue-137586.KmHRwR.rst b/Misc/NEWS.d/next/Library/2026-03-26-01-42-20.gh-issue-137586.KmHRwR.rst new file mode 100644 index 00000000000000..70122c8ceae507 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-01-42-20.gh-issue-137586.KmHRwR.rst @@ -0,0 +1,3 @@ +Add :class:`!MacOS` to :mod:`webbrowser` for macOS, which opens URLs via +``/usr/bin/open`` instead of piping AppleScript to ``osascript``. +Deprecate :class:`!MacOSXOSAScript` in favour of :class:`!MacOS`. diff --git a/Misc/NEWS.d/next/Security/2026-03-26-01-42-15.gh-issue-137586.j3SkOm.rst b/Misc/NEWS.d/next/Security/2026-03-26-01-42-15.gh-issue-137586.j3SkOm.rst new file mode 100644 index 00000000000000..ce9387adc069a8 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-03-26-01-42-15.gh-issue-137586.j3SkOm.rst @@ -0,0 +1,4 @@ +Fix a PATH-injection vulnerability in :mod:`webbrowser` on macOS where +``osascript`` was invoked without an absolute path. The new :class:`!MacOS` +class uses ``/usr/bin/open`` directly, eliminating the dependency on +``osascript`` entirely. From 9274d969f3b8d66a01ae704cac866ceb9e207d0e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 6 May 2026 16:28:11 +0200 Subject: [PATCH 08/15] gh-148675: Optimize arraydescr structure: use char[3] (GH-149455) Replace "const char *typecode;" with "char typecode[3];" to make the arraydescr structure smaller and avoids an indirection. --- Modules/arraymodule.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index a8347123e6496a..472c59ea8c9882 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -33,7 +33,7 @@ static struct PyModuleDef arraymodule; * functions aren't visible yet. */ struct arraydescr { - const char *typecode; + char typecode[3]; // big enough to store "Zd\0" int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); @@ -784,7 +784,7 @@ static const struct arraydescr descriptors[] = { {"d", sizeof(double), d_getitem, d_setitem, NULL, 0, 0}, {"Zf", 2*sizeof(float), cf_getitem, cf_setitem, NULL, 0, 0}, {"Zd", 2*sizeof(double), cd_getitem, cd_setitem, NULL, 0, 0}, - {NULL, 0, 0, 0, 0, 0, 0} /* Sentinel */ + {"", 0, 0, 0, 0, 0, 0} /* Sentinel */ }; /**************************************************************************** @@ -2298,12 +2298,12 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, arraytype->tp_name, state->ArrayType->tp_name); return NULL; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (strcmp(descr->typecode, typecode) == 0) { break; } } - if (descr->typecode == NULL) { + if (descr->typecode[0] == 0) { PyErr_SetString(PyExc_ValueError, "second argument must be a valid type code"); return NULL; @@ -2500,7 +2500,7 @@ array__array_reconstructor_impl(PyObject *module, PyTypeObject *arraytype, * * XXX: Is it possible to write a unit test for this? */ - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (descr->is_integer_type && (size_t)descr->itemsize == mf_descr.size && descr->is_signed == mf_descr.is_signed) @@ -3047,7 +3047,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) */ initial = NULL; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { if (strcmp(descr->typecode, s) == 0) { PyObject *a; Py_ssize_t len; @@ -3531,7 +3531,7 @@ array_modexec(PyObject *m) if (typecodes == NULL) { return -1; } - for (descr = descriptors; descr->typecode != NULL; descr++) { + for (descr = descriptors; descr->typecode[0] != 0; descr++) { PyObject *typecode = PyUnicode_DecodeASCII(descr->typecode, strlen(descr->typecode), NULL); if (typecode == NULL) { Py_DECREF(typecodes); From a5c7a7441870e045eb7589d1a1ff93d9423dea03 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 6 May 2026 17:40:10 +0300 Subject: [PATCH 09/15] gh-139489: Add xml.is_valid_text() (GH-149412) --- Doc/library/xml.rst | 14 +++++++++++++- Doc/whatsnew/3.15.rst | 4 ++++ Lib/test/test_xml.py | 16 ++++++++++++++++ Lib/xml/utils.py | 12 ++++++++++++ ...026-05-05-13-12-58.gh-issue-139489.a8qqIM.rst | 2 ++ 5 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-05-13-12-58.gh-issue-139489.a8qqIM.rst diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index f9ffaa9a94aacc..98be50e15ff463 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -54,7 +54,19 @@ This module also defines utility functions. "!", "?", and "=" are forbidden. The name cannot start with a digit or a character like "-", ".", and "·". - ..versionadded:: next + .. versionadded:: next + + +.. function:: is_valid_text(data) + + Return ``True`` if the string is a sequence of legal XML 1.0 characters, + ``False`` otherwise. + + Almost all characters are permitted in XML 1.0 documents, except C0 control + characters (excluding TAB, CR and LF), surrogate characters and special + Unicode characters U+FFFE and U+FFFF. + + .. versionadded:: next .. _xml-security: diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 6007d772f8e2d7..698a9f88e1ee39 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1714,6 +1714,10 @@ xml whether a string can be used as an element or attribute name in XML. (Contributed by Serhiy Storchaka in :gh:`139489`.) +* Add the :func:`xml.is_valid_text` function, which allows to check + whether a string can be used in the XML document. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + xml.parsers.expat ----------------- diff --git a/Lib/test/test_xml.py b/Lib/test/test_xml.py index fd3633e43982d7..3a8b92048166f2 100644 --- a/Lib/test/test_xml.py +++ b/Lib/test/test_xml.py @@ -22,6 +22,22 @@ def test_is_valid_name(self): for c in '<>/!?=\x00\x01\x7f\ud800\udfff\ufffe\uffff\U000F0000': self.assertFalse(is_valid_name('name' + c)) + def test_is_valid_text(self): + is_valid_text = xml.is_valid_text + self.assertTrue(is_valid_text('')) + self.assertTrue(is_valid_text('!0Aa_~ \r\n\t\x85\xa0')) + self.assertTrue(is_valid_text('\ud7ff\ue000\ufffd\U00010000\U0010ffff')) + self.assertFalse(is_valid_text('\x00')) + self.assertFalse(is_valid_text('\x01')) + self.assertFalse(is_valid_text('\x1f')) + self.assertTrue(is_valid_text('\x7f')) + self.assertTrue(is_valid_text('\x80')) + self.assertTrue(is_valid_text('\x9f')) + self.assertFalse(is_valid_text('\ud800')) + self.assertFalse(is_valid_text('\udfff')) + self.assertFalse(is_valid_text('\ufffe')) + self.assertFalse(is_valid_text('\uffff')) + if __name__ == '__main__': unittest.main() diff --git a/Lib/xml/utils.py b/Lib/xml/utils.py index c9a0b260675bed..532aa224dae677 100644 --- a/Lib/xml/utils.py +++ b/Lib/xml/utils.py @@ -23,3 +23,15 @@ def is_valid_name(name): '\uF900-\uFDCF\uFDF0-\uFFFD\U00010000-\U000EFFFF' ']*+', name) is not None + +# https://www.w3.org/TR/xml/#charsets +_ILLEGAL_XML_CHAR = ( + '[' + '\x00-\x08\x0B\x0C\x0E-\x1F' # C0 controls except TAB, CR and LF + '\uD800-\uDFFF' # the surrogate blocks + '\uFFFE\uFFFF' # special Unicode characters + ']') + +def is_valid_text(data): + """Test whether a string is a sequence of legal XML 1.0 characters.""" + return _re.search(_ILLEGAL_XML_CHAR, data) is None diff --git a/Misc/NEWS.d/next/Library/2026-05-05-13-12-58.gh-issue-139489.a8qqIM.rst b/Misc/NEWS.d/next/Library/2026-05-05-13-12-58.gh-issue-139489.a8qqIM.rst new file mode 100644 index 00000000000000..c76879d3025bb6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-05-13-12-58.gh-issue-139489.a8qqIM.rst @@ -0,0 +1,2 @@ +Add the :func:`xml.is_valid_text` function, which allows to check whether +a string can be used in the XML document. From 7b6c248d61808b787f09ed3d05e4c233a5841a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Wed, 6 May 2026 16:41:26 +0200 Subject: [PATCH 10/15] gh-142307: deprecate legacy support for altering `IMAP4.file` (#142335) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/deprecations/pending-removal-in-3.19.rst | 9 ++++ Doc/library/imaplib.rst | 10 +++++ Doc/whatsnew/3.15.rst | 7 +++ Lib/imaplib.py | 44 ++++++++++++------- Lib/test/test_imaplib.py | 30 +++++++++++-- ...-12-06-11-24-25.gh-issue-142307.w8evI9.rst | 4 ++ 6 files changed, 83 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 044bb8a3934a2a..4a58c606ab7596 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -31,3 +31,12 @@ Pending removal in Python 3.19 * :meth:`http.cookies.BaseCookie.js_output` is deprecated and will be removed in Python 3.19. +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index b29b02d3cf5fe8..fabe2ca9127984 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -695,6 +695,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: next 3.19 + + .. _imap4-example: IMAP4 Example diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 698a9f88e1ee39..2ca28378e6ef73 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2107,6 +2107,13 @@ New deprecations (Contributed by kishorhange111 in :gh:`148849`.) +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + + * :mod:`re`: * :func:`re.match` and :meth:`re.Pattern.match` are now diff --git a/Lib/imaplib.py b/Lib/imaplib.py index cb3edceae0d9f1..2fafd9322c609e 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -313,25 +313,34 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): self.host = host self.port = port self.sock = self._create_socket(timeout) - self._file = self.sock.makefile('rb') - + # Since IMAP4 implements its own read() and readline() buffering, + # the '_imaplib_file' attribute is unused. Nonetheless it is kept + # and exposed solely for backward compatibility purposes. + self._imaplib_file = self.sock.makefile('rb') @property def file(self): - # The old 'file' attribute is no longer used now that we do our own - # read() and readline() buffering, with which it conflicts. - # As an undocumented interface, it should never have been accessed by - # external code, and therefore does not warrant deprecation. - # Nevertheless, we provide this property for now, to avoid suddenly - # breaking any code in the wild that might have been using it in a - # harmless way. import warnings - warnings.warn( - 'IMAP4.file is unsupported, can cause errors, and may be removed.', - RuntimeWarning, - stacklevel=2) - return self._file + warnings._deprecated("IMAP4.file", remove=(3, 19)) + return self._imaplib_file + @file.setter + def file(self, value): + import warnings + warnings._deprecated("IMAP4.file", remove=(3, 19)) + # Ideally, we would want to close the previous file, + # but since we do not know how subclasses will use + # that setter, it is probably better to leave it to + # the caller. + self._imaplib_file = value + + def _close_imaplib_file(self): + file = self._imaplib_file + if file is not None: + try: + file.close() + except OSError: + pass def read(self, size): """Read 'size' bytes from remote.""" @@ -417,7 +426,7 @@ def send(self, data): def shutdown(self): """Close I/O established in "open".""" - self._file.close() + self._close_imaplib_file() try: self.sock.shutdown(socket.SHUT_RDWR) except OSError as exc: @@ -921,9 +930,10 @@ def starttls(self, ssl_context=None): ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': + self._close_imaplib_file() self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) - self._file = self.sock.makefile('rb') + self._imaplib_file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() else: @@ -1680,7 +1690,7 @@ def open(self, host=None, port=None, timeout=None): self.host = None # For compatibility with parent class self.port = None self.sock = None - self._file = None + self._imaplib_file = None self.process = subprocess.Popen(self.command, bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index cb5454b40eccf9..0b704d62655762 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -665,11 +665,33 @@ def test_control_characters(self): # property tests - def test_file_property_should_not_be_accessed(self): + def test_file_property_getter(self): client, _ = self._setup(SimpleIMAPHandler) - # the 'file' property replaced a private attribute that is now unsafe - with self.assertWarns(RuntimeWarning): - client.file + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(client.file.raw, socket.SocketIO) + + def test_file_property_setter(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertWarns(DeprecationWarning): + # ensure that the caller closes the existing file + client.file.close() + for new_file in [mock.Mock(), None]: + with self.assertWarns(DeprecationWarning): + client.file = new_file + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, new_file) + + def test_file_property_setter_should_not_close_previous_file(self): + client, _ = self._setup(SimpleIMAPHandler) + with mock.patch.object(client, "_imaplib_file", mock.Mock()) as f: + f.close.assert_not_called() + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, f) + with self.assertWarns(DeprecationWarning): + client.file = None + with self.assertWarns(DeprecationWarning): + self.assertIsNone(client.file) + f.close.assert_not_called() class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst b/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst new file mode 100644 index 00000000000000..3c0eb0edcfba48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst @@ -0,0 +1,4 @@ +:mod:`imaplib`: deprecate support for :attr:`IMAP4.file `. +This attribute was never meant to be part of the public interface and altering +its value may result in unclosed files or other synchronization issues with +the underlying socket. Patch by Bénédikt Tran. From 646853df13492e2260befd5a13dba29af3c6be46 Mon Sep 17 00:00:00 2001 From: Alex Malyshev Date: Wed, 6 May 2026 11:01:12 -0400 Subject: [PATCH 11/15] gh-145559: Add PyUnstable_DumpTraceback() and PyUnstable_DumpTracebackThreads() (#148145) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/exceptions.rst | 64 +++++++++++++++++++ Doc/whatsnew/3.15.rst | 5 ++ Include/cpython/traceback.h | 8 +++ Include/internal/pycore_traceback.h | 50 --------------- ...-04-05-18-18-59.gh-issue-145559.qKJH9S.rst | 3 + Modules/faulthandler.c | 28 ++++---- Platforms/emscripten/node_entry.mjs | 2 +- .../web_example_pyrepl_jspi/src.mjs | 2 +- Python/pylifecycle.c | 6 +- Python/traceback.c | 13 ++-- configure | 2 +- configure.ac | 2 +- 12 files changed, 109 insertions(+), 76 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2026-04-05-18-18-59.gh-issue-145559.qKJH9S.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 7a07818b7b4d1a..2f8f108ee27f6a 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -1348,3 +1348,67 @@ Tracebacks This function returns ``0`` on success, and returns ``-1`` with an exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: next + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate, Py_ssize_t max_threads) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + + This function is meant to debug debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first *max_threads* threads, + further output is truncated with the line ``...``. If *max_threads* is 0, the + function will use a default value of 100 for the argument. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2ca28378e6ef73..f3bcdd5f1a35b9 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2352,6 +2352,11 @@ New features Python 3.14. (Contributed by Victor Stinner in :gh:`142417`.) +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) + Changed C APIs -------------- diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f2..7f42730f1b0919 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,11 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads); diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index fbf6bc2c41f51d..e016afaa5c5687 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,56 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate, - Py_ssize_t max_threads); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. diff --git a/Misc/NEWS.d/next/C_API/2026-04-05-18-18-59.gh-issue-145559.qKJH9S.rst b/Misc/NEWS.d/next/C_API/2026-04-05-18-18-59.gh-issue-145559.qKJH9S.rst new file mode 100644 index 00000000000000..9495d42160a9cd --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-04-05-18-18-59.gh-issue-145559.qKJH9S.rst @@ -0,0 +1,3 @@ +Rename ``_Py_DumpTraceback`` and ``_Py_DumpTracebackThreads`` to +:c:func:`PyUnstable_DumpTraceback` and +:c:func:`PyUnstable_DumpTracebackThreads`. diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index 923f6f5b56d32b..1b4f0c2302daae 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -7,7 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyTime_FromSecondsObject() -#include "pycore_traceback.h" // _Py_DumpTracebackThreads +#include "pycore_traceback.h" // _Py_DumpStack() #ifdef HAVE_UNISTD_H # include // _exit() #endif @@ -206,14 +206,15 @@ faulthandler_dump_traceback(int fd, int all_threads, PyThreadState *tstate = PyGILState_GetThisThreadState(); if (all_threads == 1) { - (void)_Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); + (void)PyUnstable_DumpTracebackThreads(fd, NULL, tstate, max_threads); } else { if (all_threads == FT_IGNORE_ALL_THREADS) { PUTS(fd, "\n"); } - if (tstate != NULL) - _Py_DumpTraceback(fd, tstate); + if (tstate != NULL) { + PyUnstable_DumpTraceback(fd, tstate); + } } reentrant = 0; @@ -277,17 +278,18 @@ faulthandler_dump_traceback_py_impl(PyObject *module, PyObject *file, /* gh-128400: Accessing other thread states while they're running * isn't safe if those threads are running. */ _PyEval_StopTheWorld(interp); - errmsg = _Py_DumpTracebackThreads(fd, NULL, tstate, max_threads); + errmsg = PyUnstable_DumpTracebackThreads(fd, NULL, tstate, max_threads); _PyEval_StartTheWorld(interp); - if (errmsg != NULL) { - PyErr_SetString(PyExc_RuntimeError, errmsg); - Py_XDECREF(file); - return NULL; - } } else { - _Py_DumpTraceback(fd, tstate); + errmsg = PyUnstable_DumpTraceback(fd, tstate); } + if (errmsg != NULL) { + PyErr_SetString(PyExc_RuntimeError, errmsg); + Py_XDECREF(file); + return NULL; + } + Py_XDECREF(file); if (PyErr_CheckSignals()) @@ -713,8 +715,8 @@ faulthandler_thread(void *unused) (void)_Py_write_noraise(thread.fd, thread.header, (int)thread.header_len); - errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, NULL, - thread.max_threads); + errmsg = PyUnstable_DumpTracebackThreads(thread.fd, thread.interp, NULL, + thread.max_threads); ok = (errmsg == NULL); if (thread.exit) diff --git a/Platforms/emscripten/node_entry.mjs b/Platforms/emscripten/node_entry.mjs index 9478b7714adbc8..110aadc5de1014 100644 --- a/Platforms/emscripten/node_entry.mjs +++ b/Platforms/emscripten/node_entry.mjs @@ -57,6 +57,6 @@ try { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs index 5642372c9d2472..38a622117c2a50 100644 --- a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs +++ b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs @@ -189,6 +189,6 @@ try { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback - Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + Module.PyUnstable_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); process.exit(1); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 728c0acdd4df67..8f31756f3df840 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -29,7 +29,7 @@ #include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" // _PyStats_InterpInit() #include "pycore_sysmodule.h" // _PySys_ClearAttrString() -#include "pycore_traceback.h" // _Py_DumpTracebackThreads() +#include "pycore_traceback.h" // PyUnstable_TracebackThreads() #include "pycore_tuple.h" // _PyTuple_FromPair #include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_typevarobject.h" // _Py_clear_generic_types() @@ -3348,9 +3348,9 @@ _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, /* display the current Python stack */ #ifndef Py_GIL_DISABLED - _Py_DumpTracebackThreads(fd, interp, tstate, 0); + PyUnstable_DumpTracebackThreads(fd, interp, tstate, 0); #else - _Py_DumpTraceback(fd, tstate); + PyUnstable_DumpTraceback(fd, tstate); #endif } diff --git a/Python/traceback.c b/Python/traceback.c index f0e0df7101bc21..50a79d78d2e10e 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1167,10 +1167,11 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ -void -_Py_DumpTraceback(int fd, PyThreadState *tstate) +const char* +PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) { dump_traceback(fd, tstate, 1); + return NULL; } #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) @@ -1264,16 +1265,16 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) The caller is responsible to call PyErr_CheckSignals() to call Python signal handlers if signals were received. */ const char* _Py_NO_SANITIZE_THREAD -_Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, - PyThreadState *current_tstate, - Py_ssize_t max_threads) +PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads) { if (max_threads == 0) { max_threads = DEFAULT_MAX_NTHREADS; } if (current_tstate == NULL) { - /* _Py_DumpTracebackThreads() is called from signal handlers by + /* PyUnstable_DumpTracebackThreads() is called from signal handlers by faulthandler. SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals diff --git a/configure b/configure index 627f5ee4888a1f..d4a0a9d74b7631 100755 --- a/configure +++ b/configure @@ -9788,7 +9788,7 @@ fi as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" diff --git a/configure.ac b/configure.ac index 4f968b25d91566..395bfc30651171 100644 --- a/configure.ac +++ b/configure.ac @@ -2406,7 +2406,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) From 4ed40146f17ea71e94b629600a8d540436367607 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 6 May 2026 16:03:37 +0100 Subject: [PATCH 12/15] gh-149202: Fix frame pointer unwinding on s390x and ARM (GH-149362) -fno-omit-frame-pointer is not enough to make every target walkable by the simple manual frame pointer unwinder. The helper used by test_frame_pointer_unwind used to assume the frame pointer named a two-word record where fp[0] was the previous frame pointer and fp[1] was the return address. That is only the generic layout used by some targets. This patch keeps that default, but moves the slots behind named offsets so architecture-specific layouts can describe where the backchain and return address really live. On s390x, GCC and Clang do not emit a usable backchain unless -mbackchain is enabled. Without it, the unwinder stops at the current C frame and the test reports no Python frames. Once backchains are present, the helper must also stop at the current thread's known C stack bounds; otherwise it can follow the final backchain far enough to dereference an invalid frame and segfault. For Linux s390x backchain frames, the documented z/Architecture stack-frame layout saves r14, the return-address register, at byte offset 112 from the frame pointer, so read the return address from that named slot instead of fp[1]. The 112-byte offset comes from Linux's s390 debugging documentation: its Stack Frame Layout table shows z/Architecture backchain frames with the backchain at offset 0 and saved r14 of the caller function at offset 112: https://www.kernel.org/doc/html/v5.3/s390/debugging390.html#stack-frame-layout This helper remains scoped to Linux s390x backchain frames. GNU SFrame's s390x notes state that the s390x ELF ABI does not generally mandate where RA and FP are saved, or whether they are saved at all: https://sourceware.org/binutils/docs/sframe-spec.html#s390x As Jens Remus noted, -fno-omit-frame-pointer is not needed when -mbackchain is present. On 32-bit ARM, GCC defaults to Thumb mode on common armhf toolchains. The Thumb prologue keeps the saved frame pointer and link register at offsets that depend on the generated frame, which breaks the fp[0]/fp[1] walk used by the helper. Use -marm when it is supported for frame-pointer builds, and teach the helper the GCC ARM-mode slots where the previous frame pointer is at fp[-1] and the saved LR return address is at fp[0]. Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/howto/perf_profiling.rst | 4 +- Doc/using/configure.rst | 18 +- Doc/whatsnew/3.15.rst | 13 ++ Lib/test/test_frame_pointer_unwind.py | 10 ++ ...-04-05-16-10-00.gh-issue-149202.W8sQeR.rst | 7 +- Modules/_testinternalcapi.c | 155 ++++++++++++++++-- configure | 103 ++++++++++++ configure.ac | 14 ++ pyconfig.h.in | 4 + 9 files changed, 302 insertions(+), 26 deletions(-) diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 653f28ddbabfa4..657cb287ad3d60 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -218,8 +218,8 @@ How to obtain the best results ------------------------------ For best results, keep frame pointers enabled. On supported GCC-compatible -toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and, when -available, ``-mno-omit-leaf-frame-pointer`` by default. These flags allow +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and similar +flags (see :option:`--without-frame-pointers` for details). These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 086f6bfa22ad4a..62c53c283825c8 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -784,11 +784,19 @@ also be used to improve performance. Disable frame pointers, which are enabled by default (see :pep:`831`). - By default, the build appends ``-fno-omit-frame-pointer`` (and - ``-mno-omit-leaf-frame-pointer`` when the compiler supports it) to - ``BASECFLAGS`` so profilers, debuggers, and system tracing tools - (``perf``, ``eBPF``, ``dtrace``, ``gdb``) can walk the C call stack - without DWARF metadata. The flags propagate to third-party C + By default, the build appends flags to generate frame or backchain + pointers to ``BASECFLAGS``: + + - ``-fno-omit-frame-pointer`` and/or ``-mno-omit-leaf-frame-pointer`` + are added when the compiler supports them. + - ``-marm`` is added on 32-bit ARM when supported, + - on s390x platforms, when supported, ``-mbackchain`` is added *instead*. + of the above frame pointer flags. + + Frame pointers enable profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) to walk the C call stack + without DWARF metadata. + The flags propagate to third-party C extensions through :mod:`sysconfig`. On compilers that do not understand them, the build silently skips them. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index f3bcdd5f1a35b9..98af62a412fab7 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -2552,6 +2552,19 @@ Build changes and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) +.. _whatsnew315-frame-pointers: + +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + + Authors of C extensions and native libraries built with custom build + systems should ensure the unwind chain is intact. + This is usually done by adding ``-fno-omit-frame-pointer`` and + similar flags to ``CFLAGS``. See :option:`--without-frame-pointers` + documentation for the specific flags Python uses. + + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + .. _whatsnew315-windows-tail-calling-interpreter: * 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new diff --git a/Lib/test/test_frame_pointer_unwind.py b/Lib/test/test_frame_pointer_unwind.py index 4081e1cbd8aaac..5cd94e5b27f394 100644 --- a/Lib/test/test_frame_pointer_unwind.py +++ b/Lib/test/test_frame_pointer_unwind.py @@ -21,6 +21,16 @@ def _frame_pointers_expected(machine): + _Py_WITH_FRAME_POINTERS = getattr( + _testinternalcapi, + "_Py_WITH_FRAME_POINTERS", + -1, + ) + if _Py_WITH_FRAME_POINTERS > 0: + return True + if _Py_WITH_FRAME_POINTERS == 0: + return False + cflags = " ".join( value for value in ( sysconfig.get_config_var("PY_CORE_CFLAGS"), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst index f82ca91f5ba000..aae1529547c837 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-05-16-10-00.gh-issue-149202.W8sQeR.rst @@ -1,4 +1,5 @@ Enable frame pointers by default for GCC-compatible CPython builds, including -``-mno-omit-leaf-frame-pointer`` when the compiler supports it, so profilers -and debuggers can unwind native interpreter frames more reliably. Users can pass -``--without-frame-pointers`` to opt out. +``-mno-omit-leaf-frame-pointer``, ``-marm`` on 32-bit ARM, and/or ``-mbackchain`` +on s390x platforms when the compiler supports them, so profilers and debuggers +can unwind native interpreter frames more reliably. Users can pass +:option:`--without-frame-pointers` to ``./configure`` to opt out. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 23ef4f13be3b5f..73451b5117fa8c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -63,6 +63,40 @@ static const uintptr_t min_frame_pointer_addr = 0x1000; #define MAX_UNWIND_FRAMES 200 +#ifdef __s390x__ +// Linux's s390 "Stack Frame Layout" table documents that z/Architecture +// backchain frames start with the backchain at offset 0 and store "saved r14 +// of caller function" at offset 112. The same document's register table +// identifies r14 as the return-address register, so this backchain unwinder +// reads the return address from fp + 112. +// https://www.kernel.org/doc/html/v5.3/s390/debugging390.html#stack-frame-layout +// +// This is only for Linux s390x backchain frames. The s390x ELF ABI does not +// generally mandate where RA and FP are saved, or whether they are saved at all. +// https://sourceware.org/binutils/docs/sframe-spec.html#s390x +# define S390X_FRAME_RETURN_ADDRESS_OFFSET 112 +#endif + +// The generic manual unwinder treats the frame pointer as a two-word record: +// fp[0] is the previous frame pointer and fp[1] is the return address. That is +// not true for every architecture, even with frame pointers enabled, so these +// offsets describe the actual slots used by each supported frame layout. +#if defined(__arm__) && !defined(__thumb__) && !defined(__clang__) +// GCC ARM mode keeps the caller's fp one word below fp and the saved LR at +// fp[0], so the return address is not in the generic fp[1] slot. +# define FRAME_POINTER_NEXT_OFFSET (-1) +# define FRAME_POINTER_RETURN_OFFSET 0 +#elif defined(__s390x__) +// s390x backchain frames keep the previous frame pointer at fp[0], but save the +// return-address register in the ABI register save area rather than fp[1]. +# define FRAME_POINTER_NEXT_OFFSET 0 +# define FRAME_POINTER_RETURN_OFFSET \ + (S390X_FRAME_RETURN_ADDRESS_OFFSET / (Py_ssize_t)sizeof(uintptr_t)) +#else +# define FRAME_POINTER_NEXT_OFFSET 0 +# define FRAME_POINTER_RETURN_OFFSET 1 +#endif + static PyObject * _get_current_module(void) @@ -329,15 +363,96 @@ get_jit_backend(PyObject *self, PyObject *Py_UNUSED(args)) #endif } +static int +stack_address_is_valid(uintptr_t addr, uintptr_t stack_min, uintptr_t stack_max) +{ + if (addr < min_frame_pointer_addr) { + return 0; + } + if (stack_min != 0 && (addr < stack_min || addr >= stack_max)) { + return 0; + } + return 1; +} + +static int +frame_pointer_slot_is_valid(uintptr_t *frame_pointer, Py_ssize_t offset, + uintptr_t stack_min, uintptr_t stack_max) +{ + uintptr_t fp_addr = (uintptr_t)frame_pointer; + uintptr_t slot_addr; + uintptr_t delta = (uintptr_t)Py_ABS(offset) * sizeof(uintptr_t); + if (offset < 0) { + if (fp_addr < delta) { + return 0; + } + slot_addr = fp_addr - delta; + } + else { + if (fp_addr > UINTPTR_MAX - delta) { + return 0; + } + slot_addr = fp_addr + delta; + } + if (!stack_address_is_valid(slot_addr, stack_min, stack_max)) { + return 0; + } + if (stack_max != 0) { + if (slot_addr > UINTPTR_MAX - sizeof(uintptr_t)) { + return 0; + } + if (slot_addr + sizeof(uintptr_t) > stack_max) { + return 0; + } + } + return 1; +} + +static int +next_frame_pointer_is_valid(uintptr_t *frame_pointer, uintptr_t *next_fp, + uintptr_t stack_min, uintptr_t stack_max) +{ + uintptr_t fp_addr = (uintptr_t)frame_pointer; + uintptr_t next_addr = (uintptr_t)next_fp; + if (!stack_address_is_valid(next_addr, stack_min, stack_max)) { + return 0; + } + if ((next_addr % sizeof(uintptr_t)) != 0) { + return 0; + } +#if _Py_STACK_GROWS_DOWN + return next_addr > fp_addr; +#else + return next_addr < fp_addr; +#endif +} + static PyObject * manual_unwind_from_fp(uintptr_t *frame_pointer) { - int stack_grows_down = _Py_STACK_GROWS_DOWN; + uintptr_t stack_min = 0; + uintptr_t stack_max = 0; + +#ifdef __s390x__ + Py_BUILD_ASSERT(S390X_FRAME_RETURN_ADDRESS_OFFSET % sizeof(uintptr_t) == 0); +#endif if (frame_pointer == NULL) { return PyList_New(0); } + PyThreadState *tstate = _PyThreadState_GET(); + if (tstate != NULL) { + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN + stack_min = tstate_impl->c_stack_hard_limit; + stack_max = tstate_impl->c_stack_top; +#else + stack_min = tstate_impl->c_stack_top; + stack_max = tstate_impl->c_stack_hard_limit; +#endif + } + PyObject *result = PyList_New(0); if (result == NULL) { return NULL; @@ -357,7 +472,21 @@ manual_unwind_from_fp(uintptr_t *frame_pointer) MAX_UNWIND_FRAMES); return NULL; } - uintptr_t return_addr = frame_pointer[1]; + if (!stack_address_is_valid(fp_addr, stack_min, stack_max)) { + break; + } + if (!frame_pointer_slot_is_valid(frame_pointer, + FRAME_POINTER_NEXT_OFFSET, + stack_min, stack_max)) { + break; + } + if (!frame_pointer_slot_is_valid(frame_pointer, + FRAME_POINTER_RETURN_OFFSET, + stack_min, stack_max)) { + break; + } + uintptr_t *next_fp = (uintptr_t *)frame_pointer[FRAME_POINTER_NEXT_OFFSET]; + uintptr_t return_addr = frame_pointer[FRAME_POINTER_RETURN_OFFSET]; PyObject *addr_obj = PyLong_FromUnsignedLongLong(return_addr); if (addr_obj == NULL) { @@ -372,22 +501,10 @@ manual_unwind_from_fp(uintptr_t *frame_pointer) Py_DECREF(addr_obj); depth++; - uintptr_t *next_fp = (uintptr_t *)frame_pointer[0]; - // Stop if the frame pointer is extremely low. - if ((uintptr_t)next_fp < min_frame_pointer_addr) { + if (!next_frame_pointer_is_valid(frame_pointer, next_fp, + stack_min, stack_max)) { break; } - uintptr_t next_addr = (uintptr_t)next_fp; - if (stack_grows_down) { - if (next_addr <= fp_addr) { - break; - } - } - else { - if (next_addr >= fp_addr) { - break; - } - } frame_pointer = next_fp; } @@ -3170,6 +3287,12 @@ module_exec(PyObject *module) return 1; } +#ifdef _Py_WITH_FRAME_POINTERS + if (PyModule_AddIntMacro(module, _Py_WITH_FRAME_POINTERS) < 0) { + return 1; + } +#endif + return 0; } diff --git a/configure b/configure index d4a0a9d74b7631..f970bf9b7ba3c7 100755 --- a/configure +++ b/configure @@ -10343,6 +10343,106 @@ else case e in #( esac fi + case $host_cpu in #( + arm|armv*) : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -marm" >&5 +printf %s "checking whether C compiler accepts -marm... " >&6; } +if test ${ax_cv_check_cflags__Werror__marm+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -marm" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__marm=yes +else case e in #( + e) ax_cv_check_cflags__Werror__marm=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__marm" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__marm" >&6; } +if test "x$ax_cv_check_cflags__Werror__marm" = xyes +then : + + frame_pointer_cflags="$frame_pointer_cflags -marm" + +else case e in #( + e) : ;; +esac +fi + + ;; #( + *) : + ;; +esac + case $host_cpu in #( + s390*) : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -mbackchain" >&5 +printf %s "checking whether C compiler accepts -mbackchain... " >&6; } +if test ${ax_cv_check_cflags__Werror__mbackchain+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -Werror -mbackchain" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags__Werror__mbackchain=yes +else case e in #( + e) ax_cv_check_cflags__Werror__mbackchain=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags__Werror__mbackchain" >&5 +printf "%s\n" "$ax_cv_check_cflags__Werror__mbackchain" >&6; } +if test "x$ax_cv_check_cflags__Werror__mbackchain" = xyes +then : + + frame_pointer_cflags="-mbackchain" + +else case e in #( + e) : ;; +esac +fi + + ;; #( + *) : + ;; +esac else case e in #( e) : ;; @@ -10351,6 +10451,9 @@ fi if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then BASECFLAGS="$frame_pointer_cflags $BASECFLAGS" + +printf "%s\n" "#define _Py_WITH_FRAME_POINTERS 1" >>confdefs.h + fi CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" diff --git a/configure.ac b/configure.ac index 395bfc30651171..9f91a10c2918cf 100644 --- a/configure.ac +++ b/configure.ac @@ -2548,9 +2548,23 @@ AS_VAR_IF([ac_cv_gcc_compat], [yes], [ AX_CHECK_COMPILE_FLAG([-mno-omit-leaf-frame-pointer], [ frame_pointer_cflags="$frame_pointer_cflags -mno-omit-leaf-frame-pointer" ], [], [-Werror]) + AS_CASE([$host_cpu], [arm|armv*], [ + AX_CHECK_COMPILE_FLAG([-marm], [ + frame_pointer_cflags="$frame_pointer_cflags -marm" + ], [], [-Werror]) + ]) + AS_CASE([$host_cpu], [s390*], [ + AX_CHECK_COMPILE_FLAG([-mbackchain], [ + dnl Do not use no-omit-frame-pointer; see gh-149362 + frame_pointer_cflags="-mbackchain" + ], [], [-Werror]) + ]) ], [], [-Werror]) if test -n "$frame_pointer_cflags" && test "x$with_frame_pointers" != xno; then BASECFLAGS="$frame_pointer_cflags $BASECFLAGS" + AC_DEFINE([_Py_WITH_FRAME_POINTERS], [1], + [Define to 1 if frame unwinding via pointers is expected + to work, 0 if not. Leave undefined if unknown.]) fi CFLAGS_NODIST="$CFLAGS_NODIST -std=c11" diff --git a/pyconfig.h.in b/pyconfig.h.in index 4eeec330466441..ad372255445d13 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -2073,6 +2073,10 @@ /* Define if you want to use tail-calling interpreters in CPython. */ #undef _Py_TAIL_CALL_INTERP +/* Define to 1 if frame unwinding via pointers is expected to work, 0 if not. + Leave undefined if unknown. */ +#undef _Py_WITH_FRAME_POINTERS + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT From 1dcc546d841a5e88675eb2af2c55b63f26930ccf Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 6 May 2026 10:44:47 -0500 Subject: [PATCH 13/15] Rewrite RTD configuration to use build.jobs rather than build.commands (GH-149429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of this conversion, we now ensure that we're comparing against the merge-base of the PR branch and the base branch when checking whether an RTD build is worthwhile, deepening the history of the base branch by up to 500 commits if necessary. If the merge-base can't be found or there are merge conflicts with the head of the base branch, the build is skipped since it would give a warped perception of the actual changes anyway. This unfortunately does nothing about RTD preview comments comparing against the wrong base, other than skipping builds that shouldn't produce any diff at all thus avoiding the comment. Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- .readthedocs.yml | 64 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 20 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 0a2c3f8345367f..3b8a30c0251873 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -12,23 +12,47 @@ build: tools: python: "3" - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html + jobs: + post_checkout: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there aren't changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=main; + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" From d13fc36f7319dca8d11d8e69c69adcfe919814e4 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 6 May 2026 11:04:42 -0500 Subject: [PATCH 14/15] gh-124111: Only set TCLSH_NATIVE for AMD64/ARM64 (GH-149443) The Tcl 9 makefile.vc now uses TCLSH_NATIVE during the build process, not just the installation. We had been setting it to the installed location of the x86 tclsh.exe, which does not yet exist when the x86 build process needs it. That build doesn't actually need TCLSH_NATIVE, though (there's a check specifically allowing TCLSH to be used if MACHINE is IX86 and TCLSH_NATIVE is undefined), so don't set it. --- PCbuild/tcl.vcxproj | 6 +++--- PCbuild/tcltk.props | 3 +-- PCbuild/tk.vcxproj | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/PCbuild/tcl.vcxproj b/PCbuild/tcl.vcxproj index ab68db9210fbe4..2233559dfc8f5a 100644 --- a/PCbuild/tcl.vcxproj +++ b/PCbuild/tcl.vcxproj @@ -63,8 +63,8 @@ setlocal set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tclDir)win" -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" core shell dlls -nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" install-binaries install-libraries +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) core shell dlls +nmake -f makefile.vc MACHINE=$(TclMachine) OPTS=$(TclOpts) $(TclDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) install-binaries install-libraries copy /Y ..\license.terms "$(OutDir)\tcllicense.terms" @@ -74,4 +74,4 @@ copy /Y ..\license.terms "$(OutDir)\tcllicense.terms" - \ No newline at end of file + diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 55f98be1eb7eeb..a1da1155b881fd 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -17,8 +17,7 @@ $(ExternalsDir)tcltk-$(TclVersion)\$(ArchName)\ t tcl9 - $(tcltkDir)\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix).exe - $(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix).exe + TCLSH_NATIVE="$(tcltkDir)\..\win32\bin\tclsh$(TclMajorVersion)$(TclMinorVersion)$(tcltkSuffix).exe" TCL_WITH_EXTERNAL_TOMMATH; diff --git a/PCbuild/tk.vcxproj b/PCbuild/tk.vcxproj index b111969ca5de6c..204244db0d3e4b 100644 --- a/PCbuild/tk.vcxproj +++ b/PCbuild/tk.vcxproj @@ -64,8 +64,8 @@ setlocal set VCINSTALLDIR=$(VCInstallDir) cd /D "$(tkDir)win" -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" all -nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) TCLSH_NATIVE="$(tclWin32Exe)" install-binaries install-libraries +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) all +nmake /nologo -f makefile.vc RC=rc MACHINE=$(TclMachine) OPTS=$(TkOpts) $(TkDirs) $(DebugFlags) $(WarningsFlags) $(TclshNativeFlag) install-binaries install-libraries copy /Y ..\license.terms "$(OutDir)\tklicense.terms" @@ -80,4 +80,4 @@ copy /Y ..\license.terms "$(OutDir)\tklicense.terms" - \ No newline at end of file + From 7cea70e14dac091cbd7e0601b96a59458f8c9bee Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 6 May 2026 19:07:43 +0300 Subject: [PATCH 15/15] gh-144384: Lazily import `_colorize` (#149318) --- .github/workflows/mypy.yml | 3 +- Lib/difflib.py | 2 +- Lib/doctest.py | 4 +- .../sampling/live_collector/collector.py | 2 +- Lib/profiling/sampling/pstats_collector.py | 2 +- Lib/profiling/sampling/sample.py | 2 +- Lib/test/test_difflib.py | 8 +++ Lib/test/test_traceback.py | 8 ++- Lib/traceback.py | 50 +++++++++++++++---- Lib/unittest/runner.py | 3 +- ...-05-03-17-32-24.gh-issue-144384.q-8jSr.rst | 1 + 11 files changed, 65 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-03-17-32-24.gh-issue-144384.q-8jSr.rst diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 7f6571ef954576..490c32ecfc9a62 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -71,7 +71,8 @@ jobs: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: - python-version: "3.13" + python-version: "3.15" + allow-prereleases: true cache: pip cache-dependency-path: Tools/requirements-dev.txt - run: pip install -r Tools/requirements-dev.txt diff --git a/Lib/difflib.py b/Lib/difflib.py index eb249e3e288923..7a4ff15c34267b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -30,10 +30,10 @@ 'Differ','IS_CHARACTER_JUNK', 'IS_LINE_JUNK', 'context_diff', 'unified_diff', 'diff_bytes', 'HtmlDiff', 'Match'] -from _colorize import can_colorize, get_theme from heapq import nlargest as _nlargest from collections import namedtuple as _namedtuple from types import GenericAlias +lazy from _colorize import can_colorize, get_theme Match = _namedtuple('Match', 'a b size') diff --git a/Lib/doctest.py b/Lib/doctest.py index 05acac1745ace9..be950079e396de 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -106,8 +106,8 @@ def _test(): import unittest from io import StringIO, TextIOWrapper, BytesIO from collections import namedtuple -import _colorize # Used in doctests -from _colorize import ANSIColors, can_colorize +lazy import _colorize # Used in doctests +lazy from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): diff --git a/Lib/profiling/sampling/live_collector/collector.py b/Lib/profiling/sampling/live_collector/collector.py index c03df4075277cd..a53cfc6b719a10 100644 --- a/Lib/profiling/sampling/live_collector/collector.py +++ b/Lib/profiling/sampling/live_collector/collector.py @@ -9,7 +9,7 @@ import sys import sysconfig import time -import _colorize +lazy import _colorize from ..collector import Collector, extract_lineno from ..constants import ( diff --git a/Lib/profiling/sampling/pstats_collector.py b/Lib/profiling/sampling/pstats_collector.py index 6be1d698ffaa9a..50500296c15acc 100644 --- a/Lib/profiling/sampling/pstats_collector.py +++ b/Lib/profiling/sampling/pstats_collector.py @@ -1,8 +1,8 @@ import collections import marshal import pstats +lazy from _colorize import ANSIColors -from _colorize import ANSIColors from .collector import Collector, extract_lineno from .constants import MICROSECONDS_PER_SECOND, PROFILING_MODE_CPU diff --git a/Lib/profiling/sampling/sample.py b/Lib/profiling/sampling/sample.py index 9e315c080c353d..5bbe2483581333 100644 --- a/Lib/profiling/sampling/sample.py +++ b/Lib/profiling/sampling/sample.py @@ -6,7 +6,7 @@ import sysconfig import time from collections import deque -from _colorize import ANSIColors +lazy from _colorize import ANSIColors from .pstats_collector import PstatsCollector from .stack_collector import CollapsedStackCollector, FlamegraphCollector diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 771fd46e042a41..46c9b2c1d8c9fc 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -1,5 +1,7 @@ import difflib +from test import support from test.support import findfile, force_colorized +from test.support.import_helper import ensure_lazy_imports import unittest import doctest import sys @@ -644,6 +646,12 @@ def setUpModule(): difflib.HtmlDiff._default_prefix = 0 +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("difflib", {"_colorize"}) + + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(difflib)) return tests diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 6624191f164bc1..bb64153b91c92c 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -23,7 +23,7 @@ requires_subprocess, os_helper) from test.support.os_helper import TESTFN, temp_dir, unlink from test.support.script_helper import assert_python_ok, assert_python_failure, make_script -from test.support.import_helper import forget +from test.support.import_helper import ensure_lazy_imports, forget from test.support import force_not_colorized, force_not_colorized_test_class import json @@ -5632,5 +5632,11 @@ def test_suggestion_still_works_for_non_lazy_attributes(self): self.assertNotIn(b"BAR_MODULE_LOADED", stdout) +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("traceback", {"_colorize"}) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/traceback.py b/Lib/traceback.py index 66e88d0a588af3..88529e1c259a29 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -16,9 +16,9 @@ import io import importlib.util import pathlib -import _colorize from contextlib import suppress +lazy import _colorize try: from _missing_stdlib_info import _MISSING_STDLIB_MODULE_MESSAGES @@ -32,6 +32,36 @@ 'FrameSummary', 'StackSummary', 'TracebackException', 'walk_stack', 'walk_tb', 'print_list'] + +class _ShutdownTheme: + """Empty stand-in if `_colorize` cannot be imported during late shutdown.""" + def __getattr__(self, _): return self + def __getitem__(self, _): return "" + def __format__(self, _): return "" + def __str__(self): return "" + def __add__(self, other): return other + __radd__ = __add__ + + +_shutdown_theme = _ShutdownTheme() + + +def _safe_get_theme(*, force_color=False, force_no_color=False): + try: + return _colorize.get_theme( + force_color=force_color, force_no_color=force_no_color + ) + except ImportError: + return _shutdown_theme + + +def _safe_can_colorize(*, file=None): + try: + return _colorize.can_colorize(file=file) + except ImportError: + return False + + # # Formatting and printing lists of traceback lines. # @@ -151,7 +181,7 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ def _print_exception_bltin(exc, file=None, /): if file is None: file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _colorize.can_colorize(file=file) + colorize = _safe_can_colorize(file=file) return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) @@ -199,9 +229,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= valuestr = _safe_string(value, 'exception') end_char = "\n" if insert_final_newline else "" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback if value is None or not valuestr: line = f"{theme.type}{etype}{theme.reset}{end_char}" else: @@ -555,9 +585,9 @@ def format_frame_summary(self, frame_summary, **kwargs): if frame_summary.filename.startswith("'): filename = "" if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback row.append( ' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( theme.filename, @@ -1344,9 +1374,9 @@ def format_exception_only(self, *, show_group=False, _depth=0, **kwargs): """ colorize = kwargs.get("colorize", False) if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback indent = 3 * _depth * ' ' if not self._have_exc_type: @@ -1494,9 +1524,9 @@ def _format_syntax_error(self, stype, **kwargs): # Show exactly where the problem was found. colorize = kwargs.get("colorize", False) if colorize: - theme = _colorize.get_theme(force_color=True).traceback + theme = _safe_get_theme(force_color=True).traceback else: - theme = _colorize.get_theme(force_no_color=True).traceback + theme = _safe_get_theme(force_no_color=True).traceback filename_suffix = '' if self.lineno is not None: yield ' File {}"{}"{}, line {}{}{}\n'.format( diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 5f22d91aebd05f..893fcba968c3ef 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,11 +4,10 @@ import time import warnings -from _colorize import get_theme - from . import result from .case import _SubTest from .signals import registerResult +lazy from _colorize import get_theme __unittest = True diff --git a/Misc/NEWS.d/next/Library/2026-05-03-17-32-24.gh-issue-144384.q-8jSr.rst b/Misc/NEWS.d/next/Library/2026-05-03-17-32-24.gh-issue-144384.q-8jSr.rst new file mode 100644 index 00000000000000..aad4b716e05372 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-03-17-32-24.gh-issue-144384.q-8jSr.rst @@ -0,0 +1 @@ +Lazily import :mod:`!_colorize`. Patch by Hugo van Kemenade.