From f28c0cc341c3ba29732d63253b818cff92500b75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 6 Dec 2025 13:23:01 +0100 Subject: [PATCH 1/3] gh-142307: restore legacy support for altering ``IMAP4.file`` The ``IMAP4.file`` attribute was never meant to be part of the public API. Previously, this attribute was created when opening a connection but subsequent changes may introduce a desynchronization with the underlying socket as the previous file is not closed. --- Doc/deprecations/pending-removal-in-3.19.rst | 10 +++++ Doc/library/imaplib.rst | 10 +++++ Doc/whatsnew/3.15.rst | 6 +++ Lib/imaplib.py | 44 ++++++++++++------- Lib/test/test_imaplib.py | 32 ++++++++++++-- ...-12-06-11-24-25.gh-issue-142307.w8evI9.rst | 4 ++ 6 files changed, 85 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 25f9cba390de68..502e64cf84ec94 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -22,3 +22,13 @@ Pending removal in Python 3.19 supported depending on the backend implementation of hash functions. Prefer passing the initial data as a positional argument for maximum backwards compatibility. + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 0b0537d3bbd104..4e012029b5e78d 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -703,6 +703,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: next 3.19 + + .. _imap4-example: IMAP4 Example diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 27e3f23e47c875..2d6109a452cf11 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1013,6 +1013,12 @@ New deprecations (Contributed by Bénédikt Tran in :gh:`134978`.) +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + * ``__version__`` * The ``__version__`` attribute has been deprecated in these standard library diff --git a/Lib/imaplib.py b/Lib/imaplib.py index c176736548188c..52219025c9541a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -313,25 +313,34 @@ def open(self, host='', port=IMAP4_PORT, timeout=None): self.host = host self.port = port self.sock = self._create_socket(timeout) - self._file = self.sock.makefile('rb') - + # Since IMAP4 implements its own read() and readline() buffering, + # the '_imaplib_file' attribute is unused. Nonetheless it is kept + # and exposed solely for backward compatibility purposes. + self._imaplib_file = self.sock.makefile('rb') @property def file(self): - # The old 'file' attribute is no longer used now that we do our own - # read() and readline() buffering, with which it conflicts. - # As an undocumented interface, it should never have been accessed by - # external code, and therefore does not warrant deprecation. - # Nevertheless, we provide this property for now, to avoid suddenly - # breaking any code in the wild that might have been using it in a - # harmless way. import warnings - warnings.warn( - 'IMAP4.file is unsupported, can cause errors, and may be removed.', - RuntimeWarning, - stacklevel=2) - return self._file + warnings._deprecated("IMAP4.file", remove=(3, 19)) + return self._imaplib_file + @file.setter + def file(self, value): + import warnings + warnings._deprecated("IMAP4.file", remove=(3, 19)) + # Ideally, we would want to close the previous file, + # but since we do not know how subclasses will use + # that setter, it is probably better to leave it to + # the caller. + self._imaplib_file = value + + def _close_imaplib_file(self): + file = self._imaplib_file + if file is not None: + try: + file.close() + except OSError: + pass def read(self, size): """Read 'size' bytes from remote.""" @@ -417,7 +426,7 @@ def send(self, data): def shutdown(self): """Close I/O established in "open".""" - self._file.close() + self._close_imaplib_file() try: self.sock.shutdown(socket.SHUT_RDWR) except OSError as exc: @@ -921,9 +930,10 @@ def starttls(self, ssl_context=None): ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': + self._close_imaplib_file() self.sock = ssl_context.wrap_socket(self.sock, server_hostname=self.host) - self._file = self.sock.makefile('rb') + self._imaplib_file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() else: @@ -1678,7 +1688,7 @@ def open(self, host=None, port=None, timeout=None): self.host = None # For compatibility with parent class self.port = None self.sock = None - self._file = None + self._imaplib_file = None self.process = subprocess.Popen(self.command, bufsize=DEFAULT_BUFFER_SIZE, stdin=subprocess.PIPE, stdout=subprocess.PIPE, diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 430fa71fa29f59..68a426a7d72b9c 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -1,3 +1,5 @@ +import io + from test import support from test.support import socket_helper @@ -659,11 +661,33 @@ def test_unselect(self): # property tests - def test_file_property_should_not_be_accessed(self): + def test_file_property_getter(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(client.file.raw, socket.SocketIO) + + def test_file_property_setter(self): + client, _ = self._setup(SimpleIMAPHandler) + with self.assertWarns(DeprecationWarning): + # ensure that the caller closes the existing file + client.file.close() + for new_file in [mock.Mock(), None]: + with self.assertWarns(DeprecationWarning): + client.file = new_file + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, new_file) + + def test_file_property_setter_should_not_close_previous_file(self): client, _ = self._setup(SimpleIMAPHandler) - # the 'file' property replaced a private attribute that is now unsafe - with self.assertWarns(RuntimeWarning): - client.file + with mock.patch.object(client, "_imaplib_file", mock.Mock()) as f: + f.close.assert_not_called() + with self.assertWarns(DeprecationWarning): + self.assertIs(client.file, f) + with self.assertWarns(DeprecationWarning): + client.file = None + with self.assertWarns(DeprecationWarning): + self.assertIsNone(client.file) + f.close.assert_not_called() class NewIMAPTests(NewIMAPTestsMixin, unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst b/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst new file mode 100644 index 00000000000000..3c0eb0edcfba48 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-06-11-24-25.gh-issue-142307.w8evI9.rst @@ -0,0 +1,4 @@ +:mod:`imaplib`: deprecate support for :attr:`IMAP4.file `. +This attribute was never meant to be part of the public interface and altering +its value may result in unclosed files or other synchronization issues with +the underlying socket. Patch by Bénédikt Tran. From f807145b8ddea70637b1c3be0b60e84d242a6fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 6 Dec 2025 13:28:12 +0100 Subject: [PATCH 2/3] Update Lib/test/test_imaplib.py --- Lib/test/test_imaplib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 68a426a7d72b9c..7a7230485c369a 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -1,5 +1,3 @@ -import io - from test import support from test.support import socket_helper From b1a2060c57b3945a2c23314ff1f801c4277c6bd1 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 6 May 2026 15:17:43 +0300 Subject: [PATCH 3/3] Fix merge conflict --- Doc/whatsnew/3.15.rst | 1676 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 1483 insertions(+), 193 deletions(-) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 2d6109a452cf11..aba41355a11a42 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -65,127 +65,457 @@ Summary -- Release highlights .. PEP-sized items next. +* :pep:`810`: :ref:`Explicit lazy imports for faster startup times + ` +* :pep:`814`: :ref:`Add frozendict built-in type + ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python - profiling tools ` + profiling tools ` +* :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler + ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` +* :pep:`798`: :ref:`Unpacking in comprehensions + ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` +* :pep:`829`: :ref:`Package startup configuration files ` +* :pep:`728`: :ref:`TypedDict with typed extra items ` +* :pep:`747`: :ref:`Annotating type forms with TypeForm + ` +* :pep:`800`: Disjoint bases in the type system * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object - ` + ` +* :pep:`803`, :pep:`820 <820>`, :pep:`793 <793>`: + :ref:`Stable ABI for Free-Threaded Builds ` and + related C API +* :ref:`The JIT compiler has been significantly upgraded ` * :ref:`Improved error messages ` - +* :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter + ` New features ============ +.. _whatsnew315-lazy-imports: + +:pep:`810`: Explicit lazy imports +--------------------------------- + +Large Python applications often suffer from slow startup times. A +significant contributor to this problem is the import system: when a module +is imported, Python must locate the file, read it from disk, compile it to +bytecode, and execute all top-level code. For applications with deep +dependency trees, this process can take seconds, even when most of the +imported code is never actually used during a particular run. + +Developers have worked around this by moving imports inside functions, using +:mod:`importlib` to load modules on demand, or restructuring code to avoid +unnecessary dependencies. These approaches work but make code harder to read +and maintain, scatter import statements throughout the codebase, and require +discipline to apply consistently. + +Python now provides a cleaner solution through explicit :keyword:`lazy` +imports using the new ``lazy`` soft keyword. When you mark an import as +lazy, Python defers the actual module loading until the imported name is +first used. This gives you the organizational benefits of declaring all +imports at the top of the file while only paying the loading cost for +modules you actually use. + +The ``lazy`` keyword works with both ``import`` and ``from ... import`` +statements. When you write ``lazy import heavy_module``, Python does not +immediately load the module. Instead, it creates a lightweight proxy object. +The actual module loading happens transparently when you first access the +name: + +.. code-block:: python + + lazy import json + lazy from pathlib import Path + + print("Starting up...") # json and pathlib not loaded yet + + data = json.loads('{"key": "value"}') # json loads here + p = Path(".") # pathlib loads here + +This mechanism is particularly useful for applications that import many +modules at the top level but may only use a subset of them in any given run. +The deferred loading reduces startup latency without requiring code +restructuring or conditional imports scattered throughout the codebase. + +In the case where loading a lazily imported module fails (for example, if +the module does not exist), Python raises the exception at the point of +first use rather than at import time. The associated traceback includes both +the location where the name was accessed and the original import statement, +making it straightforward to diagnose and debug the failure. + +For cases where you want to enable lazy loading globally without modifying +source code, Python provides the :option:`-X lazy_imports <-X>` command-line +option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both +accept three values: ``all`` makes all imports lazy by default, ``none`` +disables lazy imports entirely (even explicit ``lazy`` statements become +eager), and ``normal`` (the default) respects the ``lazy`` keyword in source +code. The :func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` +functions allow changing and querying this mode at runtime. + +For more selective control, :func:`sys.set_lazy_imports_filter` accepts a +callable that determines whether a specific module should be loaded lazily. +The filter receives three arguments: the importing module's name (or +``None``), the imported module's name, and the fromlist (or ``None`` for +regular imports). It should return ``True`` to allow the import to be lazy, +or ``False`` to force eager loading. This allows patterns like making only +your own application's modules lazy while keeping third-party dependencies +eager: + +.. code-block:: python + + import sys + + def myapp_filter(importing, imported, fromlist): + return imported.startswith("myapp.") + sys.set_lazy_imports_filter(myapp_filter) + sys.set_lazy_imports("all") + + import myapp.slow_module # lazy (matches filter) + import json # eager (does not match filter) + +The proxy type itself is available as :data:`types.LazyImportType` for code +that needs to detect lazy imports programmatically. + +There are some restrictions on where the ``lazy`` keyword can be used. Lazy +imports are only permitted at module scope; using ``lazy`` inside a +function, class body, or ``try``/``except``/``finally`` block raises a +:exc:`SyntaxError`. Neither star imports nor future imports can be lazy +(``lazy from module import *`` and ``lazy from __future__ import ...`` both +raise :exc:`SyntaxError`). + +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + +.. seealso:: :pep:`810` for the full specification and rationale. + +(Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) + + +.. _whatsnew315-frozendict: + +:pep:`814`: Add frozendict built-in type +---------------------------------------- + +A new :term:`immutable` type, :class:`frozendict`, is added to the :mod:`builtins` module. +It does not allow modification after creation. A :class:`!frozendict` is not a subclass of ``dict``; +it inherits directly from ``object``. A :class:`!frozendict` is :term:`hashable` +as long as all of its keys and values are hashable. A :class:`!frozendict` preserves +insertion order, but comparison does not take order into account. + +For example:: + + >>> a = frozendict(x=1, y=2) + >>> a + frozendict({'x': 1, 'y': 2}) + >>> a['z'] = 3 + Traceback (most recent call last): + File "", line 1, in + a['z'] = 3 + ~^^^^^ + TypeError: 'frozendict' object does not support item assignment + >>> b = frozendict(y=2, x=1) + >>> hash(a) == hash(b) + True + >>> a == b + True + +The following standard library modules have been updated to accept +:class:`!frozendict`: :mod:`copy`, :mod:`decimal`, :mod:`json`, :mod:`marshal`, +:mod:`plistlib` (only for serialization), :mod:`pickle`, :mod:`pprint` and +:mod:`xml.etree.ElementTree`. + +:func:`eval` and :func:`exec` accept :class:`!frozendict` for *globals*, and +:func:`type` and :meth:`str.maketrans` accept :class:`!frozendict` for *dict*. + +Code checking for :class:`dict` type using ``isinstance(arg, dict)`` can be +updated to ``isinstance(arg, (dict, frozendict))`` to accept also the +:class:`!frozendict` type, or to ``isinstance(arg, collections.abc.Mapping)`` +to accept also other mapping types such as :class:`~types.MappingProxyType`. + +.. seealso:: :pep:`814` for the full specification and rationale. + +(Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) + + +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + + +.. _whatsnew315-profiling-package: + +:pep:`799`: A dedicated profiling package +----------------------------------------- + +A new :mod:`profiling` module has been added to organize Python's built-in +profiling tools under a single, coherent namespace. This module contains: + +* :mod:`profiling.tracing`: deterministic function-call tracing (relocated from + ``cProfile``). +* :mod:`profiling.sampling`: a new statistical sampling profiler (named Tachyon). + +The ``cProfile`` module remains as an alias for backwards compatibility. +The :mod:`profile` module is deprecated and will be removed in Python 3.17. + +.. seealso:: :pep:`799` for further details. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + .. _whatsnew315-sampling-profiler: -:pep:`799`: High frequency statistical sampling profiler --------------------------------------------------------- +Tachyon: High frequency statistical sampling profiler +----------------------------------------------------- + +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png + :alt: Tachyon profiler logo + :align: center + :width: 200px -A new statistical sampling profiler has been added to the new :mod:`!profiling` module as -:mod:`!profiling.sampling`. This profiler enables low-overhead performance analysis of +A new statistical sampling profiler (Tachyon) has been added as +:mod:`profiling.sampling`. This profiler enables low-overhead performance analysis of running Python processes without requiring code modification or process restart. -Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument +Unlike deterministic profilers (such as :mod:`profiling.tracing`) that instrument every function call, the sampling profiler periodically captures stack traces from running processes. This approach provides virtually zero overhead while achieving sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging -performance issues in production environments. +performance issues in production environments. This capability is particularly +valuable for debugging performance issues in production systems where traditional +profiling approaches would be too intrusive. Key features include: * **Zero-overhead profiling**: Attach to any running Python process without - affecting its performance -* **No code modification required**: Profile existing applications without restart -* **Real-time statistics**: Monitor sampling quality during data collection -* **Multiple output formats**: Generate both detailed statistics and flamegraph data -* **Thread-aware profiling**: Option to profile all threads or just the main thread + affecting its performance. Ideal for production debugging where you can't afford + to restart or slow down your application. + +* **No code modification required**: Profile existing applications without restart. + Simply point the profiler at a running process by PID and start collecting data. + +* **Flexible target modes**: + + * Profile running processes by PID (``attach``) - attach to already-running applications + * Run and profile scripts directly (``run``) - profile from the very start of execution + * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + * Capture a one-shot snapshot of a running process (``dump``) - print a + traceback-style stack of every thread (or all asyncio tasks with + ``--async-aware``). Useful for investigating hung processes. + +* **Multiple profiling modes**: Choose what to measure based on your performance investigation: + + * **Wall-clock time** (``--mode wall``, default): Measures real elapsed time including I/O, + network waits, and blocking operations. Use this to understand where your program spends + calendar time, including when waiting for external resources. + * **CPU time** (``--mode cpu``): Measures only active CPU execution time, excluding I/O waits + and blocking. Use this to identify CPU-bound bottlenecks and optimize computational work. + * **GIL-holding time** (``--mode gil``): Measures time spent holding Python's Global Interpreter + Lock. Use this to identify which threads dominate GIL usage in multi-threaded applications. + * **Exception handling time** (``--mode exception``): Captures samples only from threads with + an active exception. Use this to analyze exception handling overhead. + +* **Thread-aware profiling**: Option to profile all threads (``-a``) or just the main thread, + essential for understanding multi-threaded application behavior. + +* **Multiple output formats**: Choose the visualization that best fits your workflow: + + * ``--pstats``: Detailed tabular statistics compatible with :mod:`pstats`. Shows function-level + timing with direct and cumulative samples. Best for detailed analysis and integration with + existing Python profiling tools. + * ``--collapsed``: Generates collapsed stack traces (one line per stack). This format is + specifically designed for creating flame graphs with external tools like Brendan Gregg's + FlameGraph scripts or speedscope. + * ``--flamegraph``: Generates a self-contained interactive HTML flame graph using D3.js. + Opens directly in your browser for immediate visual analysis. Flame graphs show the call + hierarchy where width represents time spent, making it easy to spot bottlenecks at a glance. + * ``--gecko``: Generates Gecko Profiler format compatible with `Firefox Profiler + `__. Upload the output to Firefox Profiler for advanced + timeline-based analysis with features like stack charts, markers, and network activity. + * ``--heatmap``: Generates an interactive HTML heatmap visualization with line-level sample + counts. Creates a directory with per-file heatmaps showing exactly where time is spent + at the source code level. + +* **Live interactive mode**: Real-time TUI profiler with a top-like interface (``--live``). + Monitor performance as your application runs with interactive sorting and filtering. + +* **Async-aware profiling**: Profile async/await code with task-based stack reconstruction + (``--async-aware``). See which coroutines are consuming time, with options to show only + running tasks or all tasks including those waiting. + +* **Opcode-level profiling**: Gather bytecode opcode information for instruction-level + profiling (``--opcodes``). Shows which bytecode instructions are executing, including + specializations from the adaptive interpreter. + +See :mod:`profiling.sampling` for the complete documentation, including all +available output formats, profiling modes, and configuration options. + +(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) + + +.. _whatsnew315-pep831: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. -Profile process 1234 for 10 seconds with default settings: +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. -.. code-block:: shell +.. important:: - python -m profiling.sampling 1234 + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. -Profile with custom interval and duration, save to file: +.. seealso:: :pep:`831` for further details. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, and +Savannah Ostrowski.) + + +.. _whatsnew315-unpacking-in-comprehensions: + +:pep:`798`: Unpacking in Comprehensions +--------------------------------------- + +List, set, and dictionary comprehensions, as well as generator expressions, now +support unpacking with ``*`` and ``**``. This extends the unpacking syntax +from :pep:`448` to comprehensions, providing a new syntax for combining an +arbitrary number of iterables or dictionaries into a single flat structure. +This new syntax is a direct alternative to nested comprehensions, +:func:`itertools.chain`, and :meth:`itertools.chain.from_iterable`. For +example:: + + >>> lists = [[1, 2], [3, 4], [5]] + >>> [*L for L in lists] # equivalent to [x for L in lists for x in L] + [1, 2, 3, 4, 5] -.. code-block:: shell + >>> sets = [{1, 2}, {2, 3}, {3, 4}] + >>> {*s for s in sets} # equivalent to {x for s in sets for x in s} + {1, 2, 3, 4} + + >>> dicts = [{'a': 1}, {'b': 2}, {'a': 3}] + >>> {**d for d in dicts} # equivalent to {k: v for d in dicts for k,v in d.items()} + {'a': 3, 'b': 2} - python -m profiling.sampling -i 50 -d 30 -o profile.stats 1234 +Generator expressions can similarly use unpacking to yield values from multiple +iterables:: -Generate collapsed stacks for flamegraph: + >>> gen = (*L for L in lists) # equivalent to (x for L in lists for x in L) + >>> list(gen) + [1, 2, 3, 4, 5] -.. code-block:: shell +This change also extends to asynchronous generator expressions, such that, for +example, ``(*a async for a in agen())`` is equivalent to ``(x async for a in +agen() for x in a)``. - python -m profiling.sampling --collapsed 1234 +.. seealso:: :pep:`798` for further details. -Profile all threads and sort by total time: +(Contributed by Adam Hartz in :gh:`143055`.) -.. code-block:: shell +.. _whatsnew315-startup-files: - python -m profiling.sampling -a --sort-tottime 1234 +:pep:`829`: Package startup configuration files +----------------------------------------------- -The profiler generates statistical estimates of where time is spent: +Loaded by the :mod:`site` module when ``-S`` is not given, :ref:`.pth files ` +can contain lines that both extend :data:`sys.path` and execute arbitrary code +when the line starts with ``import`` (followed by a space or tab). The latter +functionality can be problematic, since it is difficult to know exactly what +gets executed when Python starts up. -.. code-block:: text +As a step towards improving the ability to audit pre-start executable code, +Python 3.15 introduces :ref:`.start files ` which contain entry point +specifications of the form ``pkg.mod:callable`` where ``pkg.mod`` is the +import path to the given callable. When Python starts up, the callable is +located and called with no arguments. - Real-time sampling stats: Mean: 100261.5Hz (9.97µs) Min: 86333.4Hz (11.58µs) Max: 118807.2Hz (8.42µs) Samples: 400001 - Captured 498841 samples in 5.00 seconds - Sample rate: 99768.04 samples/sec - Error rate: 0.72% - Profile Stats: - nsamples sample% tottime (s) cumul% cumtime (s) filename:lineno(function) - 43/418858 0.0 0.000 87.9 4.189 case.py:667(TestCase.run) - 3293/418812 0.7 0.033 87.9 4.188 case.py:613(TestCase._callTestMethod) - 158562/158562 33.3 1.586 33.3 1.586 test_compile.py:725(TestSpecifics.test_compiler_recursion_limit..check_limit) - 129553/129553 27.2 1.296 27.2 1.296 ast.py:46(parse) - 0/128129 0.0 0.000 26.9 1.281 test_ast.py:884(AST_Tests.test_ast_recursion_limit..check_limit) - 7/67446 0.0 0.000 14.2 0.674 test_compile.py:729(TestSpecifics.test_compiler_recursion_limit) - 6/60380 0.0 0.000 12.7 0.604 test_ast.py:888(AST_Tests.test_ast_recursion_limit) - 3/50020 0.0 0.000 10.5 0.500 test_compile.py:727(TestSpecifics.test_compiler_recursion_limit) - 1/38011 0.0 0.000 8.0 0.380 test_ast.py:886(AST_Tests.test_ast_recursion_limit) - 1/25076 0.0 0.000 5.3 0.251 test_compile.py:728(TestSpecifics.test_compiler_recursion_limit) - 22361/22362 4.7 0.224 4.7 0.224 test_compile.py:1368(TestSpecifics.test_big_dict_literal) - 4/18008 0.0 0.000 3.8 0.180 test_ast.py:889(AST_Tests.test_ast_recursion_limit) - 11/17696 0.0 0.000 3.7 0.177 subprocess.py:1038(Popen.__init__) - 16968/16968 3.6 0.170 3.6 0.170 subprocess.py:1900(Popen._execute_child) - 2/16941 0.0 0.000 3.6 0.169 test_compile.py:730(TestSpecifics.test_compiler_recursion_limit) +``import`` lines in :file:`.pth` files are silently deprecated. When a +matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files +are ignored. There is no change to :data:`sys.path` extension lines in +:file:`.pth` files. - Legend: - nsamples: Direct/Cumulative samples (direct executing / on call stack) - sample%: Percentage of total samples this function was directly executing - tottime: Estimated total time spent directly in this function - cumul%: Percentage of total samples when this function was on the call stack - cumtime: Estimated cumulative time (including time in called functions) - filename:lineno(function): Function location and name +(Contributed by Barry Warsaw in :gh:`148641`.) - Summary of Interesting Functions: - Functions with Highest Direct/Cumulative Ratio (Hot Spots): - 1.000 direct/cumulative ratio, 33.3% direct samples: test_compile.py:(TestSpecifics.test_compiler_recursion_limit..check_limit) - 1.000 direct/cumulative ratio, 27.2% direct samples: ast.py:(parse) - 1.000 direct/cumulative ratio, 3.6% direct samples: subprocess.py:(Popen._execute_child) +.. _whatsnew315-abi3t: - Functions with Highest Call Frequency (Indirect Calls): - 418815 indirect calls, 87.9% total stack presence: case.py:(TestCase.run) - 415519 indirect calls, 87.9% total stack presence: case.py:(TestCase._callTestMethod) - 159470 indirect calls, 33.5% total stack presence: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) +:pep:`803` -- Stable ABI for Free-Threaded Builds +------------------------------------------------------------ - Functions with Highest Call Magnification (Cumulative/Direct): - 12267.9x call magnification, 159470 indirect calls from 13 direct: test_compile.py:(TestSpecifics.test_compiler_recursion_limit) - 10581.7x call magnification, 116388 indirect calls from 11 direct: test_ast.py:(AST_Tests.test_ast_recursion_limit) - 9740.9x call magnification, 418815 indirect calls from 43 direct: case.py:(TestCase.run) +C extensions that target the :ref:`Stable ABI ` can now be +compiled for the new *Stable ABI for Free-Threaded Builds* (also known +as ``abi3t``), which makes them compatible with +:term:`free-threaded builds ` of CPython. +This usually requires some non-trivial changes to the source code; +specifically: -The profiler automatically identifies performance bottlenecks through statistical -analysis, highlighting functions with high CPU usage and call frequency patterns. +- Switching to API introduced in :pep:`697` (Python 3.12), such as + negative :c:member:`~PyType_Spec.basicsize` and + :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject` + part of the instance struct; and +- Switching from a ``PyInit_`` function to a new export hook, + :c:func:`PyModExport_* `, introduced for this + purpose in :pep:`793`, with a new :c:type:`PySlot` structure + introduced in :pep:`820`. -This capability is particularly valuable for debugging performance issues in -production systems where traditional profiling approaches would be too intrusive. +The reference documentation for these features is complete, but currently +aimed at early adopters. +A migration guide is planned for an upcoming beta release. - .. seealso:: :pep:`799` for further details. +Note that Stable ABI does not offer all the functionality that CPython +has to offer. +Extensions that cannot switch to ``abi3t`` should continue to build for +the existing Stable ABI (``abi3``) and the version-specific ABI for +free-threading (``cp315t``) separately. -(Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953`.) +Stable ABI for Free-Threaded Builds should typically +be selected in a build tool (such as, for example, Setuptools, meson-python, +scikit-build-core, or Maturin). +At the time of writing, these tools did **not** support ``abi3t``. +If this is the case for your tool, compile for ``cp315t`` separately. +If not using a build tool -- or when writing such a tool -- you can select +``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed +in :ref:`abi3-compiling`. .. _whatsnew315-improved-error-messages: @@ -220,13 +550,76 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ - AttributeError: 'Container' object has no attribute 'area'. Did you mean: 'inner.area'? + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ + AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? + +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) Other language changes @@ -259,28 +652,6 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) - -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? - - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) - * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) @@ -398,6 +769,33 @@ Other language changes for any class. (Contributed by Serhiy Storchaka in :gh:`41779`.) +* Allowed defining any :ref:`__slots__ ` for a class derived from + :class:`tuple` (including classes created by :func:`collections.namedtuple`). + (Contributed by Serhiy Storchaka in :gh:`41779`.) + +* The :class:`slice` type now supports subscription, + making it a :term:`generic type`. + (Contributed by James Hilton-Balfe in :gh:`128335`.) + +* The class :class:`memoryview` now supports the :c:expr:`float complex` and + :c:expr:`double complex` C types: formatting characters ``'Zf'`` and ``'Zd'`` + respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Allow the *count* argument of :meth:`bytes.replace` to be a keyword. + (Contributed by Stan Ulbrych in :gh:`147856`.) + +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + New modules =========== @@ -424,17 +822,144 @@ argparse default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. + (Contributed by Savannah Ostrowski in :gh:`142390`.) + +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + + +array +----- + +* Support the :c:expr:`float complex` and :c:expr:`double complex` C types: + formatting characters ``'Zf'`` and ``'Zd'`` respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) + +* Support half-floats (16-bit IEEE 754 binary interchange format): formatting + character ``'e'``. + (Contributed by Sergey B Kirpichev in :gh:`146238`.) + +* The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` + to support type codes longer than 1 character (``Zf`` and ``Zd``). + (Contributed by Victor Stinner in :gh:`148675`.) + + +ast +--- + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + + +base64 +------ + +* Added the *pad* parameter in :func:`~base64.z85encode`. + (Contributed by Hauke Dämpfling in :gh:`143103`.) + +* Added the *padded* parameter in + :func:`~base64.b32encode`, :func:`~base64.b32decode`, + :func:`~base64.b32hexencode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64encode`, :func:`~base64.b64decode`, + :func:`~base64.urlsafe_b64encode`, and :func:`~base64.urlsafe_b64decode`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~base64.b16encode`, + :func:`~base64.b32encode`, :func:`~base64.b32hexencode`, + :func:`~base64.b64encode`, :func:`~base64.b85encode`, and + :func:`~base64.z85encode`. + (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) + +* Added the *ignorechars* parameter in :func:`~base64.b16decode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, :func:`~base64.urlsafe_b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + +binascii +-------- + +* Added functions for Base32 encoding: + + - :func:`~binascii.b2a_base32` and :func:`~binascii.a2b_base32` + + (Contributed by James Seo in :gh:`146192`.) + +* Added functions for Ascii85, Base85, and Z85 encoding: + + - :func:`~binascii.b2a_ascii85` and :func:`~binascii.a2b_ascii85` + - :func:`~binascii.b2a_base85` and :func:`~binascii.a2b_base85` + + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Added the *padded* parameter in + :func:`~binascii.b2a_base32`, :func:`~binascii.a2b_base32`, + :func:`~binascii.b2a_base64`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Added the *wrapcol* parameter in :func:`~binascii.b2a_base64`. + (Contributed by Serhiy Storchaka in :gh:`143214`.) + +* Added the *alphabet* parameter in :func:`~binascii.b2a_base64` and + :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`145980`.) + +* Added the *ignorechars* parameter in :func:`~binascii.a2b_hex`, + :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. + (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) + +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + + calendar -------- -* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support - dark mode and have been migrated to the HTML5 standard for improved accessibility. - (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) +* :mod:`calendar`'s :ref:`command-line ` text output has more + color. This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148352`.) * The :mod:`calendar`'s :ref:`command-line ` HTML output now accepts the year-month option: ``python -m calendar -t html 2009 06``. (Contributed by Pål Grønås Drange in :gh:`140212`.) +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + collections ----------- @@ -444,25 +969,6 @@ collections between :class:`~collections.Counter` objects. (Contributed by Raymond Hettinger in :gh:`138682`.) -collections.abc ---------------- - -* :class:`collections.abc.ByteString` has been removed from - ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been - deprecated since Python 3.12, and is scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from collections.abc import ByteString`` - * ``import collections.abc; collections.abc.ByteString``. - - ``DeprecationWarning``\ s were already emitted if - :class:`collections.abc.ByteString` was subclassed or used as the second - argument to :func:`isinstance` or :func:`issubclass`, but warnings were not - previously emitted if it was merely imported or accessed from the - :mod:`!collections.abc` module. - concurrent.futures ------------------ @@ -474,6 +980,25 @@ concurrent.futures (Contributed by Jonathan Berg in :gh:`139486`.) +contextlib +---------- + +* Added support for arbitrary descriptors :meth:`!__enter__`, + :meth:`!__exit__`, :meth:`!__aenter__`, and :meth:`!__aexit__` in + :class:`~contextlib.ExitStack` and :class:`contextlib.AsyncExitStack`, for + consistency with the :keyword:`with` and :keyword:`async with` statements. + (Contributed by Serhiy Storchaka in :gh:`144386`.) + +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + + dataclasses ----------- @@ -489,7 +1014,6 @@ dbm (Contributed by Andrea Oliveri in :gh:`134004`.) - difflib ------- @@ -506,6 +1030,25 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + functools --------- @@ -513,6 +1056,21 @@ functools callables. (Contributed by Serhiy Storchaka in :gh:`140873`.) +* :func:`~functools.singledispatchmethod` now dispatches on the second argument + if it wraps a regular method and is called as a class attribute. + (Contributed by Bartosz Sławecki in :gh:`143535`.) + + +gc +-- + +* Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later and Python 3.15. + hashlib ------- @@ -535,11 +1093,30 @@ http.client (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) -http.cookies ------------- +http.server +----------- + +* The logging of :mod:`~http.server.BaseHTTPRequestHandler`, + as used by the :ref:`command-line interface `, + is colored by default. + This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`146292`.) -* Allow '``"``' double quotes in cookie values. - (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + +* Add a new ``extra_response_headers`` keyword argument to + :class:`~http.server.SimpleHTTPRequestHandler` to support custom headers in + HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + +* Add a ``-H/--header`` option to the :program:`python -m http.server` + command-line interface to support custom headers in HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) inspect @@ -549,6 +1126,17 @@ inspect for :func:`~inspect.getdoc`. (Contributed by Serhiy Storchaka in :gh:`132686`.) +json +---- + +* Add the *array_hook* parameter to :func:`~json.load` and + :func:`~json.loads` functions: + allow a callback for JSON literal array types to customize Python lists in + the resulting decoded object. Passing combined :class:`frozendict` to + *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a + deeply nested immutable Python structure representing the JSON data. + (Contributed by Joao S. O. Bueno in :gh:`146440`.) + locale ------ @@ -558,6 +1146,9 @@ locale but included in the language code. (Contributed by Serhiy Storchaka in :gh:`137729`.) +* Undeprecate the :func:`locale.getdefaultlocale` function. + (Contributed by Victor Stinner in :gh:`130796`.) + math ---- @@ -572,8 +1163,11 @@ math mimetypes --------- -* Add ``application/node`` MIME type for ``.cjs`` extension. (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -588,6 +1182,11 @@ mmap not be duplicated. (Contributed by Serhiy Storchaka in :gh:`78502`.) +* Added the :meth:`mmap.mmap.set_name` method + to annotate an anonymous memory mapping + if Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + (Contributed by Donghee Na in :gh:`142419`.) + os -- @@ -610,6 +1209,56 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + +pickle +------ + +* Add support for pickling private methods and nested classes. + (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) + + +pickletools +----------- + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* :mod:`pprint` now uses modern defaults: ``indent=4, width=88``, + and the default ``compact=False`` output is now formatted similar to + pretty-printed :func:`json.dumps`. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632` and :gh:`149189`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + +re +-- + +* :func:`re.prefixmatch` and a corresponding :meth:`re.Pattern.prefixmatch` + have been added as alternate, more explicit names for the existing + and now :term:`soft deprecated` + :func:`re.match` and :meth:`re.Pattern.match` APIs. These are intended + to be used to alleviate confusion around what *match* means by following the + Zen of Python's *"Explicit is better than implicit"* mantra. Most other + language regular expression libraries use an API named *match* to mean what + Python has always called *search*. + (Contributed by Gregory P. Smith in :gh:`86519`.) + + resource -------- @@ -625,6 +1274,9 @@ shelve * Added new :meth:`!reorganize` method to :mod:`shelve` used to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) +* Add support for custom serialization and deserialization functions + in the :mod:`shelve` module. + (Contributed by Furkan Onder in :gh:`99631`.) socket @@ -639,42 +1291,42 @@ sqlite3 * The :ref:`command-line interface ` has several new features: - * SQL keyword completion on . - (Contributed by Long Tan in :gh:`133393`.) + * SQL keyword completion on . + (Contributed by Long Tan in :gh:`133393`.) - * Prompts, error messages, and help text are now colored. - This is enabled by default, see :ref:`using-on-controlling-color` for - details. - (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) + * Prompts, error messages, and help text are now colored. + This is enabled by default, see :ref:`using-on-controlling-color` for + details. + (Contributed by Stan Ulbrych and Łukasz Langa in :gh:`133461`.) - * Table, index, trigger, view, column, function, and schema completion on . - (Contributed by Long Tan in :gh:`136101`.) + * Table, index, trigger, view, column, function, and schema completion on . + (Contributed by Long Tan in :gh:`136101`.) ssl --- * Indicate through :data:`ssl.HAS_PSK_TLS13` whether the :mod:`ssl` module - supports "External PSKs" in TLSv1.3, as described in RFC 9258. + supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. (Contributed by Will Childs-Klein in :gh:`133624`.) * Added new methods for managing groups used for SSL key agreement - * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing - key agreement, extending the previous - :meth:`ssl.SSLContext.set_ecdh_curve` method. - This new API provides the ability to list multiple groups and - supports fixed-field and post-quantum groups in addition to ECDH - curves. This method can also be used to control what key shares - are sent in the TLS handshake. - * :meth:`ssl.SSLSocket.group` returns the group selected for doing key - agreement on the current connection after the TLS handshake completes. - This call requires OpenSSL 3.2 or later. - * :meth:`ssl.SSLContext.get_groups` returns a list of all available key - agreement groups compatible with the minimum and maximum TLS versions - currently set in the context. This call requires OpenSSL 3.5 or later. - - (Contributed by Ron Frederick in :gh:`136306`.) + * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing + key agreement, extending the previous + :meth:`ssl.SSLContext.set_ecdh_curve` method. + This new API provides the ability to list multiple groups and + supports fixed-field and post-quantum groups in addition to ECDH + curves. This method can also be used to control what key shares + are sent in the TLS handshake. + * :meth:`ssl.SSLSocket.group` returns the group selected for doing key + agreement on the current connection after the TLS handshake completes. + This call requires OpenSSL 3.2 or later. + * :meth:`ssl.SSLContext.get_groups` returns a list of all available key + agreement groups compatible with the minimum and maximum TLS versions + currently set in the context. This call requires OpenSSL 3.5 or later. + + (Contributed by Ron Frederick in :gh:`136306`.) * Added a new method :meth:`ssl.SSLContext.set_ciphersuites` for setting TLS 1.3 ciphers. For TLS 1.2 or earlier, :meth:`ssl.SSLContext.set_ciphers` should @@ -698,7 +1350,30 @@ ssl selected for the server to complete the TLS handshake on the current connection. This call requires OpenSSL 3.5 or later. - (Contributed by Ron Frederick in :gh:`138252`.) + (Contributed by Ron Frederick in :gh:`138252`.) + + +subprocess +---------- + +* :meth:`subprocess.Popen.wait`: when ``timeout`` is not ``None`` and the + platform supports it, an efficient event-driven mechanism is used to wait for + process termination: + + - Linux >= 5.3 uses :func:`os.pidfd_open` + :func:`select.poll`. + - macOS and other BSD variants use :func:`select.kqueue` + ``KQ_FILTER_PROC`` + ``KQ_NOTE_EXIT``. + - Windows keeps using ``WaitForSingleObject`` (unchanged). + + If none of these mechanisms are available, the function falls back to the + traditional busy loop (non-blocking call and short sleeps). + (Contributed by Giampaolo Rodola in :gh:`83069`.) + + +symtable +-------- + +* Add :meth:`symtable.Function.get_cells` and :meth:`symtable.Symbol.is_cell` methods. + (Contributed by Yashp002 in :gh:`143504`.) sys @@ -708,6 +1383,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -734,14 +1422,33 @@ tarfile (Contributed by Christoph Walcher in :gh:`57911`.) +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) + + timeit ------ +* The output of the :mod:`timeit` command-line interface is colored by default. + This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`146609`.) * The command-line interface now colorizes error tracebacks by default. This can be controlled with :ref:`environment variables `. (Contributed by Yi Hong in :gh:`139374`.) +* Make the target time of :meth:`timeit.Timer.autorange` configurable + and add ``--target-time`` option to the command-line interface. + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) + + tkinter ------- @@ -749,15 +1456,82 @@ tkinter arguments: *nolinestop* which allows the search to continue across line boundaries; and *strictlimits* which restricts the search to within the specified range. - (Contributed by Rihaan Meher in :gh:`130848`) + (Contributed by Rihaan Meher in :gh:`130848`.) * A new method :meth:`!tkinter.Text.search_all` has been introduced. This method allows for searching for all matches of a pattern using Tcl's ``-all`` and ``-overlap`` options. - (Contributed by Rihaan Meher in :gh:`130848`) + (Contributed by Rihaan Meher in :gh:`130848`.) + +* Added new methods :meth:`!pack_content`, :meth:`!place_content` and + :meth:`!grid_content` which use Tk commands with new names (introduced + in Tk 8.6) instead of :meth:`!*_slaves` methods which use Tk commands + with outdated names. + (Contributed by Serhiy Storchaka in :gh:`143754`.) + +* Added :class:`!Event` attributes :attr:`!user_data` for Tk virtual events + and :attr:`!detail` for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut``, + and ``ConfigureRequest`` events. + (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) + + +tokenize +-------- + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + +.. _whatsnew315-tomllib-1-1-0: + +tomllib +------- + +* The :mod:`tomllib` module now supports TOML 1.1.0. + This is a backwards compatible update, meaning that all valid TOML 1.0.0 + documents are parsed the same way. + + The changes, according to the `official TOML changelog`_, are: + + - Allow newlines and trailing commas in inline tables. + + Previously an inline table had to be on a single line and couldn't end + with a trailing comma. This is now relaxed so that the following is valid: + + .. code-block:: toml + + tbl = { + key = "a string", + moar-tbl = { + key = 1, + }, + } + + - Add ``\xHH`` notation to basic strings for codepoints under 255, + and the ``\e`` escape for the escape character: + + .. code-block:: toml + + null = "null byte: \x00; letter a: \x61" + csi = "\e[" + + - Seconds in datetime and time values are now optional. + The following are now valid: + + .. code-block:: toml + + dt = 2010-02-03 14:15 + t = 14:15 + + (Contributed by Taneli Hukkinen in :gh:`142956`.) + +.. _official TOML changelog: https://github.com/toml-lang/toml/blob/main/CHANGELOG.md + types ------- +----- * Expose the write-through :func:`locals` proxy type as :data:`types.FrameLocalsProxyType`. @@ -765,6 +1539,61 @@ types as described in :pep:`667`. +typing +------ + +.. _whatsnew315-typeform: + +* :pep:`747`: Add :data:`~typing.TypeForm`, a new special form for annotating + values that are themselves type expressions. + ``TypeForm[T]`` means "a type form object describing ``T`` (or a type + assignable to ``T``)". At runtime, ``TypeForm(x)`` simply returns ``x``, + which allows explicit annotation of type-form values without changing + behavior. + + This helps libraries that accept user-provided type expressions + (for example ``int``, ``str | None``, :class:`~typing.TypedDict` + classes, or ``list[int]``) expose precise signatures: + + .. code-block:: python + + from typing import Any, TypeForm + + def cast[T](typ: TypeForm[T], value: Any) -> T: ... + + (Contributed by Jelle Zijlstra in :gh:`145033`.) + +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + +* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises + a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles + type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` + as it was incorrectly inferred in runtime before. + (Contributed by Nikita Sobolev in :gh:`137191`.) + +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + + unicodedata ----------- @@ -775,14 +1604,47 @@ unicodedata `Unicode Standard Annex #31 `_ identifier. (Contributed by Stan Ulbrych in :gh:`129117`.) +* Add the :func:`~unicodedata.iter_graphemes` + function to iterate over grapheme clusters according to rules defined in + `Unicode Standard Annex #29, "Unicode Text Segmentation" + `_. + Add :func:`~unicodedata.grapheme_cluster_break`, + :func:`~unicodedata.indic_conjunct_break` and + :func:`~unicodedata.extended_pictographic` functions to get the properties + of the character which are related to the above algorithm. + (Contributed by Serhiy Storchaka and Guillaume Sanchez in :gh:`74902`.) + +* Add :func:`~unicodedata.block` function to return the `Unicode block + `_ + assigned to a character. + (Contributed by Stan Ulbrych in :gh:`66802`.) + unittest -------- -* :func:`unittest.TestCase.assertLogs` will now accept a formatter +* :meth:`unittest.TestCase.assertLogs` will now accept a formatter to control how messages are formatted. (Contributed by Garry Cairns in :gh:`134567`.) +* :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex. + Nested context managers are now supported. + (Contributed by Serhiy Storchaka in :gh:`143231`.) + + +urllib.parse +------------ + +* Add the *missing_as_none* parameter to :func:`~urllib.parse.urlsplit`, + :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. + Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and + :func:`~urllib.parse.urlunparse` functions. + This allows to distinguish between empty and not defined URI components + and preserve empty components. + (Contributed by Serhiy Storchaka in :gh:`67041`.) + venv ---- @@ -807,6 +1669,33 @@ warnings (Contributed by Serhiy Storchaka in :gh:`135801`.) +wave +---- + +* Added support for IEEE floating-point WAVE audio + (``WAVE_FORMAT_IEEE_FLOAT``) in :mod:`wave`. + +* Added :meth:`wave.Wave_read.getformat`, :meth:`wave.Wave_write.getformat`, + and :meth:`wave.Wave_write.setformat` for explicit frame format handling. + +* :meth:`wave.Wave_write.setparams` accepts both 7-item tuples including + ``format`` and 6-item tuples for backwards compatibility (defaulting to + ``WAVE_FORMAT_PCM``). + +* ``WAVE_FORMAT_IEEE_FLOAT`` output now includes a ``fact`` chunk, + as required for non-PCM WAVE formats. + +(Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) + + +xml +--- + +* Add the :func:`xml.is_valid_name` function, which allows to check + whether a string can be used as an element or attribute name in XML. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + + xml.parsers.expat ----------------- @@ -840,6 +1729,29 @@ zlib Optimizations ============= +* ``mimalloc`` is now used as the default allocator for + raw memory allocations such as via :c:func:`PyMem_RawMalloc` + for better performance on :term:`free-threaded builds `. + (Contributed by Kumar Aditya in :gh:`144914`.) + + +base64 & binascii +----------------- + +* CPython's underlying base64 implementation now encodes 2x faster and decodes 3x + faster thanks to simple CPU pipelining optimizations. + (Contributed by Gregory P. Smith and Serhiy Storchaka in :gh:`143262`.) + +* Implementation for Ascii85, Base85, and Z85 encoding has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster and consumes + two orders of magnitude less memory. + (Contributed by James Seo and Serhiy Storchaka in :gh:`101178`.) + +* Implementation for Base32 has been rewritten in C. + Encoding and decoding is now two orders of magnitude faster. + (Contributed by James Seo in :gh:`146192`.) + + csv --- @@ -847,8 +1759,122 @@ csv (Contributed by Maurycy Pawłowski-Wieroński in :gh:`137628`.) +.. _whatsnew315-jit: + +Upgraded JIT compiler +--------------------- + +Results from the `pyperformance `__ +benchmark suite report +`8-9% `__ +geometric mean performance improvement for the JIT over the standard CPython +interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 +macOS, the JIT has a +`12-13% `__ +speedup over the :ref:`tail calling interpreter ` +with all optimizations enabled. The speedups for JIT +builds versus no JIT builds range from roughly 15% slowdown to over +100% speedup (ignoring the ``unpack_sequence`` microbenchmark) on +x86-64 Linux and AArch64 macOS systems. + +.. attention:: + These results are not yet final. + +The major upgrades to the JIT are: + +* LLVM 21 build-time dependency +* New tracing frontend +* Basic register allocation in the JIT +* More JIT optimizations +* Better machine code generation + +.. rubric:: LLVM 21 build-time dependency + +The JIT compiler now uses LLVM 21 for build-time stencil generation. As +always, LLVM is only needed when building CPython with the JIT enabled; +end users running Python do not need LLVM installed. Instructions for +installing LLVM can be found in the `JIT compiler documentation +`__ +for all supported platforms. +(Contributed by Savannah Ostrowski in :gh:`140973`.) + +.. rubric:: A new tracing frontend + +The JIT compiler now supports significantly more bytecode operations and +control flow than in Python 3.14, enabling speedups on a wider variety of +code. For example, simple Python object creation is now understood by the +3.15 JIT compiler. Overloaded operations and generators are also partially +supported. This was made possible by an overhauled JIT tracing frontend +that records actual execution paths through code, rather than estimating +them as the previous implementation did. +(Contributed by Ken Jin in :gh:`139109`. Support for Windows added by +Mark Shannon in :gh:`141703`.) + +.. rubric:: Basic register allocation in the JIT + +A basic form of register allocation has been added to the JIT compiler's +optimizer. This allows the JIT compiler to avoid certain stack operations +altogether and instead operate on registers. This allows the JIT to produce +more efficient traces by avoiding reads and writes to memory. +(Contributed by Mark Shannon in :gh:`135379`.) + +.. rubric:: More JIT optimizations + +More `constant-propagation `__ +is now performed. This means when the JIT compiler detects that certain user +code results in constants, the code can be simplified by the JIT. +(Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.) + +:term:`Reference count`\ s are avoided whenever it is safe to do so. This generally +reduces the cost of most operations in Python. +(Contributed by Ken Jin, Donghee Na, Zheao Li, Hai Zhu, Savannah Ostrowski, +Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) + +By tracking unique references to objects, the JIT optimizer can now eliminate +reference count updates and perform inplace operations on ints and floats. +(Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) + +The JIT optimizer now supports significantly more operations than in 3.14. +(Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) + +.. rubric:: Better machine code generation + +The JIT compiler's machine code generator now produces better machine code +for x86-64 and AArch64 macOS and Linux targets. In general, users should +experience lower memory usage for generated machine code and more efficient +machine code versus 3.14. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. +Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. +Additional optimizations for AArch64 contributed by Mark Shannon and +Diego Russo in :gh:`140683` and :gh:`142305`.) + +.. rubric:: Maintainability + +The JIT optimizer's operations have been simplified. +This was made possible by a refactoring of JIT data structures. +(Contributed by Zhongtian Zheng in :gh:`148211` and Hai Zhu in :gh:`143421`.) + + Removed -======= +======== + +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + +collections.abc +--------------- + +* :class:`collections.abc.ByteString` has been removed from + ``collections.abc.__all__``. :class:`!collections.abc.ByteString` has been + deprecated since Python 3.12, and is scheduled for removal in Python 3.17. + ctypes ------ @@ -857,6 +1883,21 @@ ctypes which has been deprecated since Python 3.13. (Contributed by Bénédikt Tran in :gh:`133866`.) +* Change the :py:attr:`~ctypes._SimpleCData._type_` of + :class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and + :class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, + ``Zd`` and ``Zg`` for compatibility with numpy. + (Contributed by Victor Stinner in :gh:`148675`.) + + +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + glob ---- @@ -881,7 +1922,7 @@ importlib.resources * Removed deprecated ``package`` parameter from :func:`importlib.resources.files` function. - (Contributed by Semyon Moroz in :gh:`138044`) + (Contributed by Semyon Moroz in :gh:`138044`.) pathlib @@ -923,9 +1964,21 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed deprecated in :pep:`626` since Python 3.12 + :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ +* :class:`typing.ByteString` has been removed from ``typing.__all__``. + :class:`!typing.ByteString` has been deprecated since Python 3.9, and is + scheduled for removal in Python 3.17. + * The undocumented keyword argument syntax for creating :class:`~typing.NamedTuple` classes (for example, ``Point = NamedTuple("Point", x=int, y=int)``) is no longer supported. @@ -938,30 +1991,6 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) -* Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises - a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles - type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly inferred in runtime before. - (Contributed by Nikita Sobolev in :gh:`137191`.) - -* :class:`typing.ByteString` has been removed from ``typing.__all__``. - :class:`!typing.ByteString` has been deprecated since Python 3.9, and is - scheduled for removal in Python 3.17. - -* The following statements now cause ``DeprecationWarning``\ s to be emitted at - runtime: - - * ``from typing import ByteString`` - * ``import typing; typing.ByteString``. - - ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` - was subclassed or used as the second argument to :func:`isinstance` or - :func:`issubclass`, but warnings were not previously emitted if it was merely - imported or accessed from the :mod:`!typing` module. - * Deprecated :func:`!typing.no_type_check_decorator` has been removed. (Contributed by Nikita Sobolev in :gh:`133601`.) @@ -989,6 +2018,22 @@ Deprecated New deprecations ---------------- +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + +* :mod:`base64`: + + * Accepting the ``+`` and ``/`` characters with an alternative alphabet in + :func:`~base64.b64decode` and :func:`~base64.urlsafe_b64decode` is now + deprecated. + In future Python versions they will be errors in the strict mode and + discarded in the non-strict mode. + (Contributed by Serhiy Storchaka in :gh:`125346`.) + * CLI: * Deprecate :option:`-b` and :option:`!-bb` command-line options @@ -999,6 +2044,21 @@ New deprecations (Contributed by Nikita Sobolev in :gh:`136355`.) +* :mod:`collections.abc` + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from collections.abc import ByteString`` + * ``import collections.abc; collections.abc.ByteString``. + + ``DeprecationWarning``\ s were already emitted if + :class:`collections.abc.ByteString` was subclassed or used as the second + argument to :func:`isinstance` or :func:`issubclass`, but warnings were not + previously emitted if it was merely imported or accessed from the + :mod:`!collections.abc` module. + + * :mod:`hashlib`: * In hash function constructors such as :func:`~hashlib.new` or the @@ -1013,22 +2073,79 @@ New deprecations (Contributed by Bénédikt Tran in :gh:`134978`.) + +* :mod:`http.cookies`: + + * :meth:`Morsel.js_output ` and + :meth:`BaseCookie.js_output ` are + deprecated and will be removed in Python 3.19. Use + :meth:`Morsel.output ` or + :meth:`BaseCookie.output ` instead. + (Contributed by kishorhange111 in :gh:`148849`.) + + * :mod:`imaplib`: * Altering :attr:`IMAP4.file ` is now deprecated and slated for removal in Python 3.19. This property is now unused and changing its value does *not* explicitly close the current file. + +* :mod:`re`: + + * :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) + + +* :mod:`struct`: + + * Calling the ``Struct.__new__()`` without required argument now is + deprecated and will be removed in Python 3.20. Calling + :meth:`~object.__init__` method on initialized :class:`~struct.Struct` + objects is deprecated and will be removed in Python 3.20. + + (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + +* :mod:`typing`: + + * The following statements now cause ``DeprecationWarning``\ s to be emitted + at runtime: + + * ``from typing import ByteString`` + * ``import typing; typing.ByteString``. + + ``DeprecationWarning``\ s were already emitted if :class:`typing.ByteString` + was subclassed or used as the second argument to :func:`isinstance` or + :func:`issubclass`, but warnings were not previously emitted if it was + merely imported or accessed from the :mod:`!typing` module. + + * ``__version__`` - * The ``__version__`` attribute has been deprecated in these standard library - modules and will be removed in Python 3.20. - Use :py:data:`sys.version_info` instead. + * The ``__version__``, ``version`` and ``VERSION`` attributes have been + deprecated in these standard library modules and will be removed in + Python 3.20. Use :py:data:`sys.version_info` instead. - :mod:`argparse` - :mod:`csv` + - :mod:`ctypes` - :mod:`!ctypes.macholib` - :mod:`decimal` (use :data:`decimal.SPEC_VERSION` instead) + - :mod:`http.server` - :mod:`imaplib` - :mod:`ipaddress` - :mod:`json` @@ -1039,8 +2156,13 @@ New deprecations - :mod:`re` - :mod:`socketserver` - :mod:`tabnanny` + - :mod:`tarfile` - :mod:`tkinter.font` - :mod:`tkinter.ttk` + - :mod:`wsgiref.simple_server` + - :mod:`xml.etree.ElementTree` + - :mod:`!xml.sax.expatreader` + - :mod:`xml.sax.handler` - :mod:`zlib` (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) @@ -1051,12 +2173,16 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + .. include:: ../deprecations/pending-removal-in-3.19.rst .. include:: ../deprecations/pending-removal-in-3.20.rst .. include:: ../deprecations/pending-removal-in-future.rst +.. include:: ../deprecations/soft-deprecations.rst + C API changes ============= @@ -1064,6 +2190,25 @@ C API changes New features ------------ +* Add :c:func:`PyArg_ParseArray` and :c:func:`PyArg_ParseArrayAndKeywords` + functions to parse arguments of functions using the :c:macro:`METH_FASTCALL` + calling convention. + (Contributed by Victor Stinner in :gh:`144175`.) + +* Add the following functions for the new :class:`frozendict` type: + + * :c:func:`PyAnyDict_Check` + * :c:func:`PyAnyDict_CheckExact` + * :c:func:`PyFrozenDict_Check` + * :c:func:`PyFrozenDict_CheckExact` + * :c:func:`PyFrozenDict_New` + + (Contributed by Victor Stinner in :gh:`141510`.) + +* Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C + API. + (Contributed by Victor Stinner in :gh:`146063`.) + * Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` functions as replacements for :c:func:`PySys_GetObject`. @@ -1078,7 +2223,7 @@ New features and :c:data:`Py_mod_abi`. (Contributed by Petr Viktorin in :gh:`137210`.) -.. _whatsnew315-pep782: +.. _whatsnew315-pybyteswriter: * Implement :pep:`782`, the :ref:`PyBytesWriter API `. Add functions: @@ -1098,6 +2243,11 @@ New features (Contributed by Victor Stinner in :gh:`129813`.) +* :c:type:`PyCriticalSection` and related functions are added to the Stable + ABI. + + (Contributed in :gh:`149227`.) + * Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a module from a *spec* and *initfunc*. (Contributed by Itamar Oren in :gh:`116146`.) @@ -1105,24 +2255,84 @@ New features * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) -* Add :c:func:`PyUnstable_Object_Dump` to dump an object to ``stderr``. +* Add a new module export hook, + :c:func:`PyModExport_* `. + + (Contributed by Petr Viktorin in :pep:`793` and :gh:`140550`.) + +* Add functions that are guaranteed to be safe for use in + :c:member:`~PyTypeObject.tp_traverse` handlers: + :c:func:`PyObject_GetTypeData_DuringGC`, + :c:func:`PyObject_GetItemData_DuringGC`, + :c:func:`PyType_GetModuleState_DuringGC`, + :c:func:`PyModule_GetState_DuringGC`, :c:func:`PyModule_GetToken_DuringGC`, + :c:func:`PyType_GetBaseByToken_DuringGC`, + :c:func:`PyType_GetModule_DuringGC`, + :c:func:`PyType_GetModuleByToken_DuringGC`. + (Contributed by Petr Viktorin in :gh:`145925`.) + +* Add :c:func:`PyObject_Dump` to dump an object to ``stderr``. It should only be used for debugging. (Contributed by Victor Stinner in :gh:`141070`.) +* Implement :pep:`820`: ``PySlot`` -- Unified slot system for the C API. + See :ref:`capi-slots` for documentation. + + This adds: + + * The :c:type:`PySlot` struct; + * the :c:func:`PyType_FromSlots` function; + * new slot IDs: :c:macro:`Py_slot_end`, :c:macro:`Py_slot_invalid`; + :c:macro:`Py_slot_subslots`, :c:macro:`Py_tp_slots` + :c:macro:`Py_mod_slots`; + :c:macro:`Py_tp_name`, :c:macro:`Py_tp_basicsize`, + :c:macro:`Py_tp_extra_basicsize`, :c:macro:`Py_tp_itemsize`, + :c:macro:`Py_tp_flags`, :c:macro:`Py_tp_metaclass`, + :c:macro:`Py_tp_module`, :c:macro:`Py_tp_flags`; + * convenience macros: :c:macro:`PySlot_DATA`, :c:macro:`PySlot_FUNC`, + :c:macro:`PySlot_SIZE` :c:macro:`PySlot_INT64`, :c:macro:`PySlot_UINT64`, + :c:macro:`PySlot_STATIC_DATA`, :c:macro:`PySlot_END`, + :c:macro:`PySlot_PTR`, :c:macro:`PySlot_PTR_STATIC`. + + The :c:func:`PyModule_FromSlotsAndSpec` function and + ``PyModExport`` :ref:`module export hook ` also + use the new :c:type:`!PySlot` struct. + + These following functions are :term:`soft deprecated`: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + * :c:func:`PyType_FromMetaclass` + * :c:func:`PyModule_FromDefAndSpec` + * :c:func:`PyModule_FromDefAndSpec2` + * :c:func:`PyModule_ExecDef` + + (Contributed by Petr Viktorin in :gh:`149044`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add :c:func:`PyUnstable_SetImmortal` C-API function to mark objects as :term:`immortal`. + (Contributed by Kumar Aditya in :gh:`143300`.) + +* Restore private provisional ``_Py_InitializeMain()`` function removed in + Python 3.14. + (Contributed by Victor Stinner in :gh:`142417`.) Changed C APIs -------------- * If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` or :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` flag is set then :c:macro:`Py_TPFLAGS_HAVE_GC` must be set too. - (Contributed by Sergey Miryanov in :gh:`134786`) + (Contributed by Sergey Miryanov in :gh:`134786`.) +* :c:macro:`PyDateTime_IMPORT` is now thread safe. Code that directly checks ``PyDateTimeAPI`` + for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. + (Contributed by Kumar Aditya in :gh:`141563`.) Porting to Python 3.15 ---------------------- @@ -1206,6 +2416,16 @@ on Python 3.13 and older. Deprecated C APIs ----------------- +* Deprecate :pep:`456` support for providing an external definition + of the string hashing scheme. Removal is scheduled for Python 3.19. + + Previously, embedders could define :c:macro:`Py_HASH_ALGORITHM` to be + ``Py_HASH_EXTERNAL`` to indicate that the hashing scheme was provided + externally but this feature was undocumented, untested and most likely + unused. + + (Contributed by Bénédikt Tran in :gh:`141226`.) + * For unsigned integer formats in :c:func:`PyArg_ParseTuple`, accepting Python integers with value that is larger than the maximal value for the C type or less than the minimal value for the corresponding @@ -1217,6 +2437,13 @@ Deprecated C APIs use the :c:type:`PyBytesWriter` API instead. (Contributed by Victor Stinner in :gh:`129813`.) +* :c:func:`!_PyObject_CallMethodId`, :c:func:`!_PyObject_GetAttrId` and + :c:func:`!_PyUnicode_FromId` are deprecated since 3.15 and will be removed in + 3.20. Instead, use :c:func:`PyUnicode_InternFromString()` and cache the result in + the module state, then call :c:func:`PyObject_CallMethod` or + :c:func:`PyObject_GetAttr`. + (Contributed by Victor Stinner in :gh:`141049`.) + * Deprecate :c:member:`~PyComplexObject.cval` field of the :c:type:`PyComplexObject` type. Use :c:func:`PyComplex_AsCComplex` and :c:func:`PyComplex_FromCComplex` @@ -1237,6 +2464,24 @@ Deprecated C APIs use the C11 standard ```` :c:macro:`!INFINITY` instead. (Contributed by Sergey B Kirpichev in :gh:`141004`.) +* The following macros are :term:`soft deprecated`: + + - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. + - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. + - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, + :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, + :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: + Use C99 types/limits. + - :c:macro:`Py_UNICODE_SIZE`: Use ``sizeof(wchar_t)`` directly. + - :c:macro:`Py_VA_COPY`: Use ``va_copy`` directly. + + The macro :c:macro:`Py_UNICODE_WIDE`, which was scheduled for removal, + is :term:`soft deprecated` instead. + + (Contributed by Petr Viktorin in :gh:`146175`.) + * :c:macro:`!Py_MATH_El` and :c:macro:`!Py_MATH_PIl` are deprecated since 3.15 and will be removed in 3.20. (Contributed by Sergey B Kirpichev in :gh:`141004`.) @@ -1259,6 +2504,37 @@ Build changes modules that are missing or packaged separately. (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.) +* The new configure option :option:`--with-pymalloc-hugepages` enables huge + page support for :ref:`pymalloc ` arenas. When enabled, arena size + increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or + ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages. + On Windows, use ``build.bat --pymalloc-hugepages``. + At runtime, huge pages must be explicitly enabled by setting the + :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable to ``1``. + +* Annotating anonymous mmap usage is now supported if Linux kernel supports + :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). + Annotations are visible in ``/proc//maps`` if the kernel supports the feature + and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + (Contributed by Donghee Na in :gh:`141770`.) + +.. _whatsnew315-windows-tail-calling-interpreter: + +* 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new + :ref:`tail-calling interpreter `. + Results on Visual Studio 18.1.1 report between + `15-20% `__ + speedup on the geometric mean of pyperformance on Windows x86-64 over + the switch-case interpreter on an AMD Ryzen 7 5800X. We have + observed speedups ranging from 14% for large pure-Python libraries + to 40% for long-running small pure-Python scripts on Windows. + This was made possible by a new feature introduced in MSVC 18, + which the official Windows 64-bit binaries on python.org__ now use. + (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`. + Special thanks to Steve Dower, and the MSVC team including Hulon Jenkins.) + + __ https://www.python.org/downloads/windows/ + Porting to Python 3.15 ====================== @@ -1283,7 +2559,7 @@ that may require changes to your code. :func:`resource.setrlimit` and :func:`resource.prlimit` is now deprecated. (Contributed by Serhiy Storchaka in :gh:`137044`.) -* :meth:`~mmap.mmap.resize` has been removed on platforms that don't support the +* :meth:`mmap.mmap.resize` has been removed on platforms that don't support the underlying syscall, instead of raising a :exc:`SystemError`. * A resource warning is now emitted for an unclosed @@ -1298,3 +2574,17 @@ that may require changes to your code. *dest* is now ``'foo'`` instead of ``'f'``. Pass an explicit *dest* argument to preserve the old behavior. (Contributed by Serhiy Storchaka in :gh:`138697`.) + +* Padding of input no longer required in :func:`base64.urlsafe_b64decode`. + Pass a new argument ``padded=True`` or use :func:`base64.b64decode` + with argument ``altchars=b'-_'`` (this works with older Python versions) + to make padding required. + (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Since :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex, your tests may start leaking + some warnings that were previously masked. + Use warning filters to silence them or additional :meth:`!assertWarns*` + to catch and check them. + (Contributed by Serhiy Storchaka in :gh:`143231`.)