From 9d07159cda9390c5714d46447ca918bae9fb8dcb Mon Sep 17 00:00:00 2001 From: Stan Ulbrych Date: Thu, 7 May 2026 20:06:37 +0100 Subject: [PATCH 1/6] gh-149499: Fixes for 3.16 bump (GH-149500) Also fixes gh-149507, regenerating `configure` for 3.16. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Zachary Ware --- Doc/whatsnew/3.16.rst | 10 +++-- Include/internal/pycore_magic_number.h | 5 ++- Lib/sysconfig/__init__.py | 38 ------------------- ...-05-07-17-35-47.gh-issue-149499.G2ER-R.rst | 3 ++ PC/launcher.c | 1 + PC/pyconfig.h | 8 ++-- configure | 26 ++++++------- 7 files changed, 30 insertions(+), 61 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 5931586d5a1d9c..cc88a694608233 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -106,10 +106,13 @@ module_name Removed ======= -module_name ------------ +sysconfig +--------- + +* The :func:`!sysconfig.expand_makefile_vars` function + which has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. -* TODO .. Add removals above alphabetically, not here at the end. @@ -156,4 +159,3 @@ Deprecated C APIs Removed C APIs -------------- - diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 177938e3cdb5eb..0f1af8ba338865 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -297,9 +297,10 @@ Known values: Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations) + Python 3.16a0 3700 (Initial version) - Python 3.16 will start with 3700 + Python 3.17 will start with 3750 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out @@ -310,7 +311,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3666 +#define PYC_MAGIC_NUMBER 3700 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 47415adce04c2c..298256a5b23a9c 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -753,41 +753,3 @@ def get_python_version(): def _get_python_version_abi(): return _PY_VERSION_SHORT + get_config_var("abi_thread") - - -def expand_makefile_vars(s, vars): - """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in - 'string' according to 'vars' (a dictionary mapping variable names to - values). Variables not present in 'vars' are silently expanded to the - empty string. The variable values in 'vars' should not contain further - variable expansions; if 'vars' is the output of 'parse_makefile()', - you're fine. Returns a variable-expanded version of 's'. - """ - - import warnings - warnings.warn( - 'sysconfig.expand_makefile_vars is deprecated and will be removed in ' - 'Python 3.16. Use sysconfig.get_paths(vars=...) instead.', - DeprecationWarning, - stacklevel=2, - ) - - import re - - _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" - _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - - # This algorithm does multiple expansion, so if vars['foo'] contains - # "${bar}", it will expand ${foo} to ${bar}, and then expand - # ${bar}... and so forth. This is fine as long as 'vars' comes from - # 'parse_makefile()', which takes care of such expansions eagerly, - # according to make's variable expansion semantics. - - while True: - m = re.search(_findvar1_rx, s) or re.search(_findvar2_rx, s) - if m: - (beg, end) = m.span() - s = s[0:beg] + vars.get(m.group(1)) + s[end:] - else: - break - return s diff --git a/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst b/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst new file mode 100644 index 00000000000000..796d39f5ba05bf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-07-17-35-47.gh-issue-149499.G2ER-R.rst @@ -0,0 +1,3 @@ +Removed the :func:`!sysconfig.expand_makefile_vars` function which has been +deprecated since Python 3.14. Use the ``vars`` argument of +:func:`sysconfig.get_paths` instead. diff --git a/PC/launcher.c b/PC/launcher.c index fed5e156b92cb3..5667fc851aa590 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1273,6 +1273,7 @@ static PYC_MAGIC magic_values[] = { { 3550, 3599, L"3.13" }, { 3600, 3649, L"3.14" }, { 3650, 3699, L"3.15" }, + { 3700, 3749, L"3.16" }, { 0 } }; diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 72a475777b7ad0..2381ed3772b109 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -330,21 +330,21 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ the linking is explicitly handled */ # if defined(Py_GIL_DISABLED) # if defined(Py_DEBUG) -# pragma comment(lib,"python315t_d.lib") +# pragma comment(lib,"python316t_d.lib") # elif defined(Py_LIMITED_API) || defined(Py_TARGET_ABI3T) # pragma comment(lib,"python3t.lib") # else -# pragma comment(lib,"python315t.lib") +# pragma comment(lib,"python316t.lib") # endif /* Py_DEBUG */ # else /* Py_GIL_DISABLED */ # if defined(Py_DEBUG) -# pragma comment(lib,"python315_d.lib") +# pragma comment(lib,"python316_d.lib") # elif defined(Py_TARGET_ABI3T) # pragma comment(lib,"python3t.lib") # elif defined(Py_LIMITED_API) # pragma comment(lib,"python3.lib") # else -# pragma comment(lib,"python315.lib") +# pragma comment(lib,"python316.lib") # endif /* Py_DEBUG */ # endif /* Py_GIL_DISABLED */ # endif /* _MSC_VER && !Py_NO_LINK_LIB */ diff --git a/configure b/configure index cff7dfbfba8b9a..ecdd6095669ddc 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.72 for python 3.15. +# Generated by GNU Autoconf 2.72 for python 3.16. # # Report bugs to . # @@ -604,8 +604,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='python' PACKAGE_TARNAME='python' -PACKAGE_VERSION='3.15' -PACKAGE_STRING='python 3.15' +PACKAGE_VERSION='3.16' +PACKAGE_STRING='python 3.16' PACKAGE_BUGREPORT='https://github.com/python/cpython/issues/' PACKAGE_URL='' @@ -1750,7 +1750,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -'configure' configures python 3.15 to adapt to many kinds of systems. +'configure' configures python 3.16 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1816,7 +1816,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of python 3.15:";; + short | recursive ) echo "Configuration of python 3.16:";; esac cat <<\_ACEOF @@ -1875,9 +1875,9 @@ Optional Features: Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) - --with-build-python=python3.15 + --with-build-python=python3.16 path to build python binary for cross compiling - (default: _bootstrap_python or python3.15) + (default: _bootstrap_python or python3.16) --with-pkg-config=[yes|no|check] use pkg-config to detect build options (default is check) @@ -2133,7 +2133,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -python configure 3.15 +python configure 3.16 generated by GNU Autoconf 2.72 Copyright (C) 2023 Free Software Foundation, Inc. @@ -2811,7 +2811,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by python $as_me 3.15, which was +It was created by python $as_me 3.16, which was generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -3846,7 +3846,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.16 python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -3922,7 +3922,7 @@ rm confdefs.h mv confdefs.h.new confdefs.h -VERSION=3.15 +VERSION=3.16 # Version number of Python's own shared library file. @@ -35911,7 +35911,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by python $as_me 3.15, which was +This file was extended by python $as_me 3.16, which was generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -35975,7 +35975,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -python config.status 3.15 +python config.status 3.16 configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" From fbba343622c9e4a38c8ef0f0b0e311164394d76a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 7 May 2026 20:06:53 +0100 Subject: [PATCH 2/6] gh-106693: Explicitly mark ob_sval as unsigned char to avoid UB (#106826) Signed-off-by: Pablo Galindo --- Include/cpython/bytesobject.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 85bc2b827df8fb..550b5fcb952e68 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -5,7 +5,7 @@ typedef struct { PyObject_VAR_HEAD Py_DEPRECATED(3.11) Py_hash_t ob_shash; - char ob_sval[1]; + unsigned char ob_sval[1]; /* Invariants: * ob_sval contains space for 'ob_size+1' elements. @@ -20,7 +20,7 @@ PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); #define _PyBytes_CAST(op) \ (assert(PyBytes_Check(op)), _Py_CAST(PyBytesObject*, op)) -static inline char* PyBytes_AS_STRING(PyObject *op) +static inline unsigned char* PyBytes_AS_STRING(PyObject *op) { return _PyBytes_CAST(op)->ob_sval; } From 310fe88994249a5a02e20d1211b8fc067e34aa78 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 7 May 2026 22:06:57 +0300 Subject: [PATCH 3/6] gh-79638: Treat an unreachable robots.txt as "disallow all" (GH-138555) Disallow all access in urllib.robotparser if the robots.txt file is unreachable due to server or network errors. --- Lib/test/test_robotparser.py | 66 ++++++++++++++----- Lib/urllib/robotparser.py | 10 ++- ...5-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst | 2 + 3 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 3ea0ec66fbfbe9..65bfe815705e0a 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -646,26 +646,23 @@ def test_group_without_user_agent(self): ) class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() - - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) SAMPLE_ROBOTS_TXT = b'''\ @@ -687,7 +684,6 @@ def do_GET(self): def log_message(self, format, *args): pass - @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -709,17 +705,21 @@ def testRead(self): self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(403, "Forbidden access") + self.send_error(self.server.return_code) def log_message(self, format, *args): pass - @threading_helper.reap_threads + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + def testPasswordProtectedSite(self): + self.server.return_code = 403 addr = self.server.server_address url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) robots_url = url + "/robots.txt" @@ -727,6 +727,40 @@ def testPasswordProtectedSite(self): parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) + + def testServiceUnavailable(self): + self.server.return_code = 503 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index e70eae80036784..0c3e5d92890935 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,9 +65,17 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 00000000000000..bd9fff0bc2e31b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors. From f0daba1652cbf2eb04feaf21f9c913023f286e7e Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Thu, 7 May 2026 23:39:08 +0300 Subject: [PATCH 4/6] gh-106693: Revert "Explicitly mark ob_sval as unsigned char to avoid UB (#106826)" (#149514) --- Include/cpython/bytesobject.h | 4 +- Lib/test/test_robotparser.py | 66 +++++-------------- Lib/urllib/robotparser.py | 10 +-- ...5-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst | 2 - 4 files changed, 19 insertions(+), 63 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst diff --git a/Include/cpython/bytesobject.h b/Include/cpython/bytesobject.h index 550b5fcb952e68..85bc2b827df8fb 100644 --- a/Include/cpython/bytesobject.h +++ b/Include/cpython/bytesobject.h @@ -5,7 +5,7 @@ typedef struct { PyObject_VAR_HEAD Py_DEPRECATED(3.11) Py_hash_t ob_shash; - unsigned char ob_sval[1]; + char ob_sval[1]; /* Invariants: * ob_sval contains space for 'ob_size+1' elements. @@ -20,7 +20,7 @@ PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); #define _PyBytes_CAST(op) \ (assert(PyBytes_Check(op)), _Py_CAST(PyBytesObject*, op)) -static inline unsigned char* PyBytes_AS_STRING(PyObject *op) +static inline char* PyBytes_AS_STRING(PyObject *op) { return _PyBytes_CAST(op)->ob_sval; } diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 65bfe815705e0a..3ea0ec66fbfbe9 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -646,23 +646,26 @@ def test_group_without_user_agent(self): ) class BaseLocalNetworkTestCase: - @classmethod - def setUpClass(cls): + def setUp(self): # clear _opener global variable - cls.addClassCleanup(urllib.request.urlcleanup) + self.addCleanup(urllib.request.urlcleanup) - cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) - cls.addClassCleanup(cls.server.server_close) + self.server = HTTPServer((socket_helper.HOST, 0), self.RobotHandler) - t = threading.Thread( + self.t = threading.Thread( name='HTTPServer serving', - target=cls.server.serve_forever, + target=self.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - cls.enterClassContext(threading_helper.start_threads([t])) - cls.addClassCleanup(cls.server.shutdown) + self.t.daemon = True # In case this function raises. + self.t.start() + + def tearDown(self): + self.server.shutdown() + self.t.join() + self.server.server_close() SAMPLE_ROBOTS_TXT = b'''\ @@ -684,6 +687,7 @@ def do_GET(self): def log_message(self, format, *args): pass + @threading_helper.reap_threads def testRead(self): # Test that reading a weird robots.txt doesn't fail. addr = self.server.server_address @@ -705,21 +709,17 @@ def testRead(self): self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) -class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): +class PasswordProtectedSiteTestCase(BaseLocalNetworkTestCase, unittest.TestCase): class RobotHandler(BaseHTTPRequestHandler): def do_GET(self): - self.send_error(self.server.return_code) + self.send_error(403, "Forbidden access") def log_message(self, format, *args): pass - def setUp(self): - # Make sure that a valid code is set in the test. - self.server.return_code = None - + @threading_helper.reap_threads def testPasswordProtectedSite(self): - self.server.return_code = 403 addr = self.server.server_address url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) robots_url = url + "/robots.txt" @@ -727,40 +727,6 @@ def testPasswordProtectedSite(self): parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) - self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) - - def testNotFound(self): - self.server.return_code = 404 - addr = self.server.server_address - url = f'http://{socket_helper.HOST}:{addr[1]}' - robots_url = url + "/robots.txt" - parser = urllib.robotparser.RobotFileParser() - parser.set_url(url) - parser.read() - self.assertTrue(parser.can_fetch("*", robots_url)) - self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) - - def testTeapot(self): - self.server.return_code = 418 - addr = self.server.server_address - url = f'http://{socket_helper.HOST}:{addr[1]}' - robots_url = url + "/robots.txt" - parser = urllib.robotparser.RobotFileParser() - parser.set_url(url) - parser.read() - self.assertTrue(parser.can_fetch("*", robots_url)) - self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) - - def testServiceUnavailable(self): - self.server.return_code = 503 - addr = self.server.server_address - url = f'http://{socket_helper.HOST}:{addr[1]}' - robots_url = url + "/robots.txt" - parser = urllib.robotparser.RobotFileParser() - parser.set_url(url) - parser.read() - self.assertFalse(parser.can_fetch("*", robots_url)) - self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 0c3e5d92890935..e70eae80036784 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -65,17 +65,9 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): - # If access to robot.txt has the status Unauthorized/Forbidden, - # then most likely this applies to the entire site. self.disallow_all = True - elif 400 <= err.code < 500: - # RFC 9309, Section 2.3.1.3: the crawler MAY access any - # resources on the server. + elif err.code >= 400 and err.code < 500: self.allow_all = True - elif 500 <= err.code < 600: - # RFC 9309, Section 2.3.1.4: the crawler MUST assume - # complete disallow. - self.disallow_all = True err.close() else: raw = f.read() diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst deleted file mode 100644 index bd9fff0bc2e31b..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst +++ /dev/null @@ -1,2 +0,0 @@ -Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file -is unreachable due to server or network errors. From b142878db1e54149feba62b08df1236432793bf0 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 7 May 2026 15:20:35 -0700 Subject: [PATCH 5/6] Improve error messages when the WASI SDK can't be found (GH-149508) --- Platforms/WASI/_build.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 76d2853163baa9..c1a91a9c833b8e 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -222,10 +222,8 @@ def wasi_sdk(context): if wasi_sdk_path := context.wasi_sdk_path: if not wasi_sdk_path.exists(): raise ValueError( - "WASI SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk" + "WASI SDK not found at " + f"{os.fsdecode(wasi_sdk_path)!r} (via --wasi-sdk)" ) return wasi_sdk_path @@ -237,7 +235,8 @@ def wasi_sdk(context): wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) if not wasi_sdk_path.exists(): raise ValueError( - f"WASI SDK not found at $WASI_SDK_PATH ({wasi_sdk_path})" + f"WASI SDK not found at {os.fsdecode(wasi_sdk_path)!r} " + "(via $WASI_SDK_PATH)" ) else: opt_path = pathlib.Path("/opt") @@ -272,6 +271,14 @@ def wasi_sdk(context): f" Found WASI SDK {major_version}, " f"but WASI SDK {wasi_sdk_version} is the supported version", ) + elif not wasi_sdk_path: + raise ValueError( + f"WASI SDK {wasi_sdk_version} not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and install in " + f"{os.fsdecode(opt_path)!r} or specify the SDK via " + "$WASI_SDK_PATH or --wasi-sdk" + ) # Cache the result. context.wasi_sdk_path = wasi_sdk_path From 49918f5b0ceb1950c3222fd4fd6be872d2e15c6f Mon Sep 17 00:00:00 2001 From: Neko Asakura Date: Fri, 8 May 2026 07:02:04 +0800 Subject: [PATCH 6/6] gh-149481: skip `FOR_ITER` inline specialization for Python `__next__` (#149491) Co-authored-by: Savannah Ostrowski Co-authored-by: Stan Ulbrych --- Include/internal/pycore_typeobject.h | 2 ++ Lib/test/test_capi/test_opt.py | 3 ++- Objects/typeobject.c | 6 ++++++ Python/optimizer_bytecodes.c | 4 +++- Python/optimizer_cases.c.h | 3 ++- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8d48cf6605ca7e..785b77d3e3be81 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -122,6 +122,8 @@ extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index d80fec9a8a0d2b..aaa5050208ced9 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -598,7 +598,8 @@ def testfunc(n, m): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_ITER_NEXT_INLINE", uops) + self.assertIn("_FOR_ITER_TIER_TWO", uops) + self.assertNotIn("_ITER_NEXT_INLINE", uops) @requires_specialization diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4f43747ba83fd9..9a18ca72516da7 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -11079,6 +11079,12 @@ slot_tp_iternext(PyObject *self) return vectorcall_method(&_Py_ID(__next__), stack, 1); } +int +_PyType_HasSlotTpIternext(PyTypeObject *type) +{ + return type->tp_iternext == slot_tp_iternext; +} + static PyObject * slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e10a096baa3318..39cc36ae79fead 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -2,6 +2,7 @@ #include "pycore_long.h" #include "pycore_opcode_utils.h" #include "pycore_optimizer.h" +#include "pycore_typeobject.h" #include "pycore_uops.h" #include "pycore_uop_ids.h" #include "internal/pycore_moduleobject.h" @@ -1459,7 +1460,8 @@ dummy_func(void) { type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 01ecb3790aa2cd..db3dcbb97b2645 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -3706,7 +3706,8 @@ type = sym_get_probable_type(iter); definite = false; } - if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL) { + if (type != NULL && type != &PyGen_Type && type->tp_iternext != NULL + && !_PyType_HasSlotTpIternext(type)) { PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); _Py_BloomFilter_Add(dependencies, type); if (!definite) {