From ef1930f3e3bbb92d45ec956bc5774c1d1e9ee86f Mon Sep 17 00:00:00 2001 From: Ivana Date: Fri, 28 Aug 2026 02:36:24 +0000 Subject: [PATCH 1/3] Move to a PEP 517 pyproject.toml build, drop setup.py Closes #53. The only thing setup.py still needed dynamic logic for was appending unicodedata2/pypy_ctype.c to the source list on PyPy. That file existed solely to supply _Py_ctype_toupper[256] for Py_TOUPPER(), which CPython exports from libpython but PyPy's cpyext headers do not define. Replace it with capability detection in _unicodedata2_compat.h. That header is included after Python.h, which is what makes "#ifdef Py_TOUPPER" meaningful: CPython defines the macro via cpython/pyctype.h (pulled in by Python.h on every non-limited-API build, 3.8 through main), PyPy does not. When it is missing we define UNICODEDATA2_TOUPPER() over a private, static 256-byte table with the same mapping. Detecting the macro rather than _Py_ctype_toupper matters: the latter is a linker symbol the preprocessor cannot see. The table is deliberately not named _Py_ctype_toupper, so nothing is added to CPython's reserved namespace, and it stays static so it is not exported from the extension. Both branches of the macro apply Py_CHARMASK() to their argument, as Py_TOUPPER() always did, so the call sites no longer mask it themselves. The mask is idempotent and the compiler already folded the duplicate: the .text section is byte identical either way, on both interpreters. Uppercasing stays table-based on both implementations, so name lookup performance is unchanged; on PyPy the table is now a same-translation-unit static const rather than an extern from a separate object file. With no implementation-specific source, the extension can be declared statically via setuptools' ext-modules table, and setup.py can go. The build now needs setuptools >= 77.0.1 (declarative ext-modules landed in 74.1, PEP 639 license metadata in 77; 77.0.0 was never published). Also: - long_description keeps its rendered form: setup.py injected a "Changelog" setext heading between README.md and CHANGELOG.md, so CHANGELOG.md now carries that heading itself and the two files are concatenated via tool.setuptools.dynamic. - deploy job builds the sdist with "python -m build --sdist". - tox envlist points at PyPy versions that still exist. - CIBW_ENABLE=pypy restores PyPy wheels: cibuildwheel 3.0 moved PyPy into an opt-in enable group, so 17.0.0 and 17.0.1 shipped none, where 16.0.0 and earlier shipped 11-12. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Fyhi9MNffq3B9JLVSukhEa --- .github/workflows/ci.yml | 8 +++- CHANGELOG.md | 2 + pyproject.toml | 48 +++++++++++++++++++ setup.py | 49 ------------------- tox.ini | 2 +- unicodedata2/_unicodedata2_compat.h | 74 ++++++++++++++++++++++++++++- unicodedata2/pypy_ctype.c | 35 -------------- unicodedata2/pypy_ctype.h | 11 ----- unicodedata2/unicodedata.c | 4 +- 9 files changed, 132 insertions(+), 101 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py delete mode 100644 unicodedata2/pypy_ctype.c delete mode 100644 unicodedata2/pypy_ctype.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80319a7..f31d001 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,10 @@ on: branches: [master] env: + # cibuildwheel 3.0 moved PyPy into an opt-in "enable group", which silently + # dropped the PyPy wheels this project had shipped up to 16.0.0 (17.0.0 and + # 17.0.1 published none). Ask for them back explicitly. + CIBW_ENABLE: pypy CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_I686_IMAGE: manylinux2014 CIBW_MANYLINUX_PYPY_X86_64_IMAGE: manylinux2014 @@ -98,7 +102,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install --upgrade setuptools wheel twine + pip install --upgrade build twine - name: Download artifacts from build jobs uses: actions/download-artifact@v4 with: @@ -134,7 +138,7 @@ jobs: draft: false prerelease: false - name: Build sdist - run: python setup.py sdist + run: python -m build --sdist - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e02b340..d726f4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +# Changelog + ## 17.0.0 - Upgrade to Unicode 17.0.0 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a97a755 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,48 @@ +[build-system] +# setuptools 77.0.1 is the first release that combines declarative ``ext-modules`` +# (added in 74.1) with PEP 639 license metadata, which is what lets this +# project drop setup.py entirely. +requires = ["setuptools>=77.0.1"] +build-backend = "setuptools.build_meta" + +[project] +name = "unicodedata2" +version = "17.0.1" +description = "Unicodedata backport updated to the latest Unicode version." +authors = [ + { name = "Mike Kaplinskiy", email = "mike.kaplinskiy@gmail.com" }, +] +license = "Apache-2.0" +license-files = ["LICENSE"] +# README.md and CHANGELOG.md are concatenated so both are shown on PyPI. +dynamic = ["readme"] + +[project.optional-dependencies] +testing = [ + "pytest", + "coverage", + "pytest-xdist", + "pytest-randomly", +] + +[project.urls] +Homepage = "http://github.com/fonttools/unicodedata2" +Download = "http://github.com/fonttools/unicodedata2" + +[tool.setuptools] +platforms = ["any"] +# The distribution is a single top-level extension module: there are no Python +# packages or modules to ship. Saying so explicitly keeps auto-discovery from +# looking at the unicodedata2/ source directory or at tests/. +packages = [] + +[tool.setuptools.dynamic] +readme = { file = ["README.md", "CHANGELOG.md"], content-type = "text/markdown" } + +[[tool.setuptools.ext-modules]] +name = "unicodedata2" +sources = [ + "unicodedata2/unicodedata.c", + "unicodedata2/unicodectype.c", +] +include-dirs = ["unicodedata2"] diff --git a/setup.py b/setup.py deleted file mode 100644 index 2e34dac..0000000 --- a/setup.py +++ /dev/null @@ -1,49 +0,0 @@ -import sys -from setuptools import setup, Extension - -# concatenate README.md and CHANGELOG.md into long_description so they are -# displayed on the unicodedata2 project page on PyPI -with open("README.md", "r", encoding="utf-8") as readme: - long_description = readme.read() -long_description += "\nChangelog\n=========\n" -with open("CHANGELOG.md", "r", encoding="utf-8") as changelog: - long_description += changelog.read() - -module_sources = [ - "./unicodedata2/unicodedata.c", - "./unicodedata2/unicodectype.c", -] - -is_pypy = hasattr(sys, "pypy_version_info") -if is_pypy: - module_sources.append("./unicodedata2/pypy_ctype.c") - -module1 = Extension( - "unicodedata2", - sources=module_sources, - include_dirs=["./unicodedata2/"], -) - -setup( - name="unicodedata2", - version="17.0.1", - description="Unicodedata backport updated to the latest Unicode version.", - long_description=long_description, - long_description_content_type="text/markdown", - ext_modules=[module1], - author="Mike Kaplinskiy", - author_email="mike.kaplinskiy@gmail.com", - download_url="http://github.com/fonttools/unicodedata2", - license="Apache License 2.0", - platforms=["any"], - url="http://github.com/fonttools/unicodedata2", - test_suite="tests", - extras_require={ - "testing": [ - "pytest", - "coverage", - "pytest-xdist", - "pytest-randomly", - ], - }, -) diff --git a/tox.ini b/tox.ini index 73813cd..d1b49f4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,39,310,311,312,313,314}, pypy{37,38,39} +envlist = py{38,39,310,311,312,313,314}, pypy{39,310,311} skip_missing_interpreters = true [testenv] diff --git a/unicodedata2/_unicodedata2_compat.h b/unicodedata2/_unicodedata2_compat.h index c9b1fb2..1d1de38 100644 --- a/unicodedata2/_unicodedata2_compat.h +++ b/unicodedata2/_unicodedata2_compat.h @@ -3,6 +3,9 @@ /* * Compatibility shims + * + * This header must be included *after* "Python.h": it detects what the host + * Python's headers already provide and only fills in the gaps. */ @@ -18,8 +21,77 @@ static inline void _Py_SET_TYPE(PyObject *ob, PyTypeObject *type) #ifdef PYPY_VERSION -#include "pypy_ctype.h" typedef Py_ssize_t Py_ssize_clean_t; #endif + +/* ----------------------------------------------------------------------- * + * ASCII upper-casing for the character name lookup code. + * + * _gethash() and _cmpname() fold the ASCII case of character names so that + * lookup() is case insensitive. CPython does that with a 256 byte table, + * reachable as Py_TOUPPER() because "Python.h" pulls in + * on every non-limited-API build. PyPy's cpyext headers define neither the + * macro nor the table, so we provide an equivalent one ourselves. + * + * The detection tests Py_TOUPPER, the macro, rather than _Py_ctype_toupper, + * which is a linker symbol the preprocessor cannot see. Because this file + * is included after "Python.h" and nothing else defines Py_TOUPPER first, + * "is the macro defined here?" is exactly "does the host Python provide + * upper-casing?". + * + * Either way the argument is masked with Py_CHARMASK() inside the macro, so + * callers pass a plain char and must not mask it again. + * ----------------------------------------------------------------------- */ +#ifdef Py_TOUPPER + +#define UNICODEDATA2_TOUPPER(c) Py_TOUPPER(c) + +#else + +/* Same mapping as CPython's _Py_ctype_toupper: ASCII 'a'-'z' fold to 'A'-'Z', + every other byte value is passed through unchanged. Kept private to + unicodedata2 on purpose -- it deliberately does not define a symbol in + CPython's reserved _Py_ namespace, and nothing outside this extension can + link against it. */ +static const unsigned char unicodedata2_toupper_table[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + +#define UNICODEDATA2_TOUPPER(c) \ + (unicodedata2_toupper_table[Py_CHARMASK(c)]) + +#endif /* Py_TOUPPER */ + #endif diff --git a/unicodedata2/pypy_ctype.c b/unicodedata2/pypy_ctype.c deleted file mode 100644 index d29656b..0000000 --- a/unicodedata2/pypy_ctype.c +++ /dev/null @@ -1,35 +0,0 @@ -// Copied from https://github.com/python/cpython/blob/bb3e0c240bc60fe08d332ff5955d54197f79751c/Python/pyctype.c#L180-L213 -const unsigned char _Py_ctype_toupper[256] = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, - 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, - 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, - 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, - 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, - 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, - 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, - 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, - 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, - 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, - 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, - 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, - 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, - 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, - 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, - 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, - 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, - 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, - 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, - 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, - 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, - 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, - 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, - 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, -}; diff --git a/unicodedata2/pypy_ctype.h b/unicodedata2/pypy_ctype.h deleted file mode 100644 index dd37305..0000000 --- a/unicodedata2/pypy_ctype.h +++ /dev/null @@ -1,11 +0,0 @@ -#ifndef __PYPY_SHIMS_H__ -#define __PYPY_SHIMS_H__ - -#ifdef PYPY_VERSION - -PyAPI_DATA(const unsigned char) _Py_ctype_toupper[256]; -#define Py_TOUPPER(c) (_Py_ctype_toupper[Py_CHARMASK(c)]) - -#endif - -#endif diff --git a/unicodedata2/unicodedata.c b/unicodedata2/unicodedata.c index ec627e2..5c22061 100644 --- a/unicodedata2/unicodedata.c +++ b/unicodedata2/unicodedata.c @@ -890,7 +890,7 @@ _gethash(const char *s, int len, int scale) unsigned long h = 0; unsigned long ix; for (i = 0; i < len; i++) { - h = (h * scale) + (unsigned char) Py_TOUPPER(Py_CHARMASK(s[i])); + h = (h * scale) + (unsigned char) UNICODEDATA2_TOUPPER(s[i]); ix = h & 0xff000000; if (ix) h = (h ^ ((ix>>24) & 0xff)) & 0x00ffffff; @@ -1049,7 +1049,7 @@ _cmpname(PyObject *self, int code, const char* name, int namelen) if (!_getucname(self, code, buffer, NAME_MAXLEN, 1)) return 0; for (i = 0; i < namelen; i++) { - if (Py_TOUPPER(Py_CHARMASK(name[i])) != buffer[i]) + if (UNICODEDATA2_TOUPPER(name[i]) != buffer[i]) return 0; } return buffer[namelen] == '\0'; From 4df9d106fd16287e863dde347e22c4176d9da4d0 Mon Sep 17 00:00:00 2001 From: Ivana Date: Fri, 28 Aug 2026 11:27:17 +0000 Subject: [PATCH 2/3] Drop unused coverage and pytest-xdist from the testing extra CIBW_TEST_EXTRAS installs this extra into every cibuildwheel test environment, then CIBW_TEST_COMMAND runs plain "pytest {project}/tests". Nothing in the repo sets addopts, passes -n, or passes --cov: there is no conftest.py, no pytest.ini, no setup.cfg, and neither pyproject.toml nor tox.ini configures pytest. So pytest-xdist loaded as a plugin and sat idle, and coverage was never reachable at all -- pytest integration would need pytest-cov, and nothing invokes "coverage run" either. Both were installed for every wheel in the matrix across three operating systems, including the aarch64 jobs that run under QEMU emulation. pytest-randomly stays: it is a plugin that shuffles test order simply by being installed, so it does affect the run. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Fyhi9MNffq3B9JLVSukhEa --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a97a755..ed138ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,10 +18,12 @@ license-files = ["LICENSE"] dynamic = ["readme"] [project.optional-dependencies] +# Installed into every cibuildwheel test environment via CIBW_TEST_EXTRAS, so +# keep it to what "pytest tests/" actually loads. pytest-randomly is a plugin +# and shuffles test order just by being installed; coverage and pytest-xdist +# needed flags nothing passes. testing = [ "pytest", - "coverage", - "pytest-xdist", "pytest-randomly", ] From 4b820025208715c2cd9ffa3fb61e61a1f8ba74b0 Mon Sep 17 00:00:00 2001 From: Ivana Date: Fri, 28 Aug 2026 11:29:02 +0000 Subject: [PATCH 3/3] Declare tox in a PEP 735 dev dependency group tox is an orchestrator, not a test dependency: it builds the environment that installs the package, so it does not belong in the testing extra, which CIBW_TEST_EXTRAS installs into every wheel test environment and which tox itself does not even use (tox.ini declares deps = pytest). Dependency groups are the right home. They are local-only: nothing about them reaches the published wheel or sdist metadata, so this adds no install-time cost or risk for anyone depending on unicodedata2, while giving contributors "pip install --group dev" (pip >= 25.1) alongside the existing "pip install tox". Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01Fyhi9MNffq3B9JLVSukhEa --- README.md | 3 ++- pyproject.toml | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2be6d76..65f1891 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ Pre-compiled wheel packages are available on [PyPI] and can be installed via pip Testing ======= -We run the tests using `tox`. This can be installed as usual with `pip install tox`. +We run the tests using `tox`. This can be installed as usual with `pip install tox`, +or with `pip install --group dev` to pick it up from `pyproject.toml`. Without any options, `tox` will run the tests against all of the library's target Python versions. Any missing versions will be skipped. diff --git a/pyproject.toml b/pyproject.toml index ed138ec..61bd8e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,12 @@ testing = [ "pytest-randomly", ] +# Local development only. PEP 735 dependency groups are never written into +# the published wheel/sdist metadata, so this adds nothing for installers of +# unicodedata2 -- it just gives contributors "pip install --group dev". +[dependency-groups] +dev = ["tox"] + [project.urls] Homepage = "http://github.com/fonttools/unicodedata2" Download = "http://github.com/fonttools/unicodedata2"