Fix Unicode normalization on PyPy

.github/workflows/ci.yml

@@ -46,6 +46,8 @@ jobs:
         python-version: "3.x"
     - name: Install dependencies
       run: pip install cibuildwheel
+    - name: Download normalization test data
+      run: python tests/download_test_data.py
     - name: Build and Test Wheels
       run: python -m cibuildwheel --output-dir wheelhouse
     - uses: actions/upload-artifact@v4
@@ -72,6 +74,8 @@ jobs:
         platforms: all
     - name: Install dependencies
       run: pip install cibuildwheel
+    - name: Download normalization test data
+      run: python tests/download_test_data.py
     - name: Build and Test Wheels
       run: python -m cibuildwheel --output-dir wheelhouse
     - uses: actions/upload-artifact@v4

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## Unreleased
+ - Backport CPython's faster canonical ordering, also fixing Unicode normalization on PyPy.
  - Require Python 3.9 or newer; drop Python 3.8 support.
 
 ## 17.0.0

MANIFEST.in

@@ -4,3 +4,4 @@ include LICENSE
 include unicodedata2/*.c
 include unicodedata2/*.h
 include tests/*.py
+prune tests/data

README.md

@@ -23,6 +23,11 @@ We run the tests using `tox`. This can be installed as usual with `pip install t
 or with `pip install --group dev` (pip 25.1 or newer) to pick it up from
 `pyproject.toml`.
 
+Tox and CI download the version-matched Unicode normalization data before
+running tests. Download failures stop the run. Before running `pytest` directly,
+run `python tests/download_test_data.py` once. The downloaded files are not
+included in source distributions or wheels.
+
 Without any options, `tox` will run the tests against all of the library's
 target Python versions. Any missing versions will be skipped.
 

tests/download_test_data.py

@@ -0,0 +1,35 @@
+"""Download version-matched normalization data before running the tests."""
+
+from pathlib import Path
+import re
+from urllib.request import Request, urlopen
+
+
+def main():
+    tests_dir = Path(__file__).resolve().parent
+    header = tests_dir.parent / 'unicodedata2' / 'unicodedata_db.h'
+    match = re.search(r'^#define UNIDATA_VERSION "([^"]+)"',
+                      header.read_text(encoding='utf-8'), re.MULTILINE)
+    if match is None:
+        raise ValueError('Cannot find UNIDATA_VERSION in %s' % header)
+
+    data_dir = tests_dir / 'data'
+    data_dir.mkdir(exist_ok=True)
+    for version in (match.group(1), '3.2.0'):
+        filename = 'NormalizationTest-%s.txt' % version
+        if version == '3.2.0':
+            url = 'https://www.unicode.org/Public/3.2-Update/' + filename
+        else:
+            url = ('https://www.unicode.org/Public/%s/ucd/NormalizationTest.txt'
+                   % version)
+        request = Request(url, headers={'User-Agent': 'unicodedata2'})
+        with urlopen(request, timeout=60) as response:
+            data = response.read()
+        if data.decode('utf-8').splitlines()[:1] != ['# ' + filename]:
+            raise ValueError('Unexpected version header in %s' % url)
+        (data_dir / filename).write_bytes(data)
+        print('Downloaded %s' % filename)
+
+
+if __name__ == '__main__':
+    main()