update to Unicode 18.0.0

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
-## Unreleased
+## 18.0.0
+ - Upgrade to Unicode 18.0.0
+ - Backport algorithmic character names from CPython, including Tangut, Jurchen, and Small Seal.
  - Backport CPython's faster canonical ordering, also fixing Unicode normalization on PyPy.
  - Require Python 3.9 or newer; drop Python 3.8 support.
 

README.md

@@ -4,7 +4,7 @@
 unicodedata2
 ============
 
-[unicodedata] backport/updates. Currently supports Unicode 17.0.0.
+[unicodedata] backport/updates. Currently supports Unicode 18.0.0.
 
 Requires Python 3.9 or newer.
 

makeunicodedata.py

@@ -44,7 +44,7 @@
 #   * Doc/library/stdtypes.rst, and
 #   * Doc/library/unicodedata.rst
 #   * Doc/reference/lexical_analysis.rst (two occurrences)
-UNIDATA_VERSION = "17.0.0"
+UNIDATA_VERSION = "18.0.0"
 UNICODE_DATA = "UnicodeData%s.txt"
 COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt"
 EASTASIAN_WIDTH = "EastAsianWidth%s.txt"
@@ -99,21 +99,15 @@
 CASED_MASK = 0x2000
 EXTENDED_CASE_MASK = 0x4000
 
-# CJK Unified Ideograph ranges.
-# makeunicodecjk() generates unicodedata_cjk.h from these, which is
-# included by unicodedata.c (is_unified_ideograph function).
-cjk_ranges = [
-    ('3400', '4DBF', 'CJK Ideograph Extension A'),
-    ('4E00', '9FFF', 'CJK Ideograph'),
-    ('20000', '2A6DF', 'CJK Ideograph Extension B'),
-    ('2A700', '2B73F', 'CJK Ideograph Extension C'),
-    ('2B740', '2B81D', 'CJK Ideograph Extension D'),
-    ('2B820', '2CEAD', 'CJK Ideograph Extension E'),
-    ('2CEB0', '2EBE0', 'CJK Ideograph Extension F'),
-    ('2EBF0', '2EE5D', 'CJK Ideograph Extension I'),
-    ('30000', '3134A', 'CJK Ideograph Extension G'),
-    ('31350', '323AF', 'CJK Ideograph Extension H'),
-    ('323B0', '33479', 'CJK Ideograph Extension J'),
+# Maps the range names in UnicodeData.txt to prefixes for
+# derived names specified by rule NR2.
+# Hangul should always be at index 0, since it uses special format.
+derived_name_range_names = [
+    ("Hangul Syllable", "HANGUL SYLLABLE "),
+    ("CJK Ideograph", "CJK UNIFIED IDEOGRAPH-"),
+    ("Tangut Ideograph", "TANGUT IDEOGRAPH-"),
+    ("Jurchen Character", "JURCHEN CHARACTER-"),
+    ("Seal Character", "SMALL SEAL CHARACTER-"),
 ]
 
 
@@ -127,47 +121,15 @@ def maketables(trace=0):
 
     for version in old_versions:
         print("--- Reading", UNICODE_DATA % ("-"+version), "...")
-        old_unicode = UnicodeData(version, cjk_check=False)
+        old_unicode = UnicodeData(version)
         print(len(list(filter(None, old_unicode.table))), "characters")
         merge_old_version(version, unicode, old_unicode)
 
-    makeunicodecjk(trace)
     makeunicodename(unicode, trace)
     makeunicodedata(unicode, trace)
     makeunicodetype(unicode, trace)
 
 
-# --------------------------------------------------------------------
-# CJK Unified Ideograph ranges (is_unified_ideograph function)
-
-def makeunicodecjk(trace):
-
-    FILE = "unicodedata2/unicodedata_cjk.h"
-
-    print("--- Preparing", FILE, "...")
-
-    with open(FILE, "w") as fp:
-        fprint = partial(print, file=fp)
-        fprint("/* this file was generated by %s %s */" % (SCRIPT, VERSION))
-        fprint()
-        fprint("static int")
-        fprint("is_unified_ideograph(Py_UCS4 code)")
-        fprint("{")
-        fprint("    return")
-        for i, (start, end, name) in enumerate(cjk_ranges):
-            start_hex = int(start, 16)
-            end_hex = int(end, 16)
-            if i < len(cjk_ranges) - 1:
-                fprint("        (0x%X <= code && code <= 0x%X) || /* %s */"
-                       % (start_hex, end_hex, name))
-            else:
-                fprint("        (0x%X <= code && code <= 0x%X);   /* %s */"
-                       % (start_hex, end_hex, name))
-        fprint("}")
-
-    print(len(cjk_ranges), "CJK ranges")
-
-
 # --------------------------------------------------------------------
 # unicode character properties
 
@@ -855,6 +817,23 @@ def word_key(a):
             fprint('    {%d, {%s}},' % (len(sequence), seq_str))
         fprint('};')
 
+        fprint(dedent("""
+            typedef struct {
+                Py_UCS4 first;
+                Py_UCS4 last;
+                int prefixid;
+            } derived_name_range;
+            """))
+
+        fprint('static const derived_name_range derived_name_ranges[] = {')
+        for name_range in unicode.derived_name_ranges:
+            fprint('    {0x%s, 0x%s, %d},' % name_range)
+        fprint('};')
+
+        fprint('static const char * const derived_name_prefixes[] = {')
+        for _, prefix in derived_name_range_names:
+            fprint('    "%s",' % prefix)
+        fprint('};')
 
 def merge_old_version(version, new, old):
     # Changes to exclusion file not implemented yet
@@ -1066,14 +1045,14 @@ def from_row(row: List[str]) -> UcdRecord:
 class UnicodeData:
     # table: List[Optional[UcdRecord]]  # index is codepoint; None means unassigned
 
-    def __init__(self, version, cjk_check=True):
+    def __init__(self, version):
         self.changed = []
         table = [None] * 0x110000
         for s in UcdFile(UNICODE_DATA, version):
             char = int(s[0], 16)
             table[char] = from_row(s)
 
-        cjk_ranges_found = []
+        self.derived_name_ranges = []
 
         # expand first-last ranges
         field = None
@@ -1087,18 +1066,15 @@ def __init__(self, version, cjk_check=True):
                     s.name = ""
                     field = dataclasses.astuple(s)[:15]
                 elif s.name[-5:] == "Last>":
-                    if s.name.startswith("<CJK Ideograph"):
-                        cjk_ranges_found.append((field[0],
-                                                 s.codepoint))
+                    for j, (rangename, _) in enumerate(derived_name_range_names):
+                        if s.name.startswith("<" + rangename):
+                            self.derived_name_ranges.append(
+                                (field[0], s.codepoint, j))
+                            break
                     s.name = ""
                     field = None
             elif field:
                 table[i] = from_row(('%X' % i,) + field[1:])
-        if cjk_check:
-            expected = [(s, e) for s, e, _ in cjk_ranges]
-            if expected != cjk_ranges_found:
-                raise ValueError("CJK ranges deviate: have %r" % cjk_ranges_found)
-
         # public attributes
         self.filename = UNICODE_DATA % ''
         self.table = table

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "unicodedata2"
-version = "17.0.1"
+version = "18.0.0"
 requires-python = ">=3.9"
 description = "Unicodedata backport updated to the latest Unicode version."
 authors = [

tests/download_test_data.py

@@ -1,4 +1,4 @@
-"""Download version-matched normalization data before running the tests."""
+"""Download version-matched Unicode data before running the tests."""
 
 from pathlib import Path
 import re
@@ -15,13 +15,18 @@ def main():
 
     data_dir = tests_dir / 'data'
     data_dir.mkdir(exist_ok=True)
-    for version in (match.group(1), '3.2.0'):
-        filename = 'NormalizationTest-%s.txt' % version
-        if version == '3.2.0':
-            url = 'https://www.unicode.org/Public/3.2-Update/' + filename
-        else:
-            url = ('https://www.unicode.org/Public/%s/ucd/NormalizationTest.txt'
-                   % version)
+    version = match.group(1)
+    base_url = 'https://www.unicode.org/Public/'
+    downloads = [
+        ('NormalizationTest-%s.txt' % version,
+         '%s/ucd/NormalizationTest.txt' % version),
+        ('NormalizationTest-3.2.0.txt',
+         '3.2-Update/NormalizationTest-3.2.0.txt'),
+        ('DerivedName-%s.txt' % version,
+         '%s/ucd/extracted/DerivedName.txt' % version),
+    ]
+    for filename, remote_path in downloads:
+        url = base_url + remote_path
         request = Request(url, headers={'User-Agent': 'unicodedata2'})
         with urlopen(request, timeout=60) as response:
             data = response.read()

tests/test_unicodedata2.py

@@ -39,7 +39,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest):
 
     # Update this if the database changes. Make sure to do a full rebuild
     # (e.g. 'make distclean && make') to get the correct checksum.
-    expectedchecksum = '65670ae03a324c5f9e826a4de3e25bae4d73c9b7'
+    expectedchecksum = 'f11a52558bcefd64c833f44f7ccb51faa8ec3310'
 
     def test_function_checksum(self):
         import unicodedata2
@@ -347,16 +347,15 @@ def test_issue29456(self):
 
 
     def test_cjk_unified_ideograph_names(self):
-        # Test that is_unified_ideograph covers all CJK ranges by checking
+        # Test that the generated name table covers all CJK ranges by checking
         # that name() and lookup() work for the first and last codepoint of
-        # each range. These ranges must be kept in sync between
-        # makeunicodedata.py:cjk_ranges and unicodedata_cjk.h.
+        # each range, including the Unicode 18 Extension D addition.
         cjk_ranges = [
             (0x3400, 0x4DBF),    # CJK Ideograph Extension A
             (0x4E00, 0x9FFF),    # CJK Ideograph
             (0x20000, 0x2A6DF),  # CJK Ideograph Extension B
             (0x2A700, 0x2B73F),  # CJK Ideograph Extension C
-            (0x2B740, 0x2B81D),  # CJK Ideograph Extension D
+            (0x2B740, 0x2B81E),  # CJK Ideograph Extension D
             (0x2B820, 0x2CEAD),  # CJK Ideograph Extension E
             (0x2CEB0, 0x2EBE0),  # CJK Ideograph Extension F
             (0x2EBF0, 0x2EE5D),  # CJK Ideograph Extension I
@@ -371,6 +370,120 @@ def test_cjk_unified_ideograph_names(self):
                 self.assertEqual(self.db.name(char), expected_name)
                 self.assertEqual(self.db.lookup(expected_name), char)
 
+    def test_name(self):
+        name = self.db.name
+        self.assertRaises(ValueError, name, '\0')
+        self.assertRaises(ValueError, name, '\n')
+        self.assertRaises(ValueError, name, '\x1F')
+        self.assertRaises(ValueError, name, '\x7F')
+        self.assertRaises(ValueError, name, '\x9F')
+        self.assertRaises(ValueError, name, '\uFFFE')
+        self.assertRaises(ValueError, name, '\uFFFF')
+        self.assertRaises(ValueError, name, '\U0010FFFF')
+        self.assertEqual(name('\U0010FFFF', 42), 42)
+
+        self.assertEqual(name(' '), 'SPACE')
+        self.assertEqual(name('1'), 'DIGIT ONE')
+        self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A')
+        self.assertEqual(name('\xA0'), 'NO-BREAK SPACE')
+        self.assertEqual(name('\u0221', None), 'LATIN SMALL LETTER D WITH CURL')
+        self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400')
+        self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5')
+        self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA')
+        self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH')
+        self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900')
+        self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A')
+        self.assertEqual(name('\uFBF9'),
+                         'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA '
+                         'ABOVE WITH ALEF MAKSURA ISOLATED FORM')
+        self.assertEqual(name('\U00013460', None), 'EGYPTIAN HIEROGLYPH-13460')
+        self.assertEqual(name('\U000143FA', None), 'EGYPTIAN HIEROGLYPH-143FA')
+        self.assertEqual(name('\U00017000', None), 'TANGUT IDEOGRAPH-17000')
+        self.assertEqual(name('\U00018B00', None),
+                         'KHITAN SMALL SCRIPT CHARACTER-18B00')
+        self.assertEqual(name('\U00018CD5', None),
+                         'KHITAN SMALL SCRIPT CHARACTER-18CD5')
+        self.assertEqual(name('\U00018CFF', None),
+                         'KHITAN SMALL SCRIPT CHARACTER-18CFF')
+        self.assertEqual(name('\U00018D1E', None), 'TANGUT IDEOGRAPH-18D1E')
+        self.assertEqual(name('\U0001B170', None), 'NUSHU CHARACTER-1B170')
+        self.assertEqual(name('\U0001B2FB', None), 'NUSHU CHARACTER-1B2FB')
+        self.assertEqual(name('\U0001FBA8', None),
+                         'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO '
+                         'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE')
+        self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6')
+        self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D')
+        self.assertEqual(name('\U00033479', None), 'CJK UNIFIED IDEOGRAPH-33479')
+
+    def test_lookup_nonexistant(self):
+        # just make sure that lookup can fail
+        for nonexistent in [
+            "LATIN SMLL LETR A",
+            "OPEN HANDS SIGHS",
+            "DREGS",
+            "HANDBUG",
+            "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK",
+            "???",
+            "CJK UNIFIED IDEOGRAPH-03400",
+            "CJK UNIFIED IDEOGRAPH-020000",
+            "CJK UNIFIED IDEOGRAPH-33FF",
+            "CJK UNIFIED IDEOGRAPH-F900",
+            "CJK UNIFIED IDEOGRAPH-13460",
+            "CJK UNIFIED IDEOGRAPH-17000",
+            "CJK UNIFIED IDEOGRAPH-18B00",
+            "CJK UNIFIED IDEOGRAPH-1B170",
+            "CJK COMPATIBILITY IDEOGRAPH-3400",
+            "TANGUT IDEOGRAPH-3400",
+            "HANGUL SYLLABLE AC00",
+        ]:
+            self.assertRaises(KeyError, self.db.lookup, nonexistent)
+
+    def test_tangut_ideographs(self):
+        self.assertEqual(self.db.lookup("TANGUT IDEOGRAPH-17000"), "\U00017000")
+        self.assertEqual(self.db.lookup("TANGUT IDEOGRAPH-187FF"), "\U000187ff")
+        self.assertEqual(self.db.lookup("TANGUT IDEOGRAPH-18D00"), "\U00018D00")
+        self.assertEqual(self.db.lookup("TANGUT IDEOGRAPH-18D1E"), "\U00018d1e")
+        self.assertEqual(self.db.lookup("tangut ideograph-18d1e"), "\U00018d1e")
+
+    def test_all_names(self):
+        filename = 'DerivedName-%s.txt' % self.db.unidata_version
+        path = Path(__file__).with_name('data') / filename
+        with path.open(encoding='utf-8') as testdata:
+            self.assertEqual(testdata.readline().strip(), '# ' + filename)
+            self.run_name_tests(testdata)
+
+    def run_name_tests(self, testdata):
+        names_ref = {}
+
+        def parse_cp(s):
+            return int(s, 16)
+
+        # Parse data
+        for line in testdata:
+            line = line.strip()
+            if not line or line.startswith("#"):
+                continue
+            raw_cp, name = line.split("; ")
+            # Check for a range
+            if ".." in raw_cp:
+                cp1, cp2 = map(parse_cp, raw_cp.split(".."))
+                # remove ‘*’ at the end
+                assert name[-1] == '*', (raw_cp, name)
+                name = name[:-1]
+                for cp in range(cp1, cp2 + 1):
+                    names_ref[cp] = f"{name}{cp:04X}"
+            elif name[-1] == '*':
+                cp = parse_cp(raw_cp)
+                name = name[:-1]
+                names_ref[cp] = f"{name}{cp:04X}"
+            else:
+                assert '*' not in name, (raw_cp, name)
+                cp = parse_cp(raw_cp)
+                names_ref[cp] = name
+
+        for cp in range(0, sys.maxunicode + 1):
+            self.assertEqual(self.db.name(chr(cp), None), names_ref.get(cp))
+
     def test_east_asian_width(self):
         eaw = self.db.east_asian_width
         self.assertRaises(TypeError, eaw, b'a')
@@ -388,7 +501,7 @@ def test_east_asian_width(self):
     def test_east_asian_width_unassigned(self):
         eaw = self.db.east_asian_width
         # unassigned
-        for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2':
+        for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107c0\U000115f2':
             self.assertEqual(eaw(char), 'N')
             self.assertIs(self.db.name(char, None), None)