From 561995cb8ab6786337ecef4357c06a0a7e671ee8 Mon Sep 17 00:00:00 2001 From: wj <126436871+WenJing95@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:31:47 +0900 Subject: [PATCH 1/2] Fix JavaScript extraction of escaped Unicode surrogate pairs --- babel/messages/jslexer.py | 3 ++- tests/messages/test_js_extract.py | 13 +++++++++++++ tests/messages/test_jslexer.py | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index d751b58f7..87530ff8a 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -161,7 +161,8 @@ def unquote_string(string: str) -> str: if pos < len(string): add(string[pos:]) - return ''.join(result) + # Combine UTF-16 surrogate pairs while preserving unmatched surrogates. + return ''.join(result).encode('utf-16-le', 'surrogatepass').decode('utf-16-le', 'surrogatepass') def tokenize( diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index af6dfb144..f48f2b6d0 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -20,6 +20,19 @@ def test_simple_extract(): (3, ('s', 'p'), [], None)] +@pytest.mark.parametrize('source', [ + r'gettext("\uD83D\uDE00")', + r"gettext('\uD83D\uDE00')", + r'gettext(`\uD83D\uDE00`)', + r'gettext`\uD83D\uDE00`', + 'gettext("\U0001f600")', +]) +def test_extract_surrogate_pair(source): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, '\U0001f600', [], None)] + + def test_various_calls(): buf = BytesIO(b"""\ msg1 = _(i18n_arg.replace(/"/, '"')) diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 00afc78ae..9063ac81f 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -1,3 +1,5 @@ +import pytest + from babel.messages import jslexer @@ -8,6 +10,25 @@ def test_unquote(): assert jslexer.unquote_string(r'"\xebb"') == "ëb" +@pytest.mark.parametrize(('string', 'expected'), [ + (r'"\uD83D\uDE00"', '\U0001f600'), + (r'"before \ud83d\ude00 after"', 'before \U0001f600 after'), + (r'"\uD800\uDC00\uDBFF\uDFFF"', '\U00010000\U0010ffff'), + (r'"\uD83D"', '\ud83d'), + (r'"\uDE00"', '\ude00'), + (r'"\uDE00\uD83D"', '\ude00\ud83d'), + (r'"\uD83D\uD83D\uDE00"', '\ud83d\U0001f600'), + (r'"\uD83D-\uDE00"', '\ud83d-\ude00'), + (r'"\uD83D\n\uDE00"', '\ud83d\n\ude00'), + (r'"\\uD83D\\uDE00"', r'\uD83D\uDE00'), + (r'"\uD83D\\uDE00"', '\ud83d' + r'\uDE00'), + (r'"\uFEFF\u00EB\uFFFF"', '\ufeffë\uffff'), + ('"\U0001f600"', '\U0001f600'), +]) +def test_unquote_surrogates(string, expected): + assert jslexer.unquote_string(string) == expected + + def test_dollar_in_identifier(): assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)] From e6909e037de46d78513c41262ff8e587601d5a46 Mon Sep 17 00:00:00 2001 From: wj <126436871+WenJing95@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:21:34 +0900 Subject: [PATCH 2/2] Fix surrogate pairs across concatenated JavaScript literals Combine surrogate pairs after concatenating literals without crossing message or context argument boundaries. Add regression coverage for quote styles, empty fragments, unmatched surrogates, escaped backslashes, plural messages, and contexts. Validation (Linux, CPython 3.13.5): - 51 JavaScript extraction and lexer tests passed. - The added tests produced 7 failures before the fix. - 5 extraction -> PO -> MO -> GNUTranslations round-trips passed. --- babel/messages/extract.py | 7 +++++- tests/messages/test_js_extract.py | 37 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/babel/messages/extract.py b/babel/messages/extract.py index 6fad84304..3c8279715 100644 --- a/babel/messages/extract.py +++ b/babel/messages/extract.py @@ -849,7 +849,12 @@ def extract_javascript( elif token.type in ('string', 'template_string'): new_value = unquote_string(token.value) if concatenate_next: - last_argument = (last_argument or '') + new_value + # A surrogate pair may span multiple string literals. + last_argument = ( + ((last_argument or '') + new_value) + .encode('utf-16-le', 'surrogatepass') + .decode('utf-16-le', 'surrogatepass') + ) concatenate_next = False else: last_argument = new_value diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index f48f2b6d0..f51f5dc02 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -26,6 +26,11 @@ def test_simple_extract(): r'gettext(`\uD83D\uDE00`)', r'gettext`\uD83D\uDE00`', 'gettext("\U0001f600")', + r'gettext("\uD83D" + "\uDE00")', + r"gettext('\uD83D' + '\uDE00')", + r'gettext(`\uD83D` + `\uDE00`)', + r'gettext("\uD83D" + `\uDE00`)', + r'gettext("\uD83D" + "" + "\uDE00")', ]) def test_extract_surrogate_pair(source): messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) @@ -33,6 +38,38 @@ def test_extract_surrogate_pair(source): assert messages == [(1, '\U0001f600', [], None)] +@pytest.mark.parametrize(('source', 'expected'), [ + (r'gettext("\uD83D" + "-" + "\uDE00")', '\ud83d-\ude00'), + (r'gettext("\uDE00" + "\uD83D")', '\ude00\ud83d'), + (r'gettext("\\uD83D" + "\\uDE00")', r'\uD83D\uDE00'), + (r'gettext("\uD83D" + "\\uDE00")', '\ud83d' + r'\uDE00'), +]) +def test_extract_concatenated_surrogate_boundaries(source, expected): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, expected, [], None)] + + +@pytest.mark.parametrize(('source', 'message', 'context'), [ + ( + r'ngettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03", n)', + ('\U0001f600', '\U0001f603'), + None, + ), + (r'ngettext("\uD83D", "\uDE00", n)', ('\ud83d', '\ude00'), None), + ( + r'pgettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03")', + '\U0001f603', + '\U0001f600', + ), + (r'pgettext("\uD83D", "\uDE00")', '\ude00', '\ud83d'), +]) +def test_extract_surrogate_pair_arguments(source, message, context): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, message, [], context)] + + def test_various_calls(): buf = BytesIO(b"""\ msg1 = _(i18n_arg.replace(/"/, '"'))