diff --git a/babel/messages/extract.py b/babel/messages/extract.py index 6fad84304..3c8279715 100644 --- a/babel/messages/extract.py +++ b/babel/messages/extract.py @@ -849,7 +849,12 @@ def extract_javascript( elif token.type in ('string', 'template_string'): new_value = unquote_string(token.value) if concatenate_next: - last_argument = (last_argument or '') + new_value + # A surrogate pair may span multiple string literals. + last_argument = ( + ((last_argument or '') + new_value) + .encode('utf-16-le', 'surrogatepass') + .decode('utf-16-le', 'surrogatepass') + ) concatenate_next = False else: last_argument = new_value diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index d751b58f7..87530ff8a 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -161,7 +161,8 @@ def unquote_string(string: str) -> str: if pos < len(string): add(string[pos:]) - return ''.join(result) + # Combine UTF-16 surrogate pairs while preserving unmatched surrogates. + return ''.join(result).encode('utf-16-le', 'surrogatepass').decode('utf-16-le', 'surrogatepass') def tokenize( diff --git a/tests/messages/test_js_extract.py b/tests/messages/test_js_extract.py index af6dfb144..f51f5dc02 100644 --- a/tests/messages/test_js_extract.py +++ b/tests/messages/test_js_extract.py @@ -20,6 +20,56 @@ def test_simple_extract(): (3, ('s', 'p'), [], None)] +@pytest.mark.parametrize('source', [ + r'gettext("\uD83D\uDE00")', + r"gettext('\uD83D\uDE00')", + r'gettext(`\uD83D\uDE00`)', + r'gettext`\uD83D\uDE00`', + 'gettext("\U0001f600")', + r'gettext("\uD83D" + "\uDE00")', + r"gettext('\uD83D' + '\uDE00')", + r'gettext(`\uD83D` + `\uDE00`)', + r'gettext("\uD83D" + `\uDE00`)', + r'gettext("\uD83D" + "" + "\uDE00")', +]) +def test_extract_surrogate_pair(source): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, '\U0001f600', [], None)] + + +@pytest.mark.parametrize(('source', 'expected'), [ + (r'gettext("\uD83D" + "-" + "\uDE00")', '\ud83d-\ude00'), + (r'gettext("\uDE00" + "\uD83D")', '\ude00\ud83d'), + (r'gettext("\\uD83D" + "\\uDE00")', r'\uD83D\uDE00'), + (r'gettext("\uD83D" + "\\uDE00")', '\ud83d' + r'\uDE00'), +]) +def test_extract_concatenated_surrogate_boundaries(source, expected): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, expected, [], None)] + + +@pytest.mark.parametrize(('source', 'message', 'context'), [ + ( + r'ngettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03", n)', + ('\U0001f600', '\U0001f603'), + None, + ), + (r'ngettext("\uD83D", "\uDE00", n)', ('\ud83d', '\ude00'), None), + ( + r'pgettext("\uD83D" + "\uDE00", "\uD83D" + "\uDE03")', + '\U0001f603', + '\U0001f600', + ), + (r'pgettext("\uD83D", "\uDE00")', '\ude00', '\ud83d'), +]) +def test_extract_surrogate_pair_arguments(source, message, context): + messages = list(extract.extract('javascript', BytesIO(source.encode('utf-8')))) + + assert messages == [(1, message, [], context)] + + def test_various_calls(): buf = BytesIO(b"""\ msg1 = _(i18n_arg.replace(/"/, '"')) diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 00afc78ae..9063ac81f 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -1,3 +1,5 @@ +import pytest + from babel.messages import jslexer @@ -8,6 +10,25 @@ def test_unquote(): assert jslexer.unquote_string(r'"\xebb"') == "ëb" +@pytest.mark.parametrize(('string', 'expected'), [ + (r'"\uD83D\uDE00"', '\U0001f600'), + (r'"before \ud83d\ude00 after"', 'before \U0001f600 after'), + (r'"\uD800\uDC00\uDBFF\uDFFF"', '\U00010000\U0010ffff'), + (r'"\uD83D"', '\ud83d'), + (r'"\uDE00"', '\ude00'), + (r'"\uDE00\uD83D"', '\ude00\ud83d'), + (r'"\uD83D\uD83D\uDE00"', '\ud83d\U0001f600'), + (r'"\uD83D-\uDE00"', '\ud83d-\ude00'), + (r'"\uD83D\n\uDE00"', '\ud83d\n\ude00'), + (r'"\\uD83D\\uDE00"', r'\uD83D\uDE00'), + (r'"\uD83D\\uDE00"', '\ud83d' + r'\uDE00'), + (r'"\uFEFF\u00EB\uFFFF"', '\ufeffë\uffff'), + ('"\U0001f600"', '\U0001f600'), +]) +def test_unquote_surrogates(string, expected): + assert jslexer.unquote_string(string) == expected + + def test_dollar_in_identifier(): assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]