diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc index 4969beada682..c56546f7966b 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc @@ -1245,7 +1245,7 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { // decomposed: U+0061(LATIN SMALL LETTER A) + U+0301(COMBINING ACUTE ACCENT) // composed: U+00E1(LATIN SMALL LETTER A WITH ACUTE) - const char* json_composed = "[\"foo\", \"á\"]"; + const char* json_composed = "[\"foo\", \"\xc3\xa1\"]"; const char* json_decomposed = "[\"foo\", \"a\xcc\x81\"]"; for (const auto& options : compose_options) { this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_composed, @@ -1260,6 +1260,25 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { &options); } + // Hangul composes algorithmically in utf8proc, not through the composition table. + // decomposed: U+1112(HANGUL CHOSEONG HIEUH) + U+1161(HANGUL JUNGSEONG A) + + // U+11AB(HANGUL JONGSEONG NIEUN) + // composed: U+D55C(HANGUL SYLLABLE HAN) + json_composed = "[\"\xed\x95\x9c\"]"; + json_decomposed = "[\"\xe1\x84\x92\xe1\x85\xa1\xe1\x86\xab\"]"; + for (const auto& options : compose_options) { + this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_composed, + &options); + this->CheckUnary("utf8_normalize", json_composed, this->type(), json_composed, + &options); + } + for (const auto& options : decompose_options) { + this->CheckUnary("utf8_normalize", json_composed, this->type(), json_decomposed, + &options); + this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_decomposed, + &options); + } + // canonical: U+00B2(Superscript Two) // compatibility: "2" const char* json_canonical = "[\"01\xc2\xb2!\"]"; diff --git a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc index 582d559dde2e..6d4525c28696 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc @@ -547,6 +547,16 @@ struct Utf8NormalizeBase { if (res < 0) { return Status::Invalid("Cannot normalize utf8 string: ", utf8proc_errmsg(res)); } + if (decompose_options_ & UTF8PROC_COMPOSE) { + // utf8proc_decompose() only decomposes; the canonical composition step for + // NFC and NFKC is done in-place by utf8proc_normalize_utf32(). + res = utf8proc_normalize_utf32( + reinterpret_cast(codepoints_.data()), res, + decompose_options_); + if (res < 0) { + return Status::Invalid("Cannot normalize utf8 string: ", utf8proc_errmsg(res)); + } + } return res; } diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 8b2ad2b333fc..0548be7c9740 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -3825,6 +3825,15 @@ def test_utf8_normalize(): assert pc.utf8_normalize(arr, form="NFKC") == pa.array(["0123"]) assert pc.utf8_normalize(arr, "NFD") == arr assert pc.utf8_normalize(arr, "NFKD") == pa.array(["0123"]) + # GH-51225: composing forms must compose, not only decompose + composed = pa.array(["\u00e9", "\ud55c", None]) + decomposed = pa.array(["e\u0301", "\u1112\u1161\u11ab", None]) + for form in ("NFC", "NFKC"): + assert pc.utf8_normalize(decomposed, form=form) == composed + assert pc.utf8_normalize(composed, form=form) == composed + for form in ("NFD", "NFKD"): + assert pc.utf8_normalize(composed, form=form) == decomposed + assert pc.utf8_normalize(decomposed, form=form) == decomposed with pytest.raises( ValueError, match='"NFZ" is not a valid Unicode normalization form'):