From 4735374fd14eb178ba87cac01f77c2160baf15db Mon Sep 17 00:00:00 2001 From: singhpratech Date: Tue, 8 Sep 2026 09:06:19 -0400 Subject: [PATCH 1/3] GH-51225: [C++] utf8_normalize: compose for NFC and NFKC utf8proc_decompose() only decomposes; call utf8proc_normalize_utf32() on the scratch buffer when the form asks for composition. Fix the json_composed test fixture, whose bytes were the decomposed form, and add composed/decomposed pairs to the pyarrow test. --- cpp/src/arrow/compute/kernels/scalar_string_test.cc | 2 +- cpp/src/arrow/compute/kernels/scalar_string_utf8.cc | 10 ++++++++++ python/pyarrow/tests/test_compute.py | 9 +++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc index 4969beada682..28055cb44fe9 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc @@ -1245,7 +1245,7 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { // decomposed: U+0061(LATIN SMALL LETTER A) + U+0301(COMBINING ACUTE ACCENT) // composed: U+00E1(LATIN SMALL LETTER A WITH ACUTE) - const char* json_composed = "[\"foo\", \"á\"]"; + const char* json_composed = "[\"foo\", \"\xc3\xa1\"]"; const char* json_decomposed = "[\"foo\", \"a\xcc\x81\"]"; for (const auto& options : compose_options) { this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_composed, diff --git a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc index 582d559dde2e..6d4525c28696 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_utf8.cc @@ -547,6 +547,16 @@ struct Utf8NormalizeBase { if (res < 0) { return Status::Invalid("Cannot normalize utf8 string: ", utf8proc_errmsg(res)); } + if (decompose_options_ & UTF8PROC_COMPOSE) { + // utf8proc_decompose() only decomposes; the canonical composition step for + // NFC and NFKC is done in-place by utf8proc_normalize_utf32(). + res = utf8proc_normalize_utf32( + reinterpret_cast(codepoints_.data()), res, + decompose_options_); + if (res < 0) { + return Status::Invalid("Cannot normalize utf8 string: ", utf8proc_errmsg(res)); + } + } return res; } diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 8b2ad2b333fc..0548be7c9740 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -3825,6 +3825,15 @@ def test_utf8_normalize(): assert pc.utf8_normalize(arr, form="NFKC") == pa.array(["0123"]) assert pc.utf8_normalize(arr, "NFD") == arr assert pc.utf8_normalize(arr, "NFKD") == pa.array(["0123"]) + # GH-51225: composing forms must compose, not only decompose + composed = pa.array(["\u00e9", "\ud55c", None]) + decomposed = pa.array(["e\u0301", "\u1112\u1161\u11ab", None]) + for form in ("NFC", "NFKC"): + assert pc.utf8_normalize(decomposed, form=form) == composed + assert pc.utf8_normalize(composed, form=form) == composed + for form in ("NFD", "NFKD"): + assert pc.utf8_normalize(composed, form=form) == decomposed + assert pc.utf8_normalize(decomposed, form=form) == decomposed with pytest.raises( ValueError, match='"NFZ" is not a valid Unicode normalization form'): From 87c8bfb4ebc6245b3efb5ec6fe57936d9c390d65 Mon Sep 17 00:00:00 2001 From: singhpratech Date: Wed, 9 Sep 2026 18:16:06 -0400 Subject: [PATCH 2/3] GH-51225: [C++] utf8_normalize: add a Hangul pair to the compose test --- .../compute/kernels/scalar_string_test.cc | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc index 28055cb44fe9..c56546f7966b 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc @@ -1260,6 +1260,25 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { &options); } + // Hangul composes algorithmically in utf8proc, not through the composition table. + // decomposed: U+1112(HANGUL CHOSEONG HIEUH) + U+1161(HANGUL JUNGSEONG A) + + // U+11AB(HANGUL JONGSEONG NIEUN) + // composed: U+D55C(HANGUL SYLLABLE HAN) + json_composed = "[\"\xed\x95\x9c\"]"; + json_decomposed = "[\"\xe1\x84\x92\xe1\x85\xa1\xe1\x86\xab\"]"; + for (const auto& options : compose_options) { + this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_composed, + &options); + this->CheckUnary("utf8_normalize", json_composed, this->type(), json_composed, + &options); + } + for (const auto& options : decompose_options) { + this->CheckUnary("utf8_normalize", json_composed, this->type(), json_decomposed, + &options); + this->CheckUnary("utf8_normalize", json_decomposed, this->type(), json_decomposed, + &options); + } + // canonical: U+00B2(Superscript Two) // compatibility: "2" const char* json_canonical = "[\"01\xc2\xb2!\"]"; From ba541be2be9c3dc1db74e49a05d6132672bc26f8 Mon Sep 17 00:00:00 2001 From: singhpratech Date: Thu, 10 Sep 2026 09:37:20 -0400 Subject: [PATCH 3/3] GH-51225: [C++] utf8_normalize: drop the utf8proc detail from the Hangul comment, add a singleton case --- .../arrow/compute/kernels/scalar_string_test.cc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cpp/src/arrow/compute/kernels/scalar_string_test.cc b/cpp/src/arrow/compute/kernels/scalar_string_test.cc index c56546f7966b..74650c1d76c5 100644 --- a/cpp/src/arrow/compute/kernels/scalar_string_test.cc +++ b/cpp/src/arrow/compute/kernels/scalar_string_test.cc @@ -1260,7 +1260,6 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { &options); } - // Hangul composes algorithmically in utf8proc, not through the composition table. // decomposed: U+1112(HANGUL CHOSEONG HIEUH) + U+1161(HANGUL JUNGSEONG A) + // U+11AB(HANGUL JONGSEONG NIEUN) // composed: U+D55C(HANGUL SYLLABLE HAN) @@ -1279,6 +1278,21 @@ TYPED_TEST(TestStringKernels, Utf8Normalize) { &options); } + // singleton: U+212B(ANGSTROM SIGN) decomposes to U+0041(LATIN CAPITAL LETTER A) + + // U+030A(COMBINING RING ABOVE), which composes to U+00C5(LATIN CAPITAL + // LETTER A WITH RING ABOVE), so the composed form differs from the input + const char* json_singleton = "[\"\xe2\x84\xab\"]"; + json_composed = "[\"\xc3\x85\"]"; + json_decomposed = "[\"A\xcc\x8a\"]"; + for (const auto& options : compose_options) { + this->CheckUnary("utf8_normalize", json_singleton, this->type(), json_composed, + &options); + } + for (const auto& options : decompose_options) { + this->CheckUnary("utf8_normalize", json_singleton, this->type(), json_decomposed, + &options); + } + // canonical: U+00B2(Superscript Two) // compatibility: "2" const char* json_canonical = "[\"01\xc2\xb2!\"]";