From 6b6c72fcca4fef924eb0b305dea7e727d9ec010a Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Sun, 20 Sep 2026 10:38:11 +0200 Subject: [PATCH 1/2] forms/tests: spell the bidi controls in tst_i18n.qml as escapes (fixes #610) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The morph#596 block embedded U+061C, U+200E and U+200F as raw code points while the morph#583 block sixty lines above, the corpus PR #609 added, and the C++ mirror in tests/test_render_locale_format.cpp all spell the same characters explicitly. Two idioms for one kind of value, and one of the two renders as nothing -- a reviewer cannot read what line 310 asserted without a hexdump, and a copy-paste through a tool that strips bidi controls would delete one silently while the suite stayed green, because ASCII '+' is accepted in every locale by design (morph#596). Seventeen raw code points across nine assertion lines become \uXXXX escapes. No string value changes, which is the whole point, so it is proved rather than asserted: every double- and single-quoted literal in the file was extracted and its QML/JS escapes decoded, before and after, and the two hex dumps of the decoded values are identical. As a negative control, a harness that drops one ‎ from the converted file is detected by that same comparison, so it is measuring something. The issue's own acceptance command now returns nothing: $ grep -cP '[\x{061C}\x{200E}\x{200F}]' src/qt/forms/tests/tst_i18n.qml 0 and tree-wide over the full trojan-source class (bidi controls plus the embedding, override and isolate ranges) `grep -rlP` over src/ tests/ examples/ matches no file at all. The QML suite is green: 284 passed, 0 failed. The repo-wide lint the issue floats as optional is deliberately not here; it is a policy decision and belongs in its own issue. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AbwhcguQFkhvVi2AH19sWk --- src/qt/forms/tests/tst_i18n.qml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/qt/forms/tests/tst_i18n.qml b/src/qt/forms/tests/tst_i18n.qml index 93d5a0188..796567d32 100644 --- a/src/qt/forms/tests/tst_i18n.qml +++ b/src/qt/forms/tests/tst_i18n.qml @@ -280,8 +280,8 @@ Item { function test_qtExposesAPositiveSignOnTheLocaleObject() { compare(Qt.locale("C").positiveSign, "+") compare(Qt.locale("de").positiveSign, "+") - compare(Qt.locale("ar_EG").positiveSign, "؜+") - compare(Qt.locale("az_IR").positiveSign, "‎+‎") + compare(Qt.locale("ar_EG").positiveSign, "\u061C+") + compare(Qt.locale("az_IR").positiveSign, "\u200E+\u200E") } // The defect: a leading "+" fell through to the "any other character is @@ -305,22 +305,22 @@ Item { // what the product accepts. function test_positiveSignIsMatchedAsAWholeString() { compare(localeForm.normalizeLocaleNumber("+5", ".", ""), "5") - compare(localeForm.normalizeLocaleNumber("؜+5", ".", "", "-", "؜+"), "5") // ar_EG - compare(localeForm.normalizeLocaleNumber("‎+5", ".", "", "-", "‎+"), "5") // ar_DZ - compare(localeForm.normalizeLocaleNumber("‎+‎5", ".", "", "-", "‎+‎"), "5") // az_IR - compare(localeForm.normalizeLocaleNumber("‏+5", ".", "", "-", "‏+"), "5") // ckb_IQ + compare(localeForm.normalizeLocaleNumber("\u061C+5", ".", "", "-", "\u061C+"), "5") // ar_EG + compare(localeForm.normalizeLocaleNumber("\u200E+5", ".", "", "-", "\u200E+"), "5") // ar_DZ + compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", ".", "", "-", "\u200E+\u200E"), "5") // az_IR + compare(localeForm.normalizeLocaleNumber("\u200F+5", ".", "", "-", "\u200F+"), "5") // ckb_IQ // Controls: with positiveSign left at its ASCII default, the // bidi-prefixed spellings are still rejected -- which is what makes // the parameter, and not the unconditional ASCII acceptance, the // thing under test above. - compare(localeForm.normalizeLocaleNumber("؜+5", ".", ""), null) - compare(localeForm.normalizeLocaleNumber("‎+‎5", ".", ""), null) + compare(localeForm.normalizeLocaleNumber("\u061C+5", ".", ""), null) + compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", ".", ""), null) // The ASCII "+" stays accepted in a bidi-sign locale, for the same // reason the ASCII "-" does (morph#583): the locale's own spelling // is on no keyboard. - compare(localeForm.normalizeLocaleNumber("+5", ".", "", "-", "؜+"), "5") + compare(localeForm.normalizeLocaleNumber("+5", ".", "", "-", "\u061C+"), "5") // An empty positiveSign leaves the ASCII spelling, and must not // match at every index. compare(localeForm.normalizeLocaleNumber("+5", ".", "", "-", ""), "5") From e5b266b285d6ff2b5b1fa7ce49a089edb8b0d879 Mon Sep 17 00:00:00 2001 From: Yaraslau Tamashevich Date: Sun, 20 Sep 2026 11:00:01 +0200 Subject: [PATCH 2/2] render/forms: carry the locale facts in a NumericLocale, digits included (fixes #591) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit normalizeLocaleNumber compared one byte against ['0','9'], so a user of any locale whose digits are not ASCII could not enter a number at all -- a flat rejection, not a wrong value. formatCanonicalNumber was its consistent inverse: it copied the canonical ASCII digits out unchanged and wrapped them in the locale's separators and sign, which is exactly why the pair round-tripped and the gap was invisible from either side alone. Teaching the entry edge to accept U+0665 while the display edge kept emitting '5' would satisfy the bug report and break the round trip forms.md requires, so both edges and both QML mirrors move together. Measured, QLocale::matchingLocales under Qt 6.11.2, all 711 locales: 76 report a zeroDigit other than ASCII '0', across eleven sets -- U+0660 (26), U+06F0 (19), U+1E950 (12), U+0966 (8), U+09E6 (4), U+11136 (2), and one each of U+07C0, U+0F20, U+1040, U+1C50, U+ABF0. The issue's open question was the size of that set: it could defend 24 from *sign* data and said so. 76 is the measured figure, and two of the sets are astral, so a digit is up to four UTF-8 bytes and two UTF-16 units -- both scans now decode a code point rather than comparing a unit. The locale facts travel as one aggregate rather than six positional views. That is not tidiness: five adjacent swappable string_views already needed a clang-tidy suppression for bugprone-easily-swappable-parameters, with a paragraph of justification, on each of the two functions and on the sign helper. A sixth would have weakened that argument. The aggregate deletes all three suppressions -- verified, not assumed: with every suppression removed from a copy of the header, clang-tidy 22.1.8 under this repo's .clang-tidy reported the check at three sites before and none after, and the header is now clean under the full check list with no suppression of that check in it. There is no back-compatible positional overload; two spellings of one call is how the edges drifted apart to begin with. Behaviour: entry accepts a digit in [zeroDigit, zeroDigit+9] or in ['0','9'], display emits only the locale's. That asymmetry is morph#596's rule for signs applied to digits, and the header says so next to the sign reasoning. An entry that mixes the two families ("٥5") is malformed rather than read as 55 -- a decision, so a test on each edge records it. An empty or undecodable zeroDigit reads as ASCII "0", the reading an empty negativeSign gets. zeroDigit defaults to "0", so every existing caller is byte-identical -- asserted rather than assumed. A 1680-case sweep (14 locale configurations x 60 entries x both edges) was run against the pre-change header and against this one; the two dumps are identical. Negative control: removing the decoder's overlong-encoding guard changes 13 of those rows, so the sweep measures something. That guard is load-bearing for exactly this reason -- the digit test is a range test on a decoded code point, so a lenient decoder would read the overlong C0 B5 as the digit '5' in the *default* locale. The acceptance test is the round trip over a native-digit corpus, on both edges, over all eleven sets. It asserts byte equality *and* that the display text contains no ASCII digit, because the round trip alone does not fail for the half-fix: entry accepts ASCII digits, so a display edge still emitting them round-trips perfectly. Mutation-checked rather than assumed -- reverting each edge in turn, and the mixed-family rule: C++ display edge reverted -> 2 test cases fail, 100 assertions "؜-1٬050٫25" == "؜-١٬٠٥٠٫٢٥" C++ entry edge reverted -> 4 test cases fail, 106 assertions C++ mixing allowed -> 1 test case fails QML display edge reverted -> test_theDisplayEdgeEmitsTheLocalesDigits, test_thePairRoundTripsThroughEveryMeasuredDigitSet QML entry edge reverted -> 4 functions fail QML mixing allowed -> test_anEntryMayNotMixDigitFamilies Green afterwards: morph_tests 1548 cases / 22880 assertions (1 failed as expected, pre-existing), the [locale] subset 403 assertions in 49 cases, and the QML suite 292 passed / 0 failed. Doxygen with FAIL_ON_WARNINGS builds clean, and so does qmllint bar one pre-existing WizardView warning. forms.md states the round trip in terms of the digit base, including the "and no ASCII digit in the display text" half, since that is the part a faithful-looking implementation drops. morph#583 is untouched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01AbwhcguQFkhvVi2AH19sWk --- docs/spec/forms/forms.md | 163 ++++++- include/morph/render/locale_format.hpp | 607 ++++++++++++++++++------- src/qt/forms/qml/DynamicForm.qml | 142 +++++- src/qt/forms/tests/tst_i18n.qml | 234 +++++++--- tests/test_render_locale_format.cpp | 592 +++++++++++++++++------- 5 files changed, 1332 insertions(+), 406 deletions(-) diff --git a/docs/spec/forms/forms.md b/docs/spec/forms/forms.md index dfff4038c..5e03006da 100644 --- a/docs/spec/forms/forms.md +++ b/docs/spec/forms/forms.md @@ -1316,17 +1316,138 @@ could not pre-empt. Display formatting is the renderer's duty; the wire stays canonical: -- **Numbers.** `morph::render::normalizeLocaleNumber(text, decimalSeparator, - groupSeparator)` (`include/morph/render/locale_format.hpp`) converts a - locale-formatted entry (`"1.050,25"`) to the canonical `.`-decimal text - `Quantity`'s exact digit routines already consume (`"1050.25"`); malformed - input yields `std::nullopt` rather than a best-effort guess. - `formatCanonicalNumber` is the display-direction inverse, with - display-only thousands grouping. The exact `Rational`/`Quantity` digit - arithmetic ([rational.md](../util/rational.md)) never sees a - locale-formatted string — the conversion happens at the control edge only. - - Both separators are `std::string_view`, not `char`, because a real locale's +- **Numbers.** `morph::render::normalizeLocaleNumber(text, loc)` + (`include/morph/render/locale_format.hpp`) converts a locale-formatted entry + (`"1.050,25"`) to the canonical `.`-decimal text `Quantity`'s exact digit + routines already consume (`"1050.25"`); malformed input yields `std::nullopt` + rather than a best-effort guess. `formatCanonicalNumber(canonicalText, loc)` + is the display-direction inverse, with display-only thousands grouping. The + exact `Rational`/`Quantity` digit arithmetic + ([rational.md](../util/rational.md)) never sees a locale-formatted string + — the conversion happens at the control edge only. + + **The locale facts travel as one aggregate, not as a row of positional + views** (morph#591): + + ```cpp + struct NumericLocale { + std::string_view decimalSeparator = "."; + std::string_view groupSeparator = ""; + std::string_view negativeSign = "-"; + std::string_view positiveSign = "+"; + std::string_view zeroDigit = "0"; + }; + ``` + + Both edges take it, and a call site names each fact with a designated + initialiser: `normalizeLocaleNumber(text, {.decimalSeparator = ",", + .groupSeparator = "."})`. Three things this is for, and the first is not + tidiness. The five facts were five adjacent `std::string_view` parameters, + every one silently swappable with its neighbours, and the header carried a + clang-tidy suppression for `bugprone-easily-swappable-parameters` — on each + of the two functions and on the sign helper — with a paragraph of + justification each. A sixth would have made that argument weaker, not + stronger; the aggregate deleted all three suppressions instead, and the header + is clean under `bugprone-*` with no suppression of that check anywhere in it. + Second, the next locale fact (a percent sign, an exponent separator) is a new + defaulted member rather than a seventh parameter. Third, and the reason it is + worth the churn: **the two edges take the same type**, so "these two must + agree" is structural rather than a convention a caller can get half right — + which is the drift morph#591 and morph#599 both came from. There is + deliberately no back-compatible positional overload: two spellings of one call + is how the edges drifted apart to begin with. The QML mirror takes the + parallel shape, an object literal with the same member names, so the two + mirrors stay structurally identical. + + Every member is defaulted to its `"C"`-locale spelling, so a caller that names + none of them gets the identity transform in both directions. + + **The digits are locale data too, carried as a base** (morph#591). A Unicode + decimal digit set *is* ten contiguous code points — UAX #44 assigns `Nd` + with `Numeric_Value` 0 through 9 in code point order — so a single + `zeroDigit` is sufficient and a ten-element table is not needed. Measured with + `QLocale::matchingLocales` under Qt 6.11.2, over the same 711 locales: + + | `zeroDigit` | locales | e.g. | + | --- | ---: | --- | + | U+0030 | 604 | `C` | + | U+0660 | 26 | `ar_BH` | + | U+06F0 | 19 | `fa_IR` | + | U+1E950 | 12 | `ff_Adlm_BF` | + | U+0966 | 8 | `bgc_IN` | + | U+09E6 | 4 | `as_IN` | + | U+11136 | 2 | `ccp_BD` | + | U+07C0 | 1 | `nqo_GN` | + | U+0F20 | 1 | `dz_BT` | + | U+1040 | 1 | `my_MM` | + | U+1C50 | 1 | `sat_IN` | + | U+ABF0 | 1 | `mni_IN` | + + 76 of 711, across eleven distinct sets. Two of them (Chakma U+11136, Adlam + U+1E950) are outside the BMP, so one digit is four UTF-8 bytes on the C++ + edge and two UTF-16 units on the QML edge — both scans therefore decode a + code *point* rather than comparing a unit. The figure the issue could defend + before this was 24, and it said so plainly: 24 was a count of locales whose + *negative sign* is `U+061C U+002D`, not a digit-set count. 76 is the measured + one. + + Before this, `normalizeLocaleNumber` compared one byte against `['0','9']`, + so a user of any of those 76 locales could not enter a number at all — a flat + rejection, not a wrong value. + + **Both edges move or neither does, and the round trip is what says so.** + `formatCanonicalNumber` used to copy the canonical ASCII digits out unchanged. + That is *why* the pair was self-consistent and the defect invisible from + either side alone: display emitted the locale's separators and sign around + ASCII digits, and entry accepted exactly that. Teaching entry to accept + U+0665 while display kept emitting `'5'` satisfies a naive reading of the bug + report and breaks the round trip this section requires. So the display edge + emits the digit that far above `zeroDigit`, and the property is stated in + those terms: + + > for every canonical `-?[0-9]+(\.[0-9]+)?` text and every `NumericLocale`, + > `normalizeLocaleNumber(formatCanonicalNumber(canonical, loc), loc) == + > canonical`, byte for byte, with every digit of the display text drawn from + > `[loc.zeroDigit, loc.zeroDigit + 9]`. + + The second half of that sentence is load-bearing. The round trip *alone* does + not fail if only the entry edge moved, because entry accepts ASCII digits too + — so the test that pins this asserts the display text contains no ASCII + digit as well as asserting the round trip, and it is that assertion which + fails for the half-fix. Pinned over all eleven digit sets on both edges: + `[morph591]` in `tests/test_render_locale_format.cpp`, and the + `test_thePairRoundTripsThroughEveryMeasuredDigitSet` function in + `src/qt/forms/tests/tst_i18n.qml`. + + **Entry accepts the locale's digits *and* ASCII ones; display emits only the + locale's.** That asymmetry is the rule morph#596 already set for signs, + applied to digits: an ASCII `'+'` is accepted in every locale because the + locale's own spelling is on no keyboard, and an ASCII `'5'` is accepted in an + `ar_EG` locale for the same reason — a user with an ASCII keyboard has to be + able to type a number. It costs nothing, because the canonical output spells + every digit in ASCII whatever the input spelled it, so no two accepted + spellings can produce different *values*. + + **An entry may not mix the two digit families.** `"\u06655"` — one + Arabic-Indic digit and one ASCII digit — is malformed, not `"55"`. This is a + decision rather than a consequence, so it is written here and pinned by a test + on both edges: neither a keyboard nor a display edge produces an + interleaving, and rejecting it matches the existing strictness about a sign + anywhere but the leading position. When `zeroDigit` is the ASCII `"0"` the two + families are the same set, so nothing can mix and the rule is invisible — + which is why it costs no existing caller anything. + + **An empty or undecodable `zeroDigit` reads as ASCII `"0"`**, the reading an + empty `negativeSign` gets and for the same reason: there is no locale without + digits, so empty cannot mean absence, and a base of "nothing" would reject + every entry the locale can produce. Because `zeroDigit` defaults to `"0"`, + every caller that does not name it is byte-identical to the + five-positional-parameter version — asserted rather than assumed: a 1680-case + sweep (14 locale configurations × 60 entries × both edges) run against the + pre-morph#591 header and against this one produced identical output. + + All the locale facts are `std::string_view`, not `char`, because a real + locale's separator is not always one byte: fr-FR groups with U+202F (narrow no-break space, 3 bytes in UTF-8) and several locales use U+00A0 (2 bytes). Typed as `char`, neither could be expressed at all — a caller could only pass some @@ -1335,9 +1456,9 @@ Display formatting is the renderer's duty; the wire stays canonical: malformed. An empty view means "this locale has no such separator". **So is the negative sign** (morph#583). The same argument applies to the - sign, and was missing here: both functions take a fourth - `std::string_view negativeSign = "-"`, matched and emitted as a whole string - the way the separators are. Of the 711 locales `QLocale::matchingLocales` + sign, and was missing here: both edges read `NumericLocale::negativeSign`, + matched and emitted as a whole string the way the separators are. Of the 711 + locales `QLocale::matchingLocales` reports under Qt 6.11.2, 77 spell it as something other than a bare ASCII `'-'`: @@ -1378,11 +1499,11 @@ Display formatting is the renderer's duty; the wire stays canonical: `qtLocale: Qt.locale(displayLocale)` and forwards `qtLocale.decimalPoint`/`qtLocale.groupSeparator`, and now forwards `qtLocale.negativeSign` from the same object at all three call sites. The - parameter is defaulted, so a caller that passes three arguments is unchanged. + member is defaulted, so a caller that names only the separators is unchanged. **A leading positive sign is accepted on entry and never emitted on - display** (morph#596). `normalizeLocaleNumber` takes a fifth - `std::string_view positiveSign = "+"`, matched exactly as `negativeSign` is — + display** (morph#596). `normalizeLocaleNumber` reads + `NumericLocale::positiveSign`, matched exactly as `negativeSign` is — the locale's own spelling as a whole string, plus a bare ASCII `'+'` in every locale — and **drops** what it matches: `"+5"` normalises to `"5"`, not to `"+5"`. Measured over `QLocale::positiveSign` for the same 711 locales under @@ -1404,8 +1525,8 @@ Display formatting is the renderer's duty; the wire stays canonical: included. **The two functions are deliberately not inverse across a positive sign.** - `formatCanonicalNumber` takes no `positiveSign` parameter and never emits a - positive sign, in any locale. This breaks the strict inverse relationship the + `formatCanonicalNumber` ignores `NumericLocale::positiveSign` entirely and + never emits a positive sign, in any locale. This breaks the strict inverse relationship the pair otherwise holds, on purpose, and it is written down here rather than left for the next reader to infer from a missing parameter. The reason is the asymmetry in what each direction can get wrong. Canonical text is @@ -1453,7 +1574,9 @@ Display formatting is the renderer's duty; the wire stays canonical: it the same way, and its `formatCanonicalNumber` takes none, for the reason above; the renderer forwards `qtLocale.positiveSign` at the two entry call sites that already forward `qtLocale.negativeSign`, and nothing changes at the - display call site. + display call site. All three call sites now also forward + `qtLocale.zeroDigit`, the display one included — that is what "both edges, or + neither" costs for morph#591. **The two separators are matched the same way, for consistency rather than for a locale** (morph#599). Both sign conversions above left the mirror's diff --git a/include/morph/render/locale_format.hpp b/include/morph/render/locale_format.hpp index 973c16893..e2c67d491 100644 --- a/include/morph/render/locale_format.hpp +++ b/include/morph/render/locale_format.hpp @@ -13,15 +13,34 @@ /// `.`-decimal text), and a renderer calls `normalizeLocaleNumber` once, at /// the point text leaves the control, before handing it to those routines. /// +/// @par One aggregate, not a row of swappable views (morph#591) +/// Both functions take a single `NumericLocale`. They used to take the locale +/// facts as four and five positional `std::string_view`s, every one of which +/// was silently swappable with its neighbours -- the header carried a +/// clang-tidy suppression block for `bugprone-easily-swappable-parameters`, +/// with a paragraph of justification, on each of the two functions and on the +/// sign helper below. (Spelling the marker out here would suppress nothing and +/// trip `clang-tidy-nolint`'s unmatched-begin check, which is why this +/// paragraph names the check instead.) Adding a digit base would have made +/// six adjacent views. With the aggregate a call site +/// names each fact (`{.decimalSeparator = ",", .groupSeparator = "."}`), no two +/// parameters of either function share a type, and all three suppressions are +/// deleted rather than widened. The next locale fact -- a percent sign, an +/// exponent separator -- is then a new defaulted member rather than a seventh +/// parameter. The two edges taking the *same* type is the point as much as the +/// naming is: "these two must agree" becomes structural instead of a convention +/// a caller can get half right, which is the drift morph#591 and morph#599 were +/// both about. +/// /// @par Separators are strings, not characters -/// Both functions take their separators as `std::string_view`, because a -/// real locale's separator is not always one byte. fr-FR groups with U+202F -/// (narrow no-break space) and several locales use U+00A0 — three and two -/// UTF-8 bytes respectively. Typed as `char`, those cannot be expressed at -/// all: the caller can only pass some single byte that never matches, so a -/// perfectly valid `"1 050,25"` typed by a French user normalises to -/// `std::nullopt` and the entry is reported malformed. An empty view means -/// "this locale has no such separator" (the role `'\0'` used to play). +/// The locale facts are `std::string_view`, because a real locale's separator +/// is not always one byte. fr-FR groups with U+202F (narrow no-break space) and +/// several locales use U+00A0 -- three and two UTF-8 bytes respectively. Typed +/// as `char`, those cannot be expressed at all: the caller can only pass some +/// single byte that never matches, so a perfectly valid `"1 050,25"` typed by a +/// French user normalises to `std::nullopt` and the entry is reported +/// malformed. An empty view means "this locale has no such separator" (the role +/// `'\0'` used to play). /// /// @par So is the negative sign /// For the same reason, and measured rather than assumed: of the 711 locales @@ -31,19 +50,43 @@ /// RIGHT-TO-LEFT MARK -- making it two or three code points, and ar_DZ does so /// even though its sign is the ordinary hyphen. Matched as a single `char`, /// none of those round-trips: the display edge emitted a sign the entry edge -/// then rejected. So `negativeSign` is a `std::string_view` matched as a whole -/// string too, defaulting to `"-"` so that every existing caller is unchanged -/// (morph#583). +/// then rejected. So `negativeSign` is matched as a whole string too, +/// defaulting to `"-"` so that every existing caller is unchanged (morph#583). /// /// @par And so is the positive sign, on the entry edge only -/// `normalizeLocaleNumber` takes a fifth `std::string_view positiveSign = "+"` -/// and *drops* what it matches, because canonical text has no `'+'` in it -/// (morph#596). `formatCanonicalNumber` has no such parameter and never emits -/// one: a positive number displays unsigned in every locale, and changing that -/// would alter every positive number the product shows. So the two functions -/// are inverse across the decimal separator, the grouping and the negative -/// sign, but deliberately not across a positive sign -- entry accepts a -/// spelling display never produces. +/// `normalizeLocaleNumber` reads `NumericLocale::positiveSign` and *drops* what +/// it matches, because canonical text has no `'+'` in it (morph#596). +/// `formatCanonicalNumber` never emits one: a positive number displays unsigned +/// in every locale, and changing that would alter every positive number the +/// product shows. So the two functions are inverse across the decimal +/// separator, the grouping, the digits and the negative sign, but deliberately +/// not across a positive sign -- entry accepts a spelling display never +/// produces. +/// +/// @par The digits are locale data too (morph#591) +/// `NumericLocale::zeroDigit` is the locale's DIGIT ZERO, and the ten digits +/// are the ten code points contiguous from it. One base is sufficient rather +/// than a ten-element table because a Unicode decimal digit set *is* ten +/// contiguous code points: UAX #44 assigns `Nd` with `Numeric_Value` 0 through +/// 9 in code point order. Measured over the same 711 locales under Qt 6.11.2, +/// 76 report a `zeroDigit` other than ASCII `'0'`, across eleven distinct sets: +/// U+0660 (26 locales), U+06F0 (19), U+1E950 Adlam (12), U+0966 (8), U+09E6 +/// (4), U+11136 Chakma (2), and one each of U+07C0, U+0F20, U+1040, U+1C50 and +/// U+ABF0. Two of those sets are astral, so a digit is one to four UTF-8 bytes +/// and the scan decodes a code point rather than comparing a byte. +/// +/// @par Entry accepts more digit spellings than display emits +/// `normalizeLocaleNumber` accepts a digit in `[zeroDigit, zeroDigit + 9]` *or* +/// in `['0', '9']`; `formatCanonicalNumber` emits only the former. That +/// asymmetry is the rule morph#596 already set for signs, applied to digits: an +/// ASCII `'+'` is accepted in every locale because the locale's own spelling is +/// on no keyboard, and an ASCII `'5'` is accepted in an `ar_EG` locale for +/// exactly the same reason. A user with an ASCII keyboard in a native-digit +/// locale would otherwise be unable to enter a number at all. As with the +/// positive sign, accepting a spelling display never produces costs nothing: +/// the canonical output spells every digit in ASCII whatever the input spelled +/// it, so no *value* can differ. What is not accepted is the two families in +/// one entry -- see "Digit families do not mix" on `normalizeLocaleNumber`. #include #include @@ -52,8 +95,222 @@ namespace morph::render { +/// @brief The locale facts both control-edge conversions need, in one +/// designated-initialisable aggregate. +/// +/// Every member is defaulted to its `"C"`-locale spelling, so a +/// default-constructed `NumericLocale` is the identity transform in both +/// directions and a caller naming only the members it cares about gets the +/// previous five-parameter defaults exactly. +struct NumericLocale { + /// The locale's decimal-point string, e.g. `","`. Empty means the locale + /// has no decimal separator (an integer-only entry). + std::string_view decimalSeparator = "."; + /// The locale's digit-grouping string, e.g. `"."` or U+202F. Empty means + /// the locale does not group. + std::string_view groupSeparator; + /// The locale's negative-sign string, e.g. `"\u2212"`. Empty is read as the + /// ASCII `"-"`, not as "this entry cannot be negative". + std::string_view negativeSign = "-"; + /// The locale's positive-sign string, e.g. `"\u061c+"`. Entry-edge only: + /// it is accepted and dropped, never emitted. Empty leaves the ASCII `"+"` + /// as the only accepted spelling. + std::string_view positiveSign = "+"; + /// The locale's DIGIT ZERO, e.g. `"\u0660"`. The ten digits are the ten + /// code points contiguous from it. Empty, or not a well-formed UTF-8 code + /// point, is read as the ASCII `"0"`. + std::string_view zeroDigit = "0"; +}; + namespace detail { +/// @brief One decoded UTF-8 code point: its value and how many bytes it took. +struct CodePoint { + /// The decoded scalar value; meaningless when @ref length is `0`. + char32_t value = 0; + /// The number of bytes consumed, or `0` when the input does not begin with + /// a well-formed UTF-8 sequence. + std::size_t length = 0; +}; + +/// @brief Decodes the UTF-8 sequence at the start of @p text. +/// +/// Strict: an overlong encoding, a surrogate code point, a truncated sequence +/// and a stray continuation byte all report a `CodePoint::length` of `0` +/// rather than some salvaged value. That strictness is load-bearing rather than +/// pedantic. The digit test below is a *range* test on the decoded value, so a +/// lenient decoder that let the overlong `C0 B5` through would read it as +/// U+0035 and accept it as the digit `'5'` -- in the default ASCII locale, +/// where the previous byte-range scan rejected that input. Rejecting it here is +/// what keeps the default behaviour byte-identical. +/// @param text The remainder of the entry, starting at the scan position. +/// @return The decoded code point, or a `length` of `0` when @p text does not +/// start with a well-formed sequence. +[[nodiscard]] inline CodePoint decodeUtf8(std::string_view text) { + if (text.empty()) { + return {}; + } + auto const lead = static_cast(text.front()); + if (lead < 0x80U) { + return {.value = lead, .length = 1}; + } + + std::size_t length = 0; + char32_t value = 0; + char32_t least = 0; // the smallest value this length may legally encode + if ((lead & 0xE0U) == 0xC0U) { + length = 2; + value = lead & 0x1FU; + least = 0x80; + } else if ((lead & 0xF0U) == 0xE0U) { + length = 3; + value = lead & 0x0FU; + least = 0x800; + } else if ((lead & 0xF8U) == 0xF0U) { + length = 4; + value = lead & 0x07U; + least = 0x10000; + } else { + return {}; // a continuation byte or a 5+ byte lead: not a lead byte + } + if (text.size() < length) { + return {}; // truncated + } + for (std::size_t k = 1; k < length; ++k) { + // k is bounded by the size check above. + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-avoid-unchecked-container-access) + auto const cont = static_cast(text[k]); + if ((cont & 0xC0U) != 0x80U) { + return {}; // not a continuation byte + } + value = (value << 6U) | (cont & 0x3FU); + } + constexpr char32_t kMaxScalar = 0x10FFFF; + constexpr char32_t kSurrogateFirst = 0xD800; + constexpr char32_t kSurrogateLast = 0xDFFF; + if (value < least || value > kMaxScalar || (value >= kSurrogateFirst && value <= kSurrogateLast)) { + return {}; // overlong, out of range, or a surrogate + } + return {.value = value, .length = length}; +} + +/// @brief Appends @p value to @p out as UTF-8. +/// @param out The string to append to. +/// @param value A scalar value; callers here only ever pass a digit derived +/// from a `zeroDigit` that `decodeUtf8` already validated. +inline void appendUtf8(std::string& out, char32_t value) { + constexpr char32_t kContMask = 0x3F; + constexpr unsigned kContShift = 6; + auto const byte = [&out](char32_t bits) { out += static_cast(bits); }; + if (value < 0x80) { + byte(value); + } else if (value < 0x800) { + byte(0xC0U | (value >> kContShift)); + byte(0x80U | (value & kContMask)); + } else if (value < 0x10000) { + byte(0xE0U | (value >> (2 * kContShift))); + byte(0x80U | ((value >> kContShift) & kContMask)); + byte(0x80U | (value & kContMask)); + } else { + byte(0xF0U | (value >> (3 * kContShift))); + byte(0x80U | ((value >> (2 * kContShift)) & kContMask)); + byte(0x80U | ((value >> kContShift) & kContMask)); + byte(0x80U | (value & kContMask)); + } +} + +/// @brief Appends one character of canonical text to a display string, with a +/// digit rewritten into the locale's set. +/// +/// Anything that is not an ASCII digit is copied through verbatim. That arm is +/// not dead code for a caller that honours the contract, but it is what keeps +/// `formatCanonicalNumber` byte-identical for one that does not: the function +/// has always passed non-digits out unchanged, so a `"12.34.56"` handed to the +/// display edge kept its second `'.'` rather than becoming some code point +/// below the digit base. morph#591 widened what a digit *is*, not what the +/// function does with text that has none. +/// @param out The display string to append to. +/// @param base The locale's DIGIT ZERO code point (see `digitBase`). +/// @param chr One byte of canonical text. +inline void appendDisplayDigit(std::string& out, char32_t base, char chr) { + if (chr >= '0' && chr <= '9') { + appendUtf8(out, base + static_cast(chr - '0')); + } else { + out += chr; + } +} + +/// @brief How many bytes @p value occupies in UTF-8. +/// @param value A scalar value. +/// @return `1`, `2`, `3` or `4`. +[[nodiscard]] inline std::size_t utf8Length(char32_t value) { + if (value < 0x80) { + return 1; + } + if (value < 0x800) { + return 2; + } + if (value < 0x10000) { + return 3; + } + return 4; +} + +/// @brief The code point of @p zeroDigit, or `U'0'` when it is empty or not a +/// well-formed UTF-8 code point. +/// +/// Empty reads as the ASCII `'0'` for the reason an empty `negativeSign` reads +/// as `'-'`: there is no locale without digits, so empty cannot mean absence, +/// and a base of "nothing" would reject every entry the locale can produce. +/// Only the first code point is read; anything after it is locale data this +/// function has no use for. +/// @param zeroDigit The locale's DIGIT ZERO spelling. +/// @return The base code point of the locale's digit set. +[[nodiscard]] inline char32_t digitBase(std::string_view zeroDigit) { + CodePoint const decoded = decodeUtf8(zeroDigit); + return decoded.length == 0 ? U'0' : decoded.value; +} + +/// @brief A digit matched at the start of an entry. +struct DigitMatch { + /// The number of bytes the digit occupies; `0` when there is no digit there. + std::size_t length = 0; + /// The canonical ASCII spelling of the digit's value, `'0'`-`'9'`. + char canonical = '0'; + /// Whether the match came from the locale's own digit set rather than from + /// the ASCII set every locale additionally accepts. When @p base is `U'0'` + /// the two sets coincide and this is always `true`. + bool native = false; +}; + +/// @brief Matches a digit at the start of @p rest, in the locale's set or in +/// ASCII. +/// +/// A function of its own for the reason `leadingSign` is: the normalising scan +/// reads as one statement per character class, and folding the two digit +/// families plus the UTF-8 decode into it took it over clang-tidy's cognitive +/// complexity threshold. +/// @param rest The remainder of the entry, starting at the scan position. +/// @param base The locale's DIGIT ZERO code point (see `digitBase`). +/// @return The match, or a `length` of `0` when @p rest starts with no digit. +[[nodiscard]] inline DigitMatch leadingDigit(std::string_view rest, char32_t base) { + CodePoint const decoded = decodeUtf8(rest); + if (decoded.length == 0) { + return {}; + } + if (decoded.value >= base && decoded.value < base + 10) { + return { + .length = decoded.length, .canonical = static_cast(U'0' + (decoded.value - base)), .native = true}; + } + if (decoded.value >= U'0' && decoded.value <= U'9') { + // The ASCII spelling, accepted in every locale for the reason the ASCII + // '-' and '+' are: the locale's own digits are on the user's keyboard + // only if their keyboard has them. + return {.length = decoded.length, .canonical = static_cast(decoded.value), .native = false}; + } + return {}; +} + /// @brief Whether every group separator in @p text sits where a group /// separator can legally sit. /// @@ -61,58 +318,63 @@ namespace detail { /// preceded by one to three digits (the first group), or by exactly three /// (every later one); it is followed by exactly three more digits, at each /// group and at the end of the integer part; and it never appears after the -/// decimal separator. A locale with no grouping (@p groupSeparator empty) has -/// nothing to place, so it trivially passes. +/// decimal separator. A locale with no grouping (an empty +/// `NumericLocale::groupSeparator`) has nothing to place, so it trivially +/// passes. /// /// This is a pass of its own rather than extra state inside the normalising /// scan below: "the grouping is well placed" and "the digits convert" are two /// separate statements about the entry, and reading them as one made neither /// clear. /// +/// It counts *digits*, not bytes, in the locale's set as well as in ASCII +/// (morph#591). Scanning byte by byte was correct only while a digit was one +/// byte: a two-byte U+0665 would have reset the run-length counter on its own +/// continuation byte, so `"\u0661\u066c\u0660\u0665\u0660"` -- what +/// `formatCanonicalNumber` emits for `1050` in an `ar_EG` locale -- would be +/// rejected as badly grouped and the pair would not round-trip. +/// /// Characters this function does not recognise are simply not digits — the /// normalising scan is what rejects them, and it rejects them whatever this /// pass concludes. -/// @param text The locale-formatted entry. -/// @param decimalSeparator The locale's decimal-point string; may be empty. -/// @param groupSeparator The locale's digit-grouping string; empty means the -/// locale does not group. +/// @param text The locale-formatted entry. +/// @param loc The locale facts; only the two separators and the digit base are +/// read. /// @return `true` when the grouping is well placed (or absent). -[[nodiscard]] inline bool groupingIsWellPlaced(std::string_view text, std::string_view decimalSeparator, - std::string_view groupSeparator) { - if (groupSeparator.empty()) { +[[nodiscard]] inline bool groupingIsWellPlaced(std::string_view text, NumericLocale const& loc) { + if (loc.groupSeparator.empty()) { return true; } constexpr std::size_t kGroupSize = 3; + char32_t const base = digitBase(loc.zeroDigit); std::size_t digits = 0; bool sawGroup = false; bool sawDecimal = false; for (std::size_t i = 0; i < text.size();) { const std::string_view rest = text.substr(i); - if (rest.starts_with(groupSeparator)) { + if (rest.starts_with(loc.groupSeparator)) { bool const opensAGroup = sawGroup ? digits == kGroupSize : (digits >= 1 && digits <= kGroupSize); if (sawDecimal || !opensAGroup) { return false; } sawGroup = true; digits = 0; - i += groupSeparator.size(); + i += loc.groupSeparator.size(); continue; } - if (!decimalSeparator.empty() && rest.starts_with(decimalSeparator)) { + if (!loc.decimalSeparator.empty() && rest.starts_with(loc.decimalSeparator)) { if (sawGroup && digits != kGroupSize) { return false; // the last group of the integer part is short } sawDecimal = true; digits = 0; - i += decimalSeparator.size(); + i += loc.decimalSeparator.size(); continue; } - // i is bounded by the loop condition. - // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-avoid-unchecked-container-access) - char const chr = text[i]; - digits = (chr >= '0' && chr <= '9') ? digits + 1 : 0; - ++i; + DigitMatch const digit = leadingDigit(rest, base); + digits = digit.length != 0 ? digits + 1 : 0; + i += digit.length != 0 ? digit.length : 1; } // An ungrouped fractional part ends the number, so the trailing check only @@ -120,6 +382,23 @@ namespace detail { return !sawGroup || sawDecimal || digits == kGroupSize; } +/// @brief The length of @p separator at the start of @p rest, in bytes, or `0` +/// when it is not there -- or is empty. +/// +/// The empty case is why this is a function and not an inline `starts_with`: +/// `rest.starts_with("")` is `true` at every index, so an empty separator +/// matched inline would swallow the whole entry one zero-length step at a time. +/// Every call site used to spell the guard as `!sep.empty() && ...`, and two of +/// those conjunctions were what put `normalizeLocaleNumber` over clang-tidy's +/// cognitive-complexity threshold once the digit scan arrived. +/// @param rest The remainder of the entry, starting at the scan position. +/// @param separator The locale's spelling of this separator; empty means the +/// locale has none, and matches nothing. +/// @return The number of bytes the separator occupies, or `0`. +[[nodiscard]] inline std::size_t leadingSeparatorLength(std::string_view rest, std::string_view separator) { + return !separator.empty() && rest.starts_with(separator) ? separator.size() : 0U; +} + /// @brief The length of a sign at the start of @p rest, in bytes, or `0` when /// there is none there. /// @@ -174,29 +453,20 @@ struct SignMatch { /// locale Qt 6.11.2 reports -- `starts_with` is an exact prefix match and no /// locale spells one sign as a prefix of the other -- but it is fixed here so /// that it cannot vary. -/// @param rest The remainder of the entry, starting at the scan position. -/// @param negativeSign The locale's negative-sign string; empty leaves `'-'`. -/// @param positiveSign The locale's positive-sign string; empty leaves `'+'`. +/// @param rest The remainder of the entry, starting at the scan position. +/// @param loc The locale facts; only the two signs are read. An empty +/// `negativeSign` leaves `'-'`, an empty `positiveSign` leaves +/// `'+'`. /// @return The match, or a `length` of `0` when @p rest starts with no sign. -// NOLINTBEGIN(bugprone-easily-swappable-parameters) -// The two signs are adjacent `std::string_view`s sharing the suffix `Sign`, and -// the check is right that swapping them would be silent: the entry would take -// '+' as a negative and '-' as a positive. They stay in this order anyway, -// because it is normalizeLocaleNumber's own parameter order, and its single -// call site is four lines below this one. Reordering to break the adjacency -// would put the helper out of step with the function it exists to serve, which -// trades a mistake nobody can make here for one a reader of both would. -[[nodiscard]] inline SignMatch leadingSign(std::string_view rest, std::string_view negativeSign, - std::string_view positiveSign) { - // NOLINTEND(bugprone-easily-swappable-parameters) - std::size_t const negativeLength = leadingSignLength(rest, negativeSign, '-'); +[[nodiscard]] inline SignMatch leadingSign(std::string_view rest, NumericLocale const& loc) { + std::size_t const negativeLength = leadingSignLength(rest, loc.negativeSign, '-'); if (negativeLength != 0) { // The canonical spelling, whatever the locale's is. return {.length = negativeLength, .emits = "-"}; } // Dropped, never emitted: canonical text is `-?[0-9]+(\.[0-9]+)?` and has // no `+` in it. - return {.length = leadingSignLength(rest, positiveSign, '+'), .emits = ""}; + return {.length = leadingSignLength(rest, loc.positiveSign, '+'), .emits = ""}; } } // namespace detail @@ -204,14 +474,15 @@ struct SignMatch { /// @brief Converts a locale-formatted numeric string to canonical /// (`-?[0-9]+(\.[0-9]+)?`) text. /// -/// Drops @p groupSeparator where it is correctly placed, and replaces every -/// occurrence of @p decimalSeparator with `.`. Passing `decimalSeparator == -/// "."` and an empty @p groupSeparator is the identity transform (the -/// locale-free behavior). Malformed input (a second decimal separator, a -/// sign anywhere but the leading position of the *output*, or any character that -/// is not a digit) yields `std::nullopt` rather than a best-effort guess. The -/// decimal point counts as output, so a sign placed straight after the separator -/// ("`,-5`" in a de-DE locale) is rejected -- matching the QML mirror in +/// Drops `NumericLocale::groupSeparator` where it is correctly placed, replaces +/// every occurrence of `NumericLocale::decimalSeparator` with `.`, and rewrites +/// the locale's digits as ASCII ones. A default-constructed `NumericLocale` is +/// the identity transform (the locale-free behavior). Malformed input (a second +/// decimal separator, a sign anywhere but the leading position of the *output*, +/// two digit families in one entry, or any character that is not a digit) +/// yields `std::nullopt` rather than a best-effort guess. The decimal point +/// counts as output, so a sign placed straight after the separator ("`,-5`" in +/// a de-DE locale) is rejected -- matching the QML mirror in /// `src/qt/forms/qml/DynamicForm.qml`, which has always rejected it (morph#497). /// /// @par Grouping is validated, not stripped (morph#574) @@ -225,13 +496,13 @@ struct SignMatch { /// and the en-US mirror image `"1,5"` gave `15`. /// /// @par The two separators must differ -/// When @p groupSeparator is non-empty and equal to @p decimalSeparator the -/// entry is rejected: with one string in both roles there is no reading of -/// `"1.5"` the function could defend. This is a caller (locale-configuration) -/// error rather than a user one, but it is reported through the return value -/// like any other malformed entry, deliberately not through an assertion -- -/// an assertion would make the two build configurations behave differently at -/// a control edge, and would be untestable in the one where it fires. +/// When `groupSeparator` is non-empty and equal to `decimalSeparator` the entry +/// is rejected: with one string in both roles there is no reading of `"1.5"` +/// the function could defend. This is a caller (locale-configuration) error +/// rather than a user one, but it is reported through the return value like any +/// other malformed entry, deliberately not through an assertion -- an assertion +/// would make the two build configurations behave differently at a control +/// edge, and would be untestable in the one where it fires. /// /// The result is `.`-decimal and digit-only, but is **not** narrowed to /// `-?[0-9]+(\.[0-9]+)?`: a bare "`.`", a leading "`.5`" and a trailing "`5.`" @@ -240,11 +511,11 @@ struct SignMatch { /// is documented here rather than changed. /// /// Separators are matched as whole strings, so a multi-byte one (e.g. U+202F) -/// works; matching them before the per-byte digit scan is what keeps their -/// continuation bytes from being mistaken for stray non-digit characters. +/// works; matching them before the digit scan is what keeps their continuation +/// bytes from being mistaken for stray non-digit characters. /// /// @par The negative sign is matched as a whole string too (morph#583) -/// @p negativeSign is matched the same way, which is what lets a locale whose +/// `negativeSign` is matched the same way, which is what lets a locale whose /// sign is U+2212, or is prefixed by a bidi control mark, be entered at all -- /// 77 of the 711 locales Qt 6.11.2 knows. Before this the sign was the literal /// byte `'-'`, so `formatCanonicalNumber` emitted a sign this function then @@ -252,14 +523,14 @@ struct SignMatch { /// /// @par ASCII `'-'` stays accepted whatever the locale /// A bare `'-'` is accepted in the leading position in addition to -/// @p negativeSign. U+2212 and the bidi marks are on no keyboard, so matching +/// `negativeSign`. U+2212 and the bidi marks are on no keyboard, so matching /// only the locale's own spelling would reject the sign the user can actually /// type and leave them no way to enter a negative number at all. The hyphen has /// no second reading in a numeric entry, so accepting it is not the kind of /// guess morph#574 forbids -- that was about producing a wrong *value*, and /// this produces the only value the input can mean. /// -/// @par An empty @p negativeSign means the ASCII default, not "no sign" +/// @par An empty `negativeSign` means the ASCII default, not "no sign" /// Unlike a group separator, there is no locale without a negative sign, so an /// empty view leaves the ASCII `'-'` above as the only spelling rather than /// meaning "this entry cannot be negative". `formatCanonicalNumber` reads it @@ -268,7 +539,7 @@ struct SignMatch { /// one. /// /// @par A leading positive sign is accepted and dropped (morph#596) -/// @p positiveSign is matched exactly like @p negativeSign -- the locale's own +/// `positiveSign` is matched exactly like `negativeSign` -- the locale's own /// spelling as a whole string, plus a bare ASCII `'+'` in every locale. Of the /// 711 locales Qt 6.11.2 knows, 54 spell it as more than one code point /// (U+061C, U+200E or U+200F before the `'+'`, e.g. `ar_EG`, `ar_DZ`, `az_IR`, @@ -282,64 +553,74 @@ struct SignMatch { /// @par The sign is **dropped**, and `formatCanonicalNumber` never emits one /// This is a deliberate asymmetry with the negative sign, not an oversight. /// Canonical text is `-?[0-9]+(\.[0-9]+)?`: there is no `'+'` in it, so `"+5"` -/// yields `"5"` and not `"+5"`. The display edge has no @p positiveSign -/// parameter at all, because emitting one would change what every positive -/// number in every form looks like -- `5` would become `+5` on screen. So the -/// two functions are *not* strict inverses across a positive sign: entry -/// accepts a spelling display never produces. That is the only shape that adds -/// acceptance without changing a single rendered value, and it is why morph#596 -/// is an enhancement rather than the repaired round trip morph#583 was. Written -/// down in `docs/spec/forms/forms.md` as well, under "Locale data formatting". -/// -/// @par An empty @p positiveSign leaves the ASCII `'+'` +/// yields `"5"` and not `"+5"`. The display edge reads no `positiveSign` at +/// all, because emitting one would change what every positive number in every +/// form looks like -- `5` would become `+5` on screen. So the two functions are +/// *not* strict inverses across a positive sign: entry accepts a spelling +/// display never produces. That is the only shape that adds acceptance without +/// changing a single rendered value, and it is why morph#596 is an enhancement +/// rather than the repaired round trip morph#583 was. Written down in +/// `docs/spec/forms/forms.md` as well, under "Locale data formatting". +/// +/// @par An empty `positiveSign` leaves the ASCII `'+'` /// Here empty really can mean "match nothing extra", because there is no /// display edge to get wrong: the worst an unmatched positive sign can do is /// reject an entry, never produce a value of the wrong sign. The bare ASCII /// `'+'` stays accepted regardless. /// -/// @param text The locale-formatted entry, e.g. `"1.050,25"`. -/// @param decimalSeparator The locale's decimal-point string, e.g. `","`. -/// @param groupSeparator The locale's digit-grouping string, e.g. `"."`, or -/// empty when the locale has none. -/// @param negativeSign The locale's negative-sign string, e.g. `"\u2212"`; -/// empty is read as the default `"-"`. -/// @param positiveSign The locale's positive-sign string, e.g. -/// `"\u061c+"`; empty leaves the ASCII `"+"` as the -/// only accepted spelling. +/// @par The locale's own digits are accepted, and so are ASCII ones (morph#591) +/// A digit is accepted when its code point is in +/// `[zeroDigit, zeroDigit + 9]` -- a Unicode decimal digit set is ten +/// contiguous code points by definition (UAX #44) -- *or* in `['0', '9']`. 76 +/// of the 711 locales Qt 6.11.2 knows use a non-ASCII `zeroDigit`; before this +/// their users could not enter a number at all, because the scan compared a +/// single byte against the ASCII range and the very first byte of U+0665 +/// failed it. The second acceptance is the same rule as the ASCII `'-'` and +/// `'+'` above, for the same reason: a user with an ASCII keyboard in an +/// `ar_EG` locale has to be able to type `5`. It costs nothing, because the +/// canonical output spells every digit in ASCII whatever the input spelled it, +/// so no two accepted spellings can produce different *values*. +/// +/// @par Digit families do not mix +/// `"\u06655"` -- one Arabic-Indic digit and one ASCII digit -- is malformed, +/// not `"55"`. The two families are each accepted whole; interleaving them is +/// not a spelling any keyboard or any display edge produces, and rejecting it +/// matches the existing strictness about a sign anywhere but the leading +/// position. It is a choice rather than a consequence, so it is stated here and +/// pinned by a test on both edges. Note that when `zeroDigit` is the ASCII +/// `"0"` the two families are the same set, so nothing can mix and the rule is +/// invisible -- which is why it costs no existing caller anything. +/// +/// @param text The locale-formatted entry, e.g. `"1.050,25"`. +/// @param loc The locale facts. Designated initialisers are the intended +/// spelling: `{.decimalSeparator = ",", .groupSeparator = "."}`. /// @return The canonical `.`-decimal text, or `std::nullopt` when malformed. -// NOLINTBEGIN(bugprone-easily-swappable-parameters) -// The four locale strings are one fixed order; the first three are mirrored by -// formatCanonicalNumber so the two directions read alike, and separating a sign -// from the separators to break the adjacency would put them out of step. -// positiveSign is last because it is the one parameter the display direction -// does not take -- see "The sign is dropped" above. [[nodiscard]] inline std::optional normalizeLocaleNumber(std::string_view text, - std::string_view decimalSeparator, - std::string_view groupSeparator, - std::string_view negativeSign = "-", - std::string_view positiveSign = "+") { - // NOLINTEND(bugprone-easily-swappable-parameters) - if (!groupSeparator.empty() && groupSeparator == decimalSeparator) { + NumericLocale const& loc) { + if (!loc.groupSeparator.empty() && loc.groupSeparator == loc.decimalSeparator) { return std::nullopt; // one string cannot play both roles: see above } - if (!detail::groupingIsWellPlaced(text, decimalSeparator, groupSeparator)) { + if (!detail::groupingIsWellPlaced(text, loc)) { return std::nullopt; // a separator off a group boundary: see above } + char32_t const base = detail::digitBase(loc.zeroDigit); std::string canonical; canonical.reserve(text.size()); bool sawDecimal = false; bool sawAnyOutput = false; + bool sawNativeDigit = false; + bool sawAsciiDigit = false; for (std::size_t i = 0; i < text.size();) { const std::string_view rest = text.substr(i); - if (!groupSeparator.empty() && rest.starts_with(groupSeparator)) { + if (std::size_t const group = detail::leadingSeparatorLength(rest, loc.groupSeparator); group != 0) { // Placement was settled above, so by here the separator is display // only and is never carried into the output. - i += groupSeparator.size(); + i += group; continue; } - if (!decimalSeparator.empty() && rest.starts_with(decimalSeparator)) { + if (std::size_t const point = detail::leadingSeparatorLength(rest, loc.decimalSeparator); point != 0) { if (sawDecimal) { return std::nullopt; // a second decimal separator: malformed } @@ -351,10 +632,10 @@ struct SignMatch { // ("`,-5`" in a de-DE locale) is accepted as if it were leading. // morph#497. sawAnyOutput = true; - i += decimalSeparator.size(); + i += point; continue; } - detail::SignMatch const sign = detail::leadingSign(rest, negativeSign, positiveSign); + detail::SignMatch const sign = detail::leadingSign(rest, loc); if (sign.length != 0) { // Leading position of the *output*: a stripped group separator // before the sign would otherwise make an injected sign look @@ -372,17 +653,24 @@ struct SignMatch { i += sign.length; continue; } - // i is bounded by the loop condition. - // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-avoid-unchecked-container-access) - char const chr = text[i]; - if (chr < '0' || chr > '9') { + detail::DigitMatch const digit = detail::leadingDigit(rest, base); + if (digit.length == 0) { return std::nullopt; // any other character is malformed } - canonical += chr; + // Which family each digit came from is recorded here and judged once, + // below the loop: an entry that mixes them is malformed wherever the + // second family appears, so there is nothing an early return would + // decide differently, and the scan stays one statement per character + // class. + (digit.native ? sawNativeDigit : sawAsciiDigit) = true; + canonical += digit.canonical; sawAnyOutput = true; - ++i; + i += digit.length; } + if (sawNativeDigit && sawAsciiDigit) { + return std::nullopt; // two digit families in one entry: see above + } if (canonical.empty() || canonical == "-") { return std::nullopt; } @@ -393,12 +681,11 @@ struct SignMatch { /// display text, grouping the integer part in triples. /// /// The display-direction inverse of `normalizeLocaleNumber`'s -/// decimal-separator substitution, plus thousands grouping that -/// `normalizeLocaleNumber` takes back: a grouped display round-trips, because -/// the entry direction *validates* the grouping rather than stripping it (see -/// "Grouping is validated, not stripped" on that function). Passing -/// `decimalSeparator == "."` and an empty @p groupSeparator is the identity -/// transform. +/// decimal-separator substitution and digit rewriting, plus thousands grouping +/// that `normalizeLocaleNumber` takes back: a grouped display round-trips, +/// because the entry direction *validates* the grouping rather than stripping +/// it (see "Grouping is validated, not stripped" on that function). A +/// default-constructed `NumericLocale` is the identity transform. /// /// @par What this paragraph used to say, and why it was wrong (morph#597) /// It claimed grouping was "never accepted back on entry" and that @@ -412,59 +699,67 @@ struct SignMatch { /// (`docs/spec/forms/forms.md`, "Grouping is validated, never merely /// stripped") and the code already agreed; only this comment was stale. /// -/// The sign is emitted as @p negativeSign, matching what `normalizeLocaleNumber` -/// accepts back (morph#583); an empty view is read as `"-"` rather than as "no -/// sign", because formatting a negative to no sign at all is a silently wrong -/// value. +/// The sign is emitted as `NumericLocale::negativeSign`, matching what +/// `normalizeLocaleNumber` accepts back (morph#583); an empty view is read as +/// `"-"` rather than as "no sign", because formatting a negative to no sign at +/// all is a silently wrong value. +/// +/// @par The digits are emitted in the locale's set (morph#591) +/// Each canonical `'0'`-`'9'` is emitted as the code point that far above +/// `NumericLocale::zeroDigit`, so an `ar_EG` caller sees `"\u0665"` where the +/// canonical text said `'5'`. This edge *had* to move with the entry edge: it +/// used to copy the canonical ASCII bytes out unchanged, so teaching entry to +/// accept U+0665 while display kept emitting `'5'` would have left the pair no +/// longer inverse, which is the round trip `docs/spec/forms/forms.md` requires. +/// With the default `zeroDigit` of `"0"` the offset is zero and every byte is +/// the one this function emitted before. /// -/// @par There is no positive-sign parameter, deliberately (morph#596) +/// @par There is no positive-sign emission, deliberately (morph#596) /// A positive number is displayed with no sign at all, in every locale, and -/// this function takes no `positiveSign` for the caller to change that. +/// this function ignores `NumericLocale::positiveSign` entirely. /// `normalizeLocaleNumber` *accepts* a leading positive sign and drops it, so /// the pair is not a strict inverse across one: entry takes a spelling display -/// never produces. Adding the parameter is what would be the defect -- +/// never produces. Emitting it is what would be the defect -- /// `QLocale::positiveSign()` is `'+'` in 657 of the 711 locales Qt 6.11.2 /// knows, so emitting it would turn every positive number in every form from /// `5` into `+5`, a visible product change with no reported need behind it. /// Rejecting text the display edge produced is the morph#583 shape and is not /// what happens here; producing text no display edge asked for would be. -/// @param canonicalText Canonical `-?[0-9]+(\.[0-9]+)?` text. -/// @param decimalSeparator The locale's decimal-point display string. -/// @param groupSeparator The locale's digit-grouping display string, or empty -/// to omit grouping. -/// @param negativeSign The locale's negative-sign display string; empty is -/// read as the default `"-"`. +/// @param canonicalText Canonical `-?[0-9]+(\.[0-9]+)?` text. +/// @param loc The locale facts; `positiveSign` is not read. /// @return The locale-formatted display text. -// Mirrors normalizeLocaleNumber's parameter order; the two are inverses, so -// diverging here would be the more confusing choice. -// NOLINTNEXTLINE(bugprone-easily-swappable-parameters) -[[nodiscard]] inline std::string formatCanonicalNumber(std::string_view canonicalText, - std::string_view decimalSeparator, - std::string_view groupSeparator, - std::string_view negativeSign = "-") { +[[nodiscard]] inline std::string formatCanonicalNumber(std::string_view canonicalText, NumericLocale const& loc) { + constexpr std::size_t kGroupSize = 3; + char32_t const base = detail::digitBase(loc.zeroDigit); bool const neg = !canonicalText.empty() && canonicalText.front() == '-'; std::string_view const magnitude = neg ? canonicalText.substr(1) : canonicalText; auto const dot = magnitude.find('.'); std::string_view const wholePart = dot == std::string_view::npos ? magnitude : magnitude.substr(0, dot); std::string_view const fracPart = dot == std::string_view::npos ? std::string_view{} : magnitude.substr(dot + 1); - std::string grouped; - grouped.reserve(wholePart.size() + ((wholePart.size() / 3) * groupSeparator.size())); - for (std::size_t i = 0; i < wholePart.size(); ++i) { - if (!groupSeparator.empty() && i != 0 && (wholePart.size() - i) % 3 == 0) { - grouped += groupSeparator; - } - grouped += wholePart[i]; - } - + // A digit of the locale's set is one to four UTF-8 bytes, so the reserve + // hint scales with the base rather than assuming one byte per digit. + std::size_t const perDigit = detail::utf8Length(base); std::string out; + out.reserve(loc.negativeSign.size() + (magnitude.size() * perDigit) + + ((wholePart.size() / kGroupSize) * loc.groupSeparator.size()) + loc.decimalSeparator.size()); + if (neg) { - out += negativeSign.empty() ? std::string_view{"-"} : negativeSign; + out += loc.negativeSign.empty() ? std::string_view{"-"} : loc.negativeSign; + } + std::size_t index = 0; + for (char const chr : wholePart) { + if (!loc.groupSeparator.empty() && index != 0 && (wholePart.size() - index) % kGroupSize == 0) { + out += loc.groupSeparator; + } + detail::appendDisplayDigit(out, base, chr); + ++index; } - out += grouped; if (!fracPart.empty()) { - out += decimalSeparator; - out += fracPart; + out += loc.decimalSeparator; + for (char const chr : fracPart) { + detail::appendDisplayDigit(out, base, chr); + } } return out; } diff --git a/src/qt/forms/qml/DynamicForm.qml b/src/qt/forms/qml/DynamicForm.qml index 10ee54246..f5c4b4f2d 100644 --- a/src/qt/forms/qml/DynamicForm.qml +++ b/src/qt/forms/qml/DynamicForm.qml @@ -836,6 +836,27 @@ Frame { // emits one: a positive displays unsigned in every locale, and emitting the // sign would turn every positive number in every form from "5" into "+5". // The pair is therefore deliberately not inverse across a positive sign. + // The digits are locale data too (morph#591), carried as a *base*: a + // Unicode decimal digit set is ten contiguous code points by definition + // (UAX #44), so one zeroDigit is enough and a ten-element table is not + // needed. 76 of the 711 locales Qt 6.11.2 knows report a zeroDigit other + // than ASCII "0", across eleven distinct sets -- two of them astral + // (U+11136 Chakma, U+1E950 Adlam), which is why this scans code *points* + // via codePointAt and steps two units for one digit when it has to. Entry + // accepts a digit in [zeroDigit, zeroDigit+9] or in ["0","9"]; display + // emits only the locale's. That asymmetry is the morph#596 rule applied to + // digits: the locale's own digits are on the user's keyboard only if their + // keyboard has them, and accepting an extra spelling cannot change a value + // because the canonical output always spells digits in ASCII. Mixing the + // two families in one entry is malformed -- see below. + // All five facts now travel as one object rather than as five positional + // arguments, mirroring the C++ NumericLocale aggregate: on that side the + // row of interchangeable string_views needed a clang-tidy suppression for + // bugprone-easily-swappable-parameters, and a sixth would have made the + // argument for it weaker rather than stronger. Here the gain is the same + // one a reader gets -- a call names each fact -- and it keeps the two + // mirrors structurally identical, which is the property morph#599 and + // morph#591 were both about. // The two *separators* are matched as whole strings for the same reason // (morph#599), and that this was not already true was the mixed idiom // morph#583 and morph#596 left behind: they converted the signs to @@ -848,20 +869,51 @@ Frame { // could: docs/spec/forms/forms.md, "Both edges, or neither" -- the C++ edge // has always matched separators whole (`rest.starts_with(...)`), and a // divergence between the two is a divergence in what the product accepts. - function normalizeLocaleNumber(text, decimalSeparator, groupSeparator, negativeSign, positiveSign) { + // The digit-set base of a NumericLocale-shaped object: the code point of + // its zeroDigit, or ASCII "0" when it is absent or empty. Empty reads as + // the default for the reason an empty negativeSign does -- there is no + // locale without digits, so it cannot mean "this entry has none". + function localeDigitBase(loc) { + const zero = (loc && loc.zeroDigit) ? loc.zeroDigit : "0" + return zero.codePointAt(0) + } + + // A digit at index i, in the locale's set or in ASCII, or null. Returns the + // canonical ASCII spelling, how many UTF-16 units it occupied (two for an + // astral digit) and which family it came from. When base is "0" the two + // families are the same set, so `native` is always true and nothing can mix. + function leadingDigit(text, i, base) { + const cp = text.codePointAt(i) + if (cp === undefined) + return null + const units = cp > 0xFFFF ? 2 : 1 + if (cp >= base && cp < base + 10) + return { canonical: String(cp - base), units: units, native: true } + if (cp >= 0x30 && cp <= 0x39) + return { canonical: String(cp - 0x30), units: units, native: false } + return null + } + + function normalizeLocaleNumber(text, locale) { + const loc = locale ? locale : {} + const decimalSeparator = loc.decimalSeparator !== undefined ? loc.decimalSeparator : "." + const groupSeparator = loc.groupSeparator !== undefined ? loc.groupSeparator : "" // One string cannot play both roles: there is no reading of "1.5" this // function could defend, so it reports rather than guesses. if (groupSeparator !== "" && groupSeparator === decimalSeparator) return null - const sign = negativeSign ? negativeSign : "-" - const plus = positiveSign ? positiveSign : "+" + const sign = loc.negativeSign ? loc.negativeSign : "-" + const plus = loc.positiveSign ? loc.positiveSign : "+" + const base = localeDigitBase(loc) const groupSize = 3 let canonical = "" let sawDecimal = false let sawAnyOutput = false let digitsInGroup = 0 let sawGroup = false + let sawNativeDigit = false + let sawAsciiDigit = false for (let i = 0; i < text.length; ++i) { const ch = text[i] if (groupSeparator !== "" && text.startsWith(groupSeparator, i)) { @@ -917,29 +969,62 @@ Frame { // the locale's own is a bidi-prefixed form. Dropped, as above. if (sawAnyOutput) return null - } else if (ch >= "0" && ch <= "9") { - canonical += ch - ++digitsInGroup } else { - return null + const digit = leadingDigit(text, i, base) + if (digit === null) + return null + // Which family the digit came from is recorded and judged once, + // after the loop: an entry that mixes them is malformed + // wherever the second family appears. + if (digit.native) + sawNativeDigit = true + else + sawAsciiDigit = true + canonical += digit.canonical + ++digitsInGroup + i += digit.units - 1 // the loop's ++i consumes the last unit } sawAnyOutput = true } // A grouped integer part has to end on a group boundary too. if (sawGroup && !sawDecimal && digitsInGroup !== groupSize) return null + // Two digit families in one entry is malformed, not "55": neither a + // keyboard nor a display edge produces an interleaving, and rejecting + // it matches the rule that a sign anywhere but the leading position is + // malformed. Pinned by a test on both edges, because accepting it would + // have been equally implementable. + if (sawNativeDigit && sawAsciiDigit) + return null if (canonical === "" || canonical === "-") return null return canonical } - // There is no positiveSign parameter here, deliberately (morph#596): a + // One canonical character as the locale would display it: an ASCII digit + // becomes the code point that far above the digit base, anything else is + // copied through. The pass-through arm is what keeps this byte-identical + // for text that does not honour the canonical shape, which this function + // has always passed out unchanged. + function displayDigit(ch, base) { + const cp = ch.codePointAt(0) + return (cp >= 0x30 && cp <= 0x39) ? String.fromCodePoint(base + (cp - 0x30)) : ch + } + + // The locale's positiveSign is deliberately not read here (morph#596): a // positive number displays unsigned in every locale, so the entry edge above - // accepts a leading "+" that this edge never produces. - function formatCanonicalNumber(text, decimalSeparator, groupSeparator, negativeSign) { + // accepts a leading "+" that this edge never produces. The digits, by + // contrast, *are* emitted in the locale's set (morph#591) -- this edge had + // to move with the entry edge or the pair would no longer be inverse, which + // is the round trip docs/spec/forms/forms.md requires. + function formatCanonicalNumber(text, locale) { + const loc = locale ? locale : {} + const decimalSeparator = loc.decimalSeparator !== undefined ? loc.decimalSeparator : "." + const groupSeparator = loc.groupSeparator !== undefined ? loc.groupSeparator : "" // Empty reads as "-", not as "no sign": formatting a negative to no // sign at all would be a silently wrong value, not a rejected one. - const sign = negativeSign ? negativeSign : "-" + const sign = loc.negativeSign ? loc.negativeSign : "-" + const base = localeDigitBase(loc) const neg = text.startsWith("-") const magnitude = neg ? text.slice(1) : text const dot = magnitude.indexOf(".") @@ -949,9 +1034,12 @@ Frame { for (let i = 0; i < wholePart.length; ++i) { if (groupSeparator !== "" && i !== 0 && (wholePart.length - i) % 3 === 0) grouped += groupSeparator - grouped += wholePart[i] + grouped += displayDigit(wholePart[i], base) } - return (neg ? sign : "") + grouped + (fracPart !== "" ? decimalSeparator + fracPart : "") + let fraction = "" + for (let k = 0; k < fracPart.length; ++k) + fraction += displayDigit(fracPart[k], base) + return (neg ? sign : "") + grouped + (fracPart !== "" ? decimalSeparator + fraction : "") } // --- zoned Timestamp entry -------------------------------------------- @@ -1110,8 +1198,13 @@ Frame { return utcIso === null ? null : JSON.stringify(utcIso) } if (f.isQuantity) { - const canonicalText = normalizeLocaleNumber(text, qtLocale.decimalPoint, qtLocale.groupSeparator, - qtLocale.negativeSign, qtLocale.positiveSign) + const canonicalText = normalizeLocaleNumber(text, { + decimalSeparator: qtLocale.decimalPoint, + groupSeparator: qtLocale.groupSeparator, + negativeSign: qtLocale.negativeSign, + positiveSign: qtLocale.positiveSign, + zeroDigit: qtLocale.zeroDigit + }) if (canonicalText === null || !/^-?\d+(\.\d+)?$/.test(canonicalText)) return null const unit = f.unitOptions[opt(fieldUnits[f.name], 0)] @@ -1786,15 +1879,22 @@ Frame { const toUnit = fieldColumn.modelData.unitOptions[currentIndex] form.fieldUnits[name] = currentIndex if (entry.text.trim() !== "") { - const canonicalText = form.normalizeLocaleNumber( - entry.text.trim(), form.qtLocale.decimalPoint, form.qtLocale.groupSeparator, - form.qtLocale.negativeSign, form.qtLocale.positiveSign) + const canonicalText = form.normalizeLocaleNumber(entry.text.trim(), { + decimalSeparator: form.qtLocale.decimalPoint, + groupSeparator: form.qtLocale.groupSeparator, + negativeSign: form.qtLocale.negativeSign, + positiveSign: form.qtLocale.positiveSign, + zeroDigit: form.qtLocale.zeroDigit + }) const converted = canonicalText !== null ? form.convertText(canonicalText, fromUnit, toUnit) : "" entry.text = converted !== "" - ? form.formatCanonicalNumber( - converted, form.qtLocale.decimalPoint, form.qtLocale.groupSeparator, - form.qtLocale.negativeSign) + ? form.formatCanonicalNumber(converted, { + decimalSeparator: form.qtLocale.decimalPoint, + groupSeparator: form.qtLocale.groupSeparator, + negativeSign: form.qtLocale.negativeSign, + zeroDigit: form.qtLocale.zeroDigit + }) : "" } else { form.revalidate() diff --git a/src/qt/forms/tests/tst_i18n.qml b/src/qt/forms/tests/tst_i18n.qml index 796567d32..405e2d97b 100644 --- a/src/qt/forms/tests/tst_i18n.qml +++ b/src/qt/forms/tests/tst_i18n.qml @@ -223,16 +223,16 @@ Item { // directly: these locales' own separators are Arabic-Indic, which is a // separate gap (morph#591), so the sign is isolated here. function test_bidiPrefixedSignIsMatchedAsAWholeString() { - compare(signForm.normalizeLocaleNumber("\u200E\u22125", ".", "", "\u200E\u2212"), "-5") // fa_IR - compare(signForm.normalizeLocaleNumber("\u200E-\u200E5", ".", "", "\u200E-\u200E"), "-5") // az_IR - compare(signForm.normalizeLocaleNumber("\u200E-5", ".", "", "\u200E-"), "-5") // ar_DZ - compare(signForm.normalizeLocaleNumber("\u061C-5", ".", "", "\u061C-"), "-5") // ar_EG + compare(signForm.normalizeLocaleNumber("\u200E\u22125", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u200E\u2212" }), "-5") // fa_IR + compare(signForm.normalizeLocaleNumber("\u200E-\u200E5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u200E-\u200E" }), "-5") // az_IR + compare(signForm.normalizeLocaleNumber("\u200E-5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u200E-" }), "-5") // ar_DZ + compare(signForm.normalizeLocaleNumber("\u061C-5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u061C-" }), "-5") // ar_EG // Controls: rejected with the sign left at its ASCII default, which // is exactly what the renderer passed before this change. - compare(signForm.normalizeLocaleNumber("\u200E\u22125", ".", ""), null) - compare(signForm.normalizeLocaleNumber("\u200E-5", ".", ""), null) - compare(signForm.normalizeLocaleNumber("\u22125", ".", ""), null) + compare(signForm.normalizeLocaleNumber("\u200E\u22125", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(signForm.normalizeLocaleNumber("\u200E-5", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(signForm.normalizeLocaleNumber("\u22125", { decimalSeparator: ".", groupSeparator: "" }), null) } // The C++ edge (tests/test_render_locale_format.cpp, [morph583]) pins @@ -241,14 +241,14 @@ Item { function test_displayEdgeEmitsTheLocaleSignAndEntryTakesItBack() { const signs = ["\u2212", "\u200E\u2212", "\u200E-", "\u200E-\u200E", "\u061C-"] for (let i = 0; i < signs.length; ++i) { - const display = signForm.formatCanonicalNumber("-1050.25", ",", ".", signs[i]) + const display = signForm.formatCanonicalNumber("-1050.25", { decimalSeparator: ",", groupSeparator: ".", negativeSign: signs[i] }) compare(display, signs[i] + "1.050,25") - compare(signForm.normalizeLocaleNumber(display, ",", ".", signs[i]), "-1050.25") + compare(signForm.normalizeLocaleNumber(display, { decimalSeparator: ",", groupSeparator: ".", negativeSign: signs[i] }), "-1050.25") } // A positive never carries the sign, and the ASCII default is // unchanged for every existing caller. - compare(signForm.formatCanonicalNumber("1050.25", ",", ".", "\u2212"), "1.050,25") - compare(signForm.formatCanonicalNumber("-1050.25", ",", "."), "-1.050,25") + compare(signForm.formatCanonicalNumber("1050.25", { decimalSeparator: ",", groupSeparator: ".", negativeSign: "\u2212" }), "1.050,25") + compare(signForm.formatCanonicalNumber("-1050.25", { decimalSeparator: ",", groupSeparator: "." }), "-1.050,25") } // Unlike a group separator, no locale is without a negative sign, so an @@ -256,17 +256,17 @@ Item { // -5 to "5" would be a silently wrong value, which is the morph#574 // failure mode rather than a rejection. function test_anEmptySignReadsAsTheAsciiDefault() { - compare(signForm.formatCanonicalNumber("-5", ".", "", ""), "-5") - compare(signForm.normalizeLocaleNumber("-5", ".", "", ""), "-5") - compare(signForm.normalizeLocaleNumber("123", ".", "", ""), "123") + compare(signForm.formatCanonicalNumber("-5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "" }), "-5") + compare(signForm.normalizeLocaleNumber("-5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "" }), "-5") + compare(signForm.normalizeLocaleNumber("123", { decimalSeparator: ".", groupSeparator: "", negativeSign: "" }), "123") } // The morph#497 rule is about the *output*, so it has to hold for a // multi-unit sign exactly as it does for "-". function test_aLocaleSignIsStillRejectedOffTheLeadingPosition() { - compare(signForm.normalizeLocaleNumber("1\u22122", ".", "", "\u2212"), null) - compare(signForm.normalizeLocaleNumber(",\u22125", ",", ".", "\u2212"), null) - compare(signForm.normalizeLocaleNumber("\u2212", ".", "", "\u2212"), null) + compare(signForm.normalizeLocaleNumber("1\u22122", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u2212" }), null) + compare(signForm.normalizeLocaleNumber(",\u22125", { decimalSeparator: ",", groupSeparator: ".", negativeSign: "\u2212" }), null) + compare(signForm.normalizeLocaleNumber("\u2212", { decimalSeparator: ".", groupSeparator: "", negativeSign: "\u2212" }), null) } // ── morph#596: a leading positive sign is accepted, and dropped ── @@ -304,38 +304,38 @@ Item { // the identical table; a divergence between the two is a divergence in // what the product accepts. function test_positiveSignIsMatchedAsAWholeString() { - compare(localeForm.normalizeLocaleNumber("+5", ".", ""), "5") - compare(localeForm.normalizeLocaleNumber("\u061C+5", ".", "", "-", "\u061C+"), "5") // ar_EG - compare(localeForm.normalizeLocaleNumber("\u200E+5", ".", "", "-", "\u200E+"), "5") // ar_DZ - compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", ".", "", "-", "\u200E+\u200E"), "5") // az_IR - compare(localeForm.normalizeLocaleNumber("\u200F+5", ".", "", "-", "\u200F+"), "5") // ckb_IQ + compare(localeForm.normalizeLocaleNumber("+5", { decimalSeparator: ".", groupSeparator: "" }), "5") + compare(localeForm.normalizeLocaleNumber("\u061C+5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "\u061C+" }), "5") // ar_EG + compare(localeForm.normalizeLocaleNumber("\u200E+5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "\u200E+" }), "5") // ar_DZ + compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "\u200E+\u200E" }), "5") // az_IR + compare(localeForm.normalizeLocaleNumber("\u200F+5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "\u200F+" }), "5") // ckb_IQ // Controls: with positiveSign left at its ASCII default, the // bidi-prefixed spellings are still rejected -- which is what makes // the parameter, and not the unconditional ASCII acceptance, the // thing under test above. - compare(localeForm.normalizeLocaleNumber("\u061C+5", ".", ""), null) - compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", ".", ""), null) + compare(localeForm.normalizeLocaleNumber("\u061C+5", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(localeForm.normalizeLocaleNumber("\u200E+\u200E5", { decimalSeparator: ".", groupSeparator: "" }), null) // The ASCII "+" stays accepted in a bidi-sign locale, for the same // reason the ASCII "-" does (morph#583): the locale's own spelling // is on no keyboard. - compare(localeForm.normalizeLocaleNumber("+5", ".", "", "-", "\u061C+"), "5") + compare(localeForm.normalizeLocaleNumber("+5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "\u061C+" }), "5") // An empty positiveSign leaves the ASCII spelling, and must not // match at every index. - compare(localeForm.normalizeLocaleNumber("+5", ".", "", "-", ""), "5") - compare(localeForm.normalizeLocaleNumber("123", ".", "", "-", ""), "123") + compare(localeForm.normalizeLocaleNumber("+5", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "" }), "5") + compare(localeForm.normalizeLocaleNumber("123", { decimalSeparator: ".", groupSeparator: "", negativeSign: "-", positiveSign: "" }), "123") } // morph#497's rule is about the *output*, so a new sign spelling must // not open a new way to inject one. function test_aPositiveSignObeysTheLeadingPositionRule() { - compare(localeForm.normalizeLocaleNumber("1+2", ".", ""), null) - compare(localeForm.normalizeLocaleNumber("+-5", ".", ""), null) - compare(localeForm.normalizeLocaleNumber("-+5", ".", ""), null) - compare(localeForm.normalizeLocaleNumber("++5", ".", ""), null) - compare(localeForm.normalizeLocaleNumber(",+5", ",", "."), null) - compare(localeForm.normalizeLocaleNumber("+", ".", ""), null) + compare(localeForm.normalizeLocaleNumber("1+2", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(localeForm.normalizeLocaleNumber("+-5", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(localeForm.normalizeLocaleNumber("-+5", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(localeForm.normalizeLocaleNumber("++5", { decimalSeparator: ".", groupSeparator: "" }), null) + compare(localeForm.normalizeLocaleNumber(",+5", { decimalSeparator: ",", groupSeparator: "." }), null) + compare(localeForm.normalizeLocaleNumber("+", { decimalSeparator: ".", groupSeparator: "" }), null) } // The deliberate asymmetry, pinned so that "make it symmetric" is a @@ -344,15 +344,15 @@ Item { // 711 locales, so emitting it would turn every positive number in every // form from "5" into "+5". function test_theDisplayEdgeNeverEmitsAPositiveSign() { - compare(localeForm.formatCanonicalNumber("5", ".", ""), "5") - compare(localeForm.formatCanonicalNumber("1050.25", ",", "."), "1.050,25") + compare(localeForm.formatCanonicalNumber("5", { decimalSeparator: ".", groupSeparator: "" }), "5") + compare(localeForm.formatCanonicalNumber("1050.25", { decimalSeparator: ",", groupSeparator: "." }), "1.050,25") // So the pair is not inverse across a positive sign: entry accepts // a spelling display never produces. - const canonical = localeForm.normalizeLocaleNumber("+1.050,25", ",", ".") + const canonical = localeForm.normalizeLocaleNumber("+1.050,25", { decimalSeparator: ",", groupSeparator: "." }) compare(canonical, "1050.25") - compare(localeForm.formatCanonicalNumber(canonical, ",", "."), "1.050,25") + compare(localeForm.formatCanonicalNumber(canonical, { decimalSeparator: ",", groupSeparator: "." }), "1.050,25") // ...while the negative side still round-trips exactly. - compare(localeForm.formatCanonicalNumber("-1050.25", ",", "."), "-1.050,25") + compare(localeForm.formatCanonicalNumber("-1050.25", { decimalSeparator: ",", groupSeparator: "." }), "-1.050,25") } // ── morph#599: the separators are matched as whole strings too ─── @@ -429,13 +429,13 @@ Item { const G4 = "\uD834\uDD6D" const D4 = "\uD834\uDD6E" - compare(localeForm.normalizeLocaleNumber("1" + G2 + "050" + D2 + "25", D2, G2), "1050.25") - compare(localeForm.normalizeLocaleNumber("-1" + G2 + "050" + D2 + "25", D2, G2), "-1050.25") - compare(localeForm.normalizeLocaleNumber("+1" + G2 + "050" + D2 + "25", D2, G2), "1050.25") - compare(localeForm.normalizeLocaleNumber("1" + G4 + "050" + D4 + "25", D4, G4), "1050.25") - compare(localeForm.normalizeLocaleNumber("1" + G4 + "050" + G4 + "000", "", G4), "1050000") + compare(localeForm.normalizeLocaleNumber("1" + G2 + "050" + D2 + "25", { decimalSeparator: D2, groupSeparator: G2 }), "1050.25") + compare(localeForm.normalizeLocaleNumber("-1" + G2 + "050" + D2 + "25", { decimalSeparator: D2, groupSeparator: G2 }), "-1050.25") + compare(localeForm.normalizeLocaleNumber("+1" + G2 + "050" + D2 + "25", { decimalSeparator: D2, groupSeparator: G2 }), "1050.25") + compare(localeForm.normalizeLocaleNumber("1" + G4 + "050" + D4 + "25", { decimalSeparator: D4, groupSeparator: G4 }), "1050.25") + compare(localeForm.normalizeLocaleNumber("1" + G4 + "050" + G4 + "000", { decimalSeparator: "", groupSeparator: G4 }), "1050000") // A locale with a multi-unit decimal separator and no grouping. - compare(localeForm.normalizeLocaleNumber("5" + D2 + "25", D2, ""), "5.25") + compare(localeForm.normalizeLocaleNumber("5" + D2 + "25", { decimalSeparator: D2, groupSeparator: "" }), "5.25") } // Whole-string matching must not loosen any of the rules the @@ -454,21 +454,21 @@ Item { const D2 = "\u200E," // morph#574: the last group of the integer part is short. - compare(localeForm.normalizeLocaleNumber("1" + G2 + "5", D2, G2), null) - compare(localeForm.normalizeLocaleNumber("1" + G2 + "2" + G2 + "3" + G2 + "4", D2, G2), null) + compare(localeForm.normalizeLocaleNumber("1" + G2 + "5", { decimalSeparator: D2, groupSeparator: G2 }), null) + compare(localeForm.normalizeLocaleNumber("1" + G2 + "2" + G2 + "3" + G2 + "4", { decimalSeparator: D2, groupSeparator: G2 }), null) // Grouping belongs to the integer part only. - compare(localeForm.normalizeLocaleNumber("1" + D2 + "5" + G2 + "000", D2, G2), null) + compare(localeForm.normalizeLocaleNumber("1" + D2 + "5" + G2 + "000", { decimalSeparator: D2, groupSeparator: G2 }), null) // A second decimal separator. - compare(localeForm.normalizeLocaleNumber("1" + D2 + "0" + D2 + "5", D2, G2), null) + compare(localeForm.normalizeLocaleNumber("1" + D2 + "0" + D2 + "5", { decimalSeparator: D2, groupSeparator: G2 }), null) // morph#497: a sign after the decimal separator is not leading. - compare(localeForm.normalizeLocaleNumber(D2 + "-5", D2, G2), null) + compare(localeForm.normalizeLocaleNumber(D2 + "-5", { decimalSeparator: D2, groupSeparator: G2 }), null) // One string cannot play both roles, multi-unit or not. - compare(localeForm.normalizeLocaleNumber("1" + G2 + "050", G2, G2), null) + compare(localeForm.normalizeLocaleNumber("1" + G2 + "050", { decimalSeparator: G2, groupSeparator: G2 }), null) // Controls, unchanged by this fix and rejected before and after: a // lone *prefix* of the separator is not the separator. Whole-string // matching must not degrade into "any unit of it will do". - compare(localeForm.normalizeLocaleNumber("1\u200E050" + D2 + "25", D2, G2), null) - compare(localeForm.normalizeLocaleNumber("1.050,25", D2, G2), null) + compare(localeForm.normalizeLocaleNumber("1\u200E050" + D2 + "25", { decimalSeparator: D2, groupSeparator: G2 }), null) + compare(localeForm.normalizeLocaleNumber("1.050,25", { decimalSeparator: D2, groupSeparator: G2 }), null) } // The round trip, which is where "both edges, or neither" bites: @@ -479,9 +479,135 @@ Item { function test_theDisplayEdgeEmitsAMultiUnitSeparatorAndEntryTakesItBack() { const G2 = "\u200E." const D2 = "\u200E," - const display = localeForm.formatCanonicalNumber("-1050.25", D2, G2, "\u2212") + const display = localeForm.formatCanonicalNumber("-1050.25", { decimalSeparator: D2, groupSeparator: G2, negativeSign: "\u2212" }) compare(display, "\u2212" + "1" + G2 + "050" + D2 + "25") - compare(localeForm.normalizeLocaleNumber(display, D2, G2, "\u2212"), "-1050.25") + compare(localeForm.normalizeLocaleNumber(display, { decimalSeparator: D2, groupSeparator: G2, negativeSign: "\u2212" }), "-1050.25") + } + + // --- morph#591: the digits are locale data too -------------------- + // + // The same corpus the C++ edge pins under [morph591] in + // tests/test_render_locale_format.cpp. docs/spec/forms/forms.md, + // "Both edges, or neither": a row that disagrees between the two lists + // is a divergence in what the product accepts. + // + // Measured on this revision with QLocale::matchingLocales under + // Qt 6.11.2: 76 of the 711 locales report a zeroDigit other than ASCII + // "0", across the eleven sets below. Two of them are astral, so one + // digit is two UTF-16 units here and four UTF-8 bytes there -- the case + // a code-unit scan gets wrong. + + // The premise, measured rather than assumed, on the same object the + // renderer forwards from. If Qt stops reporting these, this fails first + // and says so, rather than the tests below failing for a reason that + // looks like a regression in the renderer. + function test_qtExposesAZeroDigitOnTheLocaleObject() { + compare(Qt.locale("C").zeroDigit, "0") + compare(Qt.locale("de").zeroDigit, "0") + compare(Qt.locale("ar_EG").zeroDigit, "\u0660") + compare(Qt.locale("fa_IR").zeroDigit, "\u06F0") + // Astral: one code point, two UTF-16 units. + compare(Qt.locale("ccp_BD").zeroDigit.length, 2) + compare(Qt.locale("ccp_BD").zeroDigit.codePointAt(0), 0x11136) + compare(Qt.locale("ff_Adlm_BF").zeroDigit.codePointAt(0), 0x1E950) + } + + // The defect: the scan compared one code unit against ["0","9"], so a + // locale whose digits are not ASCII could not enter a number at all -- + // a flat rejection, not a wrong value. + function test_theLocalesOwnDigitsAreAccepted() { + compare(localeForm.normalizeLocaleNumber("\u0665", { zeroDigit: "\u0660" }), "5") + compare(localeForm.normalizeLocaleNumber("\u06F5", { zeroDigit: "\u06F0" }), "5") + compare(localeForm.normalizeLocaleNumber("\u061C-\u0665", + { negativeSign: "\u061C-", zeroDigit: "\u0660" }), "-5") + // Astral digits, one code point in two units each. + compare(localeForm.normalizeLocaleNumber("\u{1113B}", { zeroDigit: "\u{11136}" }), "5") + compare(localeForm.normalizeLocaleNumber("\u{1E955}", { zeroDigit: "\u{1E950}" }), "5") + } + + // The display edge had to move with the entry edge or the pair would no + // longer be inverse. These are the exact units Qt's own + // QLocale("ar_BH").toString(-1050.25) produces. + function test_theDisplayEdgeEmitsTheLocalesDigits() { + const display = localeForm.formatCanonicalNumber("-1050.25", { + decimalSeparator: "\u066B", groupSeparator: "\u066C", + negativeSign: "\u061C-", zeroDigit: "\u0660" + }) + compare(display, "\u061C-\u0661\u066C\u0660\u0665\u0660\u066B\u0662\u0665") + } + + // The acceptance test this ticket exists for. The round trip alone does + // not catch a fix applied to the entry edge only -- entry accepts ASCII + // digits too, so a display edge still emitting them round-trips fine. + // The "no ASCII digit in the display text" check is what fails then. + function test_thePairRoundTripsThroughEveryMeasuredDigitSet() { + const zeros = ["\u0660", "\u06F0", "\u07C0", "\u0966", "\u09E6", "\u0F20", + "\u1040", "\u1C50", "\uABF0", "\u{11136}", "\u{1E950}"] + const canonicals = ["0", "5", "-5", "1050.25", "-1050.25", "1000000.25", + "0.001", "-0.001", "1234567"] + for (let i = 0; i < zeros.length; ++i) { + const loc = { + decimalSeparator: "\u066B", groupSeparator: "\u066C", + negativeSign: "\u061C-", zeroDigit: zeros[i] + } + for (let k = 0; k < canonicals.length; ++k) { + const display = localeForm.formatCanonicalNumber(canonicals[k], loc) + verify(!/[0-9]/.test(display), + "display for " + canonicals[k] + " in set " + i + " still has an ASCII digit: " + display) + compare(localeForm.normalizeLocaleNumber(display, loc), canonicals[k], + "round trip failed for " + canonicals[k] + " in set " + i) + } + } + } + + // The morph#596 precedent, applied to digits: the locale's own digits + // are on the user's keyboard only if their keyboard has them, so entry + // accepts a spelling display never produces. + function test_asciiDigitsStayAcceptedInANativeDigitLocale() { + compare(localeForm.normalizeLocaleNumber("5", { zeroDigit: "\u0660" }), "5") + compare(localeForm.normalizeLocaleNumber("-1050.25", + { negativeSign: "\u061C-", zeroDigit: "\u0660" }), "-1050.25") + compare(localeForm.normalizeLocaleNumber("5", { zeroDigit: "\u{1E950}" }), "5") + } + + // A decision, not a consequence, so a test records it: an entry that + // interleaves the two families is malformed rather than read as 55. + function test_anEntryMayNotMixDigitFamilies() { + compare(localeForm.normalizeLocaleNumber("\u06655", { zeroDigit: "\u0660" }), null) + compare(localeForm.normalizeLocaleNumber("5\u0665", { zeroDigit: "\u0660" }), null) + compare(localeForm.normalizeLocaleNumber("1\u0660\u06650", { zeroDigit: "\u0660" }), null) + // Each family on its own is fine, which is what makes the rejection + // about the mixing rather than about either spelling. + compare(localeForm.normalizeLocaleNumber("\u0665\u0665", { zeroDigit: "\u0660" }), "55") + compare(localeForm.normalizeLocaleNumber("55", { zeroDigit: "\u0660" }), "55") + // In an ASCII-digit locale the two families are the same set, so + // nothing can mix -- which is why this costs no existing caller. + compare(localeForm.normalizeLocaleNumber("55", {}), "55") + } + + // Grouping counts digits, not units. A two-unit astral digit would + // otherwise reset the run-length counter mid-digit, so a correctly + // grouped native-digit entry -- what the display edge now emits -- would + // be rejected and the pair would not round-trip. The rule itself is + // unchanged: a short group is still malformed. + function test_groupingIsValidatedInTheLocalesDigitsToo() { + const arEg = { decimalSeparator: "\u066B", groupSeparator: "\u066C", zeroDigit: "\u0660" } + compare(localeForm.normalizeLocaleNumber("\u0661\u066C\u0660\u0665\u0660", arEg), "1050") + compare(localeForm.normalizeLocaleNumber("\u0661\u066C\u0665", arEg), null) // short group + const ccp = { decimalSeparator: ".", groupSeparator: ",", zeroDigit: "\u{11136}" } + compare(localeForm.normalizeLocaleNumber("\u{11137},\u{11136}\u{1113B}\u{11136}", ccp), "1050") + compare(localeForm.normalizeLocaleNumber("\u{11137},\u{1113B}", ccp), null) // short group + } + + // An omitted or empty zeroDigit reads as ASCII "0", the reading an + // omitted negativeSign gets, and every caller that names no zeroDigit is + // byte-identical to what it produced before morph#591. + function test_anAbsentZeroDigitReadsAsAscii() { + compare(localeForm.normalizeLocaleNumber("55", { zeroDigit: "" }), "55") + compare(localeForm.formatCanonicalNumber("55", { zeroDigit: "" }), "55") + compare(localeForm.normalizeLocaleNumber("1.050,25", { decimalSeparator: ",", groupSeparator: "." }), "1050.25") + compare(localeForm.formatCanonicalNumber("1050.25", { decimalSeparator: ",", groupSeparator: "." }), "1.050,25") + compare(localeForm.formatCanonicalNumber("-1050.25", { decimalSeparator: ",", groupSeparator: "." }), "-1.050,25") } function test_zonedTimestampRoundTripsToUtc() { diff --git a/tests/test_render_locale_format.cpp b/tests/test_render_locale_format.cpp index c659b25bc..fea5de592 100644 --- a/tests/test_render_locale_format.cpp +++ b/tests/test_render_locale_format.cpp @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 +#include #include #include #include @@ -7,35 +8,36 @@ #include #include #include +#include using morph::render::formatCanonicalNumber; using morph::render::normalizeLocaleNumber; TEST_CASE("render::normalizeLocaleNumber converts de-DE grouped/decimal-comma text", "[render][locale]") { - CHECK(normalizeLocaleNumber("1.050,25", ",", ".") == "1050.25"); - CHECK(normalizeLocaleNumber("-1.050,25", ",", ".") == "-1050.25"); + CHECK(normalizeLocaleNumber("1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050.25"); + CHECK(normalizeLocaleNumber("-1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1050.25"); } TEST_CASE("render::normalizeLocaleNumber converts fr-FR space-grouped/decimal-comma text", "[render][locale]") { - CHECK(normalizeLocaleNumber("1 050,25", ",", " ") == "1050.25"); + CHECK(normalizeLocaleNumber("1 050,25", {.decimalSeparator = ",", .groupSeparator = " "}) == "1050.25"); } TEST_CASE("render::normalizeLocaleNumber is the identity transform for plain '.'-decimal text", "[render][locale]") { - CHECK(normalizeLocaleNumber("1234.5", ".", "") == "1234.5"); - CHECK(normalizeLocaleNumber("-0.001", ".", "") == "-0.001"); + CHECK(normalizeLocaleNumber("1234.5", {.decimalSeparator = ".", .groupSeparator = ""}) == "1234.5"); + CHECK(normalizeLocaleNumber("-0.001", {.decimalSeparator = ".", .groupSeparator = ""}) == "-0.001"); } TEST_CASE("render::normalizeLocaleNumber rejects malformed input rather than guessing", "[render][locale]") { - CHECK(normalizeLocaleNumber("12.34.56", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("abc", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("-", ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber("12.34.56", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("abc", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("-", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); } TEST_CASE("render::normalizeLocaleNumber rejects empty input", "[render][locale]") { // No characters ever reach the output, so `canonical` stays empty -- the // `canonical.empty()` arm of the final malformed check, distinct from the // `canonical == "-"` arm the lone-dash case above exercises. - CHECK(normalizeLocaleNumber("", ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber("", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); } TEST_CASE("render::normalizeLocaleNumber rejects a sign that is not in the leading position", "[render][locale]") { @@ -43,7 +45,7 @@ TEST_CASE("render::normalizeLocaleNumber rejects a sign that is not in the leadi // not a leading sign -- distinct from the lone-dash case (`sawAnyOutput` // is still false there) and from the multi-decimal case (a different // malformed reason entirely). - CHECK(normalizeLocaleNumber("1-2", ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber("1-2", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); } TEST_CASE("render::normalizeLocaleNumber supports a locale with no decimal separator", "[render][locale]") { @@ -51,26 +53,26 @@ TEST_CASE("render::normalizeLocaleNumber supports a locale with no decimal separ // decimalSeparator: the decimal-separator match must short-circuit on // `decimalSeparator.empty()` rather than call `starts_with` on an empty // needle, and grouping must still work standalone. - CHECK(normalizeLocaleNumber("1.050", "", ".") == "1050"); - CHECK(normalizeLocaleNumber("-1.050", "", ".") == "-1050"); + CHECK(normalizeLocaleNumber("1.050", {.decimalSeparator = "", .groupSeparator = "."}) == "1050"); + CHECK(normalizeLocaleNumber("-1.050", {.decimalSeparator = "", .groupSeparator = "."}) == "-1050"); } TEST_CASE("render::formatCanonicalNumber groups thousands and swaps the decimal separator", "[render][locale]") { - CHECK(formatCanonicalNumber("1050.25", ",", ".") == "1.050,25"); - CHECK(formatCanonicalNumber("-1050.25", ",", ".") == "-1.050,25"); - CHECK(formatCanonicalNumber("1234.5", ".", "") == "1234.5"); + CHECK(formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1.050,25"); + CHECK(formatCanonicalNumber("-1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1.050,25"); + CHECK(formatCanonicalNumber("1234.5", {.decimalSeparator = ".", .groupSeparator = ""}) == "1234.5"); } TEST_CASE("render::formatCanonicalNumber handles empty input", "[render][locale]") { // `canonicalText.empty()` must short-circuit the sign check rather than // call `.front()` on an empty view. - CHECK(formatCanonicalNumber("", ",", ".").empty()); + CHECK(formatCanonicalNumber("", {.decimalSeparator = ",", .groupSeparator = "."}).empty()); } TEST_CASE("render locale numeric round-trip: normalize then format reproduces the original", "[render][locale]") { - auto const canonical = normalizeLocaleNumber("1.050,25", ",", "."); + auto const canonical = normalizeLocaleNumber("1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}); REQUIRE(canonical == "1050.25"); - CHECK(formatCanonicalNumber("1050.25", ",", ".") == "1.050,25"); + CHECK(formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1.050,25"); } TEST_CASE("A zoned DateTime display shift round-trips to the identical canonical instant", "[render][locale]") { @@ -112,39 +114,41 @@ TEST_CASE("render::normalizeLocaleNumber accepts a multi-byte group separator", REQUIRE(kNarrowNbsp.size() == 3); REQUIRE(kNbsp.size() == 2); - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "050,25", ",", kNarrowNbsp) == - "1050.25"); - CHECK(normalizeLocaleNumber(std::string{"-1"} + std::string{kNarrowNbsp} + "050,25", ",", kNarrowNbsp) == - "-1050.25"); - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "234", ",", kNbsp) == "1234"); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "050,25", + {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == "1050.25"); + CHECK(normalizeLocaleNumber(std::string{"-1"} + std::string{kNarrowNbsp} + "050,25", + {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == "-1050.25"); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "234", + {.decimalSeparator = ",", .groupSeparator = kNbsp}) == "1234"); } TEST_CASE("render::normalizeLocaleNumber accepts a multi-byte decimal separator", "[render][locale]") { // Not a real locale, but it pins that the decimal branch matches the whole // separator too, rather than only its first byte. - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "5", kNbsp, "") == "1.5"); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "5", + {.decimalSeparator = kNbsp, .groupSeparator = ""}) == "1.5"); // A second one is still malformed. - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "5" + std::string{kNbsp} + "2", kNbsp, "") == - std::nullopt); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNbsp} + "5" + std::string{kNbsp} + "2", + {.decimalSeparator = kNbsp, .groupSeparator = ""}) == std::nullopt); } TEST_CASE("render::normalizeLocaleNumber rejects a stray separator byte", "[render][locale]") { // A lone continuation byte of a multi-byte separator is not the separator, // and must not be silently stripped. - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp.substr(0, 1)} + "050", ",", kNarrowNbsp) == - std::nullopt); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp.substr(0, 1)} + "050", + {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == std::nullopt); } TEST_CASE("render::formatCanonicalNumber emits a multi-byte group separator", "[render][locale]") { - CHECK(formatCanonicalNumber("1050.25", ",", kNarrowNbsp) == + CHECK(formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == std::string{"1"} + std::string{kNarrowNbsp} + "050,25"); - CHECK(formatCanonicalNumber("1234567", ",", kNbsp) == + CHECK(formatCanonicalNumber("1234567", {.decimalSeparator = ",", .groupSeparator = kNbsp}) == std::string{"1"} + std::string{kNbsp} + "234" + std::string{kNbsp} + "567"); } TEST_CASE("render::locale_format round-trips through a multi-byte separator", "[render][locale]") { - auto const display = formatCanonicalNumber("1050.25", ",", kNarrowNbsp); - CHECK(normalizeLocaleNumber(display, ",", kNarrowNbsp) == "1050.25"); + auto const display = formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}); + CHECK(normalizeLocaleNumber(display, {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == "1050.25"); } // ── morph#497: a sign after the decimal separator is not "leading" ── @@ -156,18 +160,22 @@ TEST_CASE("render::locale_format round-trips through a multi-byte separator", "[ // this function) always rejected these, so the two control edges disagreed. TEST_CASE("normalizeLocaleNumber: a sign after the decimal separator is rejected", "[render][locale][morph497]") { // de-DE: comma decimal, dot grouping -- the reported shape. - REQUIRE_FALSE(morph::render::normalizeLocaleNumber(",-5", ",", ".").has_value()); + REQUIRE_FALSE( + morph::render::normalizeLocaleNumber(",-5", {.decimalSeparator = ",", .groupSeparator = "."}).has_value()); // en-US equivalent. - REQUIRE_FALSE(morph::render::normalizeLocaleNumber(".-5", ".", ",").has_value()); + REQUIRE_FALSE( + morph::render::normalizeLocaleNumber(".-5", {.decimalSeparator = ".", .groupSeparator = ","}).has_value()); // With a group separator stripped first, which is the case the guard's own // comment is about. - REQUIRE_FALSE(morph::render::normalizeLocaleNumber("1.,-5", ",", ".").has_value()); + REQUIRE_FALSE( + morph::render::normalizeLocaleNumber("1.,-5", {.decimalSeparator = ",", .groupSeparator = "."}).has_value()); // Control: the guard already worked once a digit had been emitted, and must // keep working. - REQUIRE_FALSE(morph::render::normalizeLocaleNumber("1-2", ".", ",").has_value()); + REQUIRE_FALSE( + morph::render::normalizeLocaleNumber("1-2", {.decimalSeparator = ".", .groupSeparator = ","}).has_value()); // Control: a genuinely leading sign still parses. - REQUIRE(morph::render::normalizeLocaleNumber("-1,5", ",", ".") == "-1.5"); + REQUIRE(morph::render::normalizeLocaleNumber("-1,5", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1.5"); } TEST_CASE("normalizeLocaleNumber: the loose shapes stay accepted, in step with the QML mirror", @@ -175,9 +183,9 @@ TEST_CASE("normalizeLocaleNumber: the loose shapes stay accepted, in step with t // Deliberately NOT narrowed to `-?[0-9]+(\.[0-9]+)?`: DynamicForm.qml's // normalizeLocaleNumber accepts all three, and tightening one edge alone // would put them back out of step. Documented on the function. - REQUIRE(morph::render::normalizeLocaleNumber(".5", ".", ",") == ".5"); - REQUIRE(morph::render::normalizeLocaleNumber("5.", ".", ",") == "5."); - REQUIRE(morph::render::normalizeLocaleNumber(".", ".", ",") == "."); + REQUIRE(morph::render::normalizeLocaleNumber(".5", {.decimalSeparator = ".", .groupSeparator = ","}) == ".5"); + REQUIRE(morph::render::normalizeLocaleNumber("5.", {.decimalSeparator = ".", .groupSeparator = ","}) == "5."); + REQUIRE(morph::render::normalizeLocaleNumber(".", {.decimalSeparator = ".", .groupSeparator = ","}) == "."); } // ── morph#574: a group separator is validated, not stripped ────────────────── @@ -193,76 +201,78 @@ TEST_CASE("normalizeLocaleNumber: the loose shapes stay accepted, in step with t TEST_CASE("normalizeLocaleNumber: the decimal separator of another locale is rejected, not absorbed", "[render][locale][morph574]") { // THE case. de-DE locale, US-style decimal typed: this returned "15". - CHECK(normalizeLocaleNumber("1.5", ",", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.50", ",", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.2.3.4", ",", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1.5", {.decimalSeparator = ",", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.50", {.decimalSeparator = ",", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.2.3.4", {.decimalSeparator = ",", .groupSeparator = "."}) == std::nullopt); // The mirror image: en-US locale, EU-style decimal typed. Returned "15". - CHECK(normalizeLocaleNumber("1,5", ".", ",") == std::nullopt); + CHECK(normalizeLocaleNumber("1,5", {.decimalSeparator = ".", .groupSeparator = ","}) == std::nullopt); // And with a multi-byte group separator, where the same mistake is a // narrow no-break space away from a well-formed entry. - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "5", ",", kNarrowNbsp) == std::nullopt); + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "5", + {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == std::nullopt); } TEST_CASE("normalizeLocaleNumber: equal decimal and group separators are rejected rather than guessed", "[render][locale][morph574]") { // One string in both roles has no defensible reading, and the old code // silently ate the decimal: this returned "15". - CHECK(normalizeLocaleNumber("1.5", ".", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1.5", {.decimalSeparator = ".", .groupSeparator = "."}) == std::nullopt); // Not even the shapes that would be unambiguous if you squinted: the // rejection is on the configuration, not on the text. - CHECK(normalizeLocaleNumber("1.050", ".", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1", ".", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1.050", {.decimalSeparator = ".", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1", {.decimalSeparator = ".", .groupSeparator = "."}) == std::nullopt); // Control: an empty group separator is "this locale does not group", which // is a different statement and stays legal. - CHECK(normalizeLocaleNumber("1.5", ".", "") == "1.5"); + CHECK(normalizeLocaleNumber("1.5", {.decimalSeparator = ".", .groupSeparator = ""}) == "1.5"); } TEST_CASE("normalizeLocaleNumber: a group separator must sit on a group boundary", "[render][locale][morph574]") { // Preceded by one to three digits... - CHECK(normalizeLocaleNumber("1.050", "", ".") == "1050"); - CHECK(normalizeLocaleNumber("12.050", "", ".") == "12050"); - CHECK(normalizeLocaleNumber("123.050", "", ".") == "123050"); - CHECK(normalizeLocaleNumber("1234.050", "", ".") == std::nullopt); - CHECK(normalizeLocaleNumber(".050", "", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1.050", {.decimalSeparator = "", .groupSeparator = "."}) == "1050"); + CHECK(normalizeLocaleNumber("12.050", {.decimalSeparator = "", .groupSeparator = "."}) == "12050"); + CHECK(normalizeLocaleNumber("123.050", {.decimalSeparator = "", .groupSeparator = "."}) == "123050"); + CHECK(normalizeLocaleNumber("1234.050", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber(".050", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); // ...followed by exactly three, at every group and at the end of the // integer part. - CHECK(normalizeLocaleNumber("1.05", "", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.0500", "", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.050.", "", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.000.00", "", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.000.000", "", ".") == "1000000"); + CHECK(normalizeLocaleNumber("1.05", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.0500", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.050.", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.000.00", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.000.000", {.decimalSeparator = "", .groupSeparator = "."}) == "1000000"); // ...and never after the decimal separator. - CHECK(normalizeLocaleNumber("1,050.25", ",", ".") == std::nullopt); - CHECK(normalizeLocaleNumber("1.000,250.25", ",", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1,050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.000,250.25", {.decimalSeparator = ",", .groupSeparator = "."}) == std::nullopt); // A grouping locale with no separator in the entry at all: there is no // placement to be wrong about, and the digits pass through. - CHECK(normalizeLocaleNumber("1050", ",", ".") == "1050"); + CHECK(normalizeLocaleNumber("1050", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050"); // A non-digit inside a grouping locale restarts the digit run rather than // being counted into it -- and is malformed for the ordinary reason. - CHECK(normalizeLocaleNumber("1.0x0", "", ".") == std::nullopt); + CHECK(normalizeLocaleNumber("1.0x0", {.decimalSeparator = "", .groupSeparator = "."}) == std::nullopt); } TEST_CASE("normalizeLocaleNumber: every well-formed locale entry still normalises", "[render][locale][morph574]") { // The validation must not cost a single legitimate entry -- this is the // half of the change that the rejection cases cannot show. - CHECK(normalizeLocaleNumber("1.050,25", ",", ".") == "1050.25"); - CHECK(normalizeLocaleNumber("-1.050,25", ",", ".") == "-1050.25"); - CHECK(normalizeLocaleNumber("1.000.000,25", ",", ".") == "1000000.25"); - CHECK(normalizeLocaleNumber("1050,25", ",", ".") == "1050.25"); // ungrouped - CHECK(normalizeLocaleNumber("1,050.25", ".", ",") == "1050.25"); // en-US - CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "050,25", ",", kNarrowNbsp) == - "1050.25"); // fr-FR + CHECK(normalizeLocaleNumber("1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050.25"); + CHECK(normalizeLocaleNumber("-1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1050.25"); + CHECK(normalizeLocaleNumber("1.000.000,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1000000.25"); + CHECK(normalizeLocaleNumber("1050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == + "1050.25"); // ungrouped + CHECK(normalizeLocaleNumber("1,050.25", {.decimalSeparator = ".", .groupSeparator = ","}) == "1050.25"); // en-US + CHECK(normalizeLocaleNumber(std::string{"1"} + std::string{kNarrowNbsp} + "050,25", + {.decimalSeparator = ",", .groupSeparator = kNarrowNbsp}) == "1050.25"); // fr-FR // A grouped entry round-trips through the display direction unchanged. - auto const canonical = normalizeLocaleNumber("1.000.000,25", ",", "."); + auto const canonical = normalizeLocaleNumber("1.000.000,25", {.decimalSeparator = ",", .groupSeparator = "."}); REQUIRE(canonical.has_value()); - CHECK(formatCanonicalNumber(*canonical, ",", ".") == "1.000.000,25"); + CHECK(formatCanonicalNumber(*canonical, {.decimalSeparator = ",", .groupSeparator = "."}) == "1.000.000,25"); } // ──── morph#583: the negative sign is locale data, and is not always one byte ──── @@ -307,25 +317,37 @@ TEST_CASE("normalizeLocaleNumber: a locale whose sign is U+2212 can be entered", REQUIRE(kSignAzIr.size() == 7); // eu_ES: the bare U+2212 of the issue title. - CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), ".", "", kSignEuEs) == "-5"); - CHECK(normalizeLocaleNumber(entry(kSignEuEs, "1050,25"), ",", ".", kSignEuEs) == "-1050.25"); + CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignEuEs}) == "-5"); + CHECK(normalizeLocaleNumber(entry(kSignEuEs, "1050,25"), + {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == + "-1050.25"); // The control for the defect: with the sign left at its ASCII default, the // same entry is still rejected -- that default is the whole of today's // behaviour. - CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); } TEST_CASE("normalizeLocaleNumber: a bidi-prefixed sign is matched as a whole string", "[render][locale][morph583]") { // Two and three code points respectively. A one-unit comparison cannot // match either, which is why both edges compare the whole string. - CHECK(normalizeLocaleNumber(entry(kSignFaIr, "5"), ".", "", kSignFaIr) == "-5"); // fa_IR - CHECK(normalizeLocaleNumber(entry(kSignAzIr, "5"), ".", "", kSignAzIr) == "-5"); // az_IR, 3 code points - CHECK(normalizeLocaleNumber(entry(kSignArEg, "5"), ".", "", kSignArEg) == "-5"); // ar_EG, U+061C prefix + CHECK(normalizeLocaleNumber(entry(kSignFaIr, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignFaIr}) == + "-5"); // fa_IR + CHECK(normalizeLocaleNumber(entry(kSignAzIr, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignAzIr}) == + "-5"); // az_IR, 3 code points + CHECK(normalizeLocaleNumber(entry(kSignArEg, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignArEg}) == + "-5"); // ar_EG, U+061C prefix // Same controls: rejected today, for each shape. - CHECK(normalizeLocaleNumber(entry(kSignFaIr, "5"), ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(entry(kSignAzIr, "5"), ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber(entry(kSignFaIr, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); + CHECK(normalizeLocaleNumber(entry(kSignAzIr, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); } TEST_CASE("normalizeLocaleNumber: ar_DZ fails today even though its sign is the ASCII hyphen", @@ -333,42 +355,56 @@ TEST_CASE("normalizeLocaleNumber: ar_DZ fails today even though its sign is the // The case that shows this is not "the U+2212 locales": ar_DZ's sign *is* // '-', prefixed by U+200E. The stray prefix byte is what the per-byte scan // rejects, so matching the hyphen byte-wise never helped it. - CHECK(normalizeLocaleNumber(entry(kSignArDz, "5"), ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(entry(kSignArDz, "5"), ".", "", kSignArDz) == "-5"); + CHECK(normalizeLocaleNumber(entry(kSignArDz, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); + CHECK(normalizeLocaleNumber(entry(kSignArDz, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignArDz}) == "-5"); } TEST_CASE("normalizeLocaleNumber: the ASCII hyphen stays accepted in every locale", "[render][locale][morph583]") { // U+2212 and the bidi marks are on no keyboard. Matching only the locale's // own spelling would reject the sign the user can actually type and leave // them no way to enter a negative number at all. - CHECK(normalizeLocaleNumber("-5", ".", "", kSignEuEs) == "-5"); - CHECK(normalizeLocaleNumber("-5", ".", "", kSignFaIr) == "-5"); + CHECK(normalizeLocaleNumber("-5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignEuEs}) == + "-5"); + CHECK(normalizeLocaleNumber("-5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignFaIr}) == + "-5"); // And the plain ASCII locale is untouched -- the default parameter means no // existing caller changed behaviour. - CHECK(normalizeLocaleNumber("-5", ".", "") == "-5"); - CHECK(normalizeLocaleNumber("-1.050,25", ",", ".") == "-1050.25"); + CHECK(normalizeLocaleNumber("-5", {.decimalSeparator = ".", .groupSeparator = ""}) == "-5"); + CHECK(normalizeLocaleNumber("-1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1050.25"); } TEST_CASE("normalizeLocaleNumber: a locale sign is still rejected off the leading position", "[render][locale][morph583]") { // The morph#497 rule is about the *output*, so it has to hold for a // multi-byte sign exactly as it does for '-'. - CHECK(normalizeLocaleNumber("1" + entry(kSignEuEs, "2"), ".", "", kSignEuEs) == std::nullopt); - CHECK(normalizeLocaleNumber("," + entry(kSignEuEs, "5"), ",", ".", kSignEuEs) == std::nullopt); + CHECK(normalizeLocaleNumber("1" + entry(kSignEuEs, "2"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignEuEs}) == + std::nullopt); + CHECK(normalizeLocaleNumber("," + entry(kSignEuEs, "5"), + {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == + std::nullopt); // A sign and nothing else is not a number, whatever its spelling. - CHECK(normalizeLocaleNumber(kSignEuEs, ".", "", kSignEuEs) == std::nullopt); - CHECK(normalizeLocaleNumber(kSignAzIr, ".", "", kSignAzIr) == std::nullopt); + CHECK(normalizeLocaleNumber( + kSignEuEs, {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignEuEs}) == std::nullopt); + CHECK(normalizeLocaleNumber( + kSignAzIr, {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignAzIr}) == std::nullopt); } TEST_CASE("formatCanonicalNumber: the display edge emits the locale's sign", "[render][locale][morph583]") { // Before this, the sign was a hardcoded '-' whatever the locale -- so even // a caller that knew its locale's sign could not ask for it. - CHECK(formatCanonicalNumber("-1050.25", ",", ".", kSignEuEs) == entry(kSignEuEs, "1.050,25")); - CHECK(formatCanonicalNumber("-5", ".", "", kSignFaIr) == entry(kSignFaIr, "5")); + CHECK(formatCanonicalNumber("-1050.25", + {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == + entry(kSignEuEs, "1.050,25")); + CHECK(formatCanonicalNumber("-5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignFaIr}) == + entry(kSignFaIr, "5")); // A positive is untouched: the sign string is only ever emitted for a // negative, so a locale-specific sign cannot leak into a positive display. - CHECK(formatCanonicalNumber("1050.25", ",", ".", kSignEuEs) == "1.050,25"); - CHECK(formatCanonicalNumber("-1050.25", ",", ".") == "-1.050,25"); + CHECK(formatCanonicalNumber( + "1050.25", {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == "1.050,25"); + CHECK(formatCanonicalNumber("-1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1.050,25"); } TEST_CASE("locale_format: the pair is inverse for every measured sign spelling", "[render][locale][morph583]") { @@ -376,9 +412,11 @@ TEST_CASE("locale_format: the pair is inverse for every measured sign spelling", // emits, the entry edge takes back to the identical canonical text. for (auto const& sign : {kSignEuEs, kSignFaIr, kSignArDz, kSignAzIr, kSignArEg}) { INFO("sign = " << sign); - auto const display = formatCanonicalNumber("-1050.25", ",", ".", sign); + auto const display = + formatCanonicalNumber("-1050.25", {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = sign}); CHECK(display == entry(sign, "1.050,25")); - CHECK(normalizeLocaleNumber(display, ",", ".", sign) == "-1050.25"); + CHECK(normalizeLocaleNumber(display, {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = sign}) == + "-1050.25"); } } @@ -387,11 +425,11 @@ TEST_CASE("locale_format: an empty negative sign reads as '-', not as 'no sign'" // empty cannot mean absence. On the display edge it would be a silently // wrong value: -5 formatted to "5" is a valid number of the wrong sign, // which is the morph#574 failure mode, not a rejection. - CHECK(formatCanonicalNumber("-5", ".", "", "") == "-5"); - CHECK(normalizeLocaleNumber("-5", ".", "", "") == "-5"); + CHECK(formatCanonicalNumber("-5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = ""}) == "-5"); + CHECK(normalizeLocaleNumber("-5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = ""}) == "-5"); // And an empty needle must not match at every index: a scan that treated it // as a separator would never advance. - CHECK(normalizeLocaleNumber("123", ".", "", "") == "123"); + CHECK(normalizeLocaleNumber("123", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = ""}) == "123"); } // ──── morph#596: a leading positive sign is accepted, and dropped ──────────── @@ -426,19 +464,19 @@ constexpr std::string_view kPlusCkbIq = "\xE2\x80\x8F+"; // ckb_IQ: TEST_CASE("normalizeLocaleNumber: a bare ASCII '+' is accepted and dropped", "[render][locale][morph596]") { // THE case. This was std::nullopt before, in every locale. - CHECK(normalizeLocaleNumber("+5", ".", "") == "5"); - CHECK(normalizeLocaleNumber("+1.050,25", ",", ".") == "1050.25"); - CHECK(normalizeLocaleNumber("+0.001", ".", "") == "0.001"); + CHECK(normalizeLocaleNumber("+5", {.decimalSeparator = ".", .groupSeparator = ""}) == "5"); + CHECK(normalizeLocaleNumber("+1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050.25"); + CHECK(normalizeLocaleNumber("+0.001", {.decimalSeparator = ".", .groupSeparator = ""}) == "0.001"); // Dropped, not carried: canonical text is `-?[0-9]+(\.[0-9]+)?` and has no // '+' in it. "+5" must be "5", never "+5" -- a "+"-prefixed result would // fail the renderer's own `/^-?\d+(\.\d+)?$/` gate and every exact digit // routine downstream. - auto const plus = normalizeLocaleNumber("+5", ".", ""); + auto const plus = normalizeLocaleNumber("+5", {.decimalSeparator = ".", .groupSeparator = ""}); REQUIRE(plus.has_value()); CHECK(*plus == "5"); CHECK(!plus->contains('+')); - CHECK(plus == normalizeLocaleNumber("5", ".", "")); + CHECK(plus == normalizeLocaleNumber("5", {.decimalSeparator = ".", .groupSeparator = ""})); } TEST_CASE("normalizeLocaleNumber: a locale's multi-code-point positive sign is matched whole", @@ -448,59 +486,90 @@ TEST_CASE("normalizeLocaleNumber: a locale's multi-code-point positive sign is m REQUIRE(kPlusArEg.size() == 3); REQUIRE(kPlusAzIr.size() == 7); - CHECK(normalizeLocaleNumber(entry(kPlusArEg, "5"), ".", "", "-", kPlusArEg) == "5"); - CHECK(normalizeLocaleNumber(entry(kPlusArDz, "5"), ".", "", "-", kPlusArDz) == "5"); - CHECK(normalizeLocaleNumber(entry(kPlusAzIr, "5"), ".", "", "-", kPlusAzIr) == "5"); - CHECK(normalizeLocaleNumber(entry(kPlusCkbIq, "5"), ".", "", "-", kPlusCkbIq) == "5"); - CHECK(normalizeLocaleNumber(entry(kPlusArEg, "1.050,25"), ",", ".", "-", kPlusArEg) == "1050.25"); + CHECK(normalizeLocaleNumber( + entry(kPlusArEg, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusArEg}) == "5"); + CHECK(normalizeLocaleNumber( + entry(kPlusArDz, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusArDz}) == "5"); + CHECK(normalizeLocaleNumber( + entry(kPlusAzIr, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusAzIr}) == "5"); + CHECK(normalizeLocaleNumber( + entry(kPlusCkbIq, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusCkbIq}) == + "5"); + CHECK(normalizeLocaleNumber( + entry(kPlusArEg, "1.050,25"), + {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = "-", .positiveSign = kPlusArEg}) == + "1050.25"); // Controls: with positiveSign left at its ASCII default, the bidi-prefixed // spellings are still rejected -- the prefix byte is not a digit, and the // bare '+' match cannot reach past it. This is what makes the parameter, // rather than the unconditional ASCII acceptance above, the thing under // test in this case. - CHECK(normalizeLocaleNumber(entry(kPlusArEg, "5"), ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(entry(kPlusAzIr, "5"), ".", "") == std::nullopt); + CHECK(normalizeLocaleNumber(entry(kPlusArEg, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); + CHECK(normalizeLocaleNumber(entry(kPlusAzIr, "5"), {.decimalSeparator = ".", .groupSeparator = ""}) == + std::nullopt); } TEST_CASE("normalizeLocaleNumber: the ASCII '+' stays accepted in a bidi-sign locale", "[render][locale][morph596]") { // The morph#583 precedent: the locale's own spelling is on no keyboard, so // matching only it would reject the sign the user can actually type. - CHECK(normalizeLocaleNumber("+5", ".", "", "-", kPlusArEg) == "5"); - CHECK(normalizeLocaleNumber("+5", ".", "", "-", kPlusAzIr) == "5"); + CHECK(normalizeLocaleNumber( + "+5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusArEg}) == + "5"); + CHECK(normalizeLocaleNumber( + "+5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusAzIr}) == + "5"); // An empty positiveSign leaves the ASCII spelling as the only one, rather // than disabling the sign -- and must not match at every index, which would // stall the scan. - CHECK(normalizeLocaleNumber("+5", ".", "", "-", "") == "5"); - CHECK(normalizeLocaleNumber("123", ".", "", "-", "") == "123"); + CHECK(normalizeLocaleNumber( + "+5", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = ""}) == "5"); + CHECK(normalizeLocaleNumber( + "123", {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = ""}) == + "123"); } TEST_CASE("normalizeLocaleNumber: a positive sign obeys the same leading-position rule", "[render][locale][morph596]") { // morph#497's rule is about the *output*, so accepting a new sign spelling // must not open a new way to inject one. - CHECK(normalizeLocaleNumber("1+2", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("+-5", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("-+5", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("++5", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(",+5", ",", ".") == std::nullopt); // straight after the decimal point - CHECK(normalizeLocaleNumber("1" + entry(kPlusArEg, "2"), ".", "", "-", kPlusArEg) == std::nullopt); + CHECK(normalizeLocaleNumber("1+2", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("+-5", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("-+5", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("++5", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber(",+5", {.decimalSeparator = ",", .groupSeparator = "."}) == + std::nullopt); // straight after the decimal point + CHECK(normalizeLocaleNumber( + "1" + entry(kPlusArEg, "2"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusArEg}) == + std::nullopt); // A sign and nothing else is not a number, whatever its spelling -- and for // the positive sign this is the `canonical.empty()` arm rather than the // `canonical == "-"` one, because nothing is emitted at all. - CHECK(normalizeLocaleNumber("+", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(kPlusArEg, ".", "", "-", kPlusArEg) == std::nullopt); + CHECK(normalizeLocaleNumber("+", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber( + kPlusArEg, + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = "-", .positiveSign = kPlusArEg}) == + std::nullopt); } TEST_CASE("formatCanonicalNumber: the display edge never emits a positive sign", "[render][locale][morph596]") { // The deliberate asymmetry, pinned so that "make it symmetric" is a test // failure rather than a tidy-up. positiveSign is '+' in 657 of 711 locales, // so emitting it would turn every positive number in every form into "+5". - CHECK(formatCanonicalNumber("5", ".", "") == "5"); - CHECK(formatCanonicalNumber("1050.25", ",", ".") == "1.050,25"); - CHECK(formatCanonicalNumber("1050.25", ",", ".", kSignEuEs) == "1.050,25"); + CHECK(formatCanonicalNumber("5", {.decimalSeparator = ".", .groupSeparator = ""}) == "5"); + CHECK(formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1.050,25"); + CHECK(formatCanonicalNumber( + "1050.25", {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == "1.050,25"); // The negative direction is untouched by any of this. - CHECK(formatCanonicalNumber("-1050.25", ",", ".", kSignEuEs) == entry(kSignEuEs, "1.050,25")); + CHECK(formatCanonicalNumber("-1050.25", + {.decimalSeparator = ",", .groupSeparator = ".", .negativeSign = kSignEuEs}) == + entry(kSignEuEs, "1.050,25")); } TEST_CASE("locale_format: the pair is not inverse across a positive sign, by design", "[render][locale][morph596]") { @@ -508,25 +577,28 @@ TEST_CASE("locale_format: the pair is not inverse across a positive sign, by des // the relationship is pinned rather than assumed: normalising a // "+"-prefixed entry and formatting the result back gives the *unsigned* // display, the same text a user who omitted the sign would see. - auto const canonical = normalizeLocaleNumber("+1.050,25", ",", "."); + auto const canonical = normalizeLocaleNumber("+1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}); REQUIRE(canonical == "1050.25"); - CHECK(formatCanonicalNumber(*canonical, ",", ".") == "1.050,25"); - CHECK(formatCanonicalNumber(*canonical, ",", ".") != "+1.050,25"); + CHECK(formatCanonicalNumber(*canonical, {.decimalSeparator = ",", .groupSeparator = "."}) == "1.050,25"); + CHECK(formatCanonicalNumber(*canonical, {.decimalSeparator = ",", .groupSeparator = "."}) != "+1.050,25"); // ...while the negative side still round-trips exactly, unchanged. - auto const negative = normalizeLocaleNumber("-1.050,25", ",", "."); + auto const negative = normalizeLocaleNumber("-1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}); REQUIRE(negative == "-1050.25"); - CHECK(formatCanonicalNumber(*negative, ",", ".") == "-1.050,25"); + CHECK(formatCanonicalNumber(*negative, {.decimalSeparator = ",", .groupSeparator = "."}) == "-1.050,25"); } TEST_CASE("normalizeLocaleNumber: the new parameter costs no existing behaviour", "[render][locale][morph596]") { // Four- and three-argument callers are unchanged: positiveSign is // defaulted, and nothing that was accepted or rejected before has moved. - CHECK(normalizeLocaleNumber("1.050,25", ",", ".") == "1050.25"); - CHECK(normalizeLocaleNumber("-1.050,25", ",", ".") == "-1050.25"); - CHECK(normalizeLocaleNumber("1.5", ",", ".") == std::nullopt); // morph#574 still holds - CHECK(normalizeLocaleNumber("abc", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber("", ".", "") == std::nullopt); - CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), ".", "", kSignEuEs) == "-5"); // morph#583 still holds + CHECK(normalizeLocaleNumber("1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050.25"); + CHECK(normalizeLocaleNumber("-1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1050.25"); + CHECK(normalizeLocaleNumber("1.5", {.decimalSeparator = ",", .groupSeparator = "."}) == + std::nullopt); // morph#574 still holds + CHECK(normalizeLocaleNumber("abc", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber("", {.decimalSeparator = ".", .groupSeparator = ""}) == std::nullopt); + CHECK(normalizeLocaleNumber(entry(kSignEuEs, "5"), + {.decimalSeparator = ".", .groupSeparator = "", .negativeSign = kSignEuEs}) == + "-5"); // morph#583 still holds } // ──── morph#599: the QML mirror's separators, cross-checked here ───────────── @@ -607,12 +679,18 @@ TEST_CASE("locale_format: the multi-unit separator corpus the QML mirror now sha REQUIRE(kGroup4.size() == 4); // 4 UTF-8 bytes; 2 UTF-16 units REQUIRE(kDecimal4.size() == 4); - CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "050", kDecimal2, "25"}), kDecimal2, kGroup2) == "1050.25"); - CHECK(normalizeLocaleNumber(joined({"-1", kGroup2, "050", kDecimal2, "25"}), kDecimal2, kGroup2) == "-1050.25"); - CHECK(normalizeLocaleNumber(joined({"+1", kGroup2, "050", kDecimal2, "25"}), kDecimal2, kGroup2) == "1050.25"); - CHECK(normalizeLocaleNumber(joined({"1", kGroup4, "050", kDecimal4, "25"}), kDecimal4, kGroup4) == "1050.25"); - CHECK(normalizeLocaleNumber(joined({"1", kGroup4, "050", kGroup4, "000"}), "", kGroup4) == "1050000"); - CHECK(normalizeLocaleNumber(joined({"5", kDecimal2, "25"}), kDecimal2, "") == "5.25"); + CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "050", kDecimal2, "25"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == "1050.25"); + CHECK(normalizeLocaleNumber(joined({"-1", kGroup2, "050", kDecimal2, "25"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == "-1050.25"); + CHECK(normalizeLocaleNumber(joined({"+1", kGroup2, "050", kDecimal2, "25"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == "1050.25"); + CHECK(normalizeLocaleNumber(joined({"1", kGroup4, "050", kDecimal4, "25"}), + {.decimalSeparator = kDecimal4, .groupSeparator = kGroup4}) == "1050.25"); + CHECK(normalizeLocaleNumber(joined({"1", kGroup4, "050", kGroup4, "000"}), + {.decimalSeparator = "", .groupSeparator = kGroup4}) == "1050000"); + CHECK(normalizeLocaleNumber(joined({"5", kDecimal2, "25"}), + {.decimalSeparator = kDecimal2, .groupSeparator = ""}) == "5.25"); } TEST_CASE("locale_format: a multi-unit separator is validated exactly as a one-unit one is", @@ -621,18 +699,24 @@ TEST_CASE("locale_format: a multi-unit separator is validated exactly as a one-u // stated over "the separator", so they have to hold when it is longer than // one unit -- on both edges. Same rows as the mirror's // `test_aMultiUnitSeparatorIsStillValidatedTheSameWay`. - CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "5"}), kDecimal2, kGroup2) == std::nullopt); - CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "2", kGroup2, "3", kGroup2, "4"}), kDecimal2, kGroup2) == - std::nullopt); - CHECK(normalizeLocaleNumber(joined({"1", kDecimal2, "5", kGroup2, "000"}), kDecimal2, kGroup2) == std::nullopt); - CHECK(normalizeLocaleNumber(joined({"1", kDecimal2, "0", kDecimal2, "5"}), kDecimal2, kGroup2) == std::nullopt); - CHECK(normalizeLocaleNumber(joined({kDecimal2, "-5"}), kDecimal2, kGroup2) == std::nullopt); - CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "050"}), kGroup2, kGroup2) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "5"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "2", kGroup2, "3", kGroup2, "4"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({"1", kDecimal2, "5", kGroup2, "000"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({"1", kDecimal2, "0", kDecimal2, "5"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({kDecimal2, "-5"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber(joined({"1", kGroup2, "050"}), + {.decimalSeparator = kGroup2, .groupSeparator = kGroup2}) == std::nullopt); // A lone prefix of the separator is not the separator: whole-string // matching must not degrade into "any part of it will do". - CHECK(normalizeLocaleNumber(joined({"1", "\xE2\x80\x8E", "050", kDecimal2, "25"}), kDecimal2, kGroup2) == + CHECK(normalizeLocaleNumber(joined({"1", "\xE2\x80\x8E", "050", kDecimal2, "25"}), + {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); + CHECK(normalizeLocaleNumber("1.050,25", {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2}) == std::nullopt); - CHECK(normalizeLocaleNumber("1.050,25", kDecimal2, kGroup2) == std::nullopt); } TEST_CASE("locale_format: the pair round-trips through a multi-unit separator", "[render][locale][morph599]") { @@ -641,7 +725,205 @@ TEST_CASE("locale_format: the pair round-trips through a multi-unit separator", // separator the mirror's display edge produced text its own entry edge then // rejected -- the morph#583 shape, for a locale that does not exist yet. // This side round-tripped throughout; that is what made the two disagree. - auto const display = formatCanonicalNumber("-1050.25", kDecimal2, kGroup2, kSignEuEs); + auto const display = formatCanonicalNumber( + "-1050.25", {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2, .negativeSign = kSignEuEs}); CHECK(display == joined({kSignEuEs, "1", kGroup2, "050", kDecimal2, "25"})); - CHECK(normalizeLocaleNumber(display, kDecimal2, kGroup2, kSignEuEs) == "-1050.25"); + CHECK(normalizeLocaleNumber( + display, {.decimalSeparator = kDecimal2, .groupSeparator = kGroup2, .negativeSign = kSignEuEs}) == + "-1050.25"); +} + +// ---- morph#591: the digits are locale data too ------------------------------ +// +// Measured on this revision with QLocale::matchingLocales under Qt 6.11.2, over +// all 711 locales: 76 report a zeroDigit other than ASCII '0', across eleven +// distinct digit sets. +// +// U+0030 604 locales e.g. C <- the control +// U+0660 26 locales e.g. ar_BH +// U+06F0 19 locales e.g. fa_IR +// U+1E950 12 locales e.g. ff_Adlm_BF <- astral +// U+0966 8 locales e.g. bgc_IN +// U+09E6 4 locales e.g. as_IN +// U+11136 2 locales e.g. ccp_BD <- astral +// U+07C0 1 locale nqo_GN +// U+0F20 1 locale dz_BT +// U+1040 1 locale my_MM +// U+1C50 1 locale sat_IN +// U+ABF0 1 locale mni_IN +// +// The issue recorded 24 as the figure that could be defended, from *sign* data +// rather than digit data, and said so. 76 is the measured one. Two of the sets +// are outside the BMP, which is why the scan decodes a code point instead of +// widening a byte comparison: a digit there is four UTF-8 bytes. +// +// The same corpus is pinned against the QML mirror in +// src/qt/forms/tests/tst_i18n.qml ([morph591] there). docs/spec/forms/forms.md, +// "Both edges, or neither". +namespace { +struct DigitSet { + std::string_view name; + std::string_view zero; + std::string_view five; +}; + +// One representative per distinct set, spelled as explicit UTF-8 bytes with the +// code point in the comment -- the house style for this file, and the reason +// #610 converted the QML mirror to escapes: a digit that renders as itself is +// still unreadable when a reviewer does not read that script. +constexpr std::array kDigitSets = {{ + {.name = "ar_BH", .zero = "\xD9\xA0", .five = "\xD9\xA5"}, // U+0660, U+0665 + {.name = "fa_IR", .zero = "\xDB\xB0", .five = "\xDB\xB5"}, // U+06F0, U+06F5 + {.name = "nqo_GN", .zero = "\xDF\x80", .five = "\xDF\x85"}, // U+07C0, U+07C5 + {.name = "bgc_IN", .zero = "\xE0\xA5\xA6", .five = "\xE0\xA5\xAB"}, // U+0966, U+096B + {.name = "as_IN", .zero = "\xE0\xA7\xA6", .five = "\xE0\xA7\xAB"}, // U+09E6, U+09EB + {.name = "dz_BT", .zero = "\xE0\xBC\xA0", .five = "\xE0\xBC\xA5"}, // U+0F20, U+0F25 + {.name = "my_MM", .zero = "\xE1\x81\x80", .five = "\xE1\x81\x85"}, // U+1040, U+1045 + {.name = "sat_IN", .zero = "\xE1\xB1\x90", .five = "\xE1\xB1\x95"}, // U+1C50, U+1C55 + {.name = "mni_IN", .zero = "\xEA\xAF\xB0", .five = "\xEA\xAF\xB5"}, // U+ABF0, U+ABF5 + {.name = "ccp_BD", .zero = "\xF0\x91\x84\xB6", .five = "\xF0\x91\x84\xBB"}, // U+11136, U+1113B + {.name = "ff_Adlm", .zero = "\xF0\x9E\xA5\x90", .five = "\xF0\x9E\xA5\x95"}, // U+1E950, U+1E955 +}}; + +// ar_EG/ar_BH's full set of locale facts, the one Qt reports. +constexpr std::string_view kArDecimal = "\xD9\xAB"; // U+066B +constexpr std::string_view kArGroup = "\xD9\xAC"; // U+066C +constexpr std::string_view kArZero = "\xD9\xA0"; // U+0660 +constexpr std::string_view kArNegative = "\xD8\x9C-"; // U+061C U+002D +} // namespace + +TEST_CASE("formatCanonicalNumber: the display edge emits the locale's digits", "[render][locale][morph591]") { + // The exact bytes QLocale("ar_BH").toString(-1050.25) produces under + // Qt 6.11.2, measured rather than derived: + // U+061C U+002D U+0661 U+066C U+0660 U+0665 U+0660 U+066B U+0662 U+0665 + // Before morph#591 this edge emitted "\u061c-1\u066c050\u066b25" -- the + // locale's sign and separators around ASCII digits, which is what made the + // pair self-consistent and the defect invisible from either side alone. + CHECK(formatCanonicalNumber("-1050.25", {.decimalSeparator = kArDecimal, + .groupSeparator = kArGroup, + .negativeSign = kArNegative, + .zeroDigit = kArZero}) == + "\xD8\x9C-\xD9\xA1\xD9\xAC\xD9\xA0\xD9\xA5\xD9\xA0\xD9\xAB\xD9\xA2\xD9\xA5"); +} + +TEST_CASE("normalizeLocaleNumber: the locale's own digits are accepted", "[render][locale][morph591]") { + // The issue's reproduction, in reverse. Each of these was std::nullopt on + // 0e3b8823: the scan compared one byte against ['0','9'] and the first byte + // of a two-byte digit failed it. + CHECK(normalizeLocaleNumber("\xD9\xA5", {.zeroDigit = kArZero}) == "5"); + CHECK(normalizeLocaleNumber("\xDB\xB5", {.zeroDigit = "\xDB\xB0"}) == "5"); + CHECK(normalizeLocaleNumber("\xD8\x9C-\xD9\xA5", {.negativeSign = kArNegative, .zeroDigit = kArZero}) == "-5"); + // Astral: one code point, four UTF-8 bytes. + CHECK(normalizeLocaleNumber("\xF0\x91\x84\xBB", {.zeroDigit = "\xF0\x91\x84\xB6"}) == "5"); + CHECK(normalizeLocaleNumber("\xF0\x9E\xA5\x95", {.zeroDigit = "\xF0\x9E\xA5\x90"}) == "5"); +} + +TEST_CASE("locale_format: the pair round-trips through every measured digit set", "[render][locale][morph591]") { + // The acceptance test this ticket exists for. A fix applied to one edge + // only leaves the suite green -- teaching entry to accept U+0665 while + // display keeps emitting '5' still round-trips, because entry accepts ASCII + // too. So the round trip is asserted *and* the intermediate display text is + // required to contain no ASCII digit: that second assertion is what fails + // if formatCanonicalNumber did not move. + for (auto const& set : kDigitSets) { + CAPTURE(set.name); + morph::render::NumericLocale const loc{.decimalSeparator = kArDecimal, + .groupSeparator = kArGroup, + .negativeSign = kArNegative, + .zeroDigit = set.zero}; + for (std::string_view const canonical : + {"0", "5", "-5", "1050.25", "-1050.25", "1000000.25", "0.001", "-0.001", "1234567"}) { + CAPTURE(canonical); + std::string const display = formatCanonicalNumber(canonical, loc); + CHECK(display.find_first_of("0123456789") == std::string::npos); + CHECK(normalizeLocaleNumber(display, loc) == std::string{canonical}); + } + } +} + +TEST_CASE("normalizeLocaleNumber: ASCII digits stay accepted in a native-digit locale", "[render][locale][morph591]") { + // The morph#596 precedent, applied to digits: the locale's own digits are + // on the user's keyboard only if their keyboard has them. Entry therefore + // accepts a spelling display never produces, exactly as it does for the + // ASCII '+' and '-'. + CHECK(normalizeLocaleNumber("5", {.zeroDigit = kArZero}) == "5"); + CHECK(normalizeLocaleNumber( + "-1050.25", {.decimalSeparator = ".", .negativeSign = kArNegative, .zeroDigit = kArZero}) == "-1050.25"); + CHECK(normalizeLocaleNumber("5", {.zeroDigit = "\xF0\x9E\xA5\x90"}) == "5"); +} + +TEST_CASE("normalizeLocaleNumber: an entry may not mix digit families", "[render][locale][morph591]") { + // A decision, not a consequence: "\u06655" is rejected rather than read as + // 55. Neither a keyboard nor a display edge produces an interleaving, and + // rejecting it matches the existing strictness about a sign anywhere but + // the leading position. Recorded here because the alternative -- accepting + // it -- is equally implementable, so only a test says which one morph does. + CHECK(normalizeLocaleNumber("\xD9\xA5" + "5", + {.zeroDigit = kArZero}) == std::nullopt); + CHECK(normalizeLocaleNumber("5\xD9\xA5", {.zeroDigit = kArZero}) == std::nullopt); + CHECK(normalizeLocaleNumber("1\xD9\xA0\xD9\xA5" + "0", + {.zeroDigit = kArZero}) == std::nullopt); + // Each family on its own is fine, which is what makes the rejection about + // the mixing rather than about either spelling. + CHECK(normalizeLocaleNumber("\xD9\xA5\xD9\xA5", {.zeroDigit = kArZero}) == "55"); + CHECK(normalizeLocaleNumber("55", {.zeroDigit = kArZero}) == "55"); + // In an ASCII-digit locale the two families are the same set, so nothing + // can mix and the rule is invisible -- this is why it costs no existing + // caller anything. + CHECK(normalizeLocaleNumber("55", {}) == "55"); +} + +TEST_CASE("normalizeLocaleNumber: grouping is validated in the locale's digits too", "[render][locale][morph591]") { + // groupingIsWellPlaced counts digits, and used to count bytes. A two-byte + // digit would have reset the run-length counter on its own continuation + // byte, so a correctly grouped native-digit entry -- what the display edge + // now emits -- would have been rejected as badly grouped and the pair would + // not round-trip. The grouping rule itself is unchanged: a short group is + // still malformed. + morph::render::NumericLocale const arEg{ + .decimalSeparator = kArDecimal, .groupSeparator = kArGroup, .zeroDigit = kArZero}; + CHECK(normalizeLocaleNumber("\xD9\xA1\xD9\xAC\xD9\xA0\xD9\xA5\xD9\xA0", arEg) == "1050"); + CHECK(normalizeLocaleNumber("\xD9\xA1\xD9\xAC\xD9\xA5", arEg) == std::nullopt); // short group +} + +TEST_CASE("normalizeLocaleNumber: the digit base does not widen what is well-formed UTF-8", + "[render][locale][morph591]") { + // The digit test is a range test on a decoded code point, so the decoder + // has to be strict or it would widen acceptance by accident. C0 B5 is an + // overlong encoding of U+0035: read leniently it would be the digit '5' in + // the *default* locale, where the previous byte-range scan rejected it. + // Measured: removing the overlong guard changes 13 rows of the + // before/after sweep over the default locale. + CHECK(normalizeLocaleNumber("\xC0\xB5", {}) == std::nullopt); // overlong '5' + CHECK(normalizeLocaleNumber("\xED\xA0\x80", {}) == std::nullopt); // a surrogate + CHECK(normalizeLocaleNumber("\xE2\x80", {}) == std::nullopt); // truncated + CHECK(normalizeLocaleNumber("\x80", {}) == std::nullopt); // stray continuation byte +} + +TEST_CASE("locale_format: a defaulted NumericLocale reproduces the previous defaults", "[render][locale][morph591]") { + // zeroDigit defaults to "0", so every caller that does not name it is + // byte-identical to the five-positional-parameter version. Asserted rather + // than assumed, and separately measured: a 1680-case sweep (14 locale + // configurations x 60 entries x both edges) run against the pre-#591 header + // and against this one produced identical output. + CHECK(morph::render::NumericLocale{}.decimalSeparator == "."); + CHECK(morph::render::NumericLocale{}.groupSeparator.empty()); + CHECK(morph::render::NumericLocale{}.negativeSign == "-"); + CHECK(morph::render::NumericLocale{}.positiveSign == "+"); + CHECK(morph::render::NumericLocale{}.zeroDigit == "0"); + CHECK(normalizeLocaleNumber("1.050,25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1050.25"); + CHECK(formatCanonicalNumber("1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "1.050,25"); + CHECK(formatCanonicalNumber("-1050.25", {.decimalSeparator = ",", .groupSeparator = "."}) == "-1.050,25"); +} + +TEST_CASE("locale_format: an empty or malformed zeroDigit reads as ASCII '0'", "[render][locale][morph591]") { + // The same reading an empty negativeSign gets, for the same reason: there + // is no locale without digits, so empty cannot mean absence, and a base of + // "nothing" would reject every entry the locale can produce. + CHECK(normalizeLocaleNumber("55", {.zeroDigit = ""}) == "55"); + CHECK(formatCanonicalNumber("55", {.zeroDigit = ""}) == "55"); + CHECK(normalizeLocaleNumber("55", {.zeroDigit = "\xC0"}) == "55"); // not a code point + CHECK(formatCanonicalNumber("55", {.zeroDigit = "\xC0"}) == "55"); }