From 764c4e70862572bc48c050d57017c9f834c7d3c0 Mon Sep 17 00:00:00 2001 From: Sal Date: Sat, 12 Sep 2026 08:49:37 +0100 Subject: [PATCH 1/2] fix(json): parse independent of key order and decode string escapes `@str-parse-json` selected the containing object only when the wanted key was written first, and JSON objects are unordered. A document with its members reordered left the output hash unassigned, and the caller's `${(@Q)${(@z)...}}` then aborted with "bad set of key/value pairs for associative array". Select the smallest object that declares the key. String bodies were also stored raw, so `\\`, `\t`, `\n`, `\/` and `\uXXXX` reached callers as backslashes. A caller cannot repair that itself: replacing `\"` with `"` turns the JSON `"a\\\"b"`, which denotes the literal `a\"b`, into `a"b`. Add `@str-unescape-json` and decode each double-quoted body as it is captured. Surrogate pairs, unknown escapes and malformed `\u` are left as written. The two new parser tests fail against the previous behaviour. The suite had covered the parser with one assertion on `{'a':{'b':'1'}}`, which is not valid JSON and exercised neither defect. Refs z-shell/zi#513 --- docs/README.md | 48 ++++++++++++++++++++++++++-- functions/@str-parse-json | 12 ++++--- functions/@str-unescape-json | 46 +++++++++++++++++++++++++++ tests/main.zunit | 61 ++++++++++++++++++++++++++++++++++++ zsh-string-lib.lib.zsh | 1 + 5 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 functions/@str-unescape-json diff --git a/docs/README.md b/docs/README.md index f530d23..674917f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,13 @@ A string library for Zsh. Its founding function was parsing of JSON. +> [!NOTE] +> [`z-shell/zi`](https://github.com/z-shell/zi) carries a fork of +> `@str-parse-json` as `.zi-parse-json` in its own `lib/zsh/install.zsh`. It is a +> maintained fork rather than a mirror, and the two have deliberately diverged, +> so they are not expected to be byte-identical. Fixes should be carried across +> by hand in both directions. + ## List Of The Functions ### @str-parse-json @@ -83,10 +90,47 @@ Arguments: 1. The buffer with JSON. 2. The key in the JSON that should be mapped to the result (i.e.: it's possible - to map only a subset of the input). It must be the first key in the object to - map. + to map only a subset of the input). The smallest object declaring that key is + the one mapped; it does not have to be written first, because JSON objects + are unordered. 3. The name of the output hash parameter. +String escapes are decoded as they are read, so the values in the output hash +are the characters the JSON denotes, not the backslashes that spelled them. See +[`@str-unescape-json`](#str-unescape-json) for the exact coverage. + +### @str-unescape-json + +Translates the escape sequences of a JSON string body (`$1`) into the characters +they denote and returns the result in `$REPLY`. `@str-parse-json` uses it on +every double-quoted value it reads. + +Covers the eight two-character escapes of RFC 8259 (`\"` `\\` `\/` `\b` `\f` +`\n` `\r` `\t`) and `\uXXXX` in the basic multilingual plane. A surrogate +pair, an unrecognized escape such as `\q`, and a malformed `\u` are left +exactly as written, so nothing is silently discarded. + +Arguments: + +1. The string body, without its surrounding quotes. + +Example: + +```zsh +@str-unescape-json 'say \"hi\"' +print -r -- $REPLY +``` + +Output: + +```sh +say "hi" +``` + +Note that a naive replacement of `\"` with `"` is not equivalent: the JSON +`"a\\\"b"` denotes the literal `a\"b`, which such a replacement corrupts +into `a"b`. + ### @str-read-all Consumes whole data from given file descriptor and stores the string under the diff --git a/functions/@str-parse-json b/functions/@str-parse-json index aff81ce..353aab3 100644 --- a/functions/@str-parse-json +++ b/functions/@str-parse-json @@ -112,7 +112,10 @@ [[ ${match[1]} = \" && $__quoting != \' ]] && \ if [[ $__quoting = '"' ]]; then - __Strings[$__level/${__Counts[$__level]}]+=" ${(q)__input[__sidx,__idx-1]}" + # A JSON string body carries escapes. Store what they denote, so callers + # receive the actual value instead of having to re-implement a decoder. + @str-unescape-json "${__input[__sidx,__idx-1]}" + __Strings[$__level/${__Counts[$__level]}]+=" ${(q)REPLY}" __quoting="" else __had_quoted_value=1 @@ -153,10 +156,11 @@ for __pair_a ( "${__pair_order[@]}" ) { __pair_b="${__final_pairs[$__pair_a]}" __text="${__input[__pair_b,__pair_a]}" - if [[ $__text = [[:space:]]#\{[[:space:]]#[\"\']${__key}[\"\']* ]]; then + # JSON objects are unordered, so the wanted object is the smallest one that + # declares the key, not merely one that happens to open with it. + if [[ $__text = [[:space:]]#\{*[\"\']${__key}[\"\'][[:space:]]#:* ]]; then if (( __nest != 2 )) { - __found="$__text" - break + [[ -z $__found || $#__text -lt $#__found ]] && __found="$__text" } fi } diff --git a/functions/@str-unescape-json b/functions/@str-unescape-json new file mode 100644 index 0000000..8dae014 --- /dev/null +++ b/functions/@str-unescape-json @@ -0,0 +1,46 @@ +# Copyright (c) 2018 Sebastian Gniazdowski +# Copyright (c) 2021 Salvydas Lukosius +# +# @str-unescape-json +# +# Translates the escape sequences of a JSON string body ($1) into the +# characters they denote and returns the result in $REPLY. +# +# Handles the eight two-character escapes defined by RFC 8259 +# (\" \\ \/ \b \f \n \r \t) and \uXXXX in the basic multilingual plane. +# A surrogate pair, an unrecognized escape such as \q, and a malformed +# \u are left exactly as written, so nothing is silently discarded. +# +# $1 - the string body, without its surrounding quotes +# +# Return value: $REPLY - the decoded string +# +# Example: +# +# @str-unescape-json 'say \"hi\"' +# print -r -- $REPLY +# +# Output: say "hi" + +@str-unescape-json() { + emulate -LR zsh -o extendedglob -o warncreateglobal -o typesetsilent + + local __rest=$1 __out= __esc __tail + local -a match mbegin mend + local -A __map=( \" \" \\ \\ / / b $'\b' f $'\f' n $'\n' r $'\r' t $'\t' ) + integer __code + + while [[ $__rest = (#b)([^\\]#)\\(?)(*) ]]; do + __out+=$match[1] __esc=$match[2] __tail=$match[3] + if [[ $__esc == u && $__tail == (#b)([0-9a-fA-F](#c4))(*) ]] { + __code=16#$match[1] + __out+=${(#)__code} __rest=$match[2] + } elif (( ${+__map[$__esc]} )) { + __out+=$__map[$__esc] __rest=$__tail + } else { + __out+="\\$__esc" __rest=$__tail + } + done + + typeset -g REPLY=$__out$__rest +} diff --git a/tests/main.zunit b/tests/main.zunit index e12b04d..1506a4b 100644 --- a/tests/main.zunit +++ b/tests/main.zunit @@ -13,6 +13,67 @@ assert "$Strings[2/1]" same_as " b 1" } +@test 'JSON parser is independent of key order' { + + # JSON objects are unordered. The same document written the other way round + # must resolve to the same values. + local canonical='{"d":{"info":{"user":"u"},"ices":{"default":{"as":"program"}}}}' + local reordered='{"d":{"ices":{"default":{"as":"program"}},"info":{"user":"u"}}}' + + local -A A B + @str-parse-json "$canonical" info A + @str-parse-json "$reordered" info B + + local -a la=( "${(@Q)${(@z)A[1/1]}}" ) lb=( "${(@Q)${(@z)B[1/1]}}" ) + integer ia=${la[(I)ices]} ib=${lb[(I)ices]} + + assert "$ia" is_positive + assert "$ib" is_positive + + local -A ia_ices=( "${(@Q)${(@z)A[3/1]}}" ) ib_ices=( "${(@Q)${(@z)B[3/1]}}" ) + assert "$ia_ices[as]" same_as "program" + assert "$ib_ices[as]" same_as "$ia_ices[as]" +} + +@test 'JSON parser decodes string escapes' { + + # Every backslash is built at run time; a literal one in this file is at the + # mercy of whatever quoting layer reads it. + local b=$'\134' tab=$'\t' nl=$'\n' + local json='{"d":{"k":{"esc":"a'$b$b'b","quo":"say '$b'"hi'$b'"","tab":"p'$b'tq","uni":"x'$b'u0041y","slash":"a'$b'/b","keep":"a'$b'qb","tricky":"a'$b$b$b'"b"}}}' + + local -A S + @str-parse-json "$json" k S + local -A v=( "${(@Q)${(@z)S[2/1]}}" ) + + assert "$v[esc]" same_as "a${b}b" + assert "$v[quo]" same_as 'say "hi"' + assert "$v[tab]" same_as "p${tab}q" + assert "$v[uni]" same_as "xAy" + assert "$v[slash]" same_as "a/b" + # An unknown escape is not JSON; preserve it rather than guess. + assert "$v[keep]" same_as "a${b}qb" + # JSON "a\\\"b" denotes the literal a\"b, which a naive \" replacement breaks. + assert "$v[tricky]" same_as "a${b}\"b" +} + +@test 'unescape-json' { + + local b=$'\134' + @str-unescape-json "x${b}u0041y" + assert "$REPLY" same_as "xAy" + + @str-unescape-json "trail${b}${b}" + assert "$REPLY" same_as "trail${b}" + + # Malformed \u is left as written. + @str-unescape-json "${b}u12" + assert "$REPLY" same_as "${b}u12" + + @str-unescape-json "plain" + assert "$REPLY" same_as "plain" +} + @test 'read-all' { integer FD=13371337 i diff --git a/zsh-string-lib.lib.zsh b/zsh-string-lib.lib.zsh index 637a122..a45f51c 100644 --- a/zsh-string-lib.lib.zsh +++ b/zsh-string-lib.lib.zsh @@ -15,6 +15,7 @@ zmodload zsh/system 2>/dev/null # https://wiki.zshell.dev/community/zsh_plugin_standard#the-proposed-function-name-prefixes autoload -Uz \ @str-parse-json \ + @str-unescape-json \ @str-read-all \ @str-ng-match \ @str-ng-matches \ From 40ae7201b2aef3e8e2c80e13424241d9fd8aea1e Mon Sep 17 00:00:00 2001 From: Sal Date: Sat, 12 Sep 2026 09:03:01 +0100 Subject: [PATCH 2/2] docs(readme): use a plain note for the fork callout prettier 2.8.3, pinned by .trunk/trunk.yaml, predates GitHub alert syntax and folds the GitHub alert marker onto the following line, which stops the callout rendering at all. Write it as a bold lead-in instead of fighting the formatter. --- docs/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/README.md b/docs/README.md index 674917f..f8c2d27 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,12 +4,11 @@ A string library for Zsh. Its founding function was parsing of JSON. -> [!NOTE] -> [`z-shell/zi`](https://github.com/z-shell/zi) carries a fork of -> `@str-parse-json` as `.zi-parse-json` in its own `lib/zsh/install.zsh`. It is a -> maintained fork rather than a mirror, and the two have deliberately diverged, -> so they are not expected to be byte-identical. Fixes should be carried across -> by hand in both directions. +**Note on the fork.** [`z-shell/zi`](https://github.com/z-shell/zi) carries a +fork of `@str-parse-json` as `.zi-parse-json` in its own `lib/zsh/install.zsh`. +It is a maintained fork rather than a mirror, and the two have deliberately +diverged, so they are not expected to be byte-identical. Fixes should be carried +across by hand in both directions. ## List Of The Functions