From 5ec74b91ddcea6743e3d408d8696a8e737bb975f Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 15:42:46 -0400 Subject: [PATCH 01/24] =?UTF-8?q?v2=20spec:=20spec=20batch=202,=20ABI=20v2?= =?UTF-8?q?=20fingerprint=20d5323993=E2=80=A641c2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maintainers' decided cell for the second v2 spec batch, with body_read added to every refused_at enum (the vocabulary already had it): - input JSON is strict everywhere; - the export stays JSONCompactEachRow with one export_columns list, and a body mixing deferred and server-filled defaults declines with defer-defaults-mixed-body; - column_defaults: where the server would fill a column from its DEFAULT, the caller's value is used; - records, graphite_rollup, identity and session are absent; - r8 documents the known-crash list (known-crash-declined at create); - errors carry a reason (chs_error_reason; sdk.json errors.fields). spec/abi-v2/abi.json, its schema, docs.md, sdk.json and the generator are the inputs; every generated file is the generator's output: gen.py --write leaves no diff, --check passes for both majors, and gen.py --major 2 --fingerprint prints sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2. The bindings' hand-written surface for the new members follows in this branch before it lands. Co-Authored-By: Claude Opus 5.5 --- docs/reference/abi-v2.md | 812 ++++++++---- go/chtypes/vocab_gen.go | 109 +- go/internal/abi2/abi2_table.h | 14 +- go/internal/abi2/abi_gen.go | 48 +- go/internal/abi2/errmap_gen.go | 2 +- go/internal/abi2/invoke_gen_test.go | 73 +- go/internal/abi2/linked_gen.go | 3 +- go/internal/ocifetch/abi_fingerprint_gen.go | 4 +- include/v2/chtypes.h | 807 +++++++++--- python/src/chtypes/_abi2/_decls.py | 24 +- python/src/chtypes/_abi2/_errmap.py | 2 +- python/src/chtypes/_abi2/_vocab.py | 92 +- rust/src/abi2/calls_gen.rs | 20 +- rust/src/abi2/decls.rs | 21 +- rust/src/abi2/errmap_gen.rs | 2 +- rust/src/abi2/invoke_gen.rs | 20 +- rust/src/abi2/vocab_gen.rs | 201 ++- rust/src/ocifetch/abi_fingerprint.rs | 4 +- scripts/abi-v1/emit/_stubshared.py | 28 +- scripts/abi-v1/emit/cases.py | 75 +- scripts/abi-v1/emit/go_vocab.py | 65 +- scripts/abi-v1/emit/header.py | 11 +- scripts/abi-v1/emit/python.py | 52 +- scripts/abi-v1/emit/reference.py | 17 +- scripts/abi-v1/emit/rust.py | 73 +- scripts/abi-v1/emit/setup_cases.py | 15 +- scripts/abi-v1/emit/stub.py | 34 +- scripts/abi-v1/gen.py | 101 ++ scripts/abi-v1/model.py | 153 ++- spec/abi-v2/abi.json | 882 +++++++++++++- spec/abi-v2/docs.md | 176 ++- spec/abi-v2/generated/exports.txt | 3 +- spec/abi-v2/schema/abi.schema.json | 5 +- spec/abi-v2/sdk.json | 2 +- spec/abi-v2/where-settings/README.md | 8 +- tests/fixtures/abi-v2/cases.json | 1220 ++++++++++++------- tests/fixtures/abi-v2/setup-cases.json | 15 +- ts/src/abi2/calls.gen.ts | 22 +- ts/src/abi2/decls.gen.ts | 99 +- ts/src/abi2/errmap.gen.ts | 2 +- ts/src/abi2/libc.gen.ts | 2 +- ts/src/abi2/vocab.gen.ts | 102 +- 42 files changed, 4306 insertions(+), 1114 deletions(-) diff --git a/docs/reference/abi-v2.md b/docs/reference/abi-v2.md index f15cda09..17bd90bf 100644 --- a/docs/reference/abi-v2.md +++ b/docs/reference/abi-v2.md @@ -37,17 +37,17 @@ The v2 header has its own include guard and refuses, with `#error`, to follow AB ## Reference - + ### Identity | item | value | | --------------------- | ----------------------------------------------------------------------------------- | | `CHS_ABI_VERSION` | 2 | -| `CHS_ABI_FINGERPRINT` | `sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4` | +| `CHS_ABI_FINGERPRINT` | `sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2` | | library | `libchtypes`, exporting only `chs_*` | -| functions | 40: 33 api, 3 handshake, 1 tombstone, 3 tooling | -| FIRM functions | 40 | +| functions | 41: 34 api, 3 handshake, 1 tombstone, 3 tooling | +| FIRM functions | 41 | | provisional functions | 0 | | `reuse_v0_names` | true | | stability | UNSTABLE: the fingerprint moves with every change to the description until the lock | @@ -63,12 +63,12 @@ These rules bind generation 2 from its first draft: the library, every binding a **(r3) Every enum has an `unknown(n)` member.** Every enum the description defines, an `int32` enum and a string vocabulary alike (`chs_status`, `chs_format`, `value_src`, `default_kind`, the three outcomes, `filter_verdict`, `transform_reason`, every one), MUST have in every binding an explicit member `unknown(n)` carrying the raw value `n`: the integer of an `int32` enum, the exact string of a vocabulary. A reader MUST map a value its description does not list to that member, for that field alone, and go on decoding. An unlisted value never fails the document, the row or the batch that carries it. Checked by `gen.py`: no described value is spelled `unknown`, `CHS_UNKNOWN` or `CHS__UNKNOWN`, the name readers reserve for that member. - **A `fallback` changes meaning.** It no longer names the value an unlisted one is read as: the reader keeps `unknown(n)`. It names whose per-value facts `unknown(n)` reports (`lossy` for `transform_reason`, `answered` for `filter_verdict`) and how a question about the value is answered, so the fail-closed reading stays: an unknown outcome is never accepted, and an unknown verdict is never answered. -- **Facts without a fallback.** An enum whose values carry facts and which names no `fallback` MUST say what `unknown(n)` reports before the description is locked. Today that is `value_src`, whose `is_stored` has no answer for an unknown source. +- **Facts need a fallback.** An enum whose values carry facts MUST name a `fallback`, so that what `unknown(n)` reports is decided before the description is locked. `value_src` names `skipped`: an unknown source reports `is_stored` false, which is how every dev binding already read it. - **A call status.** A status outside the closed set reads as `unknown(n)` too. The call still fails, as an internal error naming `n` (the `unknown` entry of `errors.status` in `spec/abi-v2/sdk.json`), because a status a binding cannot read is never a success. - **Schemas constrain the writer.** An `enum` inside a document schema (`default_kind` in `schema_description`, `framing.container` in `batch`) constrains what the library writes, never what a reader accepts. - **Why.** Measured on the released 1.0.4 bindings, all four on 12 CI legs ([`v1-abi` run 37506595472](https://github.com/Wave-RF/chtypes/actions/runs/37506595472), the conformance legs of public pull request #509): a reader survives an unlisted value only where its vocabulary has a fallback (an outcome reads as `unsupported`, a verdict as `d`, a transform reason stays raw and reads as lossy), and an unknown call status is an internal error. An unlisted `value_src` fails the WHOLE document with an internal error, so one row's unknown source loses the entire batch; an unlisted `default_kind` fails `schema_description` the same way. -**(r4) Error codes are stable once published.** An error code a caller can match on keeps its name and its number from the first release that carries it, a `2.0.0-dev.N` pre-release included, in every later release of every generation, and is never reused for another meaning. That covers the `chs_status` values, the `CHTYPES_*` codes (`errors.codes` and `errors.fetch_codes` in `sdk.json`, and the fetch layer's table in `docs/guides/fetch-v1.md` §8) and the exit status each code maps to. A new failure gets a new code, with an exit status no published code has. Checked by `gen.py`: every `chs_status` value generation 1 published keeps its name and number, and every `errors.codes` entry of `spec/abi-v1/sdk.json` is kept and none is reused under another key. The fetch layer's table is checked against `spec/fetch-v1/constants.json`, the file §8 is generated from: `errors.fetch_codes` lists every code in it with the same exit status and no code it lacks, every `errors.codes` entry names a listed code, and every code in `spec/abi-v2/published-fetch-codes.json` (what releases have published) is kept under its exit status in both files, and a new code takes a status none of them has. The baseline is a committed file the check reads, never git history. +**(r4) Error codes are stable once published.** An error code a caller can match on keeps its name and its number from the first release that carries it, a `2.0.0-dev.N` pre-release included, in every later release of every generation, and is never reused for another meaning. That covers the `chs_status` values, the `CHTYPES_*` codes (`errors.codes` and `errors.fetch_codes` in `sdk.json`, and the fetch layer's table in `docs/guides/fetch-v1.md` §8) and the exit status each code maps to. A new failure gets a new code, with an exit status no published code has. Checked by `gen.py`: every `chs_status` value generation 1 published keeps its name and number, and every `errors.codes` entry of `spec/abi-v1/sdk.json` is kept and none is reused under another key. The fetch layer's table is checked against `spec/fetch-v1/constants.json`, the file §8 is generated from: `errors.fetch_codes` lists every code in it with the same exit status and no code it lacks, every `errors.codes` entry names a listed code, and every code in `spec/abi-v2/published-fetch-codes.json` (what releases have published) is kept under its exit status in both files, and a new code takes a status none of them has. The baseline is a committed file the check reads, never git history. The same holds for the decline catalog's reason ids (`spec/abi-v2/declines/declines.json`'s `id`, which `chs_error_reason` and every document's `reason` and `verdict_reason` carry) and for the sentinel `CHS_CODE_DECLINED`: an id keeps its meaning from the first release that carries it, a retired entry stays in the catalog under its id with a `retirement` naming its successors, and an id is never reused for another meaning. A new cause gets a new id. `scripts/abi-v2/check-declines.py` is where the check belongs, against the published catalog. **(r5) The cache.** A generation-2 binding's cache MUST be one no released 1.x reader ever reads. The normative fetch specification for generation 2 is [`docs/guides/fetch-v1.md` §"Generation 2 after the lock"](../../docs/guides/fetch-v1.md#generation-2-after-the-lock), and r5 and r6 point at it rather than restate it. @@ -96,6 +96,10 @@ These rules bind generation 2 from its first draft: the library, every binding a Checked by two shared conformance cases (`tests/fixtures/abi-v2/setup-cases.json`, generated by `scripts/abi-v1/emit/setup_cases.py`), which every binding that can express them runs through its public API over the stub library: `server.a_closed_server_is_never_passed_as_null` and `filter.a_closed_filter_is_never_passed_as_null`. Each compiles or previews with an open handle (the stub sees a non-NULL handle), with no handle (the stub sees NULL), and with a closed handle (the binding's usage error, where a NULL passed in its place would have succeeded). The server case reads the stub's live-handle counts; a batch call mints no handle, so the filter case runs on the stub variant `filter-observable`, which answers `rows_passed` 1 when a filter reached it and 0 when it received NULL. Rust declares both cases not expressible, with the reason, in the case itself. `gen.py` checks only that this rule is stated (`GENERATION_RULES`), since which parameters give NULL a meaning is prose, not structure. +**(r8) Crashes.** The library runs ClickHouse's own code on the caller's input, unmodified. A crash inside that code is a defect of security severity, like an over-accept, and is fixed or declined with a catalog entry in a later build; it is never documented as acceptable. Until then this generation does **not** promise that no input can terminate the process. What it promises is the following. (1) **The known-crash list.** Every build's release notes carry the producer's known-crash list: each entry names the line, the entry point, the input shape and the issue, and the producer's release gate counts those entries and fails on any crash not on the list. A list that is not empty is stated in the notes of every build that ships the line. (2) **The breaker.** A compile-class call (`chs_schema_create`, `chs_filter_create`, `chs_server_create`, `chs_set_defaults`, `chs_type_validate`, `chs_reference_type`, `chs_discover_columns`) records its input before it runs and clears the record after; a process that dies during the call leaves the record, and the next process of the same build on the same host declines that exact input with `CHS_DECLINED` and the reason id `vendored-crash-previously`, so a crashing input terminates a process at most once per host where the record can be written. Where it cannot (a read-only cache, no writable directory), the breaker is off and the library says so once at `chs_initialize`; `chs_build_info.capabilities.features` lists `crash_breaker`. An input whose shape is on the build's own known-crash list never reaches the code that crashes: `chs_schema_create` and `chs_filter_create` decline it at create, `CHS_DECLINED` with the reason id `known-crash-declined` and a message naming the list's rule id, the line, the clause and what to change. (3) **No wrong answer.** A breaker decline, like every decline, is never an accept and never a server's own refusal; an input the breaker does not know is answered exactly as without it. (4) **The exceptions** r8 does not cover in any generation: misuse the ABI forbids, exhaustion of the process's memory outside the admission budgets, and signals from outside the process. The full guarantee, no input terminates the process through any `chs_*` call, is generation 3's. `gen.py` checks only that this rule is stated. + +**(r9) The description evolves by reserve and decline.** After the lock a new input member, document member or enum value moves the fingerprint, so a feature that may follow the lock is reserved in the description before it: its input member is defined and a build that does not implement it declines it at any value but its default (Input documents, in the Preamble), and its output members are defined and absent until a build writes them. Implementing a reserved member is a behavior change, never a fingerprint change, and a build advertises each implemented reserved member in `chs_build_info`'s `capabilities.features`, spelled as the member's path (`insert_options.defer_defaults`). A reader treats an absent reserved output member as not reported. `gen.py` checks only that this rule is stated. + At the lock (public issue #511 says when), the description's `stability` becomes `locked`. That moves the fingerprint one last time, so a released 2.0.0 SDK never accepts a dev build and a dev SDK never accepts a 2.0.0 library. SDK 2.0.0 then ships from the production `chtypes/v2` repository, signed with the release key. ### Provisional markers @@ -112,6 +116,13 @@ None: every entry in the description is FIRM. | `process_once` | process setup, once per image: the first call sets process state, and a repeat with the same arguments is a no-op that is safe at any time | | `process_serial` | process-level: must not overlap any other call into the same library image | +### Init classes + +| class | when a call is valid | +| ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `any` | valid before and after `chs_initialize` | +| `required` | valid only once `chs_initialize` has succeeded: until then the call changes nothing and returns `CHS_INVALID_ARGUMENT`, ch_code 0, and with arguments that are otherwise valid its message starts `: call chs_initialize first` | + ### Parameter kinds | kind | C spelling | rule | @@ -128,18 +139,18 @@ None: every entry in the description is FIRM. Every counted input and every `chs_buf` is a byte string; the content says what it carries. -| content | carries | -| --------------------------- | ------------------------------------------------------------------------------------------- | -| `bytes` | arbitrary bytes: a body, a string value, an export | -| `name` | a ClickHouse column name: NUL and invalid UTF-8 are legal, so never assume UTF-8 | -| `sql` | SQL text (a statement, an expression, a type, a quoted name or literal): may carry any byte | -| `message` | a ClickHouse message: may quote input bytes, so may carry any byte | -| `ascii` | guaranteed ASCII | -| `json_object_string_values` | a JSON object whose values are JSON strings (settings, query parameters) | -| `json_array_names` | a JSON array of column names, each an object carrying the name as byte_strings says | -| `timezone` | a time zone name, validated by ClickHouse's own DateLUT; length 0 means UTC | -| `document:` | a JSON document, listed under Documents below | -| `input:` | a JSON object the library validates, listed under Input documents below | +| content | carries | +| ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `bytes` | arbitrary bytes: a body, a string value, an export | +| `name` | a ClickHouse column name: NUL and invalid UTF-8 are legal, so never assume UTF-8 | +| `sql` | SQL text (a statement, an expression, a type, a quoted name or literal): may carry any byte | +| `message` | a ClickHouse message: may quote input bytes, so may carry any byte | +| `ascii` | guaranteed ASCII | +| `json_object_byte_values` | a JSON object whose values are JSON strings, or the object `{"b64": ""}` for a value whose bytes are not valid UTF-8 (settings, query parameters) | +| `json_array_names` | a JSON array of column names, each an object carrying the name as byte_strings says | +| `timezone` | a time zone name, validated by ClickHouse's own DateLUT; length 0 means UTC | +| `document:` | a JSON document, listed under Documents below | +| `input:` | a JSON object the library validates, listed under Input documents below | ### Byte strings in JSON @@ -147,15 +158,15 @@ Every data-derived string a JSON document carries (a column name, a value's rend The data-derived fields and the entries carrying `value_b64`, by document (a path joins members with `.`, and `[]` steps into an array): -| document | data-derived fields | entries with `value_b64` | -| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------- | -| `live_handles` | - | - | -| `error_code_table` | - | - | -| `schema_description` | `columns[].name`, `columns[].type`, `columns[].default_expression`, `filter_declined_settings[].name`, `replicated.zookeeper_path`, `replicated.replica_name` | - | -| `row` | `err`, `verdict_err`, `partition_id`, `unknown_fields[].name`, `unsupported_settings[].name`, `cols[].name`, `cols[].type`, `cols[].base`, `cols[].input`, `cols[].stored`, `cols[].ref`, `cols[].ref_type`, `cols[].wire`, `computed[].name`, `computed[].stored`, `transformed[].column`, `transformed[].input`, `transformed[].stored` | `cols[]`, `computed[]` | -| `batch` | `err`, `export_declined`, `rows[].err`, `rows[].verdict_err`, `rows[].partition_id`, `rows[].unknown_fields[].name`, `rows[].unsupported_settings[].name`, `rows[].cols[].name`, `rows[].cols[].type`, `rows[].cols[].base`, `rows[].cols[].input`, `rows[].cols[].stored`, `rows[].cols[].ref`, `rows[].cols[].ref_type`, `rows[].cols[].wire`, `rows[].computed[].name`, `rows[].computed[].stored`, `rows[].transformed[].column`, `rows[].transformed[].input`, `rows[].transformed[].stored`, `engine_rows[][].name`, `engine_rows[][].stored`, `storage_transforms[].column`, `storage_transforms[].stored`, `at_merge[].column`, `at_merge[].stored`, `transformed[].column`, `transformed[].input`, `transformed[].stored`, `unsupported_settings[].name`, `framing.header.names[].name` | `rows[].cols[]`, `rows[].computed[]`, `engine_rows[][]`, `storage_transforms[]` | -| `filter_result` | `err`, `unsupported_settings[].name`, `errors[].err` | - | -| `discovery` | `columns[].name`, `columns[].type`, `columns[].default_kind`, `columns[].default_expression`, `columns[].declaration`, `columns_sql` | - | +| document | data-derived fields | entries with `value_b64` | +| -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| `live_handles` | - | - | +| `error_code_table` | - | - | +| `schema_description` | `columns[].name`, `columns[].type`, `columns[].default_expression`, `columns[].reads[].name`, `filter_declined_settings[].name`, `replicated.zookeeper_path`, `replicated.replica_name` | - | +| `row` | `err`, `verdict_err`, `partition_id`, `unknown_fields[].name`, `cols[].name`, `cols[].type`, `cols[].base`, `cols[].input`, `cols[].stored`, `cols[].ref`, `cols[].ref_type`, `cols[].wire`, `computed[].name`, `computed[].stored`, `transformed[].column`, `transformed[].input`, `transformed[].stored` | `cols[]`, `computed[]` | +| `batch` | `err`, `export_declined`, `rows[].err`, `rows[].verdict_err`, `rows[].partition_id`, `rows[].unknown_fields[].name`, `rows[].cols[].name`, `rows[].cols[].type`, `rows[].cols[].base`, `rows[].cols[].input`, `rows[].cols[].stored`, `rows[].cols[].ref`, `rows[].cols[].ref_type`, `rows[].cols[].wire`, `rows[].computed[].name`, `rows[].computed[].stored`, `rows[].transformed[].column`, `rows[].transformed[].input`, `rows[].transformed[].stored`, `engine_rows[][].name`, `engine_rows[][].stored`, `storage_transforms[].column`, `storage_transforms[].stored`, `at_merge[].column`, `at_merge[].stored`, `transformed[].column`, `transformed[].input`, `transformed[].stored`, `unsupported_settings[].name`, `export_columns[].name`, `framing.header.names[].name` | `rows[].cols[]`, `rows[].computed[]`, `engine_rows[][]`, `storage_transforms[]` | +| `filter_result` | `err`, `unsupported_settings[].name`, `errors[].err` | - | +| `discovery` | `columns[].name`, `columns[].type`, `columns[].default_kind`, `columns[].default_expression`, `columns[].declaration`, `columns_sql` | - | ### Handles @@ -199,8 +210,8 @@ A body parsed once under a schema, for evaluating many filters without parsing a The verdict of every fallible call, in five values whose numbering is frozen. - `CHS_OK`: the call succeeded and its outputs are set. -- `CHS_REJECTED`: ClickHouse's own refusal, with its own code, name and message, which a server would also give. `CHS_REJECTED` with ch_code 306, TOO_DEEP_RECURSION, never `CHS_INTERNAL`, is returned for a deep argument the call compiles (an expression, a type, or a settings, columns or options JSON), at `chs_set_defaults`, `chs_server_create`, `chs_schema_create`, `chs_schema_describe`, `chs_preview_row`, `chs_preview_batch`, `chs_filter_create`, `chs_filter_eval_body`, `chs_block_create`, `chs_filter_eval_block`, `chs_type_validate`, `chs_reference_type` and `chs_discover_columns`. A body's depth refusal is reported in its document (`CHS_OK`, with code 306), as every refused body is. -- `CHS_DECLINED`: this build will not answer; a server might accept. Never scored as agreement. +- `CHS_REJECTED`: ClickHouse's own refusal, with its own code, name and message, which a server would also give. `CHS_REJECTED` with ch_code 306, TOO_DEEP_RECURSION, never `CHS_INTERNAL`, is returned for a deep argument the call compiles (an expression, a type, or a settings, columns or options JSON), at `chs_set_defaults`, `chs_server_create`, `chs_schema_create`, `chs_filter_create`, `chs_type_validate`, `chs_reference_type` and `chs_discover_columns`. A call that answers with a document (`chs_preview_row`, `chs_preview_batch`, `chs_filter_eval_body`, `chs_block_create` and `chs_filter_eval_block`) returns a status other than `CHS_OK` only for caller misuse (`CHS_INVALID_ARGUMENT`) or a library bug (`CHS_INTERNAL`): every refusal or decline of the call itself, of its settings, its column list or its options, a deep argument (306) included, is reported in its document with `refused_at` `call`, and a body's refusal with `refused_at` `row` or `body_step`, as every refused body is. `chs_schema_describe` never refuses a schema `chs_schema_create` returned: the statement's depth is judged at compile. +- `CHS_DECLINED`: this build will not answer; a server might accept. Never scored as agreement. The error names the reason (`chs_error_reason`). A call that answers with a document never returns it: its decline is the document's `unsupported` with `refused_at` `call`. - `CHS_INVALID_ARGUMENT`: caller misuse, such as a NULL pointer with a nonzero length, a wrong-kind, freed or cross-library handle, or a NULL required out-parameter. - `CHS_INTERNAL`: a guarded exception inside the library, which is a library bug. @@ -285,11 +296,15 @@ FIRM. #### `value_src` -Where a column's value came from. `is_stored` says whether the row as stored carries a value for the column: a column the input named but the table never stores (an EPHEMERAL column, a skipped MATERIALIZED or ALIAS column) and a DEFAULT the library could not resolve carry none. +Where a column's value came from. `is_stored` says whether the row as stored carries a value for the column: a column the input named but the table never stores (an EPHEMERAL column, a skipped MATERIALIZED or ALIAS column) and a DEFAULT the library could not resolve carry none. A source this description does not list reports its fallback's facts, `skipped`'s: `is_stored` false (rule r3). `default_generated` marks a DEFAULT column whose expression calls a random or ID generator the build admits; a build that does this lists `default_generators` in `chs_build_info`'s `capabilities.features`. The label means: generated by chtypes, and it becomes the stored value when you insert this output. The library drew the value itself, with ClickHouse's own vendored function. So insert the library's output (the export of `chs_preview_batch`, or the stored values its documents report), not your original input: a server evaluates a DEFAULT only for a column the INSERT does not supply, and the original input, which omits the column, would make the server draw a different value. Generators are admitted only in DEFAULT expressions: a MATERIALIZED column is always computed by the server, and an EPHEMERAL column is not stored. No binding logic is involved; a binding reports the source as it reads it. -FIRM. +`default_override` marks a column whose value came from `insert_options`' `column_defaults`: the caller's value in a cell the server would have filled from the column's DEFAULT. It is stored, and the export carries it as a present value. A build that does not implement `column_defaults` never writes it. + +`default_deferred` marks a DEFAULT column the call deferred to the server (`insert_options`' `defer_defaults`): the library computed no value, the export omits the column from every line and from the batch document's `export_columns` (a body whose rows do not all defer it is declined: `defer_defaults`), and the server fills it on INSERT. A build that does not implement `defer_defaults` never writes it. + +FIRM; an unrecognized value is read as `skipped`. | value | is_stored | | ----------------------------- | --------- | @@ -304,6 +319,8 @@ FIRM. | `default_expr_unsupported` | false | | `default_volatile_unresolved` | false | | `default_pending` | false | +| `default_deferred` | false | +| `default_override` | true | #### `row_outcome` @@ -333,7 +350,7 @@ The batch answers the first of these that applies: A skipped row never changes the verdict. Steps 2 and 3 depart from "rejected if any row is refused" for one reason: never a refusal that is not definite. Both answer `unsupported`, which a caller never scores as agreement. -A filter's verdicts follow the batch. When the batch's outcome is not `accepted`, by any of steps 1 to 4, every row's filter verdict is `d`. A row the parse accepted carries `verdict_code` -2 and `verdict_err` `batch outcome is '': only a fully accepted batch has filter verdicts`; a refused row carries its own code and message. Nothing from such a batch is stored for a WHERE to read, exported or counted, so no verdict of it is an answer. +A filter's verdicts follow the batch. When the batch's outcome is not `accepted`, by any of steps 1 to 4, every row's filter verdict is `d`. A row the parse accepted carries `verdict_code` -2, its `verdict_reason`, and `verdict_err` `batch outcome is '': only a fully accepted batch has filter verdicts`; a refused row carries its own code and message. Nothing from such a batch is stored for a WHERE to read, exported or counted, so no verdict of it is an answer. FIRM; an unrecognized value is read as `unsupported`. @@ -346,9 +363,9 @@ FIRM; an unrecognized value is read as `unsupported`. #### `filter_outcome` -Whether a filter evaluation answers its body. `ok`: every row carries its own verdict, and per-row failures are verdicts, not outcomes. Any non-`ok` outcome, including `unsupported`, means: treat every verdict as `d`. A caller MUST do so whatever the verdict string holds. +Whether a filter evaluation answers its body. `ok`: every row carries its own verdict, and per-row failures are verdicts, not outcomes. Under any non-`ok` outcome, including `unsupported`, the library writes `d` for every verdict, on `chs_filter_eval_body` and `chs_filter_eval_block` alike, so the rule holds in the data; a caller MUST still treat every verdict as `d` whatever the verdict string holds. The document's `refused_at`, `refused_row` and `reason` say where the refusal sits and why. -`chs_filter_eval_body` and `chs_filter_eval_block` answer `ok` only for a body a server would accept, that is, when `chs_preview_batch`'s verdict over the same body and settings, with no filter, is `accepted`. Otherwise the outcome mirrors that verdict: `rejected` with its code and message; `unsupported` with its code and message, a call-level decline included; and `unsupported` with code -2 for `accepted_poisoned`, which this vocabulary does not hold and which is no refusal of the INSERT. A body the parse itself refuses answers `rejected` or `unsupported` with no verdicts. +`chs_filter_eval_body` and `chs_filter_eval_block` answer `ok` only for a body a server would accept, that is, when `chs_preview_batch`'s verdict over the same body, settings and column list, with no filter, is `accepted`. Otherwise the outcome mirrors that verdict: `rejected` with its code and message; `unsupported` with its code and message, a call-level decline included; and `unsupported` with code -2 for `accepted_poisoned`, which this vocabulary does not hold and which is no refusal of the INSERT. A body the parse itself refuses answers `rejected` or `unsupported`, with a `d` for each record read (`rows_read`, 0 when nothing was read). FIRM; an unrecognized value is read as `unsupported`. @@ -393,6 +410,19 @@ FIRM; an unrecognized value is read as `d`. | `e` | false | | `d` | false | +#### `refusal_site` + +Where a refusal or decline sits, as `refused_at` reports it on a batch document, a filter result and `chs_preview_row`'s row whenever the outcome is `rejected` or `unsupported` (Conditional members, in the Preamble): `call`, the call itself (its settings, its column list, its options, or a decline of the call such as a zone mismatch), before a byte of the body was read; `body_read`, the body outside any row and before the whole-body steps, so no record is to blame: a header the format reads first (a `WithNames` or `WithNamesAndTypes` header line, a header field naming a column the INSERT does not list under `input_format_skip_unknown_fields` 0), the framing (a JSON array's brackets, a separator or trailing content after the last record) or a byte-order mark the reader refuses; `row`, one row of the body, whose index is `refused_row`; `body_step`, a whole-body step over rows the parse accepted (the partition split, the writer's index expressions, the engine merge, the TTL, the projections, a CHECK over a merged block). `refused_row` is present exactly for `row`. A site added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + +FIRM. + +| value | +| ----------- | +| `call` | +| `body_read` | +| `row` | +| `body_step` | + #### `discover_query_param` The query parameters in the SQL `chs_discover_query` returns, each with its ClickHouse type. The caller binds them when it runs the query (over HTTP, as `param_database` and `param_table`), so no binding builds SQL. @@ -418,9 +448,22 @@ FIRM. | `ALIAS` | | `EPHEMERAL` | +#### `default_support` + +What this build does with a column's DEFAULT, as `schema_description` reports it per column (reserved: a build that does not report it writes no `default_support`): `computed`, the library evaluates the expression itself (a literal, a deterministic expression of the row, an admitted clock read); `generated`, the library draws the value with an admitted generator (`default_generated`); `deferred`, the library cannot compute it and `insert_options`' `defer_defaults` `server` would defer it; `declined`, the library cannot compute it, so a body that omits the column is declined, with the catalog id in `default_support_reason`. The honor set behind it is published per line as data beside `where-settings/.json`. A value added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + +FIRM. + +| value | +| ----------- | +| `computed` | +| `generated` | +| `deferred` | +| `declined` | + #### `declined_tier` -The tier of a setting a settings layer sets and this build's filters do not honor in a WHERE, as `schema_description`'s `filter_declined_settings` reports it, spelled as this build line's WHERE-settings list (`spec/abi-v2/where-settings/.json`) spells it: `predicate` (measured to change a verdict), `predicate-unflipped` (read in the WHERE's scope, though no value the flip test tried changed a verdict) or `result-content` (selects rows by content). The lists' other tiers (`execution`, `result-truncate` and `output`) never appear here: those settings are passed through and ignored. A tier added later is a new value, which a reader reads as its `unknown(n)` (rule r3). +The tier of a setting a settings layer sets and this build's filters do not honor in a WHERE, as `schema_description`'s `filter_declined_settings` reports it, spelled as this build line's WHERE-settings list (`spec/abi-v2/where-settings/.json`) spells it: `predicate` (measured to change a verdict), `predicate-unflipped` (read in the WHERE's scope, though no value the flip test tried changed a verdict) `result-content` (selects rows by content), or `server-refuses` (a cell of `where-settings/documented-pathological.tsv`: a value under which the server refuses every query, so no row verdict can stand). The lists' other tiers (`execution`, `result-truncate` and `output`) never appear here: those settings are passed through and ignored. A tier added later is a new value, which a reader reads as its `unknown(n)` (rule r3). FIRM. @@ -429,6 +472,7 @@ FIRM. | `predicate` | | `predicate-unflipped` | | `result-content` | +| `server-refuses` | #### `declined_layer` @@ -442,12 +486,24 @@ FIRM. | `server` | | `schema` | +#### `posture` + +How `chs_schema_create` compiles its statement, as `schema_options`' `posture` asks and `schema_description` reports: `create`, as a server runs a `CREATE TABLE` (ClickHouse's `LoadingStrictnessLevel::CREATE`), the default; `load`, as a server loads the stored definition of a table it already holds when it starts (`LoadingStrictnessLevel::FORCE_ATTACH`). `schema_description` carries the member only for `load`: absent is `create`. A posture added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + +FIRM. + +| value | +| -------- | +| `create` | +| `load` | + ### Constants | constant | type | value | status | | ---------------------------- | ------ | ----- | ------ | | `CHS_ABI_REVISION_TOMBSTONE` | int32 | 1001 | FIRM | | `CHS_EXPORT_NONE` | int32 | -1 | FIRM | +| `CHS_CODE_DECLINED` | int32 | -2 | FIRM | | `CHS_DOC_VALUES` | uint32 | 1 | FIRM | | `CHS_DOC_TRANSFORMS` | uint32 | 2 | FIRM | | `CHS_DOC_DEFAULTS` | uint32 | 4 | FIRM | @@ -494,11 +550,13 @@ The fields the description fixes, as JSON Schema; the document may carry more: #### document `schema_description` -A schema's columns, in declared order, with each column's canonical type, its `default_kind` (a value of the `default_kind` vocabulary) and default expression, and the facts a caller needs to build a row. The document is `{"columns": [...]}`, and each entry carries `name`, `type`, `default_kind` (a value of the `default_kind` vocabulary), `default_expression` (empty when there is none) and `default_is_literal`; the name, the type and the expression follow the rule for byte strings in JSON. +A schema's columns, in declared order, with each column's canonical type, its `default_kind` (a value of the `default_kind` vocabulary) and default expression, and the facts a caller needs to build a row. The document is `{"columns": [...]}`, and each entry carries `name`, `type` and `default_kind` (a value of the `default_kind` vocabulary), always, and `default_expression` (empty when there is none) and `default_is_literal`; the name, the type and the expression follow the rule for byte strings in JSON. `type` is present on every column: a column whose type the compile cannot settle (a column declared without a type whose own DEFAULT or ALIAS this build cannot type) is declined by `chs_schema_create` (`CHS_DECLINED`, reason `column-type-unsettled`), so no schema describes a column without its type. The decline is a limit of this build, never a claim about whether a server accepts the table. `default_support`, a `default_support` value with `default_support_reason` for `declined`, says what this build does with the column's DEFAULT; reserved, absent until a build writes it. `reads`, a list of name objects, names the columns the column's DEFAULT, MATERIALIZED, ALIAS or EPHEMERAL expression reads, as ClickHouse's own analysis of the expression's required source columns reports them, each once, in the table's declaration order: a build that writes it writes it on every column, `[]` for a column with no expression or one that reads none. Reserved (rule r9): absent until a build writes it, which that build advertises in `chs_build_info`'s `capabilities.features`; a reader treats it as not reported until then. A schema `chs_schema_create` returned always describes: the walk describing needs runs at compile, so a compiled schema is never refused here. + +`posture`, a `posture` value, is present only on a schema compiled in a posture other than `create`, so the document of a schema compiled in `create` is byte for byte what it was before postures existed. Two members report the server a schema was created on. Both are absent on a schema created with `server` NULL, so that document is byte for byte what it was before servers existed. -- `server`, on every schema created on a server, even one described by `{}`: `timezone` is the zone the schema's types bind (the profile's, or else the image zone, exactly as `chs_initialize` spelled it), so a reader never needs the image zone separately; `settings` is the server's settings as the profile gave them, `{}` when it gave none; `macros` is the server's macro set as the library holds it, present exactly when the profile carried `macros`. These are the caller's own JSON strings given back, not data-derived, so they are plain strings. The member no longer carries `filter_declined_settings`: that list is the top-level one, below, over every layer. +- `server`, on every schema created on a server, even one described by `{}`: `timezone` is the zone the schema's types bind (the profile's, or else the image zone, exactly as `chs_initialize` spelled it), so a reader never needs the image zone separately; `settings` is the server's settings as the profile gave them, `{}` when it gave none; `macros` is the server's macro set as the library holds it, present exactly when the profile carried `macros`. These are the caller's own values given back, not data-derived, so the rule for byte strings in JSON does not apply to them: the zone and the macros are plain strings, and each settings value is a JSON string when its bytes are valid UTF-8 and otherwise the object form `{"b64": …}` (Input values), so a settings value takes the object form only when its bytes are not UTF-8. The member no longer carries `filter_declined_settings`: that list is the top-level one, below, over every layer. - `replicated`, on a schema created on a server whose engine is a Replicated one: `zookeeper_path` and `replica_name`, the path and the replica name ClickHouse's own `TableZnodeInfo` resolved, fully expanded. The database and table names expanded into them are DDL bytes, so both follow the rule for byte strings in JSON. A caller can compare them with the server's own `system.replicas`. `filter_declined_settings` is required on every schema description, with or without a server: an array, `[]` when nothing is declined. It lists the settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known when `chs_schema_create` ran, and `chs_schema_create` computes it. The layers, lowest first, are the `declined_layer` values: @@ -509,14 +567,18 @@ Two members report the server a schema was created on. Both are absent on a sche Each entry is `{name, tier, layer}`: `name` follows the rule for byte strings in JSON (`name` or `name_b64`), `tier` is a `declined_tier` value, the tier this build line's WHERE-settings list gives the name, and `layer` is a `declined_layer` value. The entries are: -- every name set at any of the three layers that the line's list carries at tier `predicate`, `predicate-unflipped` or `result-content` and that is not in this build's honor set (the settings it applies to a WHERE, each with a measured case where the server's verdict moves and the library's tracks it); -- `compatibility`, at tier `predicate`, when its value at the highest layer that sets it moves a listed setting outside the honor set. +- every name set at any of the three layers that the line's list carries at tier `predicate` or `predicate-unflipped` and that is not in this build's honor set (the settings it applies to a WHERE, each with a measured case where the server's verdict moves and the library's tracks it); +- `compatibility`, at tier `predicate`, when its value at the highest layer that sets it moves a listed setting outside the honor set; +- every `result-content` name set at any of the three layers, honored or not: a setting that selects rows by content beyond the WHERE is outside a row verdict's scope whatever the library applies; +- every cell of the line's `where-settings/documented-pathological.tsv` present at any of the three layers, at tier `server-refuses`: the server refuses every query under that value, so no row verdict stands. + +So the consumer rule is one check: refuse when the list is not empty. No consumer copies the WHERE-settings lists or the pathological cells. A name the list does not carry is never an entry, and neither is a setting of the `execution`, `result-truncate` or `output` tiers: those are passed through and ignored. The test is presence, not value: any value at any layer counts, a declined name set to its default value included, because some of ClickHouse's reads are `isChanged()`, which asks only whether a setting was set. A higher layer that sets the same name never takes the entry away; it only moves the entry's `layer`. The order is deterministic. There is one entry per name, at the highest layer that sets it (`defaults`, then `server`, then `schema`). The entries are ordered by layer, `defaults` first, and within a layer by name as raw bytes, ascending; a `name_b64` entry sorts by its decoded bytes. -The list holds only the layers known at schema compile. A filter's own `settings` (`chs_filter_create`) are never in it: a declined one there is named at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`). The settings an evaluation brings (the call's, and the body's) are never in it either: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings (`chs_filter_create`). A schema whose list is not empty is one `chs_filter_create` declines. +The list holds only the layers known at schema compile. A filter's own `settings` (`chs_filter_create`) are never in it: a declined one there is named at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`), and a cell of `documented-pathological.tsv` there declines `chs_filter_create` itself (`setting-server-refuses`). The settings an evaluation brings (the call's, and the body's) are never in it either: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings (`chs_filter_create`). A schema whose list is not empty is one `chs_filter_create` declines. The fields the description fixes, as JSON Schema; the document may carry more: @@ -532,15 +594,21 @@ The fields the description fixes, as JSON Schema; the document may carry more: "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, - "tier": {"type": "string"}, - "layer": {"type": "string"} + "tier": { + "vocabulary": "declined_tier", + "enum": ["predicate", "predicate-unflipped", "result-content", "server-refuses"] + }, + "layer": {"vocabulary": "declined_layer", "enum": ["defaults", "server", "schema"]} }, "allOf": [{"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}] }, "column": { "type": "object", "properties": { - "default_kind": {"enum": ["", "DEFAULT", "MATERIALIZED", "ALIAS", "EPHEMERAL"]}, + "default_kind": { + "vocabulary": "default_kind", + "enum": ["", "DEFAULT", "MATERIALIZED", "ALIAS", "EPHEMERAL"] + }, "default_is_literal": {"type": "boolean"}, "name": {"type": "string"}, "name_b64": { @@ -556,17 +624,14 @@ The fields the description fixes, as JSON Schema; the document may carry more: "default_expression_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" - } + }, + "default_support": {"vocabulary": "default_support", "enum": ["computed", "generated", "deferred", "declined"]}, + "default_support_reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "reads": {"type": "array", "items": {"$ref": "#/$defs/named"}} }, "allOf": [ {"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}, - { - "oneOf": [ - {"required": ["type"], "properties": {"type_b64": false}}, - {"required": ["type_b64"], "properties": {"type": false}}, - {"properties": {"type": false, "type_b64": false}} - ] - }, + {"oneOf": [{"required": ["type"]}, {"required": ["type_b64"]}]}, { "oneOf": [ {"required": ["default_expression"], "properties": {"default_expression_b64": false}}, @@ -574,7 +639,29 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"default_expression": false, "default_expression_b64": false}} ] } - ] + ], + "required": ["default_kind"] + }, + "bytes_value": { + "type": "object", + "required": ["b64"], + "properties": { + "b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + } + }, + "named": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [{"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}] } }, "type": "object", @@ -587,7 +674,10 @@ The fields the description fixes, as JSON Schema; the document may carry more: "required": ["timezone", "settings"], "properties": { "timezone": {"type": "string"}, - "settings": {"type": "object", "additionalProperties": {"type": "string"}}, + "settings": { + "type": "object", + "additionalProperties": {"oneOf": [{"type": "string"}, {"$ref": "#/$defs/bytes_value"}]} + }, "macros": {"type": "object", "additionalProperties": {"type": "string"}} } }, @@ -609,14 +699,15 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"oneOf": [{"required": ["zookeeper_path"]}, {"required": ["zookeeper_path_b64"]}]}, {"oneOf": [{"required": ["replica_name"]}, {"required": ["replica_name_b64"]}]} ] - } + }, + "posture": {"vocabulary": "posture", "enum": ["load"]} } } ``` #### document `row` -One row's verdict and, as the flags ask, its columns' stored values, provenance and transformations. Every entry of `cols` carries its name, `null` (whether the stored value is NULL, poisoned cells included), its renderings (`input`, `stored`, `ref`, `wire`) and its types (`type`, `base`, `ref_type`), all by the rule for byte strings in JSON, and `value_b64` for a String or FixedString value. `unknown_fields` and `unsupported_settings` are lists of name objects; `computed` entries carry a name, a kind and a stored value; `err`, `verdict_err` and `partition_id` follow the rule too. The transformations come from the library, never from a binding: each is the `transformed` entry the SDK's parity fixtures pin, with its `reason` from `transform_reason`. `input_span` is `{off, len}`, the bytes of the input body the reader consumed for this record, read from the vendored reader's own position. +One row's verdict and, as the flags ask, its columns' stored values, provenance and transformations. `outcome` (a `row_outcome` value), `code` (Codes, in the Preamble) and `input_span` are always present, `err` names the refusal, and `reason` the catalog id of a decline. With a filter attached (`chs_preview_batch`), `verdict` (a `filter_verdict` value), `verdict_code`, `verdict_err` and, for `d`, `verdict_reason` are present on every row; with none they are absent. On `chs_preview_row`'s document, which stands for the whole call, `refused_at` and `refused_row` say where a refusal sits (`refusal_site`); the rows of a batch carry neither. Every entry of `cols` carries its name, `null` (whether the stored value is NULL, poisoned cells included), `src` (a `value_src` value), its renderings (`input`, `stored`, `ref`, `wire`) and its types (`type`, `base`, `ref_type`), all by the rule for byte strings in JSON, `value_b64` for a String or FixedString value, and the booleans `nullable` (the column's type is Nullable), `poison` (the stored value is one the server cannot read back), `null_input` (the input field was the format's NULL literal) and `dup_dropped` (a duplicate key the reader dropped), with `ref_unclassified` present, true, when the reference type could not be classified. `unknown_fields` is a list of name objects (the call's declined settings are the batch document's `unsupported_settings` only, which every row of the batch shares); `computed` entries carry a name, a `kind` (the column's default kind, lower case) and a stored value; `err`, `verdict_err` and `partition_id` follow the rule too. The transformations come from the library, never from a binding: each is the `transformed` entry the SDK's parity fixtures pin, with its `reason` from `transform_reason` and its `lossy` fact. `input_span` is `{off, len}`, the bytes of the input body the reader consumed for this record, read from the vendored reader's own position. A `rejected` row always carries a nonzero ClickHouse code, and an `unsupported` one `CHS_CODE_DECLINED` with its `reason`. The fields the description fixes, as JSON Schema; the document may carry more: @@ -644,9 +735,32 @@ The fields the description fixes, as JSON Schema; the document may carry more: }, "col": { "type": "object", - "required": ["null"], + "required": ["null", "src"], "properties": { "null": {"type": "boolean"}, + "src": { + "vocabulary": "value_src", + "enum": [ + "input", + "default", + "default_substituted", + "default_generated", + "absent", + "materialized_input", + "skipped", + "ephemeral_input", + "default_expr_unsupported", + "default_volatile_unresolved", + "default_pending", + "default_deferred", + "default_override" + ] + }, + "nullable": {"type": "boolean"}, + "poison": {"type": "boolean"}, + "null_input": {"type": "boolean"}, + "dup_dropped": {"type": "boolean"}, + "ref_unclassified": {"type": "boolean"}, "name": {"type": "string"}, "name_b64": { "type": "string", @@ -753,6 +867,7 @@ The fields the description fixes, as JSON Schema; the document may carry more: "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, + "kind": {"type": "string"}, "stored": {"type": "string"}, "stored_b64": { "type": "string", @@ -772,12 +887,43 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"stored": false, "stored_b64": false}} ] } - ] + ], + "required": ["kind"] }, "transform": { "type": "object", "properties": { "row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "reason": { + "vocabulary": "transform_reason", + "enum": [ + "overflow_wrap", + "null_to_default", + "null_loss", + "decimal_truncate", + "date_clamp", + "datetime_wrap", + "date_shift", + "uuid_mangle", + "ip_mangle", + "float_precision", + "lossy_numeric", + "fixedstring_pad", + "emptied", + "element_changed", + "enum_coerce", + "value_changed", + "poisoned", + "duplicate_key_dropped", + "reformat", + "default_filled", + "zero_filled", + "default_materialized", + "ttl_expired", + "ttl_column_expired" + ] + }, + "lossy": {"type": "boolean"}, "column": {"type": "string"}, "column_b64": { "type": "string", @@ -810,15 +956,23 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"stored": false, "stored_b64": false}} ] } - ] + ], + "required": ["reason", "lossy"] } }, "type": "object", - "required": ["input_span"], + "required": ["outcome", "code", "input_span"], "properties": { + "outcome": { + "vocabulary": "row_outcome", + "enum": ["accepted", "accepted_poisoned", "rejected", "skipped", "unsupported"] + }, + "code": {"type": "integer", "minimum": -2}, + "verdict": {"vocabulary": "filter_verdict", "enum": ["t", "f", "e", "d"]}, + "verdict_code": {"type": "integer", "minimum": -2}, + "verdict_reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, "input_span": {"$ref": "#/$defs/span"}, "unknown_fields": {"type": "array", "items": {"$ref": "#/$defs/named"}}, - "unsupported_settings": {"type": "array", "items": {"$ref": "#/$defs/named"}}, "cols": {"type": "array", "items": {"$ref": "#/$defs/col"}}, "computed": {"type": "array", "items": {"$ref": "#/$defs/computed"}}, "transformed": {"type": "array", "items": {"$ref": "#/$defs/transform"}}, @@ -827,7 +981,10 @@ The fields the description fixes, as JSON Schema; the document may carry more: "verdict_err": {"type": "string"}, "verdict_err_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"}, "partition_id": {"type": "string"}, - "partition_id_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"} + "partition_id_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"}, + "reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "refused_at": {"vocabulary": "refusal_site", "enum": ["call", "body_read", "row", "body_step"]}, + "refused_row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991} }, "allOf": [ { @@ -857,13 +1014,19 @@ The fields the description fixes, as JSON Schema; the document may carry more: #### document `batch` -A body's verdict, counts and per-row documents (each a row document, with its own `input_span`), and, when an export was asked for, where each accepted row sits in the export bytes. `engine_rows`, present when the table's engine merges rows at insert, is a list of rows, each a list of cells: `{name, stored, null}`, plus `value_b64` for a String or FixedString value. `storage_transforms` entries carry `row`, `column`, `reason` and `stored`, plus `value_b64`; `transformed` entries carry `row`, `column`, `input`, `stored`, `reason` and `lossy`. Every name, rendering, `err` and `export_declined` follows the rule for byte strings in JSON. +A body's verdict, counts and per-row documents, and, when an export ran, where each row sits in the export bytes. Always present: `outcome` (a `batch_outcome` value), `code` (Codes, in the Preamble), `err`, `rows` (each a row document, with its own `input_span`), `rows_read` (the records the reader read, skipped ones included), `rows_skipped` (the records skipped under the server's error allowance), `unsupported_settings`, `unconsumed` and `framing`. Present with a filter attached: `rows_passed` and `rows_cut`, which partition the accepted rows of an `accepted` batch by verdict (`t` against the rest) and read 0 and 0 on any other outcome. Present on a schema with a partition key: `partition_count`, the distinct partitions the stored rows span, which with each row's `partition_id` lets a caller cap the partitions one INSERT touches (`max_partitions_per_insert_block`). Present whenever the outcome is not `accepted` or `accepted_poisoned`: `refused_at` (a `refusal_site` value), `refused_row` for `row`, and `reason` for a decline; `rows_read` is 0 when nothing was read. With an export asked for, exactly one of `row_spans` and `export_declined` is present: `row_spans` when the export ran, one entry per `rows[]` entry, index-aligned, `{off, len}` into `out_export` for an exported row (its whole output line, `\n` included, so slicing the spans out of the bytes is the per-row payload and concatenating them reproduces it) and `null` for a row not exported; `export_declined`, naming the cause, with `export_declined_reason`, the cause's catalog id, when it did not (Conditional members, in the Preamble). When the export ran, `export_columns` is present too: the positional column list of every line in `out_export`, as name objects: the stored non-computed columns in declaration order, independent of the call's `columns` list, a column outside `insertable_columns` included, with the value the library drew, resolved or filled for it, less the columns every row defers (`defer_defaults`). A caller inserts the export naming `export_columns` as the column list; it never derives a list of its own. `export_columns` is the export's only column-list member, so no second list can disagree with it. + +The export's truth table, by outcome, export asked for and filter verdict: with no export asked for, `out_export` is NULL and neither `row_spans` nor `export_declined` is present. With an export asked for and the outcome `accepted`, the export runs: `out_export` is the bytes (an empty buffer when no row is exported) and `row_spans` is present, with a span for every row whose outcome is `accepted` and, with a filter attached, whose verdict is `t`, and `null` for every other row (`skipped`, and a `f`, `e` or `d` row under a filter); the rows are exported in body order. With an export asked for and any other outcome (`accepted_poisoned`, `rejected`, `unsupported`), nothing is exported: `out_export` is NULL and `export_declined` names the cause (the outcome, a poisoned value), with `export_declined_reason` `export-batch-not-accepted`. With an export asked for and the outcome `accepted` but the serialization failing, or the export's format unable to carry a stored value, `out_export` is NULL, `export_declined` names the cause and `export_declined_reason` its catalog id; the batch's `outcome` stays `accepted`, with code 0 and no `reason`, so an export decline is keyed by `export_declined_reason`, never by the batch's `reason`. So a caller tells a declined export from an empty one by `export_declined`, never by the buffer, and never slices the payload by an index other than the row's own. + +A computed value the export cannot carry. A column the export omits is recomputed by the server that inserts the export. A DEFAULT that reads an EPHEMERAL value is stored, so the export carries its value and the stored row is the same. A MATERIALIZED expression that reads an EPHEMERAL value the row supplied is not: the export carries no EPHEMERAL column, so the inserting server computes the MATERIALIZED value from the EPHEMERAL's type default instead, and the stored row differs from the one the document reports. Such a body's export is declined (`export_declined`, with `export_declined_reason` `export-computed-value-not-carried`): a caller that needs that stored row inserts the body. -`unsupported_settings`, at the top level, is a list of name objects with the same shape as a row document's: the call's own settings that the library declined. It is empty when none was, and when it is not empty the batch's outcome is at least `unsupported` (see `batch_outcome`). +Composition. A caller that concatenates accepted bodies into one INSERT (a worker's batch, async inserts) keeps every per-row verdict: a row's `outcome`, its stored values, its `partition_id` and its filter verdict are functions of the row and the schema alone. The whole-body steps are not compositional, and can refuse the concatenation after each body was accepted: the partition split (`max_partitions_per_insert_block`, code 252, bounded by the sum of the bodies' `partition_count`), an insert-time engine merge across bodies, `rowNumberInAllBlocks()` and the like, and a CHECK over a merged async block, which fails every INSERT in it. A caller that must not see those caps its INSERT by `partition_count` and keeps CHECK-bearing bodies isolated. `engine_rows`, present when the table's engine merges rows at insert, is a list of rows, each a list of cells: `{name, stored, null}`, plus `value_b64` for a String or FixedString value. `storage_transforms` entries carry `row`, `column`, `reason` and `stored`, plus `value_b64`; `transformed` entries carry `row`, `column`, `input`, `stored`, `reason` and `lossy`. Every name, rendering, `err` and `export_declined` follows the rule for byte strings in JSON. -`at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part this `INSERT` writes would do, at the call's clock instant. It is absent when nothing would happen. `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. Each entry is `{row, reason, column, stored, input_rows}`: +`unsupported_settings`, at the top level, is always present: a list of name objects, the call's own settings that the library declined, which every row of the batch shares. It is empty when none was, and when it is not empty the batch's outcome is at least `unsupported` (see `batch_outcome`). -- `row` indexes `engine_rows`: the part's row, not the input body's. An engine's insert-time merge (Collapsing, Replacing, Summing, Aggregating) runs first, inside the `INSERT` writer, so `engine_rows` already excludes the rows it merged away, and after a Summing or Aggregating merge the part's rows do not map one to one to the input's. +`at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part this `INSERT` writes would do, at the call's clock instant. It is absent when nothing would happen. `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. Each entry is `{part_row, reason, column, stored, input_rows}`: + +- `part_row` indexes `engine_rows`: the part's row, not the input body's. Every other `row` member of this document (`transformed[].row`, `storage_transforms[].row`, `refused_row`) and of `filter_result` (`errors[].row`) indexes `rows[]`, the input body's rows, and `input_rows` here does too. An engine's insert-time merge (Collapsing, Replacing, Summing, Aggregating) runs first, inside the `INSERT` writer, so `engine_rows` already excludes the rows it merged away, and after a Summing or Aggregating merge the part's rows do not map one to one to the input's. - `reason` is a `merge_reason` value: `ttl_delete` (the rows TTL removes the row) or `ttl_column_reset` (a column TTL resets the column's value). A row that is removed and also carries column resets lists only `ttl_delete`: it is removed in the same merge pass, so its column values are never observable. - `column` names the column, by the rule for byte strings in JSON, and is present only for `ttl_column_reset`. - `stored` is the column's value after the reset, by the same rule. It is present when that value is decided at the `INSERT`: the column's `DEFAULT` is a constant or a deterministic expression of the row, or the column has no `DEFAULT` (the type's default). It is absent when the `DEFAULT` reads the clock (`now()`) or a generator (`rand()`, `generateUUIDv4()`), and the entry then means that the value is reset at the merge and decided then. @@ -902,9 +1065,32 @@ The fields the description fixes, as JSON Schema; the document may carry more: }, "col": { "type": "object", - "required": ["null"], + "required": ["null", "src"], "properties": { "null": {"type": "boolean"}, + "src": { + "vocabulary": "value_src", + "enum": [ + "input", + "default", + "default_substituted", + "default_generated", + "absent", + "materialized_input", + "skipped", + "ephemeral_input", + "default_expr_unsupported", + "default_volatile_unresolved", + "default_pending", + "default_deferred", + "default_override" + ] + }, + "nullable": {"type": "boolean"}, + "poison": {"type": "boolean"}, + "null_input": {"type": "boolean"}, + "dup_dropped": {"type": "boolean"}, + "ref_unclassified": {"type": "boolean"}, "name": {"type": "string"}, "name_b64": { "type": "string", @@ -1011,6 +1197,7 @@ The fields the description fixes, as JSON Schema; the document may carry more: "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, + "kind": {"type": "string"}, "stored": {"type": "string"}, "stored_b64": { "type": "string", @@ -1030,12 +1217,43 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"stored": false, "stored_b64": false}} ] } - ] + ], + "required": ["kind"] }, "transform": { "type": "object", "properties": { "row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "reason": { + "vocabulary": "transform_reason", + "enum": [ + "overflow_wrap", + "null_to_default", + "null_loss", + "decimal_truncate", + "date_clamp", + "datetime_wrap", + "date_shift", + "uuid_mangle", + "ip_mangle", + "float_precision", + "lossy_numeric", + "fixedstring_pad", + "emptied", + "element_changed", + "enum_coerce", + "value_changed", + "poisoned", + "duplicate_key_dropped", + "reformat", + "default_filled", + "zero_filled", + "default_materialized", + "ttl_expired", + "ttl_column_expired" + ] + }, + "lossy": {"type": "boolean"}, "column": {"type": "string"}, "column_b64": { "type": "string", @@ -1068,15 +1286,23 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"stored": false, "stored_b64": false}} ] } - ] + ], + "required": ["reason", "lossy"] }, "row": { "type": "object", - "required": ["input_span"], + "required": ["outcome", "code", "input_span"], "properties": { + "outcome": { + "vocabulary": "row_outcome", + "enum": ["accepted", "accepted_poisoned", "rejected", "skipped", "unsupported"] + }, + "code": {"type": "integer", "minimum": -2}, + "verdict": {"vocabulary": "filter_verdict", "enum": ["t", "f", "e", "d"]}, + "verdict_code": {"type": "integer", "minimum": -2}, + "verdict_reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, "input_span": {"$ref": "#/$defs/span"}, "unknown_fields": {"type": "array", "items": {"$ref": "#/$defs/named"}}, - "unsupported_settings": {"type": "array", "items": {"$ref": "#/$defs/named"}}, "cols": {"type": "array", "items": {"$ref": "#/$defs/col"}}, "computed": {"type": "array", "items": {"$ref": "#/$defs/computed"}}, "transformed": {"type": "array", "items": {"$ref": "#/$defs/transform"}}, @@ -1094,7 +1320,10 @@ The fields the description fixes, as JSON Schema; the document may carry more: "partition_id_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" - } + }, + "reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "refused_at": {"vocabulary": "refusal_site", "enum": ["call", "body_read", "row", "body_step"]}, + "refused_row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991} }, "allOf": [ { @@ -1183,10 +1412,10 @@ The fields the description fixes, as JSON Schema; the document may carry more: }, "at_merge": { "type": "object", - "required": ["row", "reason"], + "required": ["part_row", "reason"], "properties": { - "row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, - "reason": {"type": "string"}, + "part_row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "reason": {"vocabulary": "merge_reason", "enum": ["ttl_delete", "ttl_column_reset"]}, "column": {"type": "string"}, "column_b64": { "type": "string", @@ -1218,8 +1447,22 @@ The fields the description fixes, as JSON Schema; the document may carry more: } }, "type": "object", - "required": ["rows", "unconsumed", "framing"], + "required": [ + "outcome", + "code", + "rows", + "rows_read", + "rows_skipped", + "unsupported_settings", + "unconsumed", + "framing" + ], "properties": { + "outcome": { + "vocabulary": "batch_outcome", + "enum": ["accepted", "accepted_poisoned", "rejected", "unsupported"] + }, + "code": {"type": "integer", "minimum": -2}, "rows": {"type": "array", "items": {"$ref": "#/$defs/row"}}, "unsupported_settings": {"type": "array", "items": {"$ref": "#/$defs/named"}}, "engine_rows": {"type": "array", "items": {"type": "array", "items": {"$ref": "#/$defs/cell"}}}, @@ -1251,8 +1494,19 @@ The fields the description fixes, as JSON Schema; the document may carry more: }, "err": {"type": "string"}, "err_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"}, + "rows_read": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "rows_skipped": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "rows_passed": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "rows_cut": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "partition_count": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "row_spans": {"type": "array", "items": {"oneOf": [{"$ref": "#/$defs/span"}, {"type": "null"}]}}, + "export_columns": {"type": "array", "items": {"$ref": "#/$defs/named"}}, + "reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "refused_at": {"vocabulary": "refusal_site", "enum": ["call", "body_read", "row", "body_step"]}, + "refused_row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, "export_declined": {"type": "string"}, - "export_declined_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"} + "export_declined_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"}, + "export_declined_reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"} }, "allOf": [ { @@ -1275,9 +1529,9 @@ The fields the description fixes, as JSON Schema; the document may carry more: #### document `filter_result` -A filter evaluation: the call's outcome, one verdict character per row (`filter_verdict`), and each error or declined row itemized as `{row, code, err}`. `err`, at the top level and per row, follows the rule for byte strings in JSON, and `unsupported_settings` is a list of name objects. +A filter evaluation. Always present: `outcome` (a `filter_outcome` value), `code` (Codes, in the Preamble), `err`, `rows_read`, `verdicts` (one `filter_verdict` character per row read, so its length is `rows_read`, and every character `d` under any outcome but `ok`), `unsupported_settings` (a list of name objects) and `errors`, each error or declined row itemized as `{row, code, reason, err}` with `row` indexing the body's rows, `code` nonzero (ClickHouse's for `e`, `CHS_CODE_DECLINED` for `d`) and `reason` the catalog id of a `d`. Present whenever the outcome is not `ok`: `refused_at` (a `refusal_site` value), `refused_row` for `row`, and `reason` for a decline. `err`, at the top level and per row, follows the rule for byte strings in JSON. -`unsupported_settings` (after `rows_read`, before `verdicts`) names the settings the parse itself declined, and then, after them and without repeats, every setting of the filter's own `settings` that this build's filters do not honor in a WHERE. When it names one of those, every verdict is `d` and every row is itemized in `errors` as `{row, code, err}` with `code` -2. `outcome` follows its own rules all the same: `ok` for a body a server would accept, and the mirror of `chs_preview_batch`'s verdict otherwise (`filter_outcome`). A declined setting from the `chs_set_defaults`, server or schema layer never reaches an evaluation, because `chs_filter_create` declines first (`schema_description`'s `filter_declined_settings`). +`unsupported_settings` (after `rows_read`, before `verdicts`) names the settings the parse itself declined, and then, after them and without repeats, every setting of the filter's own `settings` that this build's filters do not honor in a WHERE. When it names one of those, every verdict is `d` and every row is itemized in `errors` as `{row, code, reason, err}` with `code` -2. `outcome` follows its own rules all the same: `ok` for a body a server would accept, and the mirror of `chs_preview_batch`'s verdict otherwise (`filter_outcome`). A declined setting from the `chs_set_defaults`, server or schema layer never reaches an evaluation, because `chs_filter_create` declines first (`schema_description`'s `filter_declined_settings`). The fields the description fixes, as JSON Schema; the document may carry more: @@ -1299,6 +1553,8 @@ The fields the description fixes, as JSON Schema; the document may carry more: "type": "object", "properties": { "row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "code": {"type": "integer", "minimum": -2}, + "reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, "err": {"type": "string"}, "err_b64": { "type": "string", @@ -1313,15 +1569,23 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"err": false, "err_b64": false}} ] } - ] + ], + "required": ["row", "code"] } }, "type": "object", "properties": { + "outcome": {"vocabulary": "filter_outcome", "enum": ["ok", "rejected", "unsupported"]}, + "code": {"type": "integer", "minimum": -2}, + "rows_read": {"type": "integer", "minimum": 0, "maximum": 9007199254740991}, + "verdicts": {"type": "string"}, "unsupported_settings": {"type": "array", "items": {"$ref": "#/$defs/named"}}, "errors": {"type": "array", "items": {"$ref": "#/$defs/filter_error"}}, "err": {"type": "string"}, - "err_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"} + "err_b64": {"type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$"}, + "reason": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "refused_at": {"vocabulary": "refusal_site", "enum": ["call", "body_read", "row", "body_step"]}, + "refused_row": {"type": "integer", "minimum": 0, "maximum": 9007199254740991} }, "allOf": [ { @@ -1331,7 +1595,8 @@ The fields the description fixes, as JSON Schema; the document may carry more: {"properties": {"err": false, "err_b64": false}} ] } - ] + ], + "required": ["outcome", "code", "rows_read", "unsupported_settings", "verdicts", "errors"] } ``` @@ -1429,18 +1694,20 @@ The fields the description fixes, as JSON Schema; the document may carry more: Each is a JSON object a caller passes in a `bytes_in` parameter whose content is `input:`. The library refuses a key the document does not define, so each schema closes every object (rule r1). An options document is the one a growable function takes the inputs it gains in. -| input document | options | taken by | -| ---------------- | ------- | ----------------------------- | -| `server_profile` | no | `chs_server_create` `profile` | -| `server_options` | yes | `chs_server_create` `options` | -| `schema_options` | yes | `chs_schema_create` `options` | +| input document | options | taken by | +| ---------------- | ------- | -------------------------------------------------------------------------------------------------------------------------- | +| `server_profile` | no | `chs_server_create` `profile` | +| `server_options` | yes | `chs_server_create` `options` | +| `schema_options` | yes | `chs_schema_create` `options` | +| `insert_options` | yes | `chs_preview_row` `options`, `chs_preview_batch` `options`, `chs_filter_eval_body` `options`, `chs_block_create` `options` | +| `filter_options` | yes | `chs_filter_create` `options` | #### input `server_profile` A server's profile, `{"timezone": "", "settings": {"": ""}, "macros": {"": ""}}`. Every member is optional, so `{}` describes a server with nothing known about it. The profile holds server facts only, in the shape a server's own tables report them, so a caller can fill it from the server it describes; library knobs go in `chs_server_create`'s options instead. - `timezone`: the server's own zone, the zone its tables bind. An empty string, or no member, means the image zone. The name is validated by ClickHouse's own `DateLUT`, which reads the library host's time zone database. -- `settings`: the settings the server's profile applies to every query, each value a JSON string, never rewritten. The library accepts exactly what the server's own SET check accepts. `session_timezone` here is the default session zone of the server's user. +- `settings`: the settings the server's profile applies to every query, each value a JSON string or the object form for bytes (Input values), never rewritten. A `chtypes_*` knob is never one of them (`chs_set_defaults`). The library accepts exactly what the server's own SET check accepts. `session_timezone` here is the default session zone of the server's user. - `macros`: the server's macros, as its `system.macros` lists them (`macro`, `substitution`). Absent, they are unknown; present, even as `{}`, they are the complete set (see `chs_server_create`). A macro name that ClickHouse's own macro reader refuses (for example `a.b`) is refused with `CHS_INVALID_ARGUMENT` naming it. Its members, as JSON Schema: @@ -1451,12 +1718,28 @@ Its members, as JSON Schema: "additionalProperties": false, "properties": { "timezone": {"type": "string"}, - "settings": {"type": "object", "additionalProperties": {"type": "string"}}, + "settings": { + "type": "object", + "additionalProperties": {"oneOf": [{"type": "string"}, {"$ref": "#/$defs/bytes_value"}]} + }, "macros": { "type": "object", "propertyNames": {"minLength": 1}, "additionalProperties": {"type": "string"} } + }, + "$defs": { + "bytes_value": { + "type": "object", + "additionalProperties": false, + "required": ["b64"], + "properties": { + "b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + } + } } } ``` @@ -1473,12 +1756,97 @@ Its members, as JSON Schema: #### input `schema_options` -The options of `chs_schema_create` (rule r1). It defines no member yet, so `{}` (or length 0) is the only valid document, and any key is `CHS_INVALID_ARGUMENT`, naming the key. A choice about how one statement is compiled lands here, as a new member: for example, compiling a server's own `create_table_query`, read back from the server, rather than a caller's CREATE. +The options of `chs_schema_create` (rule r1): how one statement is compiled. Its member is `posture`, a `posture` value, and `{}` (or length 0) compiles in `create`, as before the member existed. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a `posture` that is not one of its values, naming the value. + +- `create`: the statement is a `CREATE TABLE` a server would run now, with every check a server's CREATE runs, under the settings layers (`chs_schema_create`). +- `load`: the statement is the stored definition of a table the server already holds, as that server reports it (`system.tables.create_table_query`, `SHOW CREATE TABLE`, or the table's metadata file), compiled as the server loads it when it starts. It is one switch, and ClickHouse's own: every check on the compile's path that reads the loading posture reads `FORCE_ATTACH` instead of `CREATE`, in ClickHouse's own code, and nothing else changes. So the checks a server applies only to a new table are not applied: the type gates (`allow_suspicious_low_cardinality_types`, `allow_suspicious_fixed_string_types` and the rest, at the compile and at every call on the schema), the codec, statistics and suspicious-index checks, the DEFAULT validation of a typed column, the TTL checks, the MergeTree settings' constraint and sanity checks, and the floating-point partition-key, nullable-key and deprecated-syntax gates. `flatten_nested` and `data_type_default_nullable` are not applied either: a stored definition is already flattened and typed. Which checks these are on each supported line is the artifact producer's enumeration from the vendored source; no list is kept here. A projection that a starting server skips because it no longer parses is declined, because whether a given server still holds it depends on that server's history. A Replicated engine's ZooKeeper path and replica name never change a row verdict, so under `load` a stored definition whose engine arguments read a macro the server's `macros` cannot expand (or carry none, as a Replicated database's table does) compiles as the server loads it: `schema_description` then carries no `replicated` member, and only an answer that depends on the path (none today) is declined. + +**The caller's obligation.** `load` asserts that the server holds a table defined by exactly this statement. The library cannot check that assertion, because it never connects to the server. A statement a server would refuse at `CREATE` compiles under `load`, and every answer on the schema is the answer for INSERTs into that stored table. If no server holds such a table, every answer describes a table that does not exist, and an accepted row is evidence of nothing. A statement a server reported for a table it holds (its `create_table_query`, `SHOW CREATE TABLE`, or its metadata file) MUST be compiled in `load`, and a statement a caller wrote or edited in `create`. A server's `create_table_query` compiled in `create` can be wrong in both directions, because the server created the table under its own CREATE's settings, which the statement does not carry: an over-reject where a gate was set (a `LowCardinality(UInt8)` column) and an over-accept where a shape setting was (a `Nested` column of a table created with `flatten_nested = 0`). Its members, as JSON Schema: ```json -{"type": "object", "additionalProperties": false} +{ + "type": "object", + "additionalProperties": false, + "properties": {"posture": {"enum": ["create", "load"]}} +} +``` + +#### input `insert_options` + +The options of every call that models one INSERT of a body: `chs_preview_row`, `chs_preview_batch`, `chs_block_create` and `chs_filter_eval_body` (rule r1). Its members: `insertable_columns`, `column_defaults` and `verdict_basis`, `defer_defaults`. + +- `insertable_columns`: the columns an INSERT may supply, as name objects: the server's column-level INSERT grant (`GRANT INSERT(a, b) ON t`), checked as the server checks it, on the INSERT's header and never on the fields a body supplies. The header is the call's `columns` list, or with none the server's own no-list header: every ordinary and DEFAULT column, in declaration order (so a positional CSV, TSV, Values or JSONCompactEachRow body with no list supplies all of them). The check runs after the list's own refusals (an unknown or ALIAS column, 16; a duplicate, 15; a MATERIALIZED column, 44, unless `insert_allow_materialized_columns` is set, when it needs the grant like any other) and before any byte of the body is read. A header naming a column outside the set is refused 497 (ACCESS_DENIED), with `refused_at` `call`, in every format and whatever the body holds, an empty or a malformed body included. So `columns` MUST be a subset of the set, and a caller whose set does not cover every ordinary and DEFAULT column passes `columns`, at every call that takes this document, `chs_filter_eval_body` included: with a list a call follows the listed INSERT's rules, and with none the no-list rule. A body field naming a column the header omits (a JSON key, a `WithNames` header field) is an unknown field, never a grant question: skipped, or refused 117 under `input_format_skip_unknown_fields` 0 (`refused_at` `row` for a JSON key, `body_read` for a header field), and never stored. An omitted column keeps its declared kind, exactly as with no set: a generator DEFAULT draws (`default_generated`), a clock DEFAULT resolves, a literal DEFAULT fills, a MATERIALIZED column computes and an EPHEMERAL column takes its type default. The export carries every stored non-computed column, those outside the set included (`document:batch`), so the INSERT of the export needs a grant that covers its column list. A Nested column is granted per subcolumn (`n.x`). The set is a grant, not a column list: like a grant it may name an EPHEMERAL or MATERIALIZED column, or a name the table lacks. The 497's message is ClickHouse's own, naming the user `default` and the table `default.t`, since a schema carries neither a user nor a table name; it is informative. Absent is every column. Implemented in the first build that carries this description. +- `column_defaults`: per-call DEFAULT overrides, a list of `{name, value}` (each by the rule for byte strings in JSON). It is one rule: in a cell the server would fill from the column's DEFAULT, the call uses the caller's `value` instead. A supplied value wins: the override fills exactly the cells ClickHouse's own input format marks missing for that column under the call's format settings, as the vendored reader decides it, and no other: a field the record omits, a `null` for a non-Nullable column under `input_format_null_as_default` 1, and an empty CSV field under `input_format_csv_empty_as_default` 1. An empty JSON string `""` is a supplied value. A Nullable column's `null` stays `NULL` where the server keeps it (the implementing build confirms this against the server's own missing mark). Such a cell reads `default_override`, and everything that reads the column sees the value as an INSERT that supplied it would: a CHECK, the sorting, primary and partition keys, a skip index, a projection, a TTL, a MATERIALIZED or DEFAULT expression that reads it, and a filter. The export carries it as a stored, present value at the column's position in `export_columns`, so the INSERT of the export needs no override, and a column the override fills is not deferred in those rows (`defer_defaults`). An entry may name an ordinary or a DEFAULT column, and an EPHEMERAL column where the implementing build measures it equal to a supplied EPHEMERAL value; an entry that names an unknown column, a column that is not insertable, or a MATERIALIZED or ALIAS column is `CHS_INVALID_ARGUMENT` at `call`, naming `column_defaults[i]`. When `insertable_columns` is given, an entry's column MUST be in it, because the INSERT the override models names the column in its header; one outside it is the same `CHS_INVALID_ARGUMENT`. `value` is read in the body format's own field encoding, as if the record supplied this field (the implementing build confirms); a value that does not read as the column's type is `CHS_INVALID_ARGUMENT` at `call`, naming `column_defaults[i]`, before any byte of the body is read. Built before the lock: the build that implements it lists `insert_options.column_defaults` in `capabilities.features` (rule r9), and a build that does not declines it as Input documents (in the Preamble) says. +- `verdict_basis`: `input`, the default, or `export`: what row a filter verdict answers for. Under `input`, a filter that reads a value the library drew or resolved itself is `d` (`drawn-value`), because the server would draw its own. Under `export`, the verdict answers for the row the export would store, so a drawn or resolved value is answerable, and the caller MUST insert the export. Reserved: declined at `export` until implemented. +- A member whose value is a list or an object is refused `CHS_INVALID_ARGUMENT` when an entry does not match its shape, naming the key. `{}` (or length 0) is the INSERT these calls modeled before the document existed, byte for byte. Each member means the same at every call that takes the document. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a value that is not one of the member's, naming the value; a member this build does not implement declines as Input documents (in the Preamble) says. + +- `defer_defaults`: `none`, the default, or `server` (public issue #479). `server` defers to the server each DEFAULT this build would otherwise decline because its value is the server's to compute (a generator or a server function the build does not admit). Such a column reads `default_deferred` in the rows that omit it, and the export omits it from those rows' lines. What reads it there is not answered (the deferred-column rule): a filter that reads it makes the row's verdict `d` (`verdict_code` `CHS_CODE_DECLINED`, `verdict_reason` `defer-defaults-reader`); a CHECK, a sorting, primary or partition key, a skip index, a projection, a TTL, or a MATERIALIZED or other DEFAULT expression that reads it declines the row (`unsupported`, `reason` `defer-defaults-reader`); and such a row carries no `partition_id`. Admitted generators keep drawing (`default_generated`), so a drawn id stays usable for deduplication. A body is mixed when its exported rows' lines would not all omit the same columns (some rows supply a deferred column and others omit it). The export has one column list, so a mixed body is declined, and only when an export is asked for: the batch is `unsupported`, `refused_at` `body_step`, `reason` `defer-defaults-mixed-body`; with no export asked for no line omits anything, and the batch is answered. A body whose rows all defer the same columns, or none, is never mixed: its export omits those columns from every line and from `export_columns`. Declined until implemented. + +Its members, as JSON Schema: + +```json +{ + "type": "object", + "additionalProperties": false, + "properties": { + "defer_defaults": {"enum": ["none", "server"]}, + "insertable_columns": {"type": "array", "items": {"$ref": "#/$defs/named"}}, + "column_defaults": {"type": "array", "items": {"$ref": "#/$defs/column_default"}}, + "verdict_basis": {"enum": ["input", "export"]} + }, + "$defs": { + "named": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": {"type": "string"}, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [{"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}] + }, + "column_default": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": {"type": "string"}, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "value": {"type": "string"}, + "value_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [ + {"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}, + {"oneOf": [{"required": ["value"]}, {"required": ["value_b64"]}]} + ] + } + } +} +``` + +#### input `filter_options` + +The options of `chs_filter_create` (rule r1): how one filter is compiled. `{}` (or length 0) compiles as before the document existed. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a value that is not one of the member's, naming the value; a member this build does not implement declines as Input documents (in the Preamble) says. + +- `param_encoding`: `escaped`, the default, or `raw`: how a query parameter's value is read. Under `escaped`, the value is decoded by ClickHouse's escaped-text reader, exactly as a server decodes an HTTP `param_` value, so a backslash, a tab or a newline must be escaped as that reader expects; the object form `{"b64": ...}` is decoded from base64 first and the bytes then read the same way. Under `raw`, the bytes are the value. Reserved: declined at `raw` until implemented (rule r9). + +Its members, as JSON Schema: + +```json +{ + "type": "object", + "additionalProperties": false, + "properties": {"param_encoding": {"enum": ["escaped", "raw"]}} +} ``` ### build_info @@ -1502,48 +1870,49 @@ Its members, as JSON Schema: ### Functions -| function | class | thread | returns | status | -| -------------------------- | --------- | -------------- | ----------------- | ------ | -| `chs_abi_version` | handshake | any | `int32_t` | FIRM | -| `chs_build_info` | handshake | any | `const char *` | FIRM | -| `chs_clickhouse_version` | handshake | any | `const char *` | FIRM | -| `chs_abi_revision` | tombstone | any | `int` | FIRM | -| `chs_buf_data` | api | shared | `const uint8_t *` | FIRM | -| `chs_buf_len` | api | shared | `size_t` | FIRM | -| `chs_buf_free` | api | handle_serial | `void` | FIRM | -| `chs_error_status` | api | shared | `chs_status` | FIRM | -| `chs_error_ch_code` | api | shared | `int32_t` | FIRM | -| `chs_error_ch_name` | api | shared | `chs_buf *` | FIRM | -| `chs_error_message` | api | shared | `chs_buf *` | FIRM | -| `chs_error_column` | api | shared | `chs_buf *` | FIRM | -| `chs_error_free` | api | handle_serial | `void` | FIRM | -| `chs_live_handles` | api | any | status | FIRM | -| `chs_error_codes` | api | any | status | FIRM | -| `chs_registered_families` | tooling | process_serial | status | FIRM | -| `chs_function_flags` | tooling | process_serial | status | FIRM | -| `chs_reference_type` | tooling | process_serial | status | FIRM | -| `chs_initialize` | api | process_once | status | FIRM | -| `chs_set_defaults` | api | process_serial | status | FIRM | -| `chs_shutdown` | api | process_serial | `void` | FIRM | -| `chs_type_validate` | api | any | status | FIRM | -| `chs_back_quote` | api | any | status | FIRM | -| `chs_back_quote_if_needed` | api | any | status | FIRM | -| `chs_quote_string` | api | any | status | FIRM | -| `chs_server_create` | api | any | status | FIRM | -| `chs_server_free` | api | handle_serial | `void` | FIRM | -| `chs_schema_create` | api | shared | status | FIRM | -| `chs_schema_free` | api | handle_serial | `void` | FIRM | -| `chs_schema_describe` | api | shared | status | FIRM | -| `chs_preview_row` | api | shared | status | FIRM | -| `chs_preview_batch` | api | shared | status | FIRM | -| `chs_filter_create` | api | shared | status | FIRM | -| `chs_filter_free` | api | handle_serial | `void` | FIRM | -| `chs_filter_eval_body` | api | shared | status | FIRM | -| `chs_block_create` | api | shared | status | FIRM | -| `chs_block_free` | api | handle_serial | `void` | FIRM | -| `chs_filter_eval_block` | api | shared | status | FIRM | -| `chs_discover_query` | api | any | status | FIRM | -| `chs_discover_columns` | api | any | status | FIRM | +| function | class | thread | init | returns | status | +| -------------------------- | --------- | -------------- | -------- | ----------------- | ------ | +| `chs_abi_version` | handshake | any | any | `int32_t` | FIRM | +| `chs_build_info` | handshake | any | any | `const char *` | FIRM | +| `chs_clickhouse_version` | handshake | any | any | `const char *` | FIRM | +| `chs_abi_revision` | tombstone | any | any | `int` | FIRM | +| `chs_buf_data` | api | shared | any | `const uint8_t *` | FIRM | +| `chs_buf_len` | api | shared | any | `size_t` | FIRM | +| `chs_buf_free` | api | handle_serial | any | `void` | FIRM | +| `chs_error_status` | api | shared | any | `chs_status` | FIRM | +| `chs_error_ch_code` | api | shared | any | `int32_t` | FIRM | +| `chs_error_ch_name` | api | shared | any | `chs_buf *` | FIRM | +| `chs_error_message` | api | shared | any | `chs_buf *` | FIRM | +| `chs_error_column` | api | shared | any | `chs_buf *` | FIRM | +| `chs_error_reason` | api | shared | any | `chs_buf *` | FIRM | +| `chs_error_free` | api | handle_serial | any | `void` | FIRM | +| `chs_live_handles` | api | any | required | status | FIRM | +| `chs_error_codes` | api | any | required | status | FIRM | +| `chs_registered_families` | tooling | process_serial | required | status | FIRM | +| `chs_function_flags` | tooling | process_serial | required | status | FIRM | +| `chs_reference_type` | tooling | process_serial | required | status | FIRM | +| `chs_initialize` | api | process_once | any | status | FIRM | +| `chs_set_defaults` | api | process_serial | required | status | FIRM | +| `chs_shutdown` | api | process_serial | any | `void` | FIRM | +| `chs_type_validate` | api | any | required | status | FIRM | +| `chs_back_quote` | api | any | required | status | FIRM | +| `chs_back_quote_if_needed` | api | any | required | status | FIRM | +| `chs_quote_string` | api | any | required | status | FIRM | +| `chs_server_create` | api | any | required | status | FIRM | +| `chs_server_free` | api | handle_serial | any | `void` | FIRM | +| `chs_schema_create` | api | shared | required | status | FIRM | +| `chs_schema_free` | api | handle_serial | any | `void` | FIRM | +| `chs_schema_describe` | api | shared | required | status | FIRM | +| `chs_preview_row` | api | shared | required | status | FIRM | +| `chs_preview_batch` | api | shared | required | status | FIRM | +| `chs_filter_create` | api | shared | required | status | FIRM | +| `chs_filter_free` | api | handle_serial | any | `void` | FIRM | +| `chs_filter_eval_body` | api | shared | required | status | FIRM | +| `chs_block_create` | api | shared | required | status | FIRM | +| `chs_block_free` | api | handle_serial | any | `void` | FIRM | +| `chs_filter_eval_block` | api | shared | required | status | FIRM | +| `chs_discover_query` | api | any | required | status | FIRM | +| `chs_discover_columns` | api | any | required | status | FIRM | #### `chs_abi_version` @@ -1551,7 +1920,7 @@ Its members, as JSON Schema: CHS_API int32_t chs_abi_version(void); ``` -- Class `handshake`, thread `any`, FIRM. +- Class `handshake`, thread `any`, init `any`, FIRM. The ABI generation this library implements: always 2 for this header. A loader resolves and calls it right after opening the library. A library without the symbol is not a chtypes ABI artifact of generation 1 or later, and any other value is refused, naming both values. An absent handshake symbol other than this one is refused as `missing_symbol:` instead, at whichever step first resolves it — only this symbol's absence means the library is not such an artifact at all. @@ -1561,7 +1930,7 @@ The ABI generation this library implements: always 2 for this header. A loader r CHS_API const char *chs_build_info(void); ``` -- Class `handshake`, thread `any`, FIRM. +- Class `handshake`, thread `any`, init `any`, FIRM. What this library is, as static, NUL-terminated, ASCII-only JSON in the image's read-only data: never NULL, never freed, the same bytes on every call, and callable straight after opening the library. Its fields are listed under `build_info` in the reference, and `abi_fingerprint` is `CHS_ABI_FINGERPRINT` as the library was built, copied, never recomputed. @@ -1575,7 +1944,7 @@ A loader parses it (ASCII only, duplicate keys refused, `schema` equal to 1, eve CHS_API const char *chs_clickhouse_version(void); ``` -- Class `handshake`, thread `any`, FIRM. +- Class `handshake`, thread `any`, init `any`, FIRM. The ClickHouse release this library was built from, spelled as every v0 library spelled it (for example `26.8.15.10-lts`): static, never freed. It keeps its v0 signature and spelling so a v0 binding reaches its own refusal. A binding of generation 1 or later reads `clickhouse_version` from `chs_build_info` instead. @@ -1585,7 +1954,7 @@ The ClickHouse release this library was built from, spelled as every v0 library CHS_API int chs_abi_revision(void); ``` -- Class `tombstone`, thread `any`, FIRM. +- Class `tombstone`, thread `any`, init `any`, FIRM. - Always returns `CHS_ABI_REVISION_TOMBSTONE`. The v0 tombstone. It always returns `CHS_ABI_REVISION_TOMBSTONE` (1001). It must never be removed, and its name must never be given another meaning. @@ -1598,7 +1967,7 @@ A sweep of every released binding (every minor from 0.1 to 0.5, in all four lang CHS_API const uint8_t *chs_buf_data(const chs_buf *buf); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - The pointer is borrowed from `buf`. - `buf`: handle, `chs_buf`. @@ -1610,7 +1979,7 @@ The buffer's bytes. The pointer may be NULL when `chs_buf_len` is 0, and a bindi CHS_API size_t chs_buf_len(const chs_buf *buf); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - `buf`: handle, `chs_buf`. The buffer's length in bytes. A NULL, freed, wrong-kind or cross-library buffer gives 0. @@ -1621,7 +1990,7 @@ The buffer's length in bytes. A NULL, freed, wrong-kind or cross-library buffer CHS_API void chs_buf_free(chs_buf *buf); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `buf`: handle, `chs_buf`, nullable. Releases the caller's reference to a buffer. Freeing NULL does nothing. @@ -1632,7 +2001,7 @@ Releases the caller's reference to a buffer. Freeing NULL does nothing. CHS_API chs_status chs_error_status(const chs_error *error); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - `error`: handle, `chs_error`. The error's status, never `CHS_OK` for an error a call set. A NULL, freed, wrong-kind or cross-library error gives `CHS_INVALID_ARGUMENT`. @@ -1643,7 +2012,7 @@ The error's status, never `CHS_OK` for an error a call set. A NULL, freed, wrong CHS_API int32_t chs_error_ch_code(const chs_error *error); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - `error`: handle, `chs_error`. ClickHouse's own error code, nonzero only when the status is `CHS_REJECTED`, and 0 otherwise or for an invalid error. @@ -1654,7 +2023,7 @@ ClickHouse's own error code, nonzero only when the status is `CHS_REJECTED`, and CHS_API chs_buf *chs_error_ch_name(const chs_error *error); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - Returns a new `chs_buf` the caller owns (ascii), freed with `chs_buf_free`. - `error`: handle, `chs_error`. @@ -1666,11 +2035,11 @@ A new buffer holding the name this build's vendored error table gives the code, CHS_API chs_buf *chs_error_message(const chs_error *error); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - Returns a new `chs_buf` the caller owns (message), freed with `chs_buf_free`. - `error`: handle, `chs_error`. -A new buffer holding the message: ClickHouse's own text, verbatim, for `CHS_REJECTED`, and the library's explanation otherwise. A message may quote input bytes, so it is a byte string, not text. +A new buffer holding the message: ClickHouse's own text, verbatim, for `CHS_REJECTED`, and the library's explanation otherwise. A message may quote input bytes, so it is a byte string, not text. It is informative: a caller keys on the status, the code and the reason (`chs_error_reason`), never on message text. #### `chs_error_column` @@ -1678,19 +2047,31 @@ A new buffer holding the message: ClickHouse's own text, verbatim, for `CHS_REJE CHS_API chs_buf *chs_error_column(const chs_error *error); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `any`, FIRM. - Returns a new `chs_buf` the caller owns (name), freed with `chs_buf_free`. - `error`: handle, `chs_error`. A new buffer holding the name of the column the error concerns, as raw bytes, empty when there is none. A column name is a byte string: NUL and invalid UTF-8 are legal in a name. +#### `chs_error_reason` + +```c +CHS_API chs_buf *chs_error_reason(const chs_error *error); +``` + +- Class `api`, thread `shared`, init `any`, FIRM. +- Returns a new `chs_buf` the caller owns (ascii), freed with `chs_buf_free`. +- `error`: handle, `chs_error`. + +A new buffer holding the reason: the catalog id (`spec/abi-v2/declines/declines.json`, ASCII) of a decline or a library-side refusal, empty when the error carries none (ClickHouse's own refusal, an internal error). It is the member a caller keys on, never the message. Like the other text accessors, it returns NULL only for a NULL or invalid error. + #### `chs_error_free` ```c CHS_API void chs_error_free(chs_error *error); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `error`: handle, `chs_error`, nullable. Releases the caller's reference to an error. Freeing NULL does nothing. @@ -1701,7 +2082,7 @@ Releases the caller's reference to an error. Freeing NULL does nothing. CHS_API chs_status chs_live_handles(chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `out`: out_handle, `chs_buf`, `document:live_handles`. - `err`: out_error. @@ -1714,7 +2095,7 @@ A JSON document counting the live handles of each kind this image has made, take CHS_API chs_status chs_error_codes(chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `out`: out_handle, `chs_buf`, `document:error_code_table`. - `err`: out_error. @@ -1727,7 +2108,7 @@ ClickHouse's own error-code table for this build, as a JSON array of `{"code", " CHS_API chs_status chs_registered_families(chs_buf **out, chs_error **err); ``` -- Class `tooling`, thread `process_serial`, FIRM. +- Class `tooling`, thread `process_serial`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `out`: out_handle, `chs_buf`, `ascii`. - `err`: out_error. @@ -1740,7 +2121,7 @@ Every type family in this build's own type registry, one per line. A tooling exp CHS_API chs_status chs_function_flags(chs_buf **out, chs_error **err); ``` -- Class `tooling`, thread `process_serial`, FIRM. +- Class `tooling`, thread `process_serial`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `out`: out_handle, `chs_buf`, `ascii`. - `err`: out_error. @@ -1753,7 +2134,7 @@ A tab-separated audit of every registered function's volatility flags, one funct CHS_API chs_status chs_reference_type(const uint8_t *type_expr, size_t type_expr_len, chs_buf **out, chs_error **err); ``` -- Class `tooling`, thread `process_serial`, FIRM. +- Class `tooling`, thread `process_serial`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `type_expr`: bytes_in, `sql`. - `out`: out_handle, `chs_buf`, `sql`. @@ -1767,7 +2148,7 @@ The widened reference type this build pairs with a type expression. A tooling ex CHS_API chs_status chs_initialize(const uint8_t *timezone, size_t timezone_len, chs_error **err); ``` -- Class `api`, thread `process_once`, FIRM. +- Class `api`, thread `process_once`, init `any`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `timezone`: bytes_in, `timezone`. - `err`: out_error. @@ -1782,12 +2163,14 @@ The zone is process state, not a per-call parameter, because ClickHouse's own Me CHS_API chs_status chs_set_defaults(const uint8_t *settings, size_t settings_len, chs_error **err); ``` -- Class `api`, thread `process_serial`, FIRM. +- Class `api`, thread `process_serial`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. - `err`: out_error. -Seeds the settings every later call starts from, as a JSON object whose values are JSON strings (a binding never rewrites a value, a boolean included). Each successful call replaces the previous defaults whole. A setting name the server would refuse is refused here, with ClickHouse's own code and message, and nothing is committed. `session_timezone` here is the default zone for later calls; it does not change the image zone that compiled types bind. +Seeds the settings every later call starts from, as a JSON object whose values are JSON strings, or the object form for bytes (Input values), and a binding never rewrites a value, a boolean included. Each successful call replaces the previous defaults whole. A setting name the server would refuse is refused here, with ClickHouse's own code and message, and nothing is committed. `session_timezone` here is the default zone for later calls; it does not change the image zone that compiled types bind. + +The library's own knobs are `chtypes_*` settings, and this is the only channel for the process-level ones. Each is an integer or a string given as a JSON string, and no server knows one, so a server profile refuses it (115) and a call's settings carry only the per-call ones. The knobs, their layer, type, default and effect: `chtypes_now_epoch_nanos` (per call, int64 nanoseconds since the epoch, default the wall clock: pins the call's clock instant); `chtypes_clock_offset_nanos` (per call, int64, default 0: shifts the instant); `chtypes_max_clock_skew_nanos` (per call, int64, default 0: the skew budget a clock-reading expression may resolve under); `chtypes_default_eval_memory_bytes` (process, int64 bytes, default 268435456, 0 disables: the admission memory budget one DEFAULT, CHECK, TTL or filter evaluation may use); `chtypes_default_eval_cpu_nanos` (process, int64, default 1000000000, 0 disables: the admission time budget, the evaluating thread's CPU time, detected not interrupted; the earlier name `chtypes_default_eval_wall_nanos` is refused on v2 with `CHS_INVALID_ARGUMENT` naming this one); `chtypes_custom_settings_prefixes` (process, string, default none: the server's `custom_settings_prefixes`, comma-separated). A process knob in a call's settings is declined, named in `unsupported_settings` (`call-setting-process-only-key`), and a clock knob whose value is not an integer likewise (`call-setting-clock-value`). An evaluation that exceeds a budget is declined with that budget's own reason id, never answered: `admission-memory-budget` for the memory budget and `admission-time-budget` for the time budget; the earlier single id `admission-envelope` is retired, its catalog entry naming both successors (rule r4). The time budget is the evaluating thread's CPU time, not wall time. Each catalog entry states its load dependence. A memory-budget decline is deterministic (`measured`). A time-budget decline depends on load only near the budget: CPU time rose up to x1.03 under load on linux-arm64 and up to x2.0 on linux-amd64 when two busy hardware threads share a core, so a time-budget decline is retry-stable when the evaluation costs more than the budget on an idle core and never happens under about half of it; between half and one times the budget the host's load can decide. Normal evaluations cost about 1000 times less than the 1 s default. a schema whose DEFAULT exceeds a budget at admission is declined at compile. The defaults are this description's, not a build's: a build reports nothing about them. Setup only. The defaults are immutable once this image has created its first `chs_schema` (a filter or a block needs one): from then on the call changes nothing and answers `CHS_INVALID_ARGUMENT`. No call ever reads defaults that change under it, so no binding needs a lock around them. @@ -1797,7 +2180,7 @@ Setup only. The defaults are immutable once this image has created its first `ch CHS_API void chs_shutdown(void); ``` -- Class `api`, thread `process_serial`, FIRM. +- Class `api`, thread `process_serial`, init `any`, FIRM. Stops what `chs_initialize` started (its background threads) and joins them. It is idempotent and safe before `chs_initialize`. After it, no call is valid except `chs_shutdown` again. Unloading the library, or initializing it again in the same process, is unsupported: a loader never unloads a library. @@ -1807,7 +2190,7 @@ Stops what `chs_initialize` started (its background threads) and joins them. It CHS_API chs_status chs_type_validate(const uint8_t *type_expr, size_t type_expr_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `type_expr`: bytes_in, `sql`. - `out`: out_handle, `chs_buf`, `sql`. @@ -1821,7 +2204,7 @@ Parses one type expression with ClickHouse's own parser and returns its canonica CHS_API chs_status chs_back_quote(const uint8_t *name, size_t name_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `name`: bytes_in, `name`. - `out`: out_handle, `chs_buf`, `sql`. @@ -1835,7 +2218,7 @@ A name quoted the way ClickHouse's own `backQuote` quotes it: always quoted, eve CHS_API chs_status chs_back_quote_if_needed(const uint8_t *name, size_t name_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `name`: bytes_in, `name`. - `out`: out_handle, `chs_buf`, `sql`. @@ -1849,7 +2232,7 @@ A name quoted only where this build's own `backQuoteIfNeed` says it must be. Whi CHS_API chs_status chs_quote_string(const uint8_t *text, size_t text_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `text`: bytes_in, `bytes`. - `out`: out_handle, `chs_buf`, `sql`. @@ -1863,7 +2246,7 @@ A byte string spelled as a ClickHouse string literal by the vendored `quoteStrin CHS_API chs_status chs_server_create(const uint8_t *profile, size_t profile_len, const uint8_t *options, size_t options_len, chs_server **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - Growable (rule r1): an input it gains is a new member of its options document, `options`. - `profile`: bytes_in, `input:server_profile`. @@ -1877,11 +2260,11 @@ Everything is validated here, once, and the server never changes after it. A key `CHS_DECLINED` is reserved: a build that will not describe a profile declines it, which is never a wrong answer. The one decline foreseen is a setting outside the set that shapes an input, at a value other than its default, once that set is generated from the vendored source. -The macros are the server's whole `system.macros` when `macros` is present, even as `{}`: a macro a Replicated engine's arguments name that the set lacks is then the server's own refusal (`CHS_REJECTED`, 139, NO_ELEMENTS_IN_CONFIG). When `macros` is absent the server's macros are unknown, and a schema whose engine reads one is declined, as it is without a server. A build that validates a profile's macros but does not yet apply them declines such a schema in both cases. +The macros are the server's whole `system.macros` when `macros` is present, even as `{}`: a macro a Replicated engine's arguments name that the set lacks is then the server's own refusal (`CHS_REJECTED`, 139, NO_ELEMENTS_IN_CONFIG). When `macros` is absent the server's macros are unknown, and a schema whose engine reads one is declined, as it is without a server, except under `schema_options`' `posture` `load`, where a stored Replicated definition compiles without expanding them (`input:schema_options`). A build that validates a profile's macros but does not yet apply them declines such a schema in both cases. A setting this build's filters do not honor in a WHERE is no decline here, and the call succeeds. So does `chs_schema_create` on the server, because a profile never changes a schema compile: it lists the setting in the schema's `filter_declined_settings` (`schema_description`), at layer `server`. -The call reads no defaults and leaves `chs_set_defaults`' latch open. Like every call that is not the handshake, it is valid only after `chs_initialize`. +The call reads no defaults and leaves `chs_set_defaults`' latch open. Its init class is `required`, so before `chs_initialize` it refuses (Initialization, in the Preamble). #### `chs_server_free` @@ -1889,7 +2272,7 @@ The call reads no defaults and leaves `chs_set_defaults`' latch open. Like every CHS_API void chs_server_free(chs_server *server); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `server`: handle, `chs_server`, nullable. Releases the caller's reference to a server. Schemas created on it keep it alive. Freeing NULL does nothing. @@ -1900,17 +2283,19 @@ Releases the caller's reference to a server. Schemas created on it keep it alive CHS_API chs_status chs_schema_create(const chs_server *server, const uint8_t *create_table, size_t create_table_len, const uint8_t *settings, size_t settings_len, const uint8_t *options, size_t options_len, chs_schema **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - Growable (rule r1): an input it gains is a new member of its options document, `options`. - `server`: handle, `chs_server`, nullable. - `create_table`: bytes_in, `sql`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. - `options`: bytes_in, `input:schema_options`. - `out`: out_handle, `chs_schema`. - `err`: out_error. -Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, settings and constraints) with ClickHouse's own parser and the checks a server's CREATE runs, on `server`, under a profile of settings given as a JSON object of string values. A trailing semicolon is allowed. A second statement is refused the way ClickHouse's own parser refuses one in a single query (`CHS_REJECTED`, with its code and message), and a statement of any other kind is `CHS_INVALID_ARGUMENT`. +Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, settings and constraints) with ClickHouse's own parser and the checks a server's CREATE runs, on `server`, under a profile of settings given as a JSON object of string values. A trailing semicolon is allowed. A second statement is refused the way ClickHouse's own parser refuses one in a single query (`CHS_REJECTED`, with its code and message), and a statement of any other kind is `CHS_INVALID_ARGUMENT`, except under `posture` `load`: there a `CREATE VIEW`, `CREATE MATERIALIZED VIEW` or `CREATE DICTIONARY` statement, the stored definition of an object a server lists in `system.tables` and whose columns it lists in `system.columns`, is `CHS_DECLINED` with the reason `engine-not-modeled`, so a caller that compiles a server's stored definitions falls back for it as it does for an engine this build does not model. The `ATTACH` spellings of those kinds that a server refuses with 62 stay `CHS_REJECTED` with that code, as the server answers. An `ATTACH TABLE` statement is not one of the other kinds (`options`, below). The library models a table, not its database's engine: `ATTACH TABLE` with a column list and an engine but no UUID is accepted under `posture` `load`, which a server's Atomic database refuses (80), and under `create` it is declined (`attach-statement-under-create`). + +A table whose engine this build does not model (a `Distributed` or `Buffer` table, for example) is `CHS_DECLINED` with the reason `engine-not-modeled`, which is returned only when the engine itself is the cause: a decline for anything else in the statement carries that thing's own reason. `server` is the server the table is on (`chs_server_create`), or NULL. On a server, the schema's home zone is the server's `timezone`: its zone-less `DateTime` and `DateTime64` columns bind it, as a restarted server binds its own, and every call on the schema that names no `session_timezone` runs in it. The server's `settings` layer under the schema's own, and a Replicated engine's ZooKeeper path and replica name expand the server's `macros`. With `server` NULL the schema is on the image's own server, exactly as before servers existed: the image zone, no server settings layer, and macros unknown. A freed server, a handle of another kind or a server from another library image is `CHS_INVALID_ARGUMENT`. The schema holds a counted reference to its server, so the caller may free the server at any time. @@ -1918,7 +2303,7 @@ Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, setti A setting this build's filters do not honor in a WHERE, from the `chs_set_defaults` snapshot this call captures, the server profile or this call's own `settings`, is no decline here: the call succeeds and lists it in the schema's `filter_declined_settings` (`schema_description`), and `chs_filter_create` on the schema declines. -`options` (`input:schema_options`) defines no member yet, so any key in it is `CHS_INVALID_ARGUMENT`, naming the key. +`options` (`input:schema_options`) sets the `posture` the statement compiles in: `create`, the default, or `load` for a server's own stored definition (see `input:schema_options`, and the caller's obligation there). A `posture` this build does not know, or any other key, is `CHS_INVALID_ARGUMENT`, naming it. An `ATTACH TABLE` statement is a stored definition, never a statement of another kind: under `create` it is `CHS_DECLINED`, ch_code 0, with exactly the message `chs_schema_create: an ATTACH TABLE statement is a stored definition; compile it with schema_options {"posture":"load"}; declined rather than answered`, and under `load` it compiles as the same definition spelled `CREATE TABLE`. A build that does not yet compile on a server declines a non-NULL `server`, and a schema whose compile needs its server's zone is declined on a host thread that already runs a ClickHouse query of its own. Each is `CHS_DECLINED`, which is never a wrong answer. @@ -1930,7 +2315,7 @@ The result is immutable: nothing changes a schema after this call, so it replace CHS_API void chs_schema_free(chs_schema *schema); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `schema`: handle, `chs_schema`, nullable. Releases the caller's reference to a schema. Filters and blocks made from it keep it alive. Freeing NULL does nothing. @@ -1941,84 +2326,94 @@ Releases the caller's reference to a schema. Filters and blocks made from it kee CHS_API chs_status chs_schema_describe(const chs_schema *schema, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `schema`: handle, `chs_schema`. - `out`: out_handle, `chs_buf`, `document:schema_description`. - `err`: out_error. A JSON document describing the schema's columns, owned by the caller; it replaces v0's borrowed column accessors. -A deep input the server's stack check refuses is `CHS_REJECTED` with ch_code 306. Here the input is the schema's own statement, which describing walks again; it is never `CHS_INTERNAL`. +A schema compiled in `load` carries `"posture": "load"`; one compiled in `create` carries no `posture`. + +A schema `chs_schema_create` returned always describes: the walk this document needs runs at compile, so a deep statement is refused there (`CHS_REJECTED`, 306) and this call never refuses or declines a compiled schema. It is never `CHS_INTERNAL`. #### `chs_preview_row` ```c -CHS_API chs_status chs_preview_row(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, chs_buf **out, chs_error **err); +CHS_API chs_status chs_preview_row(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, const uint8_t *options, size_t options_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Growable (rule r1): an input it gains is a new member of its options document, `options`. - `schema`: handle, `chs_schema`. - `format`: enum, `chs_format`. - `body`: bytes_in, `bytes`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. - `columns`: bytes_in, `json_array_names`. +- `options`: bytes_in, `input:insert_options`. - `out`: out_handle, `chs_buf`, `document:row`. - `err`: out_error. Validates and coerces one row of `body` under the schema, as a server's INSERT would, and returns the row's document. `columns` is the INSERT column list, a JSON array of name objects (`{name}` or `{name_b64}`), and empty for none. `session_timezone` in `settings` is this call's zone. -`body` holds exactly one row, and the library reads all of it, with the same reader as `chs_preview_batch`. A malformed tail, or trailing content the format's reader does not itself skip, is rejected with the reader's own code, as a server's INSERT rejects it; what the reader skips, such as a trailing newline, is fine. A body holding no row, or more than one, is `CHS_INVALID_ARGUMENT`, naming the call: `chs_preview_batch` answers a body of many rows. The row's document equals the single row of `chs_preview_batch`'s document for the same body. +`body` holds exactly one row, and the library reads all of it, with the same reader as `chs_preview_batch`. A malformed tail, or trailing content the format's reader does not itself skip, is rejected with the reader's own code, as a server's INSERT rejects it; what the reader skips, such as a trailing newline, is fine. A body holding no row, or more than one, is `CHS_INVALID_ARGUMENT`, naming the call: `chs_preview_batch` answers a body of many rows. The row's document equals the single row of `chs_preview_batch`'s document for the same body, and stands for the call: a refusal or decline of the call, its settings, its column list or its options is this document's `outcome` with `refused_at` `call`, never a status. An INSERT whose input block has no column at all (every insertable column EPHEMERAL, and no column list) is declined: a server answers it (code 90, EMPTY_LIST_OF_COLUMNS_PASSED) from a statement inside its INSERT interpreter that the library cannot call, and a decline is never a wrong answer. +`options` (`input:insert_options`) shapes the INSERT the call models, and `{}` is the INSERT it modeled before the document existed. + #### `chs_preview_batch` ```c -CHS_API chs_status chs_preview_batch(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, const chs_filter *filter, int32_t export_format, uint32_t doc_flags, chs_buf **out, chs_buf **out_export, chs_error **err); +CHS_API chs_status chs_preview_batch(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, const chs_filter *filter, int32_t export_format, uint32_t doc_flags, const uint8_t *options, size_t options_len, chs_buf **out, chs_buf **out_export, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Growable (rule r1): an input it gains is a new member of its options document, `options`. - `schema`: handle, `chs_schema`. - `format`: enum, `chs_format`. - `body`: bytes_in, `bytes`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. - `columns`: bytes_in, `json_array_names`. - `filter`: handle, `chs_filter`, nullable. - `export_format`: scalar, `int32`. - `doc_flags`: scalar, `uint32`. +- `options`: bytes_in, `input:insert_options`. - `out`: out_handle, `chs_buf`, `document:batch`. - `out_export`: out_handle, `chs_buf`, `bytes`, nullable. - `err`: out_error. -Validates and coerces a whole body, which may hold many rows, and returns the batch document, with the same declines as `chs_preview_row`. Row separation and the server's error allowance are ClickHouse's own, and the document's `unconsumed` and `framing` say what the reader skipped and how it framed the body. `filter`, when given, is evaluated over each stored row in the same parse. When that filter's WHERE would read a setting this build's filters do not honor, from the filter's own `settings`, the name joins the batch document's top-level `unsupported_settings` and the batch is `unsupported` by `batch_outcome`'s step 2: every row `unsupported`, filter rows `d`, and no export. The same call without the filter is unchanged. `export_format` is `CHS_EXPORT_NONE` or a `chs_format` the build can write; with an export that runs, `out_export` receives the accepted rows serialized once, an empty buffer when the batch holds no row. When the export is declined (the document carries `export_declined`), `out_export` is NULL, as it may be when no export is asked for, so a caller tells a declined export from an empty one by `export_declined`, never by the buffer. `doc_flags` chooses which groups the per-row documents carry. +Validates and coerces a whole body, which may hold many rows, and returns the batch document, with the same declines as `chs_preview_row`. Row separation and the server's error allowance are ClickHouse's own, and the document's `unconsumed` and `framing` say what the reader skipped and how it framed the body. `filter`, when given, is evaluated over each stored row in the same parse. When that filter's WHERE would read a setting this build's filters do not honor, from the filter's own `settings`, the name joins the batch document's top-level `unsupported_settings` and the batch is `unsupported` by `batch_outcome`'s step 2: every row `unsupported`, filter rows `d`, and no export. The same call without the filter is unchanged. `export_format` is `CHS_EXPORT_NONE` or a `chs_format` the build can write; with an export that runs, `out_export` receives the accepted rows serialized once, an empty buffer when the batch holds no row. When the export is declined (the document carries `export_declined` and `export_declined_reason`), `out_export` is NULL, as it may be when no export is asked for, so a caller tells a declined export from an empty one by `export_declined`, never by the buffer. `doc_flags` chooses which groups the per-row documents carry. `options` (`input:insert_options`) shapes the INSERT the call models, and `{}` is the INSERT it modeled before the document existed. `filter` must have been compiled on `schema`: a filter from another schema handle is `CHS_INVALID_ARGUMENT`, as it is at `chs_filter_eval_block`. The call returns a status other than `CHS_OK` only for misuse or an internal error: a decline of the call (the filter's zone differing from the batch's, `preview-batch-zone-mismatch`) and a refusal of its arguments are the document's, with `refused_at` `call` and `reason`. The export's truth table is in `document:batch`. #### `chs_filter_create` ```c -CHS_API chs_status chs_filter_create(const chs_schema *schema, const uint8_t *expr, size_t expr_len, const uint8_t *query_params, size_t query_params_len, const uint8_t *settings, size_t settings_len, chs_filter **out, chs_error **err); +CHS_API chs_status chs_filter_create(const chs_schema *schema, const uint8_t *expr, size_t expr_len, const uint8_t *query_params, size_t query_params_len, const uint8_t *settings, size_t settings_len, const uint8_t *options, size_t options_len, chs_filter **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. +- Class `api`, thread `shared`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Growable (rule r1): an input it gains is a new member of its options document, `options`. - `schema`: handle, `chs_schema`. - `expr`: bytes_in, `sql`. -- `query_params`: bytes_in, `json_object_string_values`. -- `settings`: bytes_in, `json_object_string_values`. +- `query_params`: bytes_in, `json_object_byte_values`. +- `settings`: bytes_in, `json_object_byte_values`. +- `options`: bytes_in, `input:filter_options`. - `out`: out_handle, `chs_filter`. - `err`: out_error. -Compiles a boolean SQL expression over the schema's columns, binding its query parameters (a JSON object of string values) by ClickHouse's own substitution, so a value is never SQL text. `settings` is the profile the expression is compiled under, so `session_timezone` there is the zone of its literals. Non-deterministic expressions are declined. +Compiles a boolean SQL expression over the schema's columns, binding its query parameters (a JSON object of string values) by ClickHouse's own substitution, so a value is never SQL text. A parameter's value is decoded by ClickHouse's escaped-text reader, exactly as a server decodes an HTTP `param_` value (`input:filter_options`' `param_encoding`, `escaped`): `\b` is an escape, so the parameter `a\b` reads as `a`, the byte 0x08 and `b`, and a literal backslash is written `\\`; `\N` reads as the empty value; `\x41`, `\'` and `\0` decode as that reader decodes them; a raw tab or newline is `CHS_REJECTED` with 457 and a trailing backslash with 25, as a server refuses them; and the bytes an escape produces are kept, valid UTF-8 or not. The object form `{"b64": ...}` (Input values) is decoded from base64 first, and its bytes are then read the same way. `options` (`input:filter_options`) shapes the compile, and `{}` is the compile before the document existed. `settings` is the profile the expression is compiled under, so `session_timezone` there is the zone of its literals. Non-deterministic expressions are declined. The WHERE's settings are layered, lowest first: the build's own, `chs_set_defaults`, the schema's server profile, the schema's own settings, and then this call's `settings`; `session_timezone` at every layer stays on the zone path. A type gate in the schema's own settings binds the CREATE only, so a WHERE's CAST validates under the session layers, every layer but the schema's own. -On a schema whose `filter_declined_settings` is not empty (`chs_schema_describe`), from any layer, the call is `CHS_DECLINED` with ch_code 0. It is checked after the argument checks and before the expression is parsed, because `dialect` can change the parse, and the message names each setting and its layer, in the list's order: `chs_filter_create: the settings this schema was compiled under set (), (), which this build's filters do not honor in a WHERE (listed in chs_schema_describe's filter_declined_settings); declined rather than answered`, where each `` is the entry's `declined_layer` value. The entries are joined by a comma and a space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as `b64:`; it does not occur in practice, because a setting name ClickHouse does not know is refused earlier, with 115. The server and the schema were created all the same: no settings layer changes a schema compile. +On a schema whose `filter_declined_settings` is not empty (`chs_schema_describe`), from any layer, the call is `CHS_DECLINED` with ch_code 0. It is checked after the argument checks and before the expression is parsed, because `dialect` can change the parse, and the message names each setting and its layer, in the list's order: `chs_filter_create: the settings this schema was compiled under set (), (), which this build's filters do not honor in a WHERE (listed in chs_schema_describe's filter_declined_settings); declined rather than answered`, where each `` is the entry's `declined_layer` value, with the reason `server-profile-setting`. The message is informative; the entries are joined by a comma and a space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as `b64:`; it does not occur in practice, because a setting name ClickHouse does not know is refused earlier, with 115. The server and the schema were created all the same: no settings layer changes a schema compile. **Changed (public issue #588).** A declined setting from the `chs_set_defaults` or the schema layer moves from evaluation time to create time: it was named at evaluation (`filter_result`'s `unsupported_settings`), and it now declines this call. A server-profile setting already declined it. -A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here, and it is never in `schema_description`. A key in this build's honor set is applied to the WHERE; a key passed through is accepted and ignored; and a key this build's filters do not honor is reported at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`, and `chs_preview_batch`'s). The WHERE runs under this call's layers only (the defaults, the server profile, the schema's own settings and this call's), as a subscriber's SELECT in its own session would. An evaluation's `settings` (`chs_filter_eval_body`, `chs_preview_batch` with a filter, `chs_block_create`) are the INSERT session's: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings. So a filter evaluated in the INSERT's own session, a materialized view's WHERE for example, is created with that session's WHERE-tier settings in its own `settings`. Every key still meets the server's own SET check first, so a name the server does not know is `CHS_REJECTED` with ch_code 115. +A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here but for one case, and it is never in `schema_description`. The case is a cell of the line's `where-settings/documented-pathological.tsv`, a value under which the server refuses every query: set in this call's own `settings`, it is `CHS_DECLINED` with ch_code 0 and the reason `setting-server-refuses`, because no verdict under it could stand. A key in this build's honor set is applied to the WHERE; a key passed through is accepted and ignored; and a key this build's filters do not honor is reported at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`, and `chs_preview_batch`'s). The WHERE runs under this call's layers only (the defaults, the server profile, the schema's own settings and this call's), as a subscriber's SELECT in its own session would. An evaluation's `settings` (`chs_filter_eval_body`, `chs_preview_batch` with a filter, `chs_block_create`) are the INSERT session's: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings. So a filter evaluated in the INSERT's own session, a materialized view's WHERE for example, is created with that session's WHERE-tier settings in its own `settings`. Every key still meets the server's own SET check first, so a name the server does not know is `CHS_REJECTED` with ch_code 115. A filter under a per-call `session_timezone` answers as the artifact producer has measured against live servers, including how the zone in an evaluation's settings combines with the zone the filter was compiled under. Until a path is measured to match a server it declines, which is never a wrong answer. The filter holds a counted reference to the schema. @@ -2028,7 +2423,7 @@ A filter under a per-call `session_timezone` answers as the artifact producer ha CHS_API void chs_filter_free(chs_filter *filter); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `filter`: handle, `chs_filter`, nullable. Releases the caller's reference to a filter. Freeing NULL does nothing. @@ -2036,37 +2431,42 @@ Releases the caller's reference to a filter. Freeing NULL does nothing. #### `chs_filter_eval_body` ```c -CHS_API chs_status chs_filter_eval_body(const chs_filter *filter, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, chs_buf **out, chs_error **err); +CHS_API chs_status chs_filter_eval_body(const chs_filter *filter, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, const uint8_t *options, size_t options_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Growable (rule r1): an input it gains is a new member of its options document, `options`. - `filter`: handle, `chs_filter`. - `format`: enum, `chs_format`. - `body`: bytes_in, `bytes`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. +- `columns`: bytes_in, `json_array_names`. +- `options`: bytes_in, `input:insert_options`. - `out`: out_handle, `chs_buf`, `document:filter_result`. - `err`: out_error. -Evaluates the filter over every row of a body, returning one verdict per row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. When a server would not accept an INSERT of the body, because `chs_preview_batch`'s verdict over the same body and settings, with no filter, is not `accepted`, every verdict is `d` and the outcome mirrors that verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts and that batch verdict together. +Evaluates the filter over every row of a body, returning one verdict per row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. `columns` is the INSERT column list, a JSON array of name objects (`{name}` or `{name_b64}`), and empty for none, read exactly as `chs_preview_batch` reads it: with a list the call models the INSERT that names it, and with none the INSERT that names none. When a server would not accept an INSERT of the body, because `chs_preview_batch`'s verdict over the same body, settings and column list, with no filter, is not `accepted`, every verdict is `d` and the outcome mirrors that verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts and that batch verdict together, and under any outcome but `ok` writes `d` for every verdict, answering a refusal or decline of the call itself in the document (`refused_at` `call`), never as a status. `options` (`input:insert_options`) is that INSERT's, read as `chs_preview_batch` reads it, so the verdict mirrored is `chs_preview_batch`'s over the same body, settings, columns and options. #### `chs_block_create` ```c -CHS_API chs_status chs_block_create(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, chs_block **out, chs_error **err); +CHS_API chs_status chs_block_create(const chs_schema *schema, chs_format format, const uint8_t *body, size_t body_len, const uint8_t *settings, size_t settings_len, const uint8_t *columns, size_t columns_len, const uint8_t *options, size_t options_len, chs_block **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Growable (rule r1): an input it gains is a new member of its options document, `options`. - `schema`: handle, `chs_schema`. - `format`: enum, `chs_format`. - `body`: bytes_in, `bytes`. -- `settings`: bytes_in, `json_object_string_values`. +- `settings`: bytes_in, `json_object_byte_values`. - `columns`: bytes_in, `json_array_names`. +- `options`: bytes_in, `input:insert_options`. - `out`: out_handle, `chs_block`. - `err`: out_error. -Parses a body once under the schema, for evaluating many filters over it. `settings` governs the parse, `session_timezone` there included, and `columns` is read as `chs_preview_row` reads it. The block holds a counted reference to the schema. It also records `chs_preview_batch`'s verdict over the same body, settings and columns, with no filter, which `chs_filter_eval_block` answers by. An accepted body is read once, for the block and that verdict together. +Parses a body once under the schema, for evaluating many filters over it. `settings` governs the parse, `session_timezone` there included, and `columns` is read as `chs_preview_row` reads it. The block holds a counted reference to the schema. It also records `chs_preview_batch`'s verdict over the same body, settings, columns and options, with no filter, which `chs_filter_eval_block` answers by; a refusal or decline of the call or the body is recorded there, so this call returns a status other than `CHS_OK` only for misuse or an internal error. An accepted body is read once, for the block and that verdict together. `options` (`input:insert_options`) is read as `chs_preview_batch` reads it. #### `chs_block_free` @@ -2074,7 +2474,7 @@ Parses a body once under the schema, for evaluating many filters over it. `setti CHS_API void chs_block_free(chs_block *block); ``` -- Class `api`, thread `handle_serial`, FIRM. +- Class `api`, thread `handle_serial`, init `any`, FIRM. - `block`: handle, `chs_block`, nullable. Releases the caller's reference to a block. Freeing NULL does nothing. @@ -2085,14 +2485,14 @@ Releases the caller's reference to a block. Freeing NULL does nothing. CHS_API chs_status chs_filter_eval_block(const chs_filter *filter, const chs_block *block, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `shared`, FIRM. -- May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. +- Class `api`, thread `shared`, init `required`, FIRM. +- May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `filter`: handle, `chs_filter`. - `block`: handle, `chs_block`. - `out`: out_handle, `chs_buf`, `document:filter_result`. - `err`: out_error. -Evaluates the filter over a parsed block. Over a body a server would accept, it gives the same answers `chs_filter_eval_body` gives for that body. Over one it would not (the verdict `chs_block_create` recorded is not `accepted`), the outcome mirrors that verdict as for `chs_filter_eval_body`, and every verdict from the first refused row on is `d`. The first refused row is the first row of that batch whose outcome is neither `accepted` nor `skipped`. When the refusal belongs to no row (a whole-body step, a poisoned value, the call's own settings), it is the first row of the block. The rows before it keep their evaluated verdicts. A caller treats those as `d` too: any non-`ok` outcome, including `unsupported`, means every verdict is `d`. It takes no settings: the filter brings the settings it was compiled under and the block those it was parsed under. The filter and the block must come from the same schema; a pair from two schemas is `CHS_INVALID_ARGUMENT`. +Evaluates the filter over a parsed block. Over a body a server would accept, it gives the same answers `chs_filter_eval_body` gives for that body. Over one it would not (the verdict `chs_block_create` recorded is not `accepted`), the outcome mirrors that verdict as for `chs_filter_eval_body`, and every verdict is written `d`, the rows before the refused one included: the two calls write the same document for the same body, and the rule that any non-`ok` outcome means every verdict is `d` holds in the data. `refused_at` and `refused_row` say where the refusal sat (`row` for a refused row, `body_step` for a whole-body step or a poisoned value, `call` for the block's own settings). It takes no settings: the filter brings the settings it was compiled under and the block those it was parsed under. The filter and the block must come from the same schema; a pair from two schemas is `CHS_INVALID_ARGUMENT`. #### `chs_discover_query` @@ -2100,7 +2500,7 @@ Evaluates the filter over a parsed block. Over a body a server would accept, it CHS_API chs_status chs_discover_query(chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `out`: out_handle, `chs_buf`, `sql`. - `err`: out_error. @@ -2113,7 +2513,7 @@ The query a caller runs against a server to read a table's `system.columns` rows CHS_API chs_status chs_discover_columns(const uint8_t *rows, size_t rows_len, chs_buf **out, chs_error **err); ``` -- Class `api`, thread `any`, FIRM. +- Class `api`, thread `any`, init `required`, FIRM. - May return `CHS_OK`, `CHS_REJECTED`, `CHS_DECLINED`, `CHS_INVALID_ARGUMENT`, `CHS_INTERNAL`. - `rows`: bytes_in, `bytes`. - `out`: out_handle, `chs_buf`, `document:discovery`. diff --git a/go/chtypes/vocab_gen.go b/go/chtypes/vocab_gen.go index db1c9128..00d79ec2 100644 --- a/go/chtypes/vocab_gen.go +++ b/go/chtypes/vocab_gen.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT package chtypes @@ -110,6 +110,10 @@ const TSVWithNames Format = 11 // ExportNone is CHS_EXPORT_NONE: no export was asked for. const ExportNone Format = -1 +// CodeDeclined is CHS_CODE_DECLINED: the code a declined row, outcome or verdict carries, outside +// ClickHouse's code table (the Codes paragraph of the reference). +const CodeDeclined int32 = -2 + // Known reports whether the description lists the format. A value it does not list is the // vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. func (f Format) Known() bool { @@ -304,9 +308,14 @@ const SourceDefaultVolatileUnresolved Source = "default_volatile_unresolved" // SourceDefaultPending is the Source value "default_pending". const SourceDefaultPending Source = "default_pending" +// SourceDefaultDeferred is the Source value "default_deferred". +const SourceDefaultDeferred Source = "default_deferred" + +// SourceDefaultOverride is the Source value "default_override". +const SourceDefaultOverride Source = "default_override" + // IsStored is the description's own is_stored fact for the source: whether the value is stored. -// The description does not yet say what an unknown(n) source reports (rule r3, "Facts without a -// fallback"); until it does, IsStored is false for one. +// An unknown(n) source reports its fallback's (skipped) fact (rule r3). func (x Source) IsStored() bool { switch x { case SourceInput: @@ -321,6 +330,8 @@ func (x Source) IsStored() bool { return true case SourceMaterializedInput: return true + case SourceDefaultOverride: + return true } return false } @@ -329,7 +340,7 @@ func (x Source) IsStored() bool { // vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. func (x Source) Known() bool { switch x { - case SourceInput, SourceDefault, SourceDefaultSubstituted, SourceDefaultGenerated, SourceAbsent, SourceMaterializedInput, SourceSkipped, SourceEphemeralInput, SourceDefaultExprUnsupported, SourceDefaultVolatileUnresolved, SourceDefaultPending: + case SourceInput, SourceDefault, SourceDefaultSubstituted, SourceDefaultGenerated, SourceAbsent, SourceMaterializedInput, SourceSkipped, SourceEphemeralInput, SourceDefaultExprUnsupported, SourceDefaultVolatileUnresolved, SourceDefaultPending, SourceDefaultDeferred, SourceDefaultOverride: return true } return false @@ -522,11 +533,14 @@ const TierPredicateUnflipped DeclinedTier = "predicate-unflipped" // TierResultContent is the DeclinedTier value "result-content". const TierResultContent DeclinedTier = "result-content" +// TierServerRefuses is the DeclinedTier value "server-refuses". +const TierServerRefuses DeclinedTier = "server-refuses" + // Known reports whether the description lists the tier. A value it does not list is the // vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. func (x DeclinedTier) Known() bool { switch x { - case TierPredicate, TierPredicateUnflipped, TierResultContent: + case TierPredicate, TierPredicateUnflipped, TierResultContent, TierServerRefuses: return true } return false @@ -556,9 +570,83 @@ func (x DeclinedLayer) Known() bool { return false } +// Posture is posture: how a schema's statement was compiled, as a server's CREATE TABLE (create) +// or as a server loads the stored definition of a table it holds (load). The spelling is the +// description's. +type Posture string + +// PostureCreate is the Posture value "create". +const PostureCreate Posture = "create" + +// PostureLoad is the Posture value "load". +const PostureLoad Posture = "load" + +// Known reports whether the description lists the posture. A value it does not list is the +// vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. +func (x Posture) Known() bool { + switch x { + case PostureCreate, PostureLoad: + return true + } + return false +} + +// RefusalSite is refusal_site: where a batch or filter document's refusal sits, the call (its settings +// or its column list), the body outside any row (a header, the framing), one row (RefusedRow) or a +// whole-body step. The spelling is the description's. +type RefusalSite string + +// RefusedAtCall is the RefusalSite value "call". +const RefusedAtCall RefusalSite = "call" + +// RefusedAtBodyRead is the RefusalSite value "body_read". +const RefusedAtBodyRead RefusalSite = "body_read" + +// RefusedAtRow is the RefusalSite value "row". +const RefusedAtRow RefusalSite = "row" + +// RefusedAtBodyStep is the RefusalSite value "body_step". +const RefusedAtBodyStep RefusalSite = "body_step" + +// Known reports whether the description lists the site. A value it does not list is the +// vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. +func (x RefusalSite) Known() bool { + switch x { + case RefusedAtCall, RefusedAtBodyRead, RefusedAtRow, RefusedAtBodyStep: + return true + } + return false +} + +// DefaultSupport is default_support: what this build does with a column's DEFAULT, as a schema +// description reports it per column. The spelling is the description's. +type DefaultSupport string + +// DefaultSupportComputed is the DefaultSupport value "computed". +const DefaultSupportComputed DefaultSupport = "computed" + +// DefaultSupportGenerated is the DefaultSupport value "generated". +const DefaultSupportGenerated DefaultSupport = "generated" + +// DefaultSupportDeferred is the DefaultSupport value "deferred". +const DefaultSupportDeferred DefaultSupport = "deferred" + +// DefaultSupportDeclined is the DefaultSupport value "declined". +const DefaultSupportDeclined DefaultSupport = "declined" + +// Known reports whether the description lists the support. A value it does not list is the +// vocabulary's unknown(n) member, carrying the raw value (rule r3): a reader keeps it and decodes on. +func (x DefaultSupport) Known() bool { + switch x { + case DefaultSupportComputed, DefaultSupportGenerated, DefaultSupportDeferred, DefaultSupportDeclined: + return true + } + return false +} + // describedVocabularyNames is every enum the description defines, in its own order. func describedVocabularyNames() []string { - return []string{"chs_status", "chs_format", "transform_reason", "merge_reason", "value_src", "row_outcome", "batch_outcome", "filter_outcome", "filter_verdict", "discover_query_param", "default_kind", "declined_tier", "declined_layer"} + return []string{"chs_status", "chs_format", "transform_reason", "merge_reason", "value_src", "row_outcome", "batch_outcome", "filter_outcome", "filter_verdict", "refusal_site", "discover_query_param", "default_kind", "default_support", "declined_tier", "declined_layer", "posture"} } // describedVocabulary builds the value of the enum `name` whose raw spelling is raw (an integer for an @@ -601,18 +689,27 @@ func describedVocabulary(name, raw string) (known bool, back string, ok bool) { case "filter_verdict": v := Verdict(raw) return v.Known(), string(v), true + case "refusal_site": + v := RefusalSite(raw) + return v.Known(), string(v), true case "discover_query_param": v := DiscoverQueryParam(raw) return v.Known(), string(v), true case "default_kind": v := DefaultKind(raw) return v.Known(), string(v), true + case "default_support": + v := DefaultSupport(raw) + return v.Known(), string(v), true case "declined_tier": v := DeclinedTier(raw) return v.Known(), string(v), true case "declined_layer": v := DeclinedLayer(raw) return v.Known(), string(v), true + case "posture": + v := Posture(raw) + return v.Known(), string(v), true } return false, "", false } diff --git a/go/internal/abi2/abi2_table.h b/go/internal/abi2/abi2_table.h index 74cd6d5e..a211434c 100644 --- a/go/internal/abi2/abi2_table.h +++ b/go/internal/abi2/abi2_table.h @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ #ifndef CHTYPES_ABI2_TABLE_H #define CHTYPES_ABI2_TABLE_H @@ -25,6 +25,7 @@ typedef int32_t (*chtypes_abi2_fn_chs_error_ch_code)(const void * error); typedef void * (*chtypes_abi2_fn_chs_error_ch_name)(const void * error); typedef void * (*chtypes_abi2_fn_chs_error_message)(const void * error); typedef void * (*chtypes_abi2_fn_chs_error_column)(const void * error); +typedef void * (*chtypes_abi2_fn_chs_error_reason)(const void * error); typedef void (*chtypes_abi2_fn_chs_error_free)(void * error); typedef int32_t (*chtypes_abi2_fn_chs_live_handles)(void ** out, void ** err); typedef int32_t (*chtypes_abi2_fn_chs_error_codes)(void ** out, void ** err); @@ -43,12 +44,12 @@ typedef void (*chtypes_abi2_fn_chs_server_free)(void * server); typedef int32_t (*chtypes_abi2_fn_chs_schema_create)(const void * server, const uint8_t * create_table, size_t create_table_len, const uint8_t * settings, size_t settings_len, const uint8_t * options, size_t options_len, void ** out, void ** err); typedef void (*chtypes_abi2_fn_chs_schema_free)(void * schema); typedef int32_t (*chtypes_abi2_fn_chs_schema_describe)(const void * schema, void ** out, void ** err); -typedef int32_t (*chtypes_abi2_fn_chs_preview_row)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, void ** out, void ** err); -typedef int32_t (*chtypes_abi2_fn_chs_preview_batch)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const void * filter, int32_t export_format, uint32_t doc_flags, void ** out, void ** out_export, void ** err); -typedef int32_t (*chtypes_abi2_fn_chs_filter_create)(const void * schema, const uint8_t * expr, size_t expr_len, const uint8_t * query_params, size_t query_params_len, const uint8_t * settings, size_t settings_len, void ** out, void ** err); +typedef int32_t (*chtypes_abi2_fn_chs_preview_row)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err); +typedef int32_t (*chtypes_abi2_fn_chs_preview_batch)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const void * filter, int32_t export_format, uint32_t doc_flags, const uint8_t * options, size_t options_len, void ** out, void ** out_export, void ** err); +typedef int32_t (*chtypes_abi2_fn_chs_filter_create)(const void * schema, const uint8_t * expr, size_t expr_len, const uint8_t * query_params, size_t query_params_len, const uint8_t * settings, size_t settings_len, const uint8_t * options, size_t options_len, void ** out, void ** err); typedef void (*chtypes_abi2_fn_chs_filter_free)(void * filter); -typedef int32_t (*chtypes_abi2_fn_chs_filter_eval_body)(const void * filter, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, void ** out, void ** err); -typedef int32_t (*chtypes_abi2_fn_chs_block_create)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, void ** out, void ** err); +typedef int32_t (*chtypes_abi2_fn_chs_filter_eval_body)(const void * filter, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err); +typedef int32_t (*chtypes_abi2_fn_chs_block_create)(const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err); typedef void (*chtypes_abi2_fn_chs_block_free)(void * block); typedef int32_t (*chtypes_abi2_fn_chs_filter_eval_block)(const void * filter, const void * block, void ** out, void ** err); typedef int32_t (*chtypes_abi2_fn_chs_discover_query)(void ** out, void ** err); @@ -67,6 +68,7 @@ struct chtypes_abi2_table { chtypes_abi2_fn_chs_error_ch_name chs_error_ch_name; chtypes_abi2_fn_chs_error_message chs_error_message; chtypes_abi2_fn_chs_error_column chs_error_column; + chtypes_abi2_fn_chs_error_reason chs_error_reason; chtypes_abi2_fn_chs_error_free chs_error_free; chtypes_abi2_fn_chs_live_handles chs_live_handles; chtypes_abi2_fn_chs_error_codes chs_error_codes; diff --git a/go/internal/abi2/abi_gen.go b/go/internal/abi2/abi_gen.go index cff68eb8..1e50745d 100644 --- a/go/internal/abi2/abi_gen.go +++ b/go/internal/abi2/abi_gen.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT package abi2 @@ -94,6 +94,8 @@ static const char *chtypes_abi2_resolve_all(void *h, struct chtypes_abi2_table * if (!t->chs_error_message) return "chs_error_message"; t->chs_error_column = (chtypes_abi2_fn_chs_error_column) dlsym(h, "chs_error_column"); if (!t->chs_error_column) return "chs_error_column"; + t->chs_error_reason = (chtypes_abi2_fn_chs_error_reason) dlsym(h, "chs_error_reason"); + if (!t->chs_error_reason) return "chs_error_reason"; t->chs_error_free = (chtypes_abi2_fn_chs_error_free) dlsym(h, "chs_error_free"); if (!t->chs_error_free) return "chs_error_free"; t->chs_live_handles = (chtypes_abi2_fn_chs_live_handles) dlsym(h, "chs_live_handles"); @@ -165,6 +167,7 @@ static int32_t chtypes_abi2_call_chs_error_ch_code(struct chtypes_abi2_table *t, static void * chtypes_abi2_call_chs_error_ch_name(struct chtypes_abi2_table *t, const void * error) { return t->chs_error_ch_name(error); } static void * chtypes_abi2_call_chs_error_message(struct chtypes_abi2_table *t, const void * error) { return t->chs_error_message(error); } static void * chtypes_abi2_call_chs_error_column(struct chtypes_abi2_table *t, const void * error) { return t->chs_error_column(error); } +static void * chtypes_abi2_call_chs_error_reason(struct chtypes_abi2_table *t, const void * error) { return t->chs_error_reason(error); } static void chtypes_abi2_call_chs_error_free(struct chtypes_abi2_table *t, void * error) { t->chs_error_free(error); } static int32_t chtypes_abi2_call_chs_live_handles(struct chtypes_abi2_table *t, void ** out, void ** err) { return t->chs_live_handles(out, err); } static int32_t chtypes_abi2_call_chs_error_codes(struct chtypes_abi2_table *t, void ** out, void ** err) { return t->chs_error_codes(out, err); } @@ -183,12 +186,12 @@ static void chtypes_abi2_call_chs_server_free(struct chtypes_abi2_table *t, void static int32_t chtypes_abi2_call_chs_schema_create(struct chtypes_abi2_table *t, const void * server, const uint8_t * create_table, size_t create_table_len, const uint8_t * settings, size_t settings_len, const uint8_t * options, size_t options_len, void ** out, void ** err) { return t->chs_schema_create(server, create_table, create_table_len, settings, settings_len, options, options_len, out, err); } static void chtypes_abi2_call_chs_schema_free(struct chtypes_abi2_table *t, void * schema) { t->chs_schema_free(schema); } static int32_t chtypes_abi2_call_chs_schema_describe(struct chtypes_abi2_table *t, const void * schema, void ** out, void ** err) { return t->chs_schema_describe(schema, out, err); } -static int32_t chtypes_abi2_call_chs_preview_row(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, void ** out, void ** err) { return t->chs_preview_row(schema, format, body, body_len, settings, settings_len, columns, columns_len, out, err); } -static int32_t chtypes_abi2_call_chs_preview_batch(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const void * filter, int32_t export_format, uint32_t doc_flags, void ** out, void ** out_export, void ** err) { return t->chs_preview_batch(schema, format, body, body_len, settings, settings_len, columns, columns_len, filter, export_format, doc_flags, out, out_export, err); } -static int32_t chtypes_abi2_call_chs_filter_create(struct chtypes_abi2_table *t, const void * schema, const uint8_t * expr, size_t expr_len, const uint8_t * query_params, size_t query_params_len, const uint8_t * settings, size_t settings_len, void ** out, void ** err) { return t->chs_filter_create(schema, expr, expr_len, query_params, query_params_len, settings, settings_len, out, err); } +static int32_t chtypes_abi2_call_chs_preview_row(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err) { return t->chs_preview_row(schema, format, body, body_len, settings, settings_len, columns, columns_len, options, options_len, out, err); } +static int32_t chtypes_abi2_call_chs_preview_batch(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const void * filter, int32_t export_format, uint32_t doc_flags, const uint8_t * options, size_t options_len, void ** out, void ** out_export, void ** err) { return t->chs_preview_batch(schema, format, body, body_len, settings, settings_len, columns, columns_len, filter, export_format, doc_flags, options, options_len, out, out_export, err); } +static int32_t chtypes_abi2_call_chs_filter_create(struct chtypes_abi2_table *t, const void * schema, const uint8_t * expr, size_t expr_len, const uint8_t * query_params, size_t query_params_len, const uint8_t * settings, size_t settings_len, const uint8_t * options, size_t options_len, void ** out, void ** err) { return t->chs_filter_create(schema, expr, expr_len, query_params, query_params_len, settings, settings_len, options, options_len, out, err); } static void chtypes_abi2_call_chs_filter_free(struct chtypes_abi2_table *t, void * filter) { t->chs_filter_free(filter); } -static int32_t chtypes_abi2_call_chs_filter_eval_body(struct chtypes_abi2_table *t, const void * filter, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, void ** out, void ** err) { return t->chs_filter_eval_body(filter, format, body, body_len, settings, settings_len, out, err); } -static int32_t chtypes_abi2_call_chs_block_create(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, void ** out, void ** err) { return t->chs_block_create(schema, format, body, body_len, settings, settings_len, columns, columns_len, out, err); } +static int32_t chtypes_abi2_call_chs_filter_eval_body(struct chtypes_abi2_table *t, const void * filter, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err) { return t->chs_filter_eval_body(filter, format, body, body_len, settings, settings_len, columns, columns_len, options, options_len, out, err); } +static int32_t chtypes_abi2_call_chs_block_create(struct chtypes_abi2_table *t, const void * schema, int32_t format, const uint8_t * body, size_t body_len, const uint8_t * settings, size_t settings_len, const uint8_t * columns, size_t columns_len, const uint8_t * options, size_t options_len, void ** out, void ** err) { return t->chs_block_create(schema, format, body, body_len, settings, settings_len, columns, columns_len, options, options_len, out, err); } static void chtypes_abi2_call_chs_block_free(struct chtypes_abi2_table *t, void * block) { t->chs_block_free(block); } static int32_t chtypes_abi2_call_chs_filter_eval_block(struct chtypes_abi2_table *t, const void * filter, const void * block, void ** out, void ** err) { return t->chs_filter_eval_block(filter, block, out, err); } static int32_t chtypes_abi2_call_chs_discover_query(struct chtypes_abi2_table *t, void ** out, void ** err) { return t->chs_discover_query(out, err); } @@ -208,7 +211,7 @@ const ChsAbiVersion = 2 // ChsAbiFingerprint is CHS_ABI_FINGERPRINT: sha256 over the RFC 8785 canonical // form of spec/abi-v2/abi.json. chs_build_info()'s abi_fingerprint field is // this value, copied from the compiled-in macro, never recomputed. -const ChsAbiFingerprint = "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4" +const ChsAbiFingerprint = "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2" // ChsAbiStability is the description's stability: "unstable" while this generation // is being designed, so ChsAbiFingerprint moves with every change, and "locked" @@ -689,6 +692,11 @@ func (t *Table) ErrorColumn(errorArg *ErrorHandle) *Buf { return newBuf(t, C.chtypes_abi2_call_chs_error_column(&t.tbl, errorArg.rawPtr())) } +// ErrorReason is the generated call wrapper for chs_error_reason. +func (t *Table) ErrorReason(errorArg *ErrorHandle) *Buf { + return newBuf(t, C.chtypes_abi2_call_chs_error_reason(&t.tbl, errorArg.rawPtr())) +} + // ErrorFree is the generated call wrapper for chs_error_free. func (t *Table) ErrorFree(errorArg *ErrorHandle) { C.chtypes_abi2_call_chs_error_free(&t.tbl, errorArg.rawPtr()) @@ -832,36 +840,39 @@ func (t *Table) SchemaDescribe(schema *Schema) (*Buf, *CallError) { } // PreviewRow is the generated call wrapper for chs_preview_row. -func (t *Table) PreviewRow(schema *Schema, format int32, body []byte, settings []byte, columns []byte) (*Buf, *CallError) { +func (t *Table) PreviewRow(schema *Schema, format int32, body []byte, settings []byte, columns []byte, options []byte) (*Buf, *CallError) { bodyPtr, bodyLen := bytesPtr(body) settingsPtr, settingsLen := bytesPtr(settings) columnsPtr, columnsLen := bytesPtr(columns) + optionsPtr, optionsLen := bytesPtr(options) var outOut unsafe.Pointer var errOut unsafe.Pointer - status := C.chtypes_abi2_call_chs_preview_row(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, &outOut, &errOut) + status := C.chtypes_abi2_call_chs_preview_row(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, optionsPtr, optionsLen, &outOut, &errOut) return newBuf(t, outOut), makeCallError(t, int32(status), newErrorHandle(t, errOut)) } // PreviewBatch is the generated call wrapper for chs_preview_batch. -func (t *Table) PreviewBatch(schema *Schema, format int32, body []byte, settings []byte, columns []byte, filter *Filter, exportFormat int32, docFlags uint32) (*Buf, *Buf, *CallError) { +func (t *Table) PreviewBatch(schema *Schema, format int32, body []byte, settings []byte, columns []byte, filter *Filter, exportFormat int32, docFlags uint32, options []byte) (*Buf, *Buf, *CallError) { bodyPtr, bodyLen := bytesPtr(body) settingsPtr, settingsLen := bytesPtr(settings) columnsPtr, columnsLen := bytesPtr(columns) + optionsPtr, optionsLen := bytesPtr(options) var outOut unsafe.Pointer var outExportOut unsafe.Pointer var errOut unsafe.Pointer - status := C.chtypes_abi2_call_chs_preview_batch(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, filter.rawPtr(), C.int32_t(exportFormat), C.uint32_t(docFlags), &outOut, &outExportOut, &errOut) + status := C.chtypes_abi2_call_chs_preview_batch(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, filter.rawPtr(), C.int32_t(exportFormat), C.uint32_t(docFlags), optionsPtr, optionsLen, &outOut, &outExportOut, &errOut) return newBuf(t, outOut), newBuf(t, outExportOut), makeCallError(t, int32(status), newErrorHandle(t, errOut)) } // FilterCreate is the generated call wrapper for chs_filter_create. -func (t *Table) FilterCreate(schema *Schema, expr []byte, queryParams []byte, settings []byte) (*Filter, *CallError) { +func (t *Table) FilterCreate(schema *Schema, expr []byte, queryParams []byte, settings []byte, options []byte) (*Filter, *CallError) { exprPtr, exprLen := bytesPtr(expr) queryParamsPtr, queryParamsLen := bytesPtr(queryParams) settingsPtr, settingsLen := bytesPtr(settings) + optionsPtr, optionsLen := bytesPtr(options) var outOut unsafe.Pointer var errOut unsafe.Pointer - status := C.chtypes_abi2_call_chs_filter_create(&t.tbl, schema.rawPtr(), exprPtr, exprLen, queryParamsPtr, queryParamsLen, settingsPtr, settingsLen, &outOut, &errOut) + status := C.chtypes_abi2_call_chs_filter_create(&t.tbl, schema.rawPtr(), exprPtr, exprLen, queryParamsPtr, queryParamsLen, settingsPtr, settingsLen, optionsPtr, optionsLen, &outOut, &errOut) return newFilter(t, outOut), makeCallError(t, int32(status), newErrorHandle(t, errOut)) } @@ -871,23 +882,26 @@ func (t *Table) FilterFree(filter *Filter) { } // FilterEvalBody is the generated call wrapper for chs_filter_eval_body. -func (t *Table) FilterEvalBody(filter *Filter, format int32, body []byte, settings []byte) (*Buf, *CallError) { +func (t *Table) FilterEvalBody(filter *Filter, format int32, body []byte, settings []byte, columns []byte, options []byte) (*Buf, *CallError) { bodyPtr, bodyLen := bytesPtr(body) settingsPtr, settingsLen := bytesPtr(settings) + columnsPtr, columnsLen := bytesPtr(columns) + optionsPtr, optionsLen := bytesPtr(options) var outOut unsafe.Pointer var errOut unsafe.Pointer - status := C.chtypes_abi2_call_chs_filter_eval_body(&t.tbl, filter.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, &outOut, &errOut) + status := C.chtypes_abi2_call_chs_filter_eval_body(&t.tbl, filter.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, optionsPtr, optionsLen, &outOut, &errOut) return newBuf(t, outOut), makeCallError(t, int32(status), newErrorHandle(t, errOut)) } // BlockCreate is the generated call wrapper for chs_block_create. -func (t *Table) BlockCreate(schema *Schema, format int32, body []byte, settings []byte, columns []byte) (*Block, *CallError) { +func (t *Table) BlockCreate(schema *Schema, format int32, body []byte, settings []byte, columns []byte, options []byte) (*Block, *CallError) { bodyPtr, bodyLen := bytesPtr(body) settingsPtr, settingsLen := bytesPtr(settings) columnsPtr, columnsLen := bytesPtr(columns) + optionsPtr, optionsLen := bytesPtr(options) var outOut unsafe.Pointer var errOut unsafe.Pointer - status := C.chtypes_abi2_call_chs_block_create(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, &outOut, &errOut) + status := C.chtypes_abi2_call_chs_block_create(&t.tbl, schema.rawPtr(), C.int32_t(format), bodyPtr, bodyLen, settingsPtr, settingsLen, columnsPtr, columnsLen, optionsPtr, optionsLen, &outOut, &errOut) return newBlock(t, outOut), makeCallError(t, int32(status), newErrorHandle(t, errOut)) } diff --git a/go/internal/abi2/errmap_gen.go b/go/internal/abi2/errmap_gen.go index 48efc93d..547ef000 100644 --- a/go/internal/abi2/errmap_gen.go +++ b/go/internal/abi2/errmap_gen.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT package abi2 diff --git a/go/internal/abi2/invoke_gen_test.go b/go/internal/abi2/invoke_gen_test.go index 45e4f155..cdca10df 100644 --- a/go/internal/abi2/invoke_gen_test.go +++ b/go/internal/abi2/invoke_gen_test.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT package abi2 @@ -345,6 +345,23 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { return nil, fmt.Errorf("chs_error_column: %w", err) } return &InvokeResult{Outputs: map[string]interface{}{"return": doc}}, nil + case "chs_error_reason": + if len(args) != 1 { + return nil, fmt.Errorf("chs_error_reason: want %d args, got %d", 1, len(args)) + } + h0, err := mintErrorHandle(t, args[0]) + if err != nil { + return nil, fmt.Errorf("chs_error_reason: arg %d: %w", 0, err) + } + v := t.ErrorReason(h0) + if v == nil { + return &InvokeResult{}, nil + } + doc, err := bufJSON(t, v) + if err != nil { + return nil, fmt.Errorf("chs_error_reason: %w", err) + } + return &InvokeResult{Outputs: map[string]interface{}{"return": doc}}, nil case "chs_error_free": if len(args) != 1 { return nil, fmt.Errorf("chs_error_free: want %d args, got %d", 1, len(args)) @@ -688,8 +705,8 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { } return res, nil case "chs_preview_row": - if len(args) != 5 { - return nil, fmt.Errorf("chs_preview_row: want %d args, got %d", 5, len(args)) + if len(args) != 6 { + return nil, fmt.Errorf("chs_preview_row: want %d args, got %d", 6, len(args)) } h0, err := mintSchema(t, args[0]) if err != nil { @@ -711,7 +728,11 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { if err != nil { return nil, fmt.Errorf("chs_preview_row: arg %d: %w", 4, err) } - o0, callErr := t.PreviewRow(h0, int32(v1), b2, b3, b4) + b5, err := argBytes(args[5]) + if err != nil { + return nil, fmt.Errorf("chs_preview_row: arg %d: %w", 5, err) + } + o0, callErr := t.PreviewRow(h0, int32(v1), b2, b3, b4, b5) res := &InvokeResult{Error: callErr} if callErr != nil { res.Status = callErr.Status @@ -728,8 +749,8 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { } return res, nil case "chs_preview_batch": - if len(args) != 8 { - return nil, fmt.Errorf("chs_preview_batch: want %d args, got %d", 8, len(args)) + if len(args) != 9 { + return nil, fmt.Errorf("chs_preview_batch: want %d args, got %d", 9, len(args)) } h0, err := mintSchema(t, args[0]) if err != nil { @@ -763,7 +784,11 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { if err != nil { return nil, fmt.Errorf("chs_preview_batch: arg %d: %w", 7, err) } - o0, o1, callErr := t.PreviewBatch(h0, int32(v1), b2, b3, b4, h5, int32(v6), uint32(v7)) + b8, err := argBytes(args[8]) + if err != nil { + return nil, fmt.Errorf("chs_preview_batch: arg %d: %w", 8, err) + } + o0, o1, callErr := t.PreviewBatch(h0, int32(v1), b2, b3, b4, h5, int32(v6), uint32(v7), b8) res := &InvokeResult{Error: callErr} if callErr != nil { res.Status = callErr.Status @@ -787,8 +812,8 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { } return res, nil case "chs_filter_create": - if len(args) != 4 { - return nil, fmt.Errorf("chs_filter_create: want %d args, got %d", 4, len(args)) + if len(args) != 5 { + return nil, fmt.Errorf("chs_filter_create: want %d args, got %d", 5, len(args)) } h0, err := mintSchema(t, args[0]) if err != nil { @@ -806,7 +831,11 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { if err != nil { return nil, fmt.Errorf("chs_filter_create: arg %d: %w", 3, err) } - o0, callErr := t.FilterCreate(h0, b1, b2, b3) + b4, err := argBytes(args[4]) + if err != nil { + return nil, fmt.Errorf("chs_filter_create: arg %d: %w", 4, err) + } + o0, callErr := t.FilterCreate(h0, b1, b2, b3, b4) res := &InvokeResult{Error: callErr} if callErr != nil { res.Status = callErr.Status @@ -826,8 +855,8 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { t.FilterFree(h0) return &InvokeResult{}, nil case "chs_filter_eval_body": - if len(args) != 4 { - return nil, fmt.Errorf("chs_filter_eval_body: want %d args, got %d", 4, len(args)) + if len(args) != 6 { + return nil, fmt.Errorf("chs_filter_eval_body: want %d args, got %d", 6, len(args)) } h0, err := mintFilter(t, args[0]) if err != nil { @@ -845,7 +874,15 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { if err != nil { return nil, fmt.Errorf("chs_filter_eval_body: arg %d: %w", 3, err) } - o0, callErr := t.FilterEvalBody(h0, int32(v1), b2, b3) + b4, err := argBytes(args[4]) + if err != nil { + return nil, fmt.Errorf("chs_filter_eval_body: arg %d: %w", 4, err) + } + b5, err := argBytes(args[5]) + if err != nil { + return nil, fmt.Errorf("chs_filter_eval_body: arg %d: %w", 5, err) + } + o0, callErr := t.FilterEvalBody(h0, int32(v1), b2, b3, b4, b5) res := &InvokeResult{Error: callErr} if callErr != nil { res.Status = callErr.Status @@ -862,8 +899,8 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { } return res, nil case "chs_block_create": - if len(args) != 5 { - return nil, fmt.Errorf("chs_block_create: want %d args, got %d", 5, len(args)) + if len(args) != 6 { + return nil, fmt.Errorf("chs_block_create: want %d args, got %d", 6, len(args)) } h0, err := mintSchema(t, args[0]) if err != nil { @@ -885,7 +922,11 @@ func invoke(t *Table, fn string, args []Arg) (*InvokeResult, error) { if err != nil { return nil, fmt.Errorf("chs_block_create: arg %d: %w", 4, err) } - o0, callErr := t.BlockCreate(h0, int32(v1), b2, b3, b4) + b5, err := argBytes(args[5]) + if err != nil { + return nil, fmt.Errorf("chs_block_create: arg %d: %w", 5, err) + } + o0, callErr := t.BlockCreate(h0, int32(v1), b2, b3, b4, b5) res := &InvokeResult{Error: callErr} if callErr != nil { res.Status = callErr.Status diff --git a/go/internal/abi2/linked_gen.go b/go/internal/abi2/linked_gen.go index 433f02ad..20d06030 100644 --- a/go/internal/abi2/linked_gen.go +++ b/go/internal/abi2/linked_gen.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT //go:build chtypes_linked @@ -34,6 +34,7 @@ static void chtypes_abi2_linked_fill(struct chtypes_abi2_table *t) { t->chs_error_ch_name = (chtypes_abi2_fn_chs_error_ch_name) &chs_error_ch_name; t->chs_error_message = (chtypes_abi2_fn_chs_error_message) &chs_error_message; t->chs_error_column = (chtypes_abi2_fn_chs_error_column) &chs_error_column; + t->chs_error_reason = (chtypes_abi2_fn_chs_error_reason) &chs_error_reason; t->chs_error_free = (chtypes_abi2_fn_chs_error_free) &chs_error_free; t->chs_live_handles = (chtypes_abi2_fn_chs_live_handles) &chs_live_handles; t->chs_error_codes = (chtypes_abi2_fn_chs_error_codes) &chs_error_codes; diff --git a/go/internal/ocifetch/abi_fingerprint_gen.go b/go/internal/ocifetch/abi_fingerprint_gen.go index 6f648f8e..1681c3d7 100644 --- a/go/internal/ocifetch/abi_fingerprint_gen.go +++ b/go/internal/ocifetch/abi_fingerprint_gen.go @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT package ocifetch @@ -6,4 +6,4 @@ package ocifetch // module speaks: the same value as the ABI layer's ChsAbiFingerprint. The dev // channel names its alias tags with it, as 64 lowercase hex characters // without the sha256: prefix (docs/guides/fetch-v1.md section 3). -const DevABIFingerprint = "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4" +const DevABIFingerprint = "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2" diff --git a/include/v2/chtypes.h b/include/v2/chtypes.h index a7d54e6a..ac9ef0b0 100644 --- a/include/v2/chtypes.h +++ b/include/v2/chtypes.h @@ -1,10 +1,10 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * chtypes.h: the chtypes C ABI, generation 2. * * This header is generated from `spec/abi-v2/abi.json` by `scripts/abi-v1/gen.py --major 2`. Edit * the description and regenerate; never edit this file. `docs/reference/abi-v2.md` is the normative - * reference, and the rules generation 2 is written under (r1 to r7) are in it. + * reference, and the rules generation 2 is written under (r1 to r9) are in it. * * Stability. Generation 2 is UNSTABLE: its description is still being designed, so * `CHS_ABI_FINGERPRINT` moves with every change until the description is locked. Nothing built from @@ -29,8 +29,43 @@ * drops only the caller's reference, so any free order is safe and freeing NULL does nothing. * * Errors. A fallible call returns a `chs_status` and, through an optional last `chs_error **err`, - * an owned error carrying the status, ClickHouse's own code and name, the message and the column. - * The status alone carries the verdict, so `err` may be NULL. + * an owned error carrying the status, ClickHouse's own code and name, the message, the column and, + * for a decline or a library-side refusal, the reason: the catalog id (`chs_error_reason`, + * `spec/abi-v2/declines/declines.json`) a caller keys on instead of matching message text. The + * status alone carries the verdict, so `err` may be NULL. + * + * Codes. One code space serves the error and every document's `code` and `verdict_code`: a positive + * value is ClickHouse's own error code, as the server's table names it; `CHS_CODE_DECLINED` (-2) is + * this build's decline; and 0 is written only on a success (`accepted`, `accepted_poisoned`, `ok`, + * a `t` or `f` verdict). A refused or declined row, outcome or verdict never carries 0, and no + * other negative or library-private number is ever written. Every decline and library-side refusal + * also names its reason, the catalog id, in the error and in the document (`reason`, + * `verdict_reason`), and ids and the sentinel are stable once published (rule r4). A message is + * informative: once a reason is structured no message text is normative, the pinned texts of + * generation 2's earlier drafts included. + * + * Conditional members. A document's schema states the members it always carries (`required`). These + * members are present under a condition the schema cannot see, and each condition is a MUST on the + * library, held by a conformance case in the SDK's suites (named in brackets) that drives it both + * ways; a binding MAY check them and, when it does, fails closed on a violation. + * + * - **The export.** When `chs_preview_batch` is asked for an export (`export_format` is not + * `CHS_EXPORT_NONE`), exactly one of `export_declined` and `row_spans` is present; with + * `export_declined`, `export_declined_reason`, its catalog id; and with `row_spans`, + * `export_columns`; with no export asked for, none of them is [`batch.export_members_xor`]. + * - **The filter.** A row's `verdict`, `verdict_code` and `verdict_err`, and the batch's + * `rows_passed` and `rows_cut`, are present exactly when the call carries a filter + * (`chs_preview_batch` with a non-NULL `filter`) [`batch.verdict_iff_filter`]. + * - **The reason.** `reason` is present exactly where its `code` is `CHS_CODE_DECLINED` (a row, a + * batch, a filter result, an `errors[]` entry), and `verdict_reason` exactly where its + * `verdict_code` is [`doc.reason_iff_declined`]. + * - **The site.** `refused_at` is present exactly when the outcome of a batch, a filter result or + * `chs_preview_row`'s row is a refusal or a decline (`rejected` or `unsupported`), and never for + * `accepted`, `accepted_poisoned`, `skipped` or `ok`; `refused_row` is present exactly when + * `refused_at` is `row`, and on a batch or a filter result it is less than `rows_read`. The rows + * of a batch carry neither: the batch's own say where [`doc.refused_at_iff_refused`]. + * - **The verdicts.** A filter result's `verdicts` has exactly `rows_read` characters + * [`filter.verdicts_len_is_rows_read`]. * * Threads. Every function has a thread class, listed in the reference. A handle is immutable once * made, so every call that only reads handles is `shared`: any number of threads may call it at @@ -38,6 +73,18 @@ * a handle that call uses. Process setup (`chs_initialize`, then `chs_set_defaults`) runs before * the traffic it configures. * + * Initialization. Every function has an init class, listed in the reference and in its own comment: + * `any`, valid before and after `chs_initialize`; or `required`, valid only once `chs_initialize` + * has succeeded. Until then a `required` call changes nothing and returns `CHS_INVALID_ARGUMENT`, + * ch_code 0, and when its arguments are otherwise valid its message starts `: call + * chs_initialize first`; it never crashes. The `any` calls are the handshake and the tombstone, + * `chs_initialize`, `chs_shutdown`, the error and buffer accessors, and every free. Each of them + * but `chs_initialize` returns no `chs_status`, so it has nothing to refuse with. The error and + * buffer accessors read the refusal itself (`chs_error_message` returns a `chs_buf`), and a free + * given NULL does nothing, which is all a server, schema, filter or block free can be given before + * `chs_initialize`, because nothing makes one. Every other call is `required`, unless its own + * section says it is valid before `chs_initialize`. + * * Stacks. A call's answer doesn't depend on the calling thread's stack. Every call that returns a * `chs_status` runs with an 8 MiB stack's worth of room: on the calling thread when half its stack, * less what it has used, is between 3.75 MiB and 4 MiB (a default 8 MiB thread: glibc's, a Go cgo @@ -64,20 +111,21 @@ * the file system is case-insensitive, so `DateLUT` there accepts a spelling such as `utc` that a * Linux server refuses; that difference is documented, not patched. * - * Settings. Every settings input is a JSON object whose values are JSON strings, and a binding - * never rewrites one. A call's settings are layered, highest first: the call's own `settings`; the - * schema's own (`chs_schema_create`); its server's `settings` (`chs_server_create`); the defaults - * set by `chs_set_defaults`; the build's own. A filter's WHERE takes the same layers, lowest first: - * the build's own, `chs_set_defaults`, the server profile, the schema's own, and the filter's own - * (`chs_filter_create`), with `session_timezone` at every layer staying on the zone path (Time - * zones, above). One exception: a type gate in the schema's own settings binds the CREATE only, and - * a WHERE's CAST validates under the session layers, every layer but the schema's own. Each setting - * that reaches a WHERE is honored (applied to it), passed through (accepted and ignored), or - * declined (`chs_filter_create` says how). A server resource limit (an `execution`-tier setting, - * e.g. a small `max_columns_to_read`) can make the server refuse a query the library answers; that - * is resource policy, not row access, and the library does not model it. A server's settings are - * the ones its profile applies to every query, and they never change how a schema compiles. No call - * reads a server's version or its changed settings itself: that discovery is a recorded gap in this + * Settings. Every settings input is a JSON object whose values are JSON strings, or the object form + * for bytes (Input values, below), and a binding never rewrites one. A call's settings are layered, + * highest first: the call's own `settings`; the schema's own (`chs_schema_create`); its server's + * `settings` (`chs_server_create`); the defaults set by `chs_set_defaults`; the build's own. A + * filter's WHERE takes the same layers, lowest first: the build's own, `chs_set_defaults`, the + * server profile, the schema's own, and the filter's own (`chs_filter_create`), with + * `session_timezone` at every layer staying on the zone path (Time zones, above). One exception: a + * type gate in the schema's own settings binds the CREATE only, and a WHERE's CAST validates under + * the session layers, every layer but the schema's own. Each setting that reaches a WHERE is + * honored (applied to it), passed through (accepted and ignored), or declined (`chs_filter_create` + * says how). A server resource limit (an `execution`-tier setting, e.g. a small + * `max_columns_to_read`) can make the server refuse a query the library answers; that is resource + * policy, not row access, and the library does not model it. A server's settings are the ones its + * profile applies to every query, and they never change how a schema compiles. No call reads a + * server's version or its changed settings itself: that discovery is a recorded gap in this * generation, so a caller that wants a server's settings passes them explicitly, in a server * profile, through `chs_set_defaults` or in each call's settings. * @@ -111,18 +159,38 @@ * value nested in another type (`Array(String)`, `Map`, `Tuple`, and the like) carries no * `value_b64`: that is a recorded gap in this generation, and its rendering is the only form. * - * Input values. A JSON input (`settings`, `query_params`, an input document) cannot carry a value - * that is not valid UTF-8 in this generation. The workaround is to inline the value in the SQL as - * `unhex('')`, built from hex digits only, which keeps it injection-safe because hex digits - * cannot break out of the quoted literal. A later version would add an object form, `{"": - * {"b64": ...}}`, never a `_b64` member, which could collide with a real key. + * Input values. A value in a settings or query-parameters input (every `settings` and + * `query_params`, and a server profile's `settings`: the content `json_object_byte_values`) is a + * JSON string when its bytes are valid UTF-8, and otherwise the object `{"b64": ""}`, which carries the raw bytes: `{"format_csv_delimiter": {"b64": "/w=="}}` sets the one + * byte 0xFF. It is never a `_b64` member, which could collide with a real key. The two forms + * are the same value when the bytes are valid UTF-8, so a binding may send either, and it MUST send + * the object form for any other value. Any other object, and a value of any other JSON type, is + * `CHS_INVALID_ARGUMENT`, naming the key. A key never takes the object form: setting and + * query-parameter names are ClickHouse identifiers. Every JSON input is valid UTF-8 JSON (RFC + * 8259), so a raw byte that is not UTF-8 inside a JSON string is `CHS_INVALID_ARGUMENT`, and the + * object form is the one way to carry such a value. The other inputs take no object form: an + * options document's values are closed vocabularies, a server profile's `timezone` and `macros` are + * names a server's configuration spells, and the input column list carries a name that is not UTF-8 + * as `name_b64`, by the rule for byte strings in JSON. * * Input documents. A counted input whose content is `input:` is a JSON object that the * library validates against the document of that name (rule r1). A key the document does not define * is `CHS_INVALID_ARGUMENT`, naming the key, and so is a document that is not a JSON object or a * member of the wrong JSON type. Length 0 is `{}`, and a repeated member keeps its last value. The * library never ignores a key, so a caller that sets an option a build does not implement is told - * so instead of silently getting the old behavior. + * so instead of silently getting the old behavior. The whole document is validated before any + * member is acted on, so a document with a key it does not define is `CHS_INVALID_ARGUMENT` even + * when it also sets a member this build declines. A member a document defines but this build does + * not implement, set to any value but its default, is `CHS_DECLINED` with ch_code 0 and exactly the + * message `: sets to "", which this build does not implement; + * declined rather than answered` (for example `chs_preview_batch: insert_options sets verdict_basis + * to "export", which this build does not implement; declined rather than answered`), reason + * `option-not-implemented`, which is never a wrong answer. At a call that answers with a document + * (`chs_preview_row`, `chs_preview_batch`, `chs_filter_eval_body`, `chs_block_create` and + * `chs_filter_eval_block`), that decline is the document's, not a status: outcome `unsupported`, + * `code` `CHS_CODE_DECLINED`, `refused_at` `call` and `reason` `option-not-implemented`, with the + * status `CHS_OK` (`refusal_site`). * * Teardown. `chs_shutdown` stops what `chs_initialize` started. A loader never unloads a library: * unloading or initializing an image again in one process is unsupported. Before `chs_initialize`, @@ -131,6 +199,17 @@ * NULL. After `chs_shutdown` has run, every such call refuses the same way, its message saying that * `chs_shutdown` has run. The handshake, `chs_abi_revision`, `chs_shutdown`, the buffer and error * accessors and the frees answer at any time. + * + * Images. (Pending the confirmation runs with the time budget at 0, and the remaining arm64 cells + * at the default budget and at 0.) Any number of library images, one per ClickHouse line, may be + * loaded and used concurrently in one process, from any number of threads, each initialized once + * with its own zone, and every answer is the one a single image on a single thread gives. The one + * exception is the admission time budget (`chs_set_defaults`): it measures the evaluating thread's + * CPU time, and an evaluation that costs more than about half of the budget on an idle core may be + * declined (`CHS_CODE_DECLINED`, reason `admission-time-budget`) when the process's load raises its + * CPU time, never answered differently: above the budget on an idle core it always declines, below + * about half of it never. Its catalog entry states this load dependence. A conformance case drives + * two lines, then four, concurrently in one process. */ #ifndef CHTYPES_V2_H #define CHTYPES_V2_H @@ -164,7 +243,7 @@ extern "C" { * spec/abi-v2/abi.json. chs_build_info()'s `abi_fingerprint` is this macro, copied; a loader * compares the two byte for byte and refuses any difference. */ -#define CHS_ABI_FINGERPRINT "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4" +#define CHS_ABI_FINGERPRINT "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2" /* * UNSTABLE: spec/abi-v2/abi.json declares `stability` "unstable". Generation 2 is still being @@ -188,6 +267,18 @@ extern "C" { * - process_serial: process-level: must not overlap any other call into the same library image. */ +/* --------------------------------------------------------------------------------- init classes */ + +/* + * Every function below names its init class: whether it is valid before `chs_initialize` + * (Initialization, above). + * + * - any: valid before and after `chs_initialize`. + * - required: valid only once `chs_initialize` has succeeded: until then the call changes nothing + * and returns `CHS_INVALID_ARGUMENT`, ch_code 0, and with arguments that are otherwise valid its + * message starts `: call chs_initialize first`. + */ + /* ------------------------------------------------------------------------- byte strings in JSON */ /* @@ -204,19 +295,20 @@ extern "C" { * The data-derived fields, by document: * * - schema_description: columns[].name, columns[].type, columns[].default_expression, - * filter_declined_settings[].name, replicated.zookeeper_path, replicated.replica_name. - * - row: err, verdict_err, partition_id, unknown_fields[].name, unsupported_settings[].name, - * cols[].name, cols[].type, cols[].base, cols[].input, cols[].stored, cols[].ref, - * cols[].ref_type, cols[].wire, computed[].name, computed[].stored, transformed[].column, - * transformed[].input, transformed[].stored. + * columns[].reads[].name, filter_declined_settings[].name, replicated.zookeeper_path, + * replicated.replica_name. + * - row: err, verdict_err, partition_id, unknown_fields[].name, cols[].name, cols[].type, + * cols[].base, cols[].input, cols[].stored, cols[].ref, cols[].ref_type, cols[].wire, + * computed[].name, computed[].stored, transformed[].column, transformed[].input, + * transformed[].stored. * - batch: err, export_declined, rows[].err, rows[].verdict_err, rows[].partition_id, - * rows[].unknown_fields[].name, rows[].unsupported_settings[].name, rows[].cols[].name, - * rows[].cols[].type, rows[].cols[].base, rows[].cols[].input, rows[].cols[].stored, - * rows[].cols[].ref, rows[].cols[].ref_type, rows[].cols[].wire, rows[].computed[].name, - * rows[].computed[].stored, rows[].transformed[].column, rows[].transformed[].input, - * rows[].transformed[].stored, engine_rows[][].name, engine_rows[][].stored, - * storage_transforms[].column, storage_transforms[].stored, at_merge[].column, at_merge[].stored, - * transformed[].column, transformed[].input, transformed[].stored, unsupported_settings[].name, + * rows[].unknown_fields[].name, rows[].cols[].name, rows[].cols[].type, rows[].cols[].base, + * rows[].cols[].input, rows[].cols[].stored, rows[].cols[].ref, rows[].cols[].ref_type, + * rows[].cols[].wire, rows[].computed[].name, rows[].computed[].stored, + * rows[].transformed[].column, rows[].transformed[].input, rows[].transformed[].stored, + * engine_rows[][].name, engine_rows[][].stored, storage_transforms[].column, + * storage_transforms[].stored, at_merge[].column, at_merge[].stored, transformed[].column, + * transformed[].input, transformed[].stored, unsupported_settings[].name, export_columns[].name, * framing.header.names[].name. * - filter_result: err, unsupported_settings[].name, errors[].err. * - discovery: columns[].name, columns[].type, columns[].default_kind, @@ -242,9 +334,10 @@ extern "C" { * - `timezone`: the server's own zone, the zone its tables bind. An empty string, or no member, * means the image zone. The name is validated by ClickHouse's own `DateLUT`, which reads the * library host's time zone database. - * - `settings`: the settings the server's profile applies to every query, each value a JSON string, - * never rewritten. The library accepts exactly what the server's own SET check accepts. - * `session_timezone` here is the default session zone of the server's user. + * - `settings`: the settings the server's profile applies to every query, each value a JSON string + * or the object form for bytes (Input values), never rewritten. A `chtypes_*` knob is never one + * of them (`chs_set_defaults`). The library accepts exactly what the server's own SET check + * accepts. `session_timezone` here is the default session zone of the server's user. * - `macros`: the server's macros, as its `system.macros` lists them (`macro`, `substitution`). * Absent, they are unknown; present, even as `{}`, they are the complete set (see * `chs_server_create`). A macro name that ClickHouse's own macro reader refuses (for example @@ -258,12 +351,28 @@ extern "C" { * "additionalProperties": false, * "properties": { * "timezone": {"type": "string"}, - * "settings": {"type": "object", "additionalProperties": {"type": "string"}}, + * "settings": { + * "type": "object", + * "additionalProperties": {"oneOf": [{"type": "string"}, {"$ref": "#/$defs/bytes_value"}]} + * }, * "macros": { * "type": "object", * "propertyNames": {"minLength": 1}, * "additionalProperties": {"type": "string"} * } + * }, + * "$defs": { + * "bytes_value": { + * "type": "object", + * "additionalProperties": false, + * "required": ["b64"], + * "properties": { + * "b64": { + * "type": "string", + * "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + * } + * } + * } * } * } * ``` @@ -282,16 +391,215 @@ extern "C" { */ /* - * input:schema_options: The options of `chs_schema_create` (rule r1). It defines no member yet, so - * `{}` (or length 0) is the only valid document, and any key is `CHS_INVALID_ARGUMENT`, naming the - * key. A choice about how one statement is compiled lands here, as a new member: for example, - * compiling a server's own `create_table_query`, read back from the server, rather than a caller's - * CREATE. + * input:schema_options: The options of `chs_schema_create` (rule r1): how one statement is + * compiled. Its member is `posture`, a `posture` value, and `{}` (or length 0) compiles in + * `create`, as before the member existed. A key it does not define is `CHS_INVALID_ARGUMENT`, + * naming the key, and so is a `posture` that is not one of its values, naming the value. + * + * - `create`: the statement is a `CREATE TABLE` a server would run now, with every check a server's + * CREATE runs, under the settings layers (`chs_schema_create`). + * - `load`: the statement is the stored definition of a table the server already holds, as that + * server reports it (`system.tables.create_table_query`, `SHOW CREATE TABLE`, or the table's + * metadata file), compiled as the server loads it when it starts. It is one switch, and + * ClickHouse's own: every check on the compile's path that reads the loading posture reads + * `FORCE_ATTACH` instead of `CREATE`, in ClickHouse's own code, and nothing else changes. So the + * checks a server applies only to a new table are not applied: the type gates + * (`allow_suspicious_low_cardinality_types`, `allow_suspicious_fixed_string_types` and the rest, + * at the compile and at every call on the schema), the codec, statistics and suspicious-index + * checks, the DEFAULT validation of a typed column, the TTL checks, the MergeTree settings' + * constraint and sanity checks, and the floating-point partition-key, nullable-key and + * deprecated-syntax gates. `flatten_nested` and `data_type_default_nullable` are not applied + * either: a stored definition is already flattened and typed. Which checks these are on each + * supported line is the artifact producer's enumeration from the vendored source; no list is kept + * here. A projection that a starting server skips because it no longer parses is declined, + * because whether a given server still holds it depends on that server's history. A Replicated + * engine's ZooKeeper path and replica name never change a row verdict, so under `load` a stored + * definition whose engine arguments read a macro the server's `macros` cannot expand (or carry + * none, as a Replicated database's table does) compiles as the server loads it: + * `schema_description` then carries no `replicated` member, and only an answer that depends on + * the path (none today) is declined. + * + * **The caller's obligation.** `load` asserts that the server holds a table defined by exactly this + * statement. The library cannot check that assertion, because it never connects to the server. A + * statement a server would refuse at `CREATE` compiles under `load`, and every answer on the schema + * is the answer for INSERTs into that stored table. If no server holds such a table, every answer + * describes a table that does not exist, and an accepted row is evidence of nothing. A statement a + * server reported for a table it holds (its `create_table_query`, `SHOW CREATE TABLE`, or its + * metadata file) MUST be compiled in `load`, and a statement a caller wrote or edited in `create`. + * A server's `create_table_query` compiled in `create` can be wrong in both directions, because the + * server created the table under its own CREATE's settings, which the statement does not carry: an + * over-reject where a gate was set (a `LowCardinality(UInt8)` column) and an over-accept where a + * shape setting was (a `Nested` column of a table created with `flatten_nested = 0`). * * An options document (rule r1). Its members, as JSON Schema: * * ```json - * {"type": "object", "additionalProperties": false} + * { + * "type": "object", + * "additionalProperties": false, + * "properties": {"posture": {"enum": ["create", "load"]}} + * } + * ``` + */ + +/* + * input:insert_options: The options of every call that models one INSERT of a body: + * `chs_preview_row`, `chs_preview_batch`, `chs_block_create` and `chs_filter_eval_body` (rule r1). + * Its members: `insertable_columns`, `column_defaults` and `verdict_basis`, `defer_defaults`. + * + * - `insertable_columns`: the columns an INSERT may supply, as name objects: the server's + * column-level INSERT grant (`GRANT INSERT(a, b) ON t`), checked as the server checks it, on the + * INSERT's header and never on the fields a body supplies. The header is the call's `columns` + * list, or with none the server's own no-list header: every ordinary and DEFAULT column, in + * declaration order (so a positional CSV, TSV, Values or JSONCompactEachRow body with no list + * supplies all of them). The check runs after the list's own refusals (an unknown or ALIAS + * column, 16; a duplicate, 15; a MATERIALIZED column, 44, unless + * `insert_allow_materialized_columns` is set, when it needs the grant like any other) and before + * any byte of the body is read. A header naming a column outside the set is refused 497 + * (ACCESS_DENIED), with `refused_at` `call`, in every format and whatever the body holds, an + * empty or a malformed body included. So `columns` MUST be a subset of the set, and a caller + * whose set does not cover every ordinary and DEFAULT column passes `columns`, at every call that + * takes this document, `chs_filter_eval_body` included: with a list a call follows the listed + * INSERT's rules, and with none the no-list rule. A body field naming a column the header omits + * (a JSON key, a `WithNames` header field) is an unknown field, never a grant question: skipped, + * or refused 117 under `input_format_skip_unknown_fields` 0 (`refused_at` `row` for a JSON key, + * `body_read` for a header field), and never stored. An omitted column keeps its declared kind, + * exactly as with no set: a generator DEFAULT draws (`default_generated`), a clock DEFAULT + * resolves, a literal DEFAULT fills, a MATERIALIZED column computes and an EPHEMERAL column takes + * its type default. The export carries every stored non-computed column, those outside the set + * included (`document:batch`), so the INSERT of the export needs a grant that covers its column + * list. A Nested column is granted per subcolumn (`n.x`). The set is a grant, not a column list: + * like a grant it may name an EPHEMERAL or MATERIALIZED column, or a name the table lacks. The + * 497's message is ClickHouse's own, naming the user `default` and the table `default.t`, since a + * schema carries neither a user nor a table name; it is informative. Absent is every column. + * Implemented in the first build that carries this description. + * - `column_defaults`: per-call DEFAULT overrides, a list of `{name, value}` (each by the rule for + * byte strings in JSON). It is one rule: in a cell the server would fill from the column's + * DEFAULT, the call uses the caller's `value` instead. A supplied value wins: the override fills + * exactly the cells ClickHouse's own input format marks missing for that column under the call's + * format settings, as the vendored reader decides it, and no other: a field the record omits, a + * `null` for a non-Nullable column under `input_format_null_as_default` 1, and an empty CSV field + * under `input_format_csv_empty_as_default` 1. An empty JSON string `""` is a supplied value. A + * Nullable column's `null` stays `NULL` where the server keeps it (the implementing build + * confirms this against the server's own missing mark). Such a cell reads `default_override`, and + * everything that reads the column sees the value as an INSERT that supplied it would: a CHECK, + * the sorting, primary and partition keys, a skip index, a projection, a TTL, a MATERIALIZED or + * DEFAULT expression that reads it, and a filter. The export carries it as a stored, present + * value at the column's position in `export_columns`, so the INSERT of the export needs no + * override, and a column the override fills is not deferred in those rows (`defer_defaults`). An + * entry may name an ordinary or a DEFAULT column, and an EPHEMERAL column where the implementing + * build measures it equal to a supplied EPHEMERAL value; an entry that names an unknown column, a + * column that is not insertable, or a MATERIALIZED or ALIAS column is `CHS_INVALID_ARGUMENT` at + * `call`, naming `column_defaults[i]`. When `insertable_columns` is given, an entry's column MUST + * be in it, because the INSERT the override models names the column in its header; one outside it + * is the same `CHS_INVALID_ARGUMENT`. `value` is read in the body format's own field encoding, as + * if the record supplied this field (the implementing build confirms); a value that does not read + * as the column's type is `CHS_INVALID_ARGUMENT` at `call`, naming `column_defaults[i]`, before + * any byte of the body is read. Built before the lock: the build that implements it lists + * `insert_options.column_defaults` in `capabilities.features` (rule r9), and a build that does + * not declines it as Input documents (in the Preamble) says. + * - `verdict_basis`: `input`, the default, or `export`: what row a filter verdict answers for. + * Under `input`, a filter that reads a value the library drew or resolved itself is `d` + * (`drawn-value`), because the server would draw its own. Under `export`, the verdict answers for + * the row the export would store, so a drawn or resolved value is answerable, and the caller MUST + * insert the export. Reserved: declined at `export` until implemented. + * - A member whose value is a list or an object is refused `CHS_INVALID_ARGUMENT` when an entry + * does not match its shape, naming the key. `{}` (or length 0) is the INSERT these calls modeled + * before the document existed, byte for byte. Each member means the same at every call that takes + * the document. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a + * value that is not one of the member's, naming the value; a member this build does not implement + * declines as Input documents (in the Preamble) says. + * + * - `defer_defaults`: `none`, the default, or `server` (public issue #479). `server` defers to the + * server each DEFAULT this build would otherwise decline because its value is the server's to + * compute (a generator or a server function the build does not admit). Such a column reads + * `default_deferred` in the rows that omit it, and the export omits it from those rows' lines. + * What reads it there is not answered (the deferred-column rule): a filter that reads it makes + * the row's verdict `d` (`verdict_code` `CHS_CODE_DECLINED`, `verdict_reason` + * `defer-defaults-reader`); a CHECK, a sorting, primary or partition key, a skip index, a + * projection, a TTL, or a MATERIALIZED or other DEFAULT expression that reads it declines the row + * (`unsupported`, `reason` `defer-defaults-reader`); and such a row carries no `partition_id`. + * Admitted generators keep drawing (`default_generated`), so a drawn id stays usable for + * deduplication. A body is mixed when its exported rows' lines would not all omit the same + * columns (some rows supply a deferred column and others omit it). The export has one column + * list, so a mixed body is declined, and only when an export is asked for: the batch is + * `unsupported`, `refused_at` `body_step`, `reason` `defer-defaults-mixed-body`; with no export + * asked for no line omits anything, and the batch is answered. A body whose rows all defer the + * same columns, or none, is never mixed: its export omits those columns from every line and from + * `export_columns`. Declined until implemented. + * + * An options document (rule r1). Its members, as JSON Schema: + * + * ```json + * { + * "type": "object", + * "additionalProperties": false, + * "properties": { + * "defer_defaults": {"enum": ["none", "server"]}, + * "insertable_columns": {"type": "array", "items": {"$ref": "#/$defs/named"}}, + * "column_defaults": {"type": "array", "items": {"$ref": "#/$defs/column_default"}}, + * "verdict_basis": {"enum": ["input", "export"]} + * }, + * "$defs": { + * "named": { + * "type": "object", + * "additionalProperties": false, + * "properties": { + * "name": {"type": "string"}, + * "name_b64": { + * "type": "string", + * "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + * } + * }, + * "allOf": [{"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}] + * }, + * "column_default": { + * "type": "object", + * "additionalProperties": false, + * "properties": { + * "name": {"type": "string"}, + * "name_b64": { + * "type": "string", + * "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + * }, + * "value": {"type": "string"}, + * "value_b64": { + * "type": "string", + * "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + * } + * }, + * "allOf": [ + * {"oneOf": [{"required": ["name"]}, {"required": ["name_b64"]}]}, + * {"oneOf": [{"required": ["value"]}, {"required": ["value_b64"]}]} + * ] + * } + * } + * } + * ``` + */ + +/* + * input:filter_options: The options of `chs_filter_create` (rule r1): how one filter is compiled. + * `{}` (or length 0) compiles as before the document existed. A key it does not define is + * `CHS_INVALID_ARGUMENT`, naming the key, and so is a value that is not one of the member's, naming + * the value; a member this build does not implement declines as Input documents (in the Preamble) + * says. + * + * - `param_encoding`: `escaped`, the default, or `raw`: how a query parameter's value is read. + * Under `escaped`, the value is decoded by ClickHouse's escaped-text reader, exactly as a server + * decodes an HTTP `param_` value, so a backslash, a tab or a newline must be escaped as that + * reader expects; the object form `{"b64": ...}` is decoded from base64 first and the bytes then + * read the same way. Under `raw`, the bytes are the value. Reserved: declined at `raw` until + * implemented (rule r9). + * + * An options document (rule r1). Its members, as JSON Schema: + * + * ```json + * { + * "type": "object", + * "additionalProperties": false, + * "properties": {"param_encoding": {"enum": ["escaped", "raw"]}} + * } * ``` */ @@ -361,11 +669,18 @@ typedef struct chs_block chs_block; * would also give. `CHS_REJECTED` with ch_code 306, TOO_DEEP_RECURSION, never `CHS_INTERNAL`, is * returned for a deep argument the call compiles (an expression, a type, or a settings, columns * or options JSON), at `chs_set_defaults`, `chs_server_create`, `chs_schema_create`, - * `chs_schema_describe`, `chs_preview_row`, `chs_preview_batch`, `chs_filter_create`, - * `chs_filter_eval_body`, `chs_block_create`, `chs_filter_eval_block`, `chs_type_validate`, - * `chs_reference_type` and `chs_discover_columns`. A body's depth refusal is reported in its - * document (`CHS_OK`, with code 306), as every refused body is. + * `chs_filter_create`, `chs_type_validate`, `chs_reference_type` and `chs_discover_columns`. A + * call that answers with a document (`chs_preview_row`, `chs_preview_batch`, + * `chs_filter_eval_body`, `chs_block_create` and `chs_filter_eval_block`) returns a status other + * than `CHS_OK` only for caller misuse (`CHS_INVALID_ARGUMENT`) or a library bug + * (`CHS_INTERNAL`): every refusal or decline of the call itself, of its settings, its column list + * or its options, a deep argument (306) included, is reported in its document with `refused_at` + * `call`, and a body's refusal with `refused_at` `row` or `body_step`, as every refused body is. + * `chs_schema_describe` never refuses a schema `chs_schema_create` returned: the statement's + * depth is judged at compile. * - `CHS_DECLINED`: this build will not answer; a server might accept. Never scored as agreement. + * The error names the reason (`chs_error_reason`). A call that answers with a document never + * returns it: its decline is the document's `unsupported` with `refused_at` `call`. * - `CHS_INVALID_ARGUMENT`: caller misuse, such as a NULL pointer with a nonzero length, a * wrong-kind, freed or cross-library handle, or a NULL required out-parameter. * - `CHS_INTERNAL`: a guarded exception inside the library, which is a library bug. @@ -418,6 +733,13 @@ typedef int32_t chs_format; */ #define CHS_EXPORT_NONE (-1) +/* + * CHS_CODE_DECLINED: The code a declined row, outcome or verdict carries (Codes, in the Preamble): + * -2, outside ClickHouse's code table. It is a sentinel, stable under rule r4, and never a + * ClickHouse code. + */ +#define CHS_CODE_DECLINED (-2) + /* * CHS_DOC_VALUES: A `doc_flags` bit: the per-row documents carry each column's stored value and * provenance. @@ -451,7 +773,7 @@ typedef int32_t chs_format; * instead, at whichever step first resolves it — only this symbol's absence means the library is * not such an artifact at all. * - * Class: handshake. Thread: any. + * Class: handshake. Thread: any. Init: any. */ CHS_API int32_t chs_abi_version(void); @@ -476,7 +798,7 @@ CHS_API int32_t chs_abi_version(void); * file's own size, digest and glibc floor are not in it, because the file cannot carry facts * measured on itself; they are in the signed statement. * - * Class: handshake. Thread: any. + * Class: handshake. Thread: any. Init: any. */ CHS_API const char *chs_build_info(void); @@ -486,7 +808,7 @@ CHS_API const char *chs_build_info(void); * and spelling so a v0 binding reaches its own refusal. A binding of generation 1 or later reads * `clickhouse_version` from `chs_build_info` instead. * - * Class: handshake. Thread: any. + * Class: handshake. Thread: any. Init: any. */ CHS_API const char *chs_clickhouse_version(void); @@ -503,7 +825,7 @@ CHS_API const char *chs_clickhouse_version(void); * it. A binding of generation 1 or later never calls it; the loader resolves it only to prove it is * present. * - * Class: tombstone. Thread: any. Always returns CHS_ABI_REVISION_TOMBSTONE (1001). + * Class: tombstone. Thread: any. Init: any. Always returns CHS_ABI_REVISION_TOMBSTONE (1001). */ CHS_API int chs_abi_revision(void); @@ -511,8 +833,8 @@ CHS_API int chs_abi_revision(void); * chs_buf_data: The buffer's bytes. The pointer may be NULL when `chs_buf_len` is 0, and a binding * never dereferences it then. A NULL, freed, wrong-kind or cross-library buffer gives NULL. * - * Class: api. Thread: shared. The pointer is borrowed from `buf` and valid until it is freed; copy - * before freeing. + * Class: api. Thread: shared. Init: any. The pointer is borrowed from `buf` and valid until it is + * freed; copy before freeing. * * - `buf`: required. */ @@ -522,7 +844,7 @@ CHS_API const uint8_t *chs_buf_data(const chs_buf *buf); * chs_buf_len: The buffer's length in bytes. A NULL, freed, wrong-kind or cross-library buffer * gives 0. * - * Class: api. Thread: shared. + * Class: api. Thread: shared. Init: any. * * - `buf`: required. */ @@ -531,7 +853,7 @@ CHS_API size_t chs_buf_len(const chs_buf *buf); /* * chs_buf_free: Releases the caller's reference to a buffer. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `buf`: may be NULL. */ @@ -541,7 +863,7 @@ CHS_API void chs_buf_free(chs_buf *buf); * chs_error_status: The error's status, never `CHS_OK` for an error a call set. A NULL, freed, * wrong-kind or cross-library error gives `CHS_INVALID_ARGUMENT`. * - * Class: api. Thread: shared. + * Class: api. Thread: shared. Init: any. * * - `error`: required. */ @@ -551,7 +873,7 @@ CHS_API chs_status chs_error_status(const chs_error *error); * chs_error_ch_code: ClickHouse's own error code, nonzero only when the status is `CHS_REJECTED`, * and 0 otherwise or for an invalid error. * - * Class: api. Thread: shared. + * Class: api. Thread: shared. Init: any. * * - `error`: required. */ @@ -563,8 +885,8 @@ CHS_API int32_t chs_error_ch_code(const chs_error *error); * `CHS_REJECTED`. Like the other two text accessors, it returns NULL only for a NULL or invalid * error. * - * Class: api. Thread: shared. Returns a new chs_buf the caller owns and frees with chs_buf_free; - * its content is guaranteed ASCII; NULL only for a NULL or invalid argument. + * Class: api. Thread: shared. Init: any. Returns a new chs_buf the caller owns and frees with + * chs_buf_free; its content is guaranteed ASCII; NULL only for a NULL or invalid argument. * * - `error`: required. */ @@ -573,11 +895,12 @@ CHS_API chs_buf *chs_error_ch_name(const chs_error *error); /* * chs_error_message: A new buffer holding the message: ClickHouse's own text, verbatim, for * `CHS_REJECTED`, and the library's explanation otherwise. A message may quote input bytes, so it - * is a byte string, not text. + * is a byte string, not text. It is informative: a caller keys on the status, the code and the + * reason (`chs_error_reason`), never on message text. * - * Class: api. Thread: shared. Returns a new chs_buf the caller owns and frees with chs_buf_free; - * its content is a ClickHouse message: may quote input bytes, so may carry any byte; NULL only for - * a NULL or invalid argument. + * Class: api. Thread: shared. Init: any. Returns a new chs_buf the caller owns and frees with + * chs_buf_free; its content is a ClickHouse message: may quote input bytes, so may carry any byte; + * NULL only for a NULL or invalid argument. * * - `error`: required. */ @@ -588,18 +911,32 @@ CHS_API chs_buf *chs_error_message(const chs_error *error); * empty when there is none. A column name is a byte string: NUL and invalid UTF-8 are legal in a * name. * - * Class: api. Thread: shared. Returns a new chs_buf the caller owns and frees with chs_buf_free; - * its content is a ClickHouse column name: NUL and invalid UTF-8 are legal, so never assume UTF-8; - * NULL only for a NULL or invalid argument. + * Class: api. Thread: shared. Init: any. Returns a new chs_buf the caller owns and frees with + * chs_buf_free; its content is a ClickHouse column name: NUL and invalid UTF-8 are legal, so never + * assume UTF-8; NULL only for a NULL or invalid argument. * * - `error`: required. */ CHS_API chs_buf *chs_error_column(const chs_error *error); +/* + * chs_error_reason: A new buffer holding the reason: the catalog id + * (`spec/abi-v2/declines/declines.json`, ASCII) of a decline or a library-side refusal, empty when + * the error carries none (ClickHouse's own refusal, an internal error). It is the member a caller + * keys on, never the message. Like the other text accessors, it returns NULL only for a NULL or + * invalid error. + * + * Class: api. Thread: shared. Init: any. Returns a new chs_buf the caller owns and frees with + * chs_buf_free; its content is guaranteed ASCII; NULL only for a NULL or invalid argument. + * + * - `error`: required. + */ +CHS_API chs_buf *chs_error_reason(const chs_error *error); + /* * chs_error_free: Releases the caller's reference to an error. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `error`: may be NULL. */ @@ -611,7 +948,8 @@ CHS_API void chs_error_free(chs_error *error); * them and then asserts every count is zero, which is how all four bindings prove their finalizers * release everything. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `out`: receives the JSON document `live_handles`, owned by the caller, freed with chs_buf_free; * required. @@ -626,7 +964,8 @@ CHS_API chs_status chs_live_handles(chs_buf **out, chs_error **err); * caller that needs several lines asks each library. The name is v0's, with the v1 call shape; the * tombstone keeps every v0 binding from reaching it. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `out`: receives the JSON document `error_code_table`, owned by the caller, freed with * chs_buf_free; required. @@ -639,8 +978,8 @@ CHS_API chs_status chs_error_codes(chs_buf **out, chs_error **err); * tooling export for the artifact producer's build gates: no binding wraps it, and a loader * resolves it only for presence. * - * Class: tooling. Thread: process_serial. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, - * CHS_INTERNAL. + * Class: tooling. Thread: process_serial. Init: required. Returns one of: CHS_OK, + * CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `out`: receives guaranteed ASCII, owned by the caller, freed with chs_buf_free; required. * - `err`: may be NULL; set only when the status is not CHS_OK. @@ -652,8 +991,8 @@ CHS_API chs_status chs_registered_families(chs_buf **out, chs_error **err); * function per line, read off this build's own registry. A tooling export for the artifact * producer's build gates, which derive the statelessness evidence from it; no binding wraps it. * - * Class: tooling. Thread: process_serial. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, - * CHS_INTERNAL. + * Class: tooling. Thread: process_serial. Init: required. Returns one of: CHS_OK, + * CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `out`: receives guaranteed ASCII, owned by the caller, freed with chs_buf_free; required. * - `err`: may be NULL; set only when the status is not CHS_OK. @@ -664,8 +1003,8 @@ CHS_API chs_status chs_function_flags(chs_buf **out, chs_error **err); * chs_reference_type: The widened reference type this build pairs with a type expression. A tooling * export; no binding wraps it. * - * Class: tooling. Thread: process_serial. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: tooling. Thread: process_serial. Init: required. Returns one of: CHS_OK, CHS_REJECTED, + * CHS_DECLINED, CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `type_expr`, `type_expr_len`: SQL text (a statement, an expression, a type, a quoted name or * literal): may carry any byte; length 0 is none, and the pointer may then be NULL. @@ -694,8 +1033,8 @@ CHS_API chs_status chs_reference_type( * before any other call; the list of type families the build must refuse is embedded when the * library is built, so nothing else is passed in. * - * Class: api. Thread: process_once. Returns one of: CHS_OK, CHS_REJECTED, CHS_INVALID_ARGUMENT, - * CHS_INTERNAL. + * Class: api. Thread: process_once. Init: any. Returns one of: CHS_OK, CHS_REJECTED, + * CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `timezone`, `timezone_len`: a time zone name, validated by ClickHouse's own DateLUT; length 0 * means UTC; length 0 is none, and the pointer may then be NULL. @@ -705,20 +1044,51 @@ CHS_API chs_status chs_initialize(const uint8_t *timezone, size_t timezone_len, /* * chs_set_defaults: Seeds the settings every later call starts from, as a JSON object whose values - * are JSON strings (a binding never rewrites a value, a boolean included). Each successful call - * replaces the previous defaults whole. A setting name the server would refuse is refused here, - * with ClickHouse's own code and message, and nothing is committed. `session_timezone` here is the - * default zone for later calls; it does not change the image zone that compiled types bind. + * are JSON strings, or the object form for bytes (Input values), and a binding never rewrites a + * value, a boolean included. Each successful call replaces the previous defaults whole. A setting + * name the server would refuse is refused here, with ClickHouse's own code and message, and nothing + * is committed. `session_timezone` here is the default zone for later calls; it does not change the + * image zone that compiled types bind. + * + * The library's own knobs are `chtypes_*` settings, and this is the only channel for the + * process-level ones. Each is an integer or a string given as a JSON string, and no server knows + * one, so a server profile refuses it (115) and a call's settings carry only the per-call ones. The + * knobs, their layer, type, default and effect: `chtypes_now_epoch_nanos` (per call, int64 + * nanoseconds since the epoch, default the wall clock: pins the call's clock instant); + * `chtypes_clock_offset_nanos` (per call, int64, default 0: shifts the instant); + * `chtypes_max_clock_skew_nanos` (per call, int64, default 0: the skew budget a clock-reading + * expression may resolve under); `chtypes_default_eval_memory_bytes` (process, int64 bytes, default + * 268435456, 0 disables: the admission memory budget one DEFAULT, CHECK, TTL or filter evaluation + * may use); `chtypes_default_eval_cpu_nanos` (process, int64, default 1000000000, 0 disables: the + * admission time budget, the evaluating thread's CPU time, detected not interrupted; the earlier + * name `chtypes_default_eval_wall_nanos` is refused on v2 with `CHS_INVALID_ARGUMENT` naming this + * one); `chtypes_custom_settings_prefixes` (process, string, default none: the server's + * `custom_settings_prefixes`, comma-separated). A process knob in a call's settings is declined, + * named in `unsupported_settings` (`call-setting-process-only-key`), and a clock knob whose value + * is not an integer likewise (`call-setting-clock-value`). An evaluation that exceeds a budget is + * declined with that budget's own reason id, never answered: `admission-memory-budget` for the + * memory budget and `admission-time-budget` for the time budget; the earlier single id + * `admission-envelope` is retired, its catalog entry naming both successors (rule r4). The time + * budget is the evaluating thread's CPU time, not wall time. Each catalog entry states its load + * dependence. A memory-budget decline is deterministic (`measured`). A time-budget decline depends + * on load only near the budget: CPU time rose up to x1.03 under load on linux-arm64 and up to x2.0 + * on linux-amd64 when two busy hardware threads share a core, so a time-budget decline is + * retry-stable when the evaluation costs more than the budget on an idle core and never happens + * under about half of it; between half and one times the budget the host's load can decide. Normal + * evaluations cost about 1000 times less than the 1 s default. a schema whose DEFAULT exceeds a + * budget at admission is declined at compile. The defaults are this description's, not a build's: a + * build reports nothing about them. * * Setup only. The defaults are immutable once this image has created its first `chs_schema` (a * filter or a block needs one): from then on the call changes nothing and answers * `CHS_INVALID_ARGUMENT`. No call ever reads defaults that change under it, so no binding needs a * lock around them. * - * Class: api. Thread: process_serial. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: process_serial. Init: required. Returns one of: CHS_OK, CHS_REJECTED, + * CHS_DECLINED, CHS_INVALID_ARGUMENT, CHS_INTERNAL. * - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. * - `err`: may be NULL; set only when the status is not CHS_OK. */ @@ -730,7 +1100,7 @@ CHS_API chs_status chs_set_defaults(const uint8_t *settings, size_t settings_len * again. Unloading the library, or initializing it again in the same process, is unsupported: a * loader never unloads a library. * - * Class: api. Thread: process_serial. + * Class: api. Thread: process_serial. Init: any. */ CHS_API void chs_shutdown(void); @@ -738,7 +1108,7 @@ CHS_API void chs_shutdown(void); * chs_type_validate: Parses one type expression with ClickHouse's own parser and returns its * canonical spelling, or ClickHouse's own refusal. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, * CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `type_expr`, `type_expr_len`: SQL text (a statement, an expression, a type, a quoted name or @@ -758,7 +1128,8 @@ CHS_API chs_status chs_type_validate( * every special byte escaped. A passthrough over the vendored function, and named after it; the * name is a byte string. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `name`, `name_len`: a ClickHouse column name: NUL and invalid UTF-8 are legal, so never assume * UTF-8; length 0 is none, and the pointer may then be NULL. @@ -777,7 +1148,8 @@ CHS_API chs_status chs_back_quote( * must be. Which names stay bare is a property of the ClickHouse release, so a caller that needs * one answer for several releases asks each library. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `name`, `name_len`: a ClickHouse column name: NUL and invalid UTF-8 are legal, so never assume * UTF-8; length 0 is none, and the pointer may then be NULL. @@ -795,7 +1167,8 @@ CHS_API chs_status chs_back_quote_if_needed( * chs_quote_string: A byte string spelled as a ClickHouse string literal by the vendored * `quoteString`. The input may contain NUL; the answer escapes it. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `text`, `text_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and * the pointer may then be NULL. @@ -834,18 +1207,20 @@ CHS_API chs_status chs_quote_string( * The macros are the server's whole `system.macros` when `macros` is present, even as `{}`: a macro * a Replicated engine's arguments name that the set lacks is then the server's own refusal * (`CHS_REJECTED`, 139, NO_ELEMENTS_IN_CONFIG). When `macros` is absent the server's macros are - * unknown, and a schema whose engine reads one is declined, as it is without a server. A build that - * validates a profile's macros but does not yet apply them declines such a schema in both cases. + * unknown, and a schema whose engine reads one is declined, as it is without a server, except under + * `schema_options`' `posture` `load`, where a stored Replicated definition compiles without + * expanding them (`input:schema_options`). A build that validates a profile's macros but does not + * yet apply them declines such a schema in both cases. * * A setting this build's filters do not honor in a WHERE is no decline here, and the call succeeds. * So does `chs_schema_create` on the server, because a profile never changes a schema compile: it * lists the setting in the schema's `filter_declined_settings` (`schema_description`), at layer * `server`. * - * The call reads no defaults and leaves `chs_set_defaults`' latch open. Like every call that is not - * the handshake, it is valid only after `chs_initialize`. + * The call reads no defaults and leaves `chs_set_defaults`' latch open. Its init class is + * `required`, so before `chs_initialize` it refuses (Initialization, in the Preamble). * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, * CHS_INVALID_ARGUMENT, CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its * options document, `options`. * @@ -868,7 +1243,7 @@ CHS_API chs_status chs_server_create( * chs_server_free: Releases the caller's reference to a server. Schemas created on it keep it * alive. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `server`: may be NULL. */ @@ -880,7 +1255,21 @@ CHS_API void chs_server_free(chs_server *server); * `server`, under a profile of settings given as a JSON object of string values. A trailing * semicolon is allowed. A second statement is refused the way ClickHouse's own parser refuses one * in a single query (`CHS_REJECTED`, with its code and message), and a statement of any other kind - * is `CHS_INVALID_ARGUMENT`. + * is `CHS_INVALID_ARGUMENT`, except under `posture` `load`: there a `CREATE VIEW`, `CREATE + * MATERIALIZED VIEW` or `CREATE DICTIONARY` statement, the stored definition of an object a server + * lists in `system.tables` and whose columns it lists in `system.columns`, is `CHS_DECLINED` with + * the reason `engine-not-modeled`, so a caller that compiles a server's stored definitions falls + * back for it as it does for an engine this build does not model. The `ATTACH` spellings of those + * kinds that a server refuses with 62 stay `CHS_REJECTED` with that code, as the server answers. An + * `ATTACH TABLE` statement is not one of the other kinds (`options`, below). The library models a + * table, not its database's engine: `ATTACH TABLE` with a column list and an engine but no UUID is + * accepted under `posture` `load`, which a server's Atomic database refuses (80), and under + * `create` it is declined (`attach-statement-under-create`). + * + * A table whose engine this build does not model (a `Distributed` or `Buffer` table, for example) + * is `CHS_DECLINED` with the reason `engine-not-modeled`, which is returned only when the engine + * itself is the cause: a decline for anything else in the statement carries that thing's own + * reason. * * `server` is the server the table is on (`chs_server_create`), or NULL. On a server, the schema's * home zone is the server's `timezone`: its zone-less `DateTime` and `DateTime64` columns bind it, @@ -905,8 +1294,14 @@ CHS_API void chs_server_free(chs_server *server); * succeeds and lists it in the schema's `filter_declined_settings` (`schema_description`), and * `chs_filter_create` on the schema declines. * - * `options` (`input:schema_options`) defines no member yet, so any key in it is - * `CHS_INVALID_ARGUMENT`, naming the key. + * `options` (`input:schema_options`) sets the `posture` the statement compiles in: `create`, the + * default, or `load` for a server's own stored definition (see `input:schema_options`, and the + * caller's obligation there). A `posture` this build does not know, or any other key, is + * `CHS_INVALID_ARGUMENT`, naming it. An `ATTACH TABLE` statement is a stored definition, never a + * statement of another kind: under `create` it is `CHS_DECLINED`, ch_code 0, with exactly the + * message `chs_schema_create: an ATTACH TABLE statement is a stored definition; compile it with + * schema_options {"posture":"load"}; declined rather than answered`, and under `load` it compiles + * as the same definition spelled `CREATE TABLE`. * * A build that does not yet compile on a server declines a non-NULL `server`, and a schema whose * compile needs its server's zone is declined on a host thread that already runs a ClickHouse query @@ -917,14 +1312,15 @@ CHS_API void chs_server_free(chs_server *server); * use it at once. Its types bind its home zone, never the `session_timezone` in its settings, which * is the zone of this call and of the later calls on the schema that name none of their own. * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, * CHS_INVALID_ARGUMENT, CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its * options document, `options`. * * - `server`: may be NULL. * - `create_table`, `create_table_len`: SQL text (a statement, an expression, a type, a quoted name * or literal): may carry any byte; length 0 is none, and the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. * - `options`, `options_len`: the input document `schema_options`; length 0 is `{}`, and the * pointer may then be NULL. @@ -946,7 +1342,7 @@ CHS_API chs_status chs_schema_create( * chs_schema_free: Releases the caller's reference to a schema. Filters and blocks made from it * keep it alive. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `schema`: may be NULL. */ @@ -956,10 +1352,14 @@ CHS_API void chs_schema_free(chs_schema *schema); * chs_schema_describe: A JSON document describing the schema's columns, owned by the caller; it * replaces v0's borrowed column accessors. * - * A deep input the server's stack check refuses is `CHS_REJECTED` with ch_code 306. Here the input - * is the schema's own statement, which describing walks again; it is never `CHS_INTERNAL`. + * A schema compiled in `load` carries `"posture": "load"`; one compiled in `create` carries no + * `posture`. + * + * A schema `chs_schema_create` returned always describes: the walk this document needs runs at + * compile, so a deep statement is refused there (`CHS_REJECTED`, 306) and this call never refuses + * or declines a compiled schema. It is never `CHS_INTERNAL`. * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_INVALID_ARGUMENT, + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, * CHS_INTERNAL. * * - `schema`: required. @@ -980,23 +1380,32 @@ CHS_API chs_status chs_schema_describe(const chs_schema *schema, chs_buf **out, * skip, is rejected with the reader's own code, as a server's INSERT rejects it; what the reader * skips, such as a trailing newline, is fine. A body holding no row, or more than one, is * `CHS_INVALID_ARGUMENT`, naming the call: `chs_preview_batch` answers a body of many rows. The - * row's document equals the single row of `chs_preview_batch`'s document for the same body. + * row's document equals the single row of `chs_preview_batch`'s document for the same body, and + * stands for the call: a refusal or decline of the call, its settings, its column list or its + * options is this document's `outcome` with `refused_at` `call`, never a status. * * An INSERT whose input block has no column at all (every insertable column EPHEMERAL, and no * column list) is declined: a server answers it (code 90, EMPTY_LIST_OF_COLUMNS_PASSED) from a * statement inside its INSERT interpreter that the library cannot call, and a decline is never a * wrong answer. * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * `options` (`input:insert_options`) shapes the INSERT the call models, and `{}` is the INSERT it + * modeled before the document existed. + * + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its options document, + * `options`. * * - `schema`: required. * - `body`, `body_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and * the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. * - `columns`, `columns_len`: a JSON array of column names, each an object carrying the name as * byte_strings says; length 0 is none, and the pointer may then be NULL. + * - `options`, `options_len`: the input document `insert_options`; length 0 is `{}`, and the + * pointer may then be NULL. * - `out`: receives the JSON document `row`, owned by the caller, freed with chs_buf_free; * required. * - `err`: may be NULL; set only when the status is not CHS_OK. @@ -1010,6 +1419,8 @@ CHS_API chs_status chs_preview_row( size_t settings_len, const uint8_t *columns, size_t columns_len, + const uint8_t *options, + size_t options_len, chs_buf **out, chs_error **err); @@ -1024,22 +1435,33 @@ CHS_API chs_status chs_preview_row( * `unsupported`, filter rows `d`, and no export. The same call without the filter is unchanged. * `export_format` is `CHS_EXPORT_NONE` or a `chs_format` the build can write; with an export that * runs, `out_export` receives the accepted rows serialized once, an empty buffer when the batch - * holds no row. When the export is declined (the document carries `export_declined`), `out_export` - * is NULL, as it may be when no export is asked for, so a caller tells a declined export from an - * empty one by `export_declined`, never by the buffer. `doc_flags` chooses which groups the per-row - * documents carry. - * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * holds no row. When the export is declined (the document carries `export_declined` and + * `export_declined_reason`), `out_export` is NULL, as it may be when no export is asked for, so a + * caller tells a declined export from an empty one by `export_declined`, never by the buffer. + * `doc_flags` chooses which groups the per-row documents carry. `options` (`input:insert_options`) + * shapes the INSERT the call models, and `{}` is the INSERT it modeled before the document existed. + * `filter` must have been compiled on `schema`: a filter from another schema handle is + * `CHS_INVALID_ARGUMENT`, as it is at `chs_filter_eval_block`. The call returns a status other than + * `CHS_OK` only for misuse or an internal error: a decline of the call (the filter's zone differing + * from the batch's, `preview-batch-zone-mismatch`) and a refusal of its arguments are the + * document's, with `refused_at` `call` and `reason`. The export's truth table is in + * `document:batch`. + * + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its options document, + * `options`. * * - `schema`: required. * - `body`, `body_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and * the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. * - `columns`, `columns_len`: a JSON array of column names, each an object carrying the name as * byte_strings says; length 0 is none, and the pointer may then be NULL. * - `filter`: may be NULL. + * - `options`, `options_len`: the input document `insert_options`; length 0 is `{}`, and the + * pointer may then be NULL. * - `out`: receives the JSON document `batch`, owned by the caller, freed with chs_buf_free; * required. * - `out_export`: receives arbitrary bytes: a body, a string value, an export, owned by the caller, @@ -1058,6 +1480,8 @@ CHS_API chs_status chs_preview_batch( const chs_filter *filter, int32_t export_format, uint32_t doc_flags, + const uint8_t *options, + size_t options_len, chs_buf **out, chs_buf **out_export, chs_error **err); @@ -1065,8 +1489,16 @@ CHS_API chs_status chs_preview_batch( /* * chs_filter_create: Compiles a boolean SQL expression over the schema's columns, binding its query * parameters (a JSON object of string values) by ClickHouse's own substitution, so a value is never - * SQL text. `settings` is the profile the expression is compiled under, so `session_timezone` there - * is the zone of its literals. Non-deterministic expressions are declined. + * SQL text. A parameter's value is decoded by ClickHouse's escaped-text reader, exactly as a server + * decodes an HTTP `param_` value (`input:filter_options`' `param_encoding`, `escaped`): `\b` is an + * escape, so the parameter `a\b` reads as `a`, the byte 0x08 and `b`, and a literal backslash is + * written `\\`; `\N` reads as the empty value; `\x41`, `\'` and `\0` decode as that reader decodes + * them; a raw tab or newline is `CHS_REJECTED` with 457 and a trailing backslash with 25, as a + * server refuses them; and the bytes an escape produces are kept, valid UTF-8 or not. The object + * form `{"b64": ...}` (Input values) is decoded from base64 first, and its bytes are then read the + * same way. `options` (`input:filter_options`) shapes the compile, and `{}` is the compile before + * the document existed. `settings` is the profile the expression is compiled under, so + * `session_timezone` there is the zone of its literals. Non-deterministic expressions are declined. * * The WHERE's settings are layered, lowest first: the build's own, `chs_set_defaults`, the schema's * server profile, the schema's own settings, and then this call's `settings`; `session_timezone` at @@ -1079,8 +1511,9 @@ CHS_API chs_status chs_preview_batch( * each setting and its layer, in the list's order: `chs_filter_create: the settings this schema was * compiled under set (), (), which this build's filters do not honor in a * WHERE (listed in chs_schema_describe's filter_declined_settings); declined rather than answered`, - * where each `` is the entry's `declined_layer` value. The entries are joined by a comma and - * a space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as + * where each `` is the entry's `declined_layer` value, with the reason + * `server-profile-setting`. The message is informative; the entries are joined by a comma and a + * space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as * `b64:`; it does not occur in practice, because a setting name ClickHouse does not know is * refused earlier, with 115. The server and the schema were created all the same: no settings layer * changes a schema compile. @@ -1090,34 +1523,42 @@ CHS_API chs_status chs_preview_batch( * `unsupported_settings`), and it now declines this call. A server-profile setting already declined * it. * - * A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here, - * and it is never in `schema_description`. A key in this build's honor set is applied to the WHERE; - * a key passed through is accepted and ignored; and a key this build's filters do not honor is - * reported at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`, and - * `chs_preview_batch`'s). The WHERE runs under this call's layers only (the defaults, the server - * profile, the schema's own settings and this call's), as a subscriber's SELECT in its own session - * would. An evaluation's `settings` (`chs_filter_eval_body`, `chs_preview_batch` with a filter, - * `chs_block_create`) are the INSERT session's: they govern parsing the body and never reach the - * WHERE, so they are never named as WHERE settings. So a filter evaluated in the INSERT's own - * session, a materialized view's WHERE for example, is created with that session's WHERE-tier - * settings in its own `settings`. Every key still meets the server's own SET check first, so a name - * the server does not know is `CHS_REJECTED` with ch_code 115. + * A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here + * but for one case, and it is never in `schema_description`. The case is a cell of the line's + * `where-settings/documented-pathological.tsv`, a value under which the server refuses every query: + * set in this call's own `settings`, it is `CHS_DECLINED` with ch_code 0 and the reason + * `setting-server-refuses`, because no verdict under it could stand. A key in this build's honor + * set is applied to the WHERE; a key passed through is accepted and ignored; and a key this build's + * filters do not honor is reported at evaluation (`filter_result`'s `unsupported_settings`, every + * verdict `d`, and `chs_preview_batch`'s). The WHERE runs under this call's layers only (the + * defaults, the server profile, the schema's own settings and this call's), as a subscriber's + * SELECT in its own session would. An evaluation's `settings` (`chs_filter_eval_body`, + * `chs_preview_batch` with a filter, `chs_block_create`) are the INSERT session's: they govern + * parsing the body and never reach the WHERE, so they are never named as WHERE settings. So a + * filter evaluated in the INSERT's own session, a materialized view's WHERE for example, is created + * with that session's WHERE-tier settings in its own `settings`. Every key still meets the server's + * own SET check first, so a name the server does not know is `CHS_REJECTED` with ch_code 115. * * A filter under a per-call `session_timezone` answers as the artifact producer has measured * against live servers, including how the zone in an evaluation's settings combines with the zone * the filter was compiled under. Until a path is measured to match a server it declines, which is * never a wrong answer. The filter holds a counted reference to the schema. * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, + * CHS_INVALID_ARGUMENT, CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its + * options document, `options`. * * - `schema`: required. * - `expr`, `expr_len`: SQL text (a statement, an expression, a type, a quoted name or literal): * may carry any byte; length 0 is none, and the pointer may then be NULL. - * - `query_params`, `query_params_len`: a JSON object whose values are JSON strings (settings, + * - `query_params`, `query_params_len`: a JSON object whose values are JSON strings, or the object + * `{"b64": ""}` for a value whose bytes are not valid UTF-8 (settings, * query parameters); length 0 is none, and the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. + * - `options`, `options_len`: the input document `filter_options`; length 0 is `{}`, and the + * pointer may then be NULL. * - `out`: receives a new chs_filter, owned by the caller, freed with chs_filter_free; required. * - `err`: may be NULL; set only when the status is not CHS_OK. */ @@ -1129,13 +1570,15 @@ CHS_API chs_status chs_filter_create( size_t query_params_len, const uint8_t *settings, size_t settings_len, + const uint8_t *options, + size_t options_len, chs_filter **out, chs_error **err); /* * chs_filter_free: Releases the caller's reference to a filter. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `filter`: may be NULL. */ @@ -1143,20 +1586,32 @@ CHS_API void chs_filter_free(chs_filter *filter); /* * chs_filter_eval_body: Evaluates the filter over every row of a body, returning one verdict per - * row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. When - * a server would not accept an INSERT of the body, because `chs_preview_batch`'s verdict over the - * same body and settings, with no filter, is not `accepted`, every verdict is `d` and the outcome - * mirrors that verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts - * and that batch verdict together. - * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. + * `columns` is the INSERT column list, a JSON array of name objects (`{name}` or `{name_b64}`), and + * empty for none, read exactly as `chs_preview_batch` reads it: with a list the call models the + * INSERT that names it, and with none the INSERT that names none. When a server would not accept an + * INSERT of the body, because `chs_preview_batch`'s verdict over the same body, settings and column + * list, with no filter, is not `accepted`, every verdict is `d` and the outcome mirrors that + * verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts and that batch + * verdict together, and under any outcome but `ok` writes `d` for every verdict, answering a + * refusal or decline of the call itself in the document (`refused_at` `call`), never as a status. + * `options` (`input:insert_options`) is that INSERT's, read as `chs_preview_batch` reads it, so the + * verdict mirrored is `chs_preview_batch`'s over the same body, settings, columns and options. + * + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its options document, + * `options`. * * - `filter`: required. * - `body`, `body_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and * the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. + * - `columns`, `columns_len`: a JSON array of column names, each an object carrying the name as + * byte_strings says; length 0 is none, and the pointer may then be NULL. + * - `options`, `options_len`: the input document `insert_options`; length 0 is `{}`, and the + * pointer may then be NULL. * - `out`: receives the JSON document `filter_result`, owned by the caller, freed with * chs_buf_free; required. * - `err`: may be NULL; set only when the status is not CHS_OK. @@ -1168,6 +1623,10 @@ CHS_API chs_status chs_filter_eval_body( size_t body_len, const uint8_t *settings, size_t settings_len, + const uint8_t *columns, + size_t columns_len, + const uint8_t *options, + size_t options_len, chs_buf **out, chs_error **err); @@ -1175,20 +1634,26 @@ CHS_API chs_status chs_filter_eval_body( * chs_block_create: Parses a body once under the schema, for evaluating many filters over it. * `settings` governs the parse, `session_timezone` there included, and `columns` is read as * `chs_preview_row` reads it. The block holds a counted reference to the schema. It also records - * `chs_preview_batch`'s verdict over the same body, settings and columns, with no filter, which - * `chs_filter_eval_block` answers by. An accepted body is read once, for the block and that verdict - * together. + * `chs_preview_batch`'s verdict over the same body, settings, columns and options, with no filter, + * which `chs_filter_eval_block` answers by; a refusal or decline of the call or the body is + * recorded there, so this call returns a status other than `CHS_OK` only for misuse or an internal + * error. An accepted body is read once, for the block and that verdict together. `options` + * (`input:insert_options`) is read as `chs_preview_batch` reads it. * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. Growable (rule r1): an input it gains is a new member of its options document, + * `options`. * * - `schema`: required. * - `body`, `body_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and * the pointer may then be NULL. - * - `settings`, `settings_len`: a JSON object whose values are JSON strings (settings, query + * - `settings`, `settings_len`: a JSON object whose values are JSON strings, or the object `{"b64": + * ""}` for a value whose bytes are not valid UTF-8 (settings, query * parameters); length 0 is none, and the pointer may then be NULL. * - `columns`, `columns_len`: a JSON array of column names, each an object carrying the name as * byte_strings says; length 0 is none, and the pointer may then be NULL. + * - `options`, `options_len`: the input document `insert_options`; length 0 is `{}`, and the + * pointer may then be NULL. * - `out`: receives a new chs_block, owned by the caller, freed with chs_block_free; required. * - `err`: may be NULL; set only when the status is not CHS_OK. */ @@ -1201,13 +1666,15 @@ CHS_API chs_status chs_block_create( size_t settings_len, const uint8_t *columns, size_t columns_len, + const uint8_t *options, + size_t options_len, chs_block **out, chs_error **err); /* * chs_block_free: Releases the caller's reference to a block. Freeing NULL does nothing. * - * Class: api. Thread: handle_serial. + * Class: api. Thread: handle_serial. Init: any. * * - `block`: may be NULL. */ @@ -1217,17 +1684,16 @@ CHS_API void chs_block_free(chs_block *block); * chs_filter_eval_block: Evaluates the filter over a parsed block. Over a body a server would * accept, it gives the same answers `chs_filter_eval_body` gives for that body. Over one it would * not (the verdict `chs_block_create` recorded is not `accepted`), the outcome mirrors that verdict - * as for `chs_filter_eval_body`, and every verdict from the first refused row on is `d`. The first - * refused row is the first row of that batch whose outcome is neither `accepted` nor `skipped`. - * When the refusal belongs to no row (a whole-body step, a poisoned value, the call's own - * settings), it is the first row of the block. The rows before it keep their evaluated verdicts. A - * caller treats those as `d` too: any non-`ok` outcome, including `unsupported`, means every - * verdict is `d`. It takes no settings: the filter brings the settings it was compiled under and - * the block those it was parsed under. The filter and the block must come from the same schema; a - * pair from two schemas is `CHS_INVALID_ARGUMENT`. - * - * Class: api. Thread: shared. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, - * CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * as for `chs_filter_eval_body`, and every verdict is written `d`, the rows before the refused one + * included: the two calls write the same document for the same body, and the rule that any non-`ok` + * outcome means every verdict is `d` holds in the data. `refused_at` and `refused_row` say where + * the refusal sat (`row` for a refused row, `body_step` for a whole-body step or a poisoned value, + * `call` for the block's own settings). It takes no settings: the filter brings the settings it was + * compiled under and the block those it was parsed under. The filter and the block must come from + * the same schema; a pair from two schemas is `CHS_INVALID_ARGUMENT`. + * + * Class: api. Thread: shared. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `filter`: required. * - `block`: required. @@ -1250,7 +1716,8 @@ CHS_API chs_status chs_filter_eval_block( * `{table:String}`), which the caller binds when it runs the query; it selects exactly the * `system.columns` fields `chs_discover_columns` reads, `FORMAT JSONEachRow`. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, CHS_INTERNAL. + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_INVALID_ARGUMENT, + * CHS_INTERNAL. * * - `out`: receives SQL text (a statement, an expression, a type, a quoted name or literal): may * carry any byte, owned by the caller, freed with chs_buf_free; required. @@ -1263,7 +1730,7 @@ CHS_API chs_status chs_discover_query(chs_buf **out, chs_error **err); * `chs_discover_query`'s query) with ClickHouse's own reader and returns the column declarations, * formatted by ClickHouse's own formatter. * - * Class: api. Thread: any. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, + * Class: api. Thread: any. Init: required. Returns one of: CHS_OK, CHS_REJECTED, CHS_DECLINED, * CHS_INVALID_ARGUMENT, CHS_INTERNAL. * * - `rows`, `rows_len`: arbitrary bytes: a body, a string value, an export; length 0 is none, and diff --git a/python/src/chtypes/_abi2/_decls.py b/python/src/chtypes/_abi2/_decls.py index cd1d4312..1b146878 100644 --- a/python/src/chtypes/_abi2/_decls.py +++ b/python/src/chtypes/_abi2/_decls.py @@ -1,4 +1,4 @@ -# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT # noqa: E501 +# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT # noqa: E501 """The Python binding's generated ABI v2 layer: ctypes signatures, the handle classes, the two-phase resolve functions and invoke_by_name(), the test-only dispatcher the conformance suite drives from tests/fixtures/abi-v2/cases.json. @@ -19,7 +19,7 @@ from . import _errmap CHS_ABI_VERSION = 2 -CHS_ABI_FINGERPRINT = "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4" +CHS_ABI_FINGERPRINT = "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2" # The description's stability: "unstable" while this generation is being # designed, so CHS_ABI_FINGERPRINT moves with every change, and "locked" after @@ -87,6 +87,7 @@ "chs_error_column", "chs_error_free", "chs_error_message", + "chs_error_reason", "chs_error_status", "chs_filter_create", "chs_filter_eval_block", @@ -248,6 +249,7 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _add("chs_error_ch_name", "handle", "chs_buf", _P("error", "handle", "chs_error")) _add("chs_error_message", "handle", "chs_buf", _P("error", "handle", "chs_error")) _add("chs_error_column", "handle", "chs_buf", _P("error", "handle", "chs_error")) +_add("chs_error_reason", "handle", "chs_buf", _P("error", "handle", "chs_error")) _add("chs_error_free", "void", None, _P("error", "handle", "chs_error", nullable=True)) _add( "chs_live_handles", @@ -371,6 +373,7 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _P("body", "bytes_in", None), _P("settings", "bytes_in", None), _P("columns", "bytes_in", None), + _P("options", "bytes_in", None), _P("out", "out_handle", "chs_buf"), _P("err", "out_error", "chs_error", nullable=True), ) @@ -386,6 +389,7 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _P("filter", "handle", "chs_filter", nullable=True), _P("export_format", "scalar", "int32"), _P("doc_flags", "scalar", "uint32"), + _P("options", "bytes_in", None), _P("out", "out_handle", "chs_buf"), _P("out_export", "out_handle", "chs_buf", nullable=True), _P("err", "out_error", "chs_error", nullable=True), @@ -398,6 +402,7 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _P("expr", "bytes_in", None), _P("query_params", "bytes_in", None), _P("settings", "bytes_in", None), + _P("options", "bytes_in", None), _P("out", "out_handle", "chs_filter"), _P("err", "out_error", "chs_error", nullable=True), ) @@ -410,6 +415,8 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _P("format", "enum", "chs_format"), _P("body", "bytes_in", None), _P("settings", "bytes_in", None), + _P("columns", "bytes_in", None), + _P("options", "bytes_in", None), _P("out", "out_handle", "chs_buf"), _P("err", "out_error", "chs_error", nullable=True), ) @@ -422,6 +429,7 @@ def _add(name: str, returns_kind: str, returns_type: str | None, *params: _P) -> _P("body", "bytes_in", None), _P("settings", "bytes_in", None), _P("columns", "bytes_in", None), + _P("options", "bytes_in", None), _P("out", "out_handle", "chs_block"), _P("err", "out_error", "chs_error", nullable=True), ) @@ -801,6 +809,7 @@ def preview_row( body: bytes | None, settings: bytes | None, columns: bytes | None, + options: bytes | None, ) -> bytes: """The generated call wrapper for chs_preview_row (thread class shared). @@ -813,6 +822,7 @@ def preview_row( *_bytes_in(body), *_bytes_in(settings), *_bytes_in(columns), + *_bytes_in(options), ctypes.byref(out), ctypes.byref(err), ) @@ -829,6 +839,7 @@ def preview_batch( filter: FilterHandle | None, export_format: int, doc_flags: int, + options: bytes | None, ) -> tuple[bytes, bytes | None]: """The generated call wrapper for chs_preview_batch (thread class shared). @@ -845,6 +856,7 @@ def preview_batch( None if filter is None else filter.value, int(export_format), int(doc_flags), + *_bytes_in(options), ctypes.byref(out), ctypes.byref(out_export), ctypes.byref(err), @@ -861,6 +873,7 @@ def filter_create( expr: bytes | None, query_params: bytes | None, settings: bytes | None, + options: bytes | None, ) -> FilterHandle: """The generated call wrapper for chs_filter_create (thread class shared). @@ -872,6 +885,7 @@ def filter_create( *_bytes_in(expr), *_bytes_in(query_params), *_bytes_in(settings), + *_bytes_in(options), ctypes.byref(out), ctypes.byref(err), ) @@ -884,6 +898,8 @@ def filter_eval_body( format: int, body: bytes | None, settings: bytes | None, + columns: bytes | None, + options: bytes | None, ) -> bytes: """The generated call wrapper for chs_filter_eval_body (thread class shared). @@ -895,6 +911,8 @@ def filter_eval_body( int(format), *_bytes_in(body), *_bytes_in(settings), + *_bytes_in(columns), + *_bytes_in(options), ctypes.byref(out), ctypes.byref(err), ) @@ -908,6 +926,7 @@ def block_create( body: bytes | None, settings: bytes | None, columns: bytes | None, + options: bytes | None, ) -> BlockHandle: """The generated call wrapper for chs_block_create (thread class shared). @@ -920,6 +939,7 @@ def block_create( *_bytes_in(body), *_bytes_in(settings), *_bytes_in(columns), + *_bytes_in(options), ctypes.byref(out), ctypes.byref(err), ) diff --git a/python/src/chtypes/_abi2/_errmap.py b/python/src/chtypes/_abi2/_errmap.py index 664cd253..c9c32fba 100644 --- a/python/src/chtypes/_abi2/_errmap.py +++ b/python/src/chtypes/_abi2/_errmap.py @@ -1,4 +1,4 @@ -# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT # noqa: E501 +# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT # noqa: E501 """spec/abi-v2/sdk.json's error-mapping tables, as references to the public classes (python/src/chtypes/errors.py, re-exported by python/src/chtypes/_abi2/_errors.py). Only the MAPPING is generated: the diff --git a/python/src/chtypes/_abi2/_vocab.py b/python/src/chtypes/_abi2/_vocab.py index 039cc061..1c3dee2a 100644 --- a/python/src/chtypes/_abi2/_vocab.py +++ b/python/src/chtypes/_abi2/_vocab.py @@ -1,4 +1,4 @@ -# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT # noqa: E501 +# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT # noqa: E501 """The Python binding's generated vocabularies, from spec/abi-v2/abi.json: the C enums (`Format`, `Status`), the document vocabularies (`Outcome`, `FilterOutcome`, `Verdict`, `DefaultKind`, `MergeReason`, `DeclinedTier`, `DeclinedLayer`, `Reason`, @@ -256,6 +256,7 @@ class DeclinedTier(StrEnum): PREDICATE = "predicate" PREDICATE_UNFLIPPED = "predicate-unflipped" RESULT_CONTENT = "result-content" + SERVER_REFUSES = "server-refuses" @classmethod def _missing_(cls, value: object) -> DeclinedTier | None: @@ -299,6 +300,83 @@ def of(cls, text: str) -> DeclinedLayer: return cls(text) +class Posture(StrEnum): + """How a schema's statement was compiled: `create`, as a server's CREATE TABLE, + or `load`, as a server loads the stored definition of a table it holds.""" + + CREATE = "create" + LOAD = "load" + + @classmethod + def _missing_(cls, value: object) -> Posture | None: + return _unknown_str(cls, value) + + @property + def known(self) -> bool: + """Whether the description lists this value: False for exactly the unknown(n) + member an unlisted value reads as (rule r3).""" + return type(self)._value2member_map_.get(self._value_) is self + + @classmethod + def of(cls, text: str) -> Posture: + """Map a document's spelling. One the description does not list is kept as + this vocabulary's unknown(n) member, carrying it (rule r3).""" + return cls(text) + + +class RefusalSite(StrEnum): + """Where a batch or filter document's refusal sits: `call` (the call, its settings or its + column list), `body_read` (the body outside any row: a header, the framing), `row` (one + row, `refused_row`) or `body_step` (a whole-body step).""" + + CALL = "call" + BODY_READ = "body_read" + ROW = "row" + BODY_STEP = "body_step" + + @classmethod + def _missing_(cls, value: object) -> RefusalSite | None: + return _unknown_str(cls, value) + + @property + def known(self) -> bool: + """Whether the description lists this value: False for exactly the unknown(n) + member an unlisted value reads as (rule r3).""" + return type(self)._value2member_map_.get(self._value_) is self + + @classmethod + def of(cls, text: str) -> RefusalSite: + """Map a document's spelling. One the description does not list is kept as + this vocabulary's unknown(n) member, carrying it (rule r3).""" + return cls(text) + + +class DefaultSupport(StrEnum): + """What this build does with a column's DEFAULT, as `schema_description` reports it per + column: `computed`, `generated`, `deferred` or `declined` (with a reason).""" + + COMPUTED = "computed" + GENERATED = "generated" + DEFERRED = "deferred" + DECLINED = "declined" + + @classmethod + def _missing_(cls, value: object) -> DefaultSupport | None: + return _unknown_str(cls, value) + + @property + def known(self) -> bool: + """Whether the description lists this value: False for exactly the unknown(n) + member an unlisted value reads as (rule r3).""" + return type(self)._value2member_map_.get(self._value_) is self + + @classmethod + def of(cls, text: str) -> DefaultSupport: + """Map a document's spelling. One the description does not list is kept as + this vocabulary's unknown(n) member, carrying it (rule r3).""" + return cls(text) + + class DiscoverQueryParam(StrEnum): """A query parameter of the SQL `discover_query` returns, which the caller binds when it runs the query.""" @@ -409,6 +487,8 @@ class Source: DEFAULT_EXPR_UNSUPPORTED: Final = "default_expr_unsupported" DEFAULT_VOLATILE_UNRESOLVED: Final = "default_volatile_unresolved" DEFAULT_PENDING: Final = "default_pending" + DEFAULT_DEFERRED: Final = "default_deferred" + DEFAULT_OVERRIDE: Final = "default_override" @staticmethod def known(source: str) -> bool: @@ -417,9 +497,8 @@ def known(source: str) -> bool: @staticmethod def is_stored(source: str) -> bool: - """The description's `is_stored` fact for `source`. The description does not yet - say what an unknown(n) source reports (rule r3, "Facts without a fallback"); - until it does, an unlisted source is not stored.""" + """The description's `is_stored` fact for `source`; an unknown(n) source + reports its fallback's (`skipped`), as rule r3 says.""" return _SOURCE_IS_STORED.get(source, False) @@ -435,6 +514,8 @@ def is_stored(source: str) -> bool: "default_expr_unsupported": False, "default_volatile_unresolved": False, "default_pending": False, + "default_deferred": False, + "default_override": True, } @@ -463,10 +544,13 @@ class DocFlags(IntFlag): "batch_outcome": Outcome, "filter_outcome": FilterOutcome, "filter_verdict": Verdict, + "refusal_site": RefusalSite, "discover_query_param": DiscoverQueryParam, "default_kind": DefaultKind, + "default_support": DefaultSupport, "declined_tier": DeclinedTier, "declined_layer": DeclinedLayer, + "posture": Posture, } diff --git a/rust/src/abi2/calls_gen.rs b/rust/src/abi2/calls_gen.rs index ff006ac6..a2016b89 100644 --- a/rust/src/abi2/calls_gen.rs +++ b/rust/src/abi2/calls_gen.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT // // The typed, memory-safe call layer over `decls.rs`: one method per // status-returning `api` call, named without the `chs_` prefix, that copies @@ -466,6 +466,7 @@ impl Api { body: &[u8], settings: &[u8], columns: &[u8], + options: &[u8], ) -> Result, RawCallError> { let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); @@ -481,6 +482,8 @@ impl Api { settings.len(), columns.as_ptr(), columns.len(), + options.as_ptr(), + options.len(), &mut out_out, &mut err, ) @@ -506,6 +509,7 @@ impl Api { filter: Option<&FilterHandle>, export_format: i32, doc_flags: u32, + options: &[u8], ) -> Result<(Vec, Option>), RawCallError> { let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut out_out_export: *mut ChsBuf = std::ptr::null_mut(); @@ -525,6 +529,8 @@ impl Api { filter.map_or(std::ptr::null(), |h| h.as_ptr()), export_format, doc_flags, + options.as_ptr(), + options.len(), &mut out_out, &mut out_out_export, &mut err, @@ -553,6 +559,7 @@ impl Api { expr: &[u8], query_params: &[u8], settings: &[u8], + options: &[u8], ) -> Result { let mut out_out: *mut ChsFilter = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); @@ -567,6 +574,8 @@ impl Api { query_params.len(), settings.as_ptr(), settings.len(), + options.as_ptr(), + options.len(), &mut out_out, &mut err, ) @@ -592,6 +601,8 @@ impl Api { format: i32, body: &[u8], settings: &[u8], + columns: &[u8], + options: &[u8], ) -> Result, RawCallError> { let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); @@ -605,6 +616,10 @@ impl Api { body.len(), settings.as_ptr(), settings.len(), + columns.as_ptr(), + columns.len(), + options.as_ptr(), + options.len(), &mut out_out, &mut err, ) @@ -627,6 +642,7 @@ impl Api { body: &[u8], settings: &[u8], columns: &[u8], + options: &[u8], ) -> Result { let mut out_out: *mut ChsBlock = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); @@ -642,6 +658,8 @@ impl Api { settings.len(), columns.as_ptr(), columns.len(), + options.as_ptr(), + options.len(), &mut out_out, &mut err, ) diff --git a/rust/src/abi2/decls.rs b/rust/src/abi2/decls.rs index 229ea88c..22647b71 100644 --- a/rust/src/abi2/decls.rs +++ b/rust/src/abi2/decls.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT // // The opaque handle types, the `extern "C"` function-pointer type for // every described function, the resolved symbol tables (`Handshake`, the @@ -21,7 +21,7 @@ pub(crate) const CHS_ABI_VERSION: i32 = 2; /// form of spec/abi-v2/abi.json (D1.1). Compared byte for byte against /// `chs_build_info()`'s `abi_fingerprint` field at loader step 4. pub(crate) const CHS_ABI_FINGERPRINT: &str = - "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4"; + "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2"; /// `CHS_ABI_STABILITY`: the description's stability, `"unstable"` while this /// generation is being designed (so `CHS_ABI_FINGERPRINT` moves with every @@ -174,6 +174,7 @@ pub(crate) type FnChsErrorChCode = unsafe extern "C" fn(*const ChsError) -> i32; pub(crate) type FnChsErrorChName = unsafe extern "C" fn(*const ChsError) -> *mut ChsBuf; pub(crate) type FnChsErrorMessage = unsafe extern "C" fn(*const ChsError) -> *mut ChsBuf; pub(crate) type FnChsErrorColumn = unsafe extern "C" fn(*const ChsError) -> *mut ChsBuf; +pub(crate) type FnChsErrorReason = unsafe extern "C" fn(*const ChsError) -> *mut ChsBuf; pub(crate) type FnChsErrorFree = unsafe extern "C" fn(*mut ChsError); pub(crate) type FnChsLiveHandles = unsafe extern "C" fn(*mut *mut ChsBuf, *mut *mut ChsError) -> i32; @@ -228,6 +229,8 @@ pub(crate) type FnChsPreviewRow = unsafe extern "C" fn( usize, *const u8, usize, + *const u8, + usize, *mut *mut ChsBuf, *mut *mut ChsError, ) -> i32; @@ -243,6 +246,8 @@ pub(crate) type FnChsPreviewBatch = unsafe extern "C" fn( *const ChsFilter, i32, u32, + *const u8, + usize, *mut *mut ChsBuf, *mut *mut ChsBuf, *mut *mut ChsError, @@ -255,6 +260,8 @@ pub(crate) type FnChsFilterCreate = unsafe extern "C" fn( usize, *const u8, usize, + *const u8, + usize, *mut *mut ChsFilter, *mut *mut ChsError, ) -> i32; @@ -266,6 +273,10 @@ pub(crate) type FnChsFilterEvalBody = unsafe extern "C" fn( usize, *const u8, usize, + *const u8, + usize, + *const u8, + usize, *mut *mut ChsBuf, *mut *mut ChsError, ) -> i32; @@ -278,6 +289,8 @@ pub(crate) type FnChsBlockCreate = unsafe extern "C" fn( usize, *const u8, usize, + *const u8, + usize, *mut *mut ChsBlock, *mut *mut ChsError, ) -> i32; @@ -346,6 +359,7 @@ pub(crate) struct Api { pub(crate) chs_error_ch_name: Symbol, pub(crate) chs_error_message: Symbol, pub(crate) chs_error_column: Symbol, + pub(crate) chs_error_reason: Symbol, pub(crate) chs_error_free: Symbol, pub(crate) chs_live_handles: Symbol, pub(crate) chs_error_codes: Symbol, @@ -423,6 +437,9 @@ impl Api { chs_error_column: lib .get(concat!("chs_error_column", "\0").as_bytes()) .map_err(|_| "chs_error_column")?, + chs_error_reason: lib + .get(concat!("chs_error_reason", "\0").as_bytes()) + .map_err(|_| "chs_error_reason")?, chs_error_free: lib .get(concat!("chs_error_free", "\0").as_bytes()) .map_err(|_| "chs_error_free")?, diff --git a/rust/src/abi2/errmap_gen.rs b/rust/src/abi2/errmap_gen.rs index da776a0e..6dc1545d 100644 --- a/rust/src/abi2/errmap_gen.rs +++ b/rust/src/abi2/errmap_gen.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT // // spec/abi-v2/sdk.json's error table: which public error class each call // status and each loader refusal maps to. See scripts/abi-v1/emit/rust.py. diff --git a/rust/src/abi2/invoke_gen.rs b/rust/src/abi2/invoke_gen.rs index 5a67f211..ce8a1694 100644 --- a/rust/src/abi2/invoke_gen.rs +++ b/rust/src/abi2/invoke_gen.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT // // The test-only invoke-by-name dispatcher: `invoke(api, name, args)` marshals // an already-resolved argument list (`ResolvedArg`, rust/src/abi2/conformance.rs) @@ -517,6 +517,7 @@ pub(crate) unsafe fn invoke( let a_body = cur.bytes()?; let a_settings = cur.bytes()?; let a_columns = cur.bytes()?; + let a_options = cur.bytes()?; let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); // SAFETY: every input is marshaled from the case's own recipe above, matching chs_preview_row's declared shape; every output pointer is this call's own local. @@ -530,6 +531,8 @@ pub(crate) unsafe fn invoke( a_settings.len(), a_columns.as_ptr(), a_columns.len(), + a_options.as_ptr(), + a_options.len(), &mut out_out, &mut err, ) @@ -551,6 +554,7 @@ pub(crate) unsafe fn invoke( a_filter.map_or(std::ptr::null_mut(), |p| p as *mut ChsFilter) as *const ChsFilter; let a_export_format = cur.int()? as i32; let a_doc_flags = cur.int()? as u32; + let a_options = cur.bytes()?; let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut out_out_export: *mut ChsBuf = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); @@ -568,6 +572,8 @@ pub(crate) unsafe fn invoke( a_filter, a_export_format, a_doc_flags, + a_options.as_ptr(), + a_options.len(), &mut out_out, &mut out_out_export, &mut err, @@ -585,6 +591,7 @@ pub(crate) unsafe fn invoke( let a_expr = cur.bytes()?; let a_query_params = cur.bytes()?; let a_settings = cur.bytes()?; + let a_options = cur.bytes()?; let mut out_out: *mut ChsFilter = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); // SAFETY: every input is marshaled from the case's own recipe above, matching chs_filter_create's declared shape; every output pointer is this call's own local. @@ -597,6 +604,8 @@ pub(crate) unsafe fn invoke( a_query_params.len(), a_settings.as_ptr(), a_settings.len(), + a_options.as_ptr(), + a_options.len(), &mut out_out, &mut err, ) @@ -612,6 +621,8 @@ pub(crate) unsafe fn invoke( let a_format = cur.int()? as i32; let a_body = cur.bytes()?; let a_settings = cur.bytes()?; + let a_columns = cur.bytes()?; + let a_options = cur.bytes()?; let mut out_out: *mut ChsBuf = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); // SAFETY: every input is marshaled from the case's own recipe above, matching chs_filter_eval_body's declared shape; every output pointer is this call's own local. @@ -623,6 +634,10 @@ pub(crate) unsafe fn invoke( a_body.len(), a_settings.as_ptr(), a_settings.len(), + a_columns.as_ptr(), + a_columns.len(), + a_options.as_ptr(), + a_options.len(), &mut out_out, &mut err, ) @@ -639,6 +654,7 @@ pub(crate) unsafe fn invoke( let a_body = cur.bytes()?; let a_settings = cur.bytes()?; let a_columns = cur.bytes()?; + let a_options = cur.bytes()?; let mut out_out: *mut ChsBlock = std::ptr::null_mut(); let mut err: *mut ChsError = std::ptr::null_mut(); // SAFETY: every input is marshaled from the case's own recipe above, matching chs_block_create's declared shape; every output pointer is this call's own local. @@ -652,6 +668,8 @@ pub(crate) unsafe fn invoke( a_settings.len(), a_columns.as_ptr(), a_columns.len(), + a_options.as_ptr(), + a_options.len(), &mut out_out, &mut err, ) diff --git a/rust/src/abi2/vocab_gen.rs b/rust/src/abi2/vocab_gen.rs index b0544f8a..75812e44 100644 --- a/rust/src/abi2/vocab_gen.rs +++ b/rust/src/abi2/vocab_gen.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT // // Every vocabulary of the description, as Rust: the `chs_format` and // `chs_status` numbers, the document vocabularies (outcomes, verdicts, @@ -534,6 +534,8 @@ pub enum DeclinedTier { PredicateUnflipped, /// The wire value `result-content`. ResultContent, + /// The wire value `server-refuses`. + ServerRefuses, /// A value the description does not list, carrying its exact spelling (rule r3, /// spec/abi-v2/docs.md): a reader keeps it, for this field alone, and decodes on. Unknown(String), @@ -546,6 +548,7 @@ impl DeclinedTier { DeclinedTier::Predicate => "predicate", DeclinedTier::PredicateUnflipped => "predicate-unflipped", DeclinedTier::ResultContent => "result-content", + DeclinedTier::ServerRefuses => "server-refuses", DeclinedTier::Unknown(raw) => raw.as_str(), } } @@ -557,6 +560,7 @@ impl DeclinedTier { "predicate" => DeclinedTier::Predicate, "predicate-unflipped" => DeclinedTier::PredicateUnflipped, "result-content" => DeclinedTier::ResultContent, + "server-refuses" => DeclinedTier::ServerRefuses, _ => DeclinedTier::Unknown(s.to_string()), } } @@ -626,6 +630,165 @@ impl std::fmt::Display for DeclinedLayer { } } +/// How a schema's statement was compiled: `create`, as a server's CREATE TABLE, or `load`, as a server +/// loads the stored definition of a table it holds. A posture the description does not list is `Unknown`, +/// and the document decodes on (rule r3). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Posture { + /// The wire value `create`. + Create, + /// The wire value `load`. + Load, + /// A value the description does not list, carrying its exact spelling (rule r3, + /// spec/abi-v2/docs.md): a reader keeps it, for this field alone, and decodes on. + Unknown(String), +} + +impl Posture { + /// The wire spelling: the description's, or the raw value `Unknown` carries. + pub fn as_str(&self) -> &str { + match self { + Posture::Create => "create", + Posture::Load => "load", + Posture::Unknown(raw) => raw.as_str(), + } + } + + /// Read one wire value. One the description does not list is `Unknown`, + /// carrying it (rule r3): it never fails the document. + pub fn from_wire(s: &str) -> Self { + match s { + "create" => Posture::Create, + "load" => Posture::Load, + _ => Posture::Unknown(s.to_string()), + } + } + + /// Whether the description lists this value: false exactly for `Unknown`. + pub fn is_known(&self) -> bool { + !matches!(self, Posture::Unknown(_)) + } +} + +/// The wire spelling (`as_str`), padded to a width the format asks for. +impl std::fmt::Display for Posture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(self.as_str()) + } +} + +/// Where a batch or filter document's refusal sits: the call, the body outside any row (a header, the +/// framing), one row (`refused_row`) or a whole-body step. A site the description does not list is +/// `Unknown`, and the document decodes on (rule r3). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RefusalSite { + /// The wire value `call`. + Call, + /// The wire value `body_read`. + BodyRead, + /// The wire value `row`. + Row, + /// The wire value `body_step`. + BodyStep, + /// A value the description does not list, carrying its exact spelling (rule r3, + /// spec/abi-v2/docs.md): a reader keeps it, for this field alone, and decodes on. + Unknown(String), +} + +impl RefusalSite { + /// The wire spelling: the description's, or the raw value `Unknown` carries. + pub fn as_str(&self) -> &str { + match self { + RefusalSite::Call => "call", + RefusalSite::BodyRead => "body_read", + RefusalSite::Row => "row", + RefusalSite::BodyStep => "body_step", + RefusalSite::Unknown(raw) => raw.as_str(), + } + } + + /// Read one wire value. One the description does not list is `Unknown`, + /// carrying it (rule r3): it never fails the document. + pub fn from_wire(s: &str) -> Self { + match s { + "call" => RefusalSite::Call, + "body_read" => RefusalSite::BodyRead, + "row" => RefusalSite::Row, + "body_step" => RefusalSite::BodyStep, + _ => RefusalSite::Unknown(s.to_string()), + } + } + + /// Whether the description lists this value: false exactly for `Unknown`. + pub fn is_known(&self) -> bool { + !matches!(self, RefusalSite::Unknown(_)) + } +} + +/// The wire spelling (`as_str`), padded to a width the format asks for. +impl std::fmt::Display for RefusalSite { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(self.as_str()) + } +} + +/// What this build does with a column's DEFAULT, as a schema description reports it per column. A value +/// the description does not list is `Unknown`, and the column decodes on (rule r3). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum DefaultSupport { + /// The wire value `computed`. + Computed, + /// The wire value `generated`. + Generated, + /// The wire value `deferred`. + Deferred, + /// The wire value `declined`. + Declined, + /// A value the description does not list, carrying its exact spelling (rule r3, + /// spec/abi-v2/docs.md): a reader keeps it, for this field alone, and decodes on. + Unknown(String), +} + +impl DefaultSupport { + /// The wire spelling: the description's, or the raw value `Unknown` carries. + pub fn as_str(&self) -> &str { + match self { + DefaultSupport::Computed => "computed", + DefaultSupport::Generated => "generated", + DefaultSupport::Deferred => "deferred", + DefaultSupport::Declined => "declined", + DefaultSupport::Unknown(raw) => raw.as_str(), + } + } + + /// Read one wire value. One the description does not list is `Unknown`, + /// carrying it (rule r3): it never fails the document. + pub fn from_wire(s: &str) -> Self { + match s { + "computed" => DefaultSupport::Computed, + "generated" => DefaultSupport::Generated, + "deferred" => DefaultSupport::Deferred, + "declined" => DefaultSupport::Declined, + _ => DefaultSupport::Unknown(s.to_string()), + } + } + + /// Whether the description lists this value: false exactly for `Unknown`. + pub fn is_known(&self) -> bool { + !matches!(self, DefaultSupport::Unknown(_)) + } +} + +/// The wire spelling (`as_str`), padded to a width the format asks for. +impl std::fmt::Display for DefaultSupport { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.pad(self.as_str()) + } +} + /// The `transform_reason` vocabulary: why a stored value differs from the supplied one, and the `lossy` /// fact the description gives each. A reason the description does not list is `Unknown`, and reports /// the fallback's (`value_changed`) facts, so it reads as lossy. @@ -855,8 +1018,8 @@ pub mod reason { } /// The `value_src` vocabulary: where a column's value came from, and whether the description says it is -/// stored. A source the description does not list is `Unknown`; the description does not yet say what -/// its `is_stored` reports (rule r3, "Facts without a fallback"), and until it does, `false`. +/// stored. A source the description does not list is `Unknown`, and reports its fallback's (`skipped`) +/// `is_stored` (rule r3). #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum Source { @@ -882,6 +1045,10 @@ pub enum Source { DefaultVolatileUnresolved, /// The wire value `default_pending`. DefaultPending, + /// The wire value `default_deferred`. + DefaultDeferred, + /// The wire value `default_override`. + DefaultOverride, /// A value the description does not list, carrying its exact spelling (rule r3, /// spec/abi-v2/docs.md): a reader keeps it, for this field alone, and decodes on. Unknown(String), @@ -902,6 +1069,8 @@ impl Source { Source::DefaultExprUnsupported => "default_expr_unsupported", Source::DefaultVolatileUnresolved => "default_volatile_unresolved", Source::DefaultPending => "default_pending", + Source::DefaultDeferred => "default_deferred", + Source::DefaultOverride => "default_override", Source::Unknown(raw) => raw.as_str(), } } @@ -921,6 +1090,8 @@ impl Source { "default_expr_unsupported" => Source::DefaultExprUnsupported, "default_volatile_unresolved" => Source::DefaultVolatileUnresolved, "default_pending" => Source::DefaultPending, + "default_deferred" => Source::DefaultDeferred, + "default_override" => Source::DefaultOverride, _ => Source::Unknown(s.to_string()), } } @@ -930,8 +1101,7 @@ impl Source { !matches!(self, Source::Unknown(_)) } - /// The description's `is_stored` fact for this source; `false` for `Unknown` until the description - /// says otherwise. + /// The description's `is_stored` fact for this source; `Unknown` reports the fallback's (`skipped`). pub fn is_stored(&self) -> bool { match self { Source::Input => true, @@ -945,6 +1115,8 @@ impl Source { Source::DefaultExprUnsupported => false, Source::DefaultVolatileUnresolved => false, Source::DefaultPending => false, + Source::DefaultDeferred => false, + Source::DefaultOverride => true, Source::Unknown(_) => false, } } @@ -983,6 +1155,10 @@ pub mod source { pub const DEFAULT_VOLATILE_UNRESOLVED: &str = "default_volatile_unresolved"; /// `default_pending`. pub const DEFAULT_PENDING: &str = "default_pending"; + /// `default_deferred`. + pub const DEFAULT_DEFERRED: &str = "default_deferred"; + /// `default_override`. + pub const DEFAULT_OVERRIDE: &str = "default_override"; /// The description's `is_stored` fact for a source spelling; one the /// description does not list is `Source::Unknown`, `false` until the @@ -1116,10 +1292,13 @@ pub(crate) const DESCRIBED_VOCABULARIES: &[&str] = &[ "batch_outcome", "filter_outcome", "filter_verdict", + "refusal_site", "discover_query_param", "default_kind", + "default_support", "declined_tier", "declined_layer", + "posture", ]; /// Build the value of the described enum `name` whose raw spelling is `raw` (an @@ -1164,6 +1343,10 @@ pub(crate) fn described_vocabulary(name: &str, raw: &str) -> Option<(bool, Strin let v = Verdict::from_wire(raw); Some((v.is_known(), v.as_str().to_string())) } + "refusal_site" => { + let v = RefusalSite::from_wire(raw); + Some((v.is_known(), v.as_str().to_string())) + } "discover_query_param" => { let v = DiscoverQueryParam::from_wire(raw); Some((v.is_known(), v.as_str().to_string())) @@ -1172,6 +1355,10 @@ pub(crate) fn described_vocabulary(name: &str, raw: &str) -> Option<(bool, Strin let v = DefaultKind::from_wire(raw); Some((v.is_known(), v.as_str().to_string())) } + "default_support" => { + let v = DefaultSupport::from_wire(raw); + Some((v.is_known(), v.as_str().to_string())) + } "declined_tier" => { let v = DeclinedTier::from_wire(raw); Some((v.is_known(), v.as_str().to_string())) @@ -1180,6 +1367,10 @@ pub(crate) fn described_vocabulary(name: &str, raw: &str) -> Option<(bool, Strin let v = DeclinedLayer::from_wire(raw); Some((v.is_known(), v.as_str().to_string())) } + "posture" => { + let v = Posture::from_wire(raw); + Some((v.is_known(), v.as_str().to_string())) + } _ => None, } } diff --git a/rust/src/ocifetch/abi_fingerprint.rs b/rust/src/ocifetch/abi_fingerprint.rs index cc7ef00d..2a30090f 100644 --- a/rust/src/ocifetch/abi_fingerprint.rs +++ b/rust/src/ocifetch/abi_fingerprint.rs @@ -1,4 +1,4 @@ -// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +// GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT //! `CHS_ABI_FINGERPRINT` of ABI v2, the generation this crate speaks: the //! same value as the ABI layer's own constant. The dev channel names its @@ -7,4 +7,4 @@ /// The ABI fingerprint this crate speaks, `sha256:` and 64 lowercase hex. pub const DEV_ABI_FINGERPRINT: &str = - "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4"; + "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2"; diff --git a/scripts/abi-v1/emit/_stubshared.py b/scripts/abi-v1/emit/_stubshared.py index 9808931f..d0f4fcfa 100644 --- a/scripts/abi-v1/emit/_stubshared.py +++ b/scripts/abi-v1/emit/_stubshared.py @@ -104,6 +104,23 @@ "chs_preview_row": ("body", "row"), "chs_discover_columns": ("rows", "discovery"), } +# Generation 2's row document: `code` is required, beside `outcome`, and the call's declined settings are the batch +# document's only (no row-level `unsupported_settings`). Generation 1's template above is +# unchanged, so its fixtures are too. +DOC_TEMPLATES_V2 = { + **DOC_TEMPLATES, + "row": [ + ("raw", '{"outcome":"accepted","code":0,"input_span":{"off":0,"len":'), + ("len",), + ("raw", '},"unknown_fields":[],"cols":[{'), + *DOC_TEMPLATES["row"][3:], + ], +} + + +def doc_templates(major: int) -> dict: + """The templates of one generation: emit/stub.py compiles them and doc_render() renders them.""" + return DOC_TEMPLATES_V2 if major >= 2 else DOC_TEMPLATES # -------------------------------------------------------------------------- @@ -145,9 +162,8 @@ } ], "computed": [{"name": "m", "kind": "materialized", "stored": "5", **_UNK}], - "transformed": [{"column": "s", "input": "1", "stored": "2", "reason": "value_changed", "row": 0, **_UNK}], + "transformed": [{"column": "s", "input": "1", "stored": "2", "reason": "value_changed", "lossy": True, "row": 0, **_UNK}], "unknown_fields": [{"name": "u", **_UNK}], - "unsupported_settings": [{"name": "st", **_UNK}], "partition_id": "all", "verdict_code": 0, **_UNK_TOP, @@ -165,7 +181,7 @@ "storage_transforms": [{"row": 0, "column": "s", "stored": "2", "reason": "date_clamp", **_UNK}], "unsupported_settings": [{"name": "st", **_UNK}], "at_merge": [ - {"row": 0, "reason": "ttl_column_reset", "column": "s", "stored": "", "input_rows": [0], **_UNK} + {"part_row": 0, "reason": "ttl_column_reset", "column": "s", "stored": "", "input_rows": [0], **_UNK} ], "engine_rows": [[{"name": "s", "stored": "abc", "null": False, "value_b64": "YWJj", **_UNK}]], "row_spans": [{"off": 0, "len": 3, **_UNK}], @@ -327,7 +343,7 @@ def _strip_unknown(v): # the message is the stub's stand-in, not ClickHouse's text. DEEP = { "id": "deep", - "fn": "chs_schema_describe", + "fn": "chs_schema_create", "status": "CHS_REJECTED", "ch_code": 306, "ch_name": "TOO_DEEP_RECURSION", @@ -443,7 +459,7 @@ def _doc_src(src: tuple, payload: bytes) -> bytes: return src[1] if src[0] == "lit" else payload + src[1] -def doc_render(template: str, param: bytes) -> dict: +def doc_render(template: str, param: bytes, major: int = 1) -> dict: """The document the stub returns for `param` (which starts DOC_PREFIX), rendered in Python from the same template the stub's C is compiled from.""" import base64 @@ -451,7 +467,7 @@ def doc_render(template: str, param: bytes) -> dict: payload = param[len(DOC_PREFIX) :] out: list[str] = [] - for step in DOC_TEMPLATES[template]: + for step in doc_templates(major)[template]: op = step[0] if op == "raw": out.append(step[1]) diff --git a/scripts/abi-v1/emit/cases.py b/scripts/abi-v1/emit/cases.py index fd90f8e1..7779837e 100644 --- a/scripts/abi-v1/emit/cases.py +++ b/scripts/abi-v1/emit/cases.py @@ -167,6 +167,20 @@ def _schema_args(model, create_table: bytes, server: dict | None = None) -> list return out +def _pad(model, fn_name: str, args: list) -> list: + """`args` with an empty input appended for each trailing bytes_in input + parameter the hand-written list does not reach: a growable call's options + document (rule r1), so a recipe written for generation 1 stays valid for + generation 2's longer signature.""" + inputs = [p for p in model.function(fn_name).params if not p.is_out] + out = list(args) + for p in inputs[len(out) :]: + if p.kind != "bytes_in": + raise ValueError(f"_pad: {fn_name}'s input {p.name!r} is not bytes_in; extend the recipe by hand") + out.append({"bytes_hex": ""}) + return out + + def _schema_recipe(model) -> dict: return {"fn": "chs_schema_create", "args": _schema_args(model, b"CREATE TABLE t (x Int32)")} @@ -179,25 +193,33 @@ def _server_recipe(model) -> dict: # noqa: ARG001 - every recipe takes the mode def _filter_recipe(model) -> dict: return { "fn": "chs_filter_create", - "args": [ - {"handle": _schema_recipe(model)}, - {"bytes_hex": b"x = 1".hex()}, - {"bytes_hex": "{}".encode().hex()}, - {"bytes_hex": ""}, - ], + "args": _pad( + model, + "chs_filter_create", + [ + {"handle": _schema_recipe(model)}, + {"bytes_hex": b"x = 1".hex()}, + {"bytes_hex": "{}".encode().hex()}, + {"bytes_hex": ""}, + ], + ), } def _block_recipe(model) -> dict: return { "fn": "chs_block_create", - "args": [ - {"handle": _schema_recipe(model)}, - {"int": 0}, # CHS_JSON_EACH_ROW - {"bytes_hex": b"{}".hex()}, - {"bytes_hex": ""}, - {"bytes_hex": ""}, - ], + "args": _pad( + model, + "chs_block_create", + [ + {"handle": _schema_recipe(model)}, + {"int": 0}, # CHS_JSON_EACH_ROW + {"bytes_hex": b"{}".hex()}, + {"bytes_hex": ""}, + {"bytes_hex": ""}, + ], + ), } @@ -222,6 +244,7 @@ def _block_recipe(model) -> dict: "message": b"msg", "ascii": b"abc", "json_object_string_values": b"{}", + "json_object_byte_values": b"{}", "json_array_names": b"[]", "timezone": b"UTC", } @@ -456,7 +479,12 @@ def _input_document_cases(model) -> list[dict]: d = model.input_of(p) if d is None: continue - known = {k: ({} if s.get("type") == "object" else "") for k, s in d.schema.get("properties", {}).items()} + # Every key the document defines, at a value its schema admits (its first enum value, or {} for an + # object), so the unknown key is the document's only fault. + known = { + k: ({} if s.get("type") == "object" else [] if s.get("type") == "array" else s["enum"][0] if "enum" in s else "") + for k, s in d.schema.get("properties", {}).items() + } doc = json.dumps({**known, UNKNOWN_INPUT_KEY: "1"}, separators=(",", ":")).encode() for case_id, data, key in ( (f"input_document.{fn.name}.{p.name}.unknown_key_refused", doc, UNKNOWN_INPUT_KEY), @@ -569,7 +597,7 @@ def _document_cases(model) -> list[dict]: param, template = _stubshared.DOC_FUNCTIONS[fn_name] data = _stubshared.DOC_PREFIX + payload case = _overridden_echo_case(model, case_id, fn_name, {param: data}) - doc = _stubshared.doc_render(template, data) + doc = _stubshared.doc_render(template, data, model.major) problems = abimodel.validate(doc, model.documents[template].schema) if problems: raise ValueError(f"{case_id}: the expected document fails documents.{template}.schema: {problems}") @@ -606,24 +634,27 @@ def _lifecycle_cases(model) -> list[dict]: schema = _schema_recipe(model) filt = { "fn": "chs_filter_create", - "args": [{"ref": "s"}, {"bytes_hex": b"x = 1".hex()}, {"bytes_hex": b"{}".hex()}, {"bytes_hex": ""}], + "args": _pad( + model, "chs_filter_create", [{"ref": "s"}, {"bytes_hex": b"x = 1".hex()}, {"bytes_hex": b"{}".hex()}, {"bytes_hex": ""}] + ), } block = { "fn": "chs_block_create", - "args": [{"ref": "s"}, {"int": 0}, {"bytes_hex": b"{}".hex()}, {"bytes_hex": ""}, {"bytes_hex": ""}], + "args": _pad( + model, "chs_block_create", [{"ref": "s"}, {"int": 0}, {"bytes_hex": b"{}".hex()}, {"bytes_hex": ""}, {"bytes_hex": ""}] + ), } body = b'{"x":1}' - eval_body = { - "fn": "chs_filter_eval_body", - "args": [{"ref": "f"}, {"int": 0}, {"bytes_hex": body.hex()}, {"bytes_hex": ""}], - } + eval_body_written = [{"ref": "f"}, {"int": 0}, {"bytes_hex": body.hex()}, {"bytes_hex": ""}] + eval_body = {"fn": "chs_filter_eval_body", "args": _pad(model, "chs_filter_eval_body", eval_body_written)} eval_body_expect = { "status": "CHS_OK", "outputs": { "out": { "fn": "chs_filter_eval_body", "out": "out", - "args": [{"kind": "chs_filter"}, 0, _stubshared.echo_bytes_field(body), _stubshared.echo_bytes_field(b"")], + "args": [{"kind": "chs_filter"}, 0, _stubshared.echo_bytes_field(body), _stubshared.echo_bytes_field(b"")] + + [_stubshared.echo_bytes_field(b"")] * (len(eval_body["args"]) - len(eval_body_written)), } }, } diff --git a/scripts/abi-v1/emit/go_vocab.py b/scripts/abi-v1/emit/go_vocab.py index f776be5a..416fe1c1 100644 --- a/scripts/abi-v1/emit/go_vocab.py +++ b/scripts/abi-v1/emit/go_vocab.py @@ -176,6 +176,13 @@ def _format(model) -> list[str]: out += _doc("ExportNone is CHS_EXPORT_NONE: no export was asked for.") out.append(f"const ExportNone Format = {const.value}") out.append("") + elif name == "CHS_CODE_DECLINED": + out += _doc( + "CodeDeclined is CHS_CODE_DECLINED: the code a declined row, outcome or verdict carries, outside\n" + "ClickHouse's code table (the Codes paragraph of the reference)." + ) + out.append(f"const CodeDeclined int32 = {const.value}") + out.append("") if model.major >= 2: out += _known_method("Format", [v.fields["ch_name"] for v in enum.values], "format", "f") out += _doc( @@ -206,7 +213,7 @@ def _doc_flags(model) -> list[str]: out += _doc(f"{ident} is {name}.") out.append(f"const {ident} DocFlags = {const.value}") out.append("") - elif name not in ("CHS_EXPORT_NONE", "CHS_ABI_REVISION_TOMBSTONE"): + elif name not in ("CHS_EXPORT_NONE", "CHS_ABI_REVISION_TOMBSTONE", "CHS_CODE_DECLINED"): raise VocabError(f"go_vocab.py: constant {name} has no Go spelling; teach emit/go_vocab.py about it") return out @@ -260,6 +267,9 @@ def _bool_method(type_name: str, method: str, doc: str, enum, field: str, fallba "merge_reason": "Merge", "declined_tier": "Tier", "declined_layer": "Layer", + "posture": "Posture", + "refusal_site": "RefusedAt", + "default_support": "DefaultSupport", } @@ -312,12 +322,14 @@ def _source(model) -> list[str]: + ( "\nThe description does not yet say what an unknown(n) source reports (rule r3, \"Facts without a\n" "fallback\"); until it does, IsStored is false for one." + if model.major >= 2 and enum.fallback is None + else f"\nAn unknown(n) source reports its fallback's ({enum.fallback}) fact (rule r3)." if model.major >= 2 else "" ), enum, "is_stored", - None, + enum.fallback if model.major >= 2 else None, ) if model.major >= 2: out += _known_method("Source", [i for i, _ in values], "source") @@ -506,6 +518,50 @@ def _declined_layer(model) -> list[str]: return out +def _posture(model) -> list[str]: + """ABI v2: posture, how a schema's statement was compiled (r3: its unknown(n) too).""" + enum = model.enums["posture"] + values = [(f"Posture{_value_ident(enum.name, v.value)}", v.value) for v in enum.values] + out = _string_type( + "Posture", + "Posture is posture: how a schema's statement was compiled, as a server's CREATE TABLE (create)\n" + "or as a server loads the stored definition of a table it holds (load). The spelling is the\n" + "description's.", + values, + ) + out += _known_method("Posture", [i for i, _ in values], "posture") + return out + + +def _refusal_site(model) -> list[str]: + """ABI v2: refusal_site, where a batch or filter document's refusal sits (r3: its unknown(n) too).""" + enum = model.enums["refusal_site"] + values = [(f"RefusedAt{_value_ident(enum.name, v.value)}", v.value) for v in enum.values] + out = _string_type( + "RefusalSite", + "RefusalSite is refusal_site: where a batch or filter document's refusal sits, the call (its settings\n" + "or its column list), the body outside any row (a header, the framing), one row (RefusedRow) or a\n" + "whole-body step. The spelling is the description's.", + values, + ) + out += _known_method("RefusalSite", [i for i, _ in values], "site") + return out + + +def _default_support(model) -> list[str]: + """ABI v2: default_support, what this build does with a column's DEFAULT (r3: its unknown(n) too).""" + enum = model.enums["default_support"] + values = [(f"DefaultSupport{_value_ident(enum.name, v.value)}", v.value) for v in enum.values] + out = _string_type( + "DefaultSupport", + "DefaultSupport is default_support: what this build does with a column's DEFAULT, as a schema\n" + "description reports it per column. The spelling is the description's.", + values, + ) + out += _known_method("DefaultSupport", [i for i, _ in values], "support") + return out + + def _discover_query_param(model) -> list[str]: """ABI v2: discover_query_param's type (r3: every enum has its unknown(n)).""" enum = model.enums["discover_query_param"] @@ -536,6 +592,9 @@ def _discover_query_param(model) -> list[str]: "merge_reason": ("MergeReason", "string"), "declined_tier": ("DeclinedTier", "string"), "declined_layer": ("DeclinedLayer", "string"), + "posture": ("Posture", "string"), + "refusal_site": ("RefusalSite", "string"), + "default_support": ("DefaultSupport", "string"), } @@ -593,7 +652,7 @@ def render_vocab_gen(model) -> str: _default_kind, ] if model.major >= 2: - parts += [_discover_query_param, _merge_reason, _declined_tier, _declined_layer, _described] + parts += [_discover_query_param, _merge_reason, _declined_tier, _declined_layer, _posture, _refusal_site, _default_support, _described] for part in parts: out += part(model) while out and out[-1] == "": diff --git a/scripts/abi-v1/emit/header.py b/scripts/abi-v1/emit/header.py index 92f0c7b4..2f806d3e 100644 --- a/scripts/abi-v1/emit/header.py +++ b/scripts/abi-v1/emit/header.py @@ -26,7 +26,7 @@ import textwrap -from model import CONTENTS, DOCS_MD, THREADS, input_name +from model import CONTENTS, DOCS_MD, INITS, THREADS, input_name from . import Output, banner, marker_key from .reference import compact_json @@ -120,7 +120,7 @@ def _section(title: str) -> list[str]: def _function_facts(model, fn) -> str: - facts = [f"Class: {fn.cls}. Thread: {fn.thread}."] + facts = [f"Class: {fn.cls}. Thread: {fn.thread}." + (f" Init: {fn.init}." if fn.init else "")] if fn.returns.kind == "status": facts.append("Returns one of: " + ", ".join(fn.may_return) + ".") r = fn.returns @@ -261,6 +261,13 @@ def comment(text: str, where: str) -> list[str]: + "\n".join(f"- {k}: {v}." for k, v in THREADS.items()), "thread classes", ) + if any(f.init for f in model.functions): + out += _section("init classes") + out += comment( + "Every function below names its init class: whether it is valid before `chs_initialize` " + "(Initialization, above).\n\n" + "\n".join(f"- {k}: {v}." for k, v in INITS.items()), + "init classes", + ) bs = model.byte_strings out += _section("byte strings in JSON") diff --git a/scripts/abi-v1/emit/python.py b/scripts/abi-v1/emit/python.py index 31097db0..1366c162 100644 --- a/scripts/abi-v1/emit/python.py +++ b/scripts/abi-v1/emit/python.py @@ -1088,6 +1088,9 @@ def render_vocab(model: Model) -> str: "merge_reason": "MergeReason", "declined_tier": "DeclinedTier", "declined_layer": "DeclinedLayer", + "posture": "Posture", + "refusal_site": "RefusalSite", + "default_support": "DefaultSupport", } _KNOWN_PROPERTY = ( @@ -1346,6 +1349,40 @@ def render_vocab_v2(model: Model) -> str: ) parts.append("") parts.append("") + parts.append( + _str_enum_v2( + model, + "Posture", + "posture", + "How a schema's statement was compiled: `create`, as a server's CREATE TABLE,\n" + " or `load`, as a server loads the stored definition of a table it holds.", + ) + ) + parts.append("") + parts.append("") + parts.append( + _str_enum_v2( + model, + "RefusalSite", + "refusal_site", + "Where a batch or filter document's refusal sits: `call` (the call, its settings or its\n" + " column list), `body_read` (the body outside any row: a header, the framing), `row` (one\n" + " row, `refused_row`) or `body_step` (a whole-body step).", + ) + ) + parts.append("") + parts.append("") + parts.append( + _str_enum_v2( + model, + "DefaultSupport", + "default_support", + "What this build does with a column's DEFAULT, as `schema_description` reports it per\n" + " column: `computed`, `generated`, `deferred` or `declined` (with a reason).", + ) + ) + parts.append("") + parts.append("") parts.append( _str_enum_v2( model, @@ -1396,10 +1433,17 @@ def render_vocab_v2(model: Model) -> str: parts.append("") parts.append(" @staticmethod") parts.append(" def is_stored(source: str) -> bool:") - parts.append(' """The description\'s `is_stored` fact for `source`. The description does not yet') - parts.append(' say what an unknown(n) source reports (rule r3, "Facts without a fallback");') - parts.append(' until it does, an unlisted source is not stored."""') - parts.append(" return _SOURCE_IS_STORED.get(source, False)") + src_fb = model.enums["value_src"].fallback + if src_fb is None: + parts.append(' """The description\'s `is_stored` fact for `source`. The description does not yet') + parts.append(' say what an unknown(n) source reports (rule r3, "Facts without a fallback");') + parts.append(' until it does, an unlisted source is not stored."""') + parts.append(" return _SOURCE_IS_STORED.get(source, False)") + else: + src_fb_stored = next(f["is_stored"] for _, v, f in sources if v == src_fb) + parts.append(' """The description\'s `is_stored` fact for `source`; an unknown(n) source') + parts.append(f' reports its fallback\'s (`{src_fb}`), as rule r3 says."""') + parts.append(f" return _SOURCE_IS_STORED.get(source, {_lit(src_fb_stored)})") parts.append("") parts.append("") parts.append( diff --git a/scripts/abi-v1/emit/reference.py b/scripts/abi-v1/emit/reference.py index acffb35c..650a932e 100644 --- a/scripts/abi-v1/emit/reference.py +++ b/scripts/abi-v1/emit/reference.py @@ -24,7 +24,7 @@ import json -from model import CONTENTS, INPUT_PREFIX, THREADS +from model import CONTENTS, INITS, INPUT_PREFIX, THREADS, content_in_generation from . import Output, banner, marker_key @@ -157,6 +157,10 @@ def note(ms: tuple[str, ...], what: str) -> None: out += ["", "### Thread classes", ""] out += table(["class", "what a caller may run at the same time"], [[_code(k), v] for k, v in THREADS.items()]) + has_init = any(f.init for f in model.functions) + if has_init: + out += ["", "### Init classes", ""] + out += table(["class", "when a call is valid"], [[_code(k), v] for k, v in INITS.items()]) out += ["", "### Parameter kinds", ""] out += table(["kind", "C spelling", "rule"], [[_code(k), c, r] for k, c, r in PARAM_KINDS]) @@ -177,7 +181,7 @@ def note(ms: tuple[str, ...], what: str) -> None: [ [_code(k), v] for k, v in [ - *CONTENTS.items(), + *((k, v) for k, v in CONTENTS.items() if content_in_generation(k, model.major)), ("document:", "a JSON document, listed under Documents below"), *inputs_row, ] @@ -339,12 +343,13 @@ def note(ms: tuple[str, ...], what: str) -> None: out += ["", "### Functions", ""] out += table( - ["function", "class", "thread", "returns", "status"], + ["function", "class", "thread", *(["init"] if has_init else []), "returns", "status"], [ [ _code(f.name), f.cls, f.thread, + *([f.init or ""] if has_init else []), "status" if f.returns.kind == "status" else _code(f.returns.c_type.strip()), _marks(f.provisional), ] @@ -353,7 +358,11 @@ def note(ms: tuple[str, ...], what: str) -> None: ) for f in model.functions: out += ["", f"#### `{f.name}`", "", "```c", f.prototype(), "```", ""] - facts = [f"- Class `{f.cls}`, thread `{f.thread}`, {_marks(f.provisional)}."] + facts = [ + f"- Class `{f.cls}`, thread `{f.thread}`, " + + (f"init `{f.init}`, " if f.init else "") + + f"{_marks(f.provisional)}." + ] if f.returns.kind == "status": facts.append("- May return " + ", ".join(_code(s) for s in f.may_return) + ".") if f.returns.kind == "handle": diff --git a/scripts/abi-v1/emit/rust.py b/scripts/abi-v1/emit/rust.py index 9eefa0f9..fede99b2 100644 --- a/scripts/abi-v1/emit/rust.py +++ b/scripts/abi-v1/emit/rust.py @@ -1383,6 +1383,9 @@ def render_vocab(model) -> str: "merge_reason": ("MergeReason", "string"), "declined_tier": ("DeclinedTier", "string"), "declined_layer": ("DeclinedLayer", "string"), + "posture": ("Posture", "string"), + "refusal_site": ("RefusalSite", "string"), + "default_support": ("DefaultSupport", "string"), } _UNKNOWN_STRING_DOC = ( @@ -1780,6 +1783,41 @@ def render_vocab_v2(model) -> str: ), ) + out += _string_enum_v2( + model, + "posture", + "Posture", + variant=_words, + doc=( + "How a schema's statement was compiled: `create`, as a server's CREATE TABLE, or `load`, as a server\n" + "loads the stored definition of a table it holds. A posture the description does not list is `Unknown`,\n" + "and the document decodes on (rule r3)." + ), + ) + + out += _string_enum_v2( + model, + "refusal_site", + "RefusalSite", + variant=_words, + doc=( + "Where a batch or filter document's refusal sits: the call, the body outside any row (a header, the\n" + "framing), one row (`refused_row`) or a whole-body step. A site the description does not list is\n" + "`Unknown`, and the document decodes on (rule r3)." + ), + ) + + out += _string_enum_v2( + model, + "default_support", + "DefaultSupport", + variant=_words, + doc=( + "What this build does with a column's DEFAULT, as a schema description reports it per column. A value\n" + "the description does not list is `Unknown`, and the column decodes on (rule r3)." + ), + ) + reason = model.enums["transform_reason"] rfb = next(v for v in reason.values if v.value == reason.fallback) out += _string_enum_v2( @@ -1821,24 +1859,33 @@ def render_vocab_v2(model) -> str: "", ] + src_enum = model.enums["value_src"] + src_fb = src_enum.fallback + if src_fb is None: + src_doc = ( + "The `value_src` vocabulary: where a column's value came from, and whether the description says it is\n" + "stored. A source the description does not list is `Unknown`; the description does not yet say what\n" + "its `is_stored` reports (rule r3, \"Facts without a fallback\"), and until it does, `false`." + ) + src_unknown, src_fact_doc = False, ( + "The description's `is_stored` fact for this source; `false` for `Unknown` until the description\n" + "says otherwise." + ) + else: + src_unknown = next(v.fields["is_stored"] for v in src_enum.values if v.value == src_fb) + src_doc = ( + "The `value_src` vocabulary: where a column's value came from, and whether the description says it is\n" + f"stored. A source the description does not list is `Unknown`, and reports its fallback's (`{src_fb}`)\n" + "`is_stored` (rule r3)." + ) + src_fact_doc = f"The description's `is_stored` fact for this source; `Unknown` reports the fallback's (`{src_fb}`)." out += _string_enum_v2( model, "value_src", "Source", variant=_words, - doc=( - "The `value_src` vocabulary: where a column's value came from, and whether the description says it is\n" - "stored. A source the description does not list is `Unknown`; the description does not yet say what\n" - "its `is_stored` reports (rule r3, \"Facts without a fallback\"), and until it does, `false`." - ), - facts=[ - ( - "is_stored", - False, - "The description's `is_stored` fact for this source; `false` for `Unknown` until the description\n" - "says otherwise.", - ) - ], + doc=src_doc, + facts=[("is_stored", src_unknown, src_fact_doc)], ) src = model.enums["value_src"] diff --git a/scripts/abi-v1/emit/setup_cases.py b/scripts/abi-v1/emit/setup_cases.py index ee177f52..b2415293 100644 --- a/scripts/abi-v1/emit/setup_cases.py +++ b/scripts/abi-v1/emit/setup_cases.py @@ -415,8 +415,8 @@ def _declined_cases(model) -> list[dict]: whose list holds one schema-layer setting, a filter compile surfaces as the binding's declined error (sdk.json's class for CHS_DECLINED) carrying the stub's status, ch_code and message; on a schema whose list is empty, the - same compile succeeds. Last, describe's deep-input refusal surfaces as the - binding's rejected error (sdk.json's class for CHS_REJECTED), ch_code 306.""" + same compile succeeds. Last, the deep input's refusal at compile surfaces as + the binding's rejected error (sdk.json's class for CHS_REJECTED), ch_code 306.""" if model.major < 2: # generation 1 is frozen and has no server return [] rule = _stubshared.DECLINED @@ -489,14 +489,15 @@ def _declined_cases(model) -> list[dict]: } cases.append( { - "id": "schema.describe_rejects_a_deep_input_306", + "id": "schema.compile_rejects_a_deep_input_306", "variant": DECLINED_VARIANT, "steps": [ {"op": "open", "expect": OK}, - {"op": "compile", "statement": rule["prefix"] + deep["id"], "as": "schema", "expect": OK}, - {"op": "describe", "schema": "schema", "expect": rejected}, - # The control: the same describe of a schema the deep input - # did not select answers its document. + # A schema chs_schema_create returned describes (F21): the statement's depth is judged + # at compile, so the deep input is refused here and never at describe. + {"op": "compile", "statement": rule["prefix"] + deep["id"], "as": "schema", "expect": rejected}, + # The control: a schema the deep input did not select compiles + # and describes as its document. *_declined_steps(model, "none", "plain", open_first=False), ], } diff --git a/scripts/abi-v1/emit/stub.py b/scripts/abi-v1/emit/stub.py index 9d559937..149f0108 100644 --- a/scripts/abi-v1/emit/stub.py +++ b/scripts/abi-v1/emit/stub.py @@ -1157,7 +1157,7 @@ def src_code(src) -> tuple[list[str], str, str, bool]: True, ) - for step in _stubshared.DOC_TEMPLATES[template]: + for step in _stubshared.doc_templates(model.major)[template]: op = step[0] if op == "raw": lines.append(f" chs_sb_cat(&sb, {_c_str(step[1])});") @@ -1432,6 +1432,8 @@ def _declined_settings(model, fn) -> list[str]: ids = list(_stubshared.DECLINED_LISTS) deep_sel = len(ids) # the deep input's selection, after every list's if fn.name == _DECLINED_CREATE: + if fn.name != deep["fn"] or deep["status"] not in model.function(fn.name).may_return: + raise ValueError(f"{fn.name}: the deep input is refused here, with {deep['status']}; the shape has changed") b = next(p for p in fn.params if p.kind == "bytes_in").name lines = ["#if defined(CHS_STUB_DECLINED_SETTINGS)", " chs_stub_declined_sel = -1;"] for i, sel in enumerate([*ids, deep["id"]]): @@ -1440,26 +1442,26 @@ def _declined_settings(model, fn) -> list[str]: f" if ({b}_len == {len(key)} && memcmp({b}, {_c_str(key.decode('ascii'))}, {len(key)}) == 0) " f"chs_stub_declined_sel = {i};" ) + name_lit, msg_lit = _c_str(deep["ch_name"]), _c_str(deep["message"]) + lines += [ + f" if (chs_stub_declined_sel == {deep_sel}) {{", + f" static const char name[] = {name_lit};", + f" static const char msg[] = {msg_lit};", + f" chs_stub_set_err(err, chs_stub_make_error({deep['status']}, {deep['ch_code']}, name, sizeof(name) - 1, msg, sizeof(msg) - 1));", + f" return {deep['status']};", + " }", + ] return [*lines, "#endif"] if fn.name == _DECLINED_DESCRIBE: out, others = _doc_out(fn) - if out is None or out.content != "document:schema_description" or fn.name != deep["fn"]: + if out is None or out.content != "document:schema_description": raise ValueError(f"{fn.name}: the shape _declined_settings answers has changed") - if deep["status"] not in model.function(fn.name).may_return: - raise ValueError(f"{fn.name}: {deep['status']} is not in its may_return") docs = ", ".join( _c_str(json.dumps(_stubshared.declined_document(sel), separators=(",", ":"), ensure_ascii=True)) for sel in ids ) - name_lit, msg_lit = _c_str(deep["ch_name"]), _c_str(deep["message"]) return [ "#if defined(CHS_STUB_DECLINED_SETTINGS)", - f" if (chs_stub_declined_sel == {deep_sel}) {{", - f" static const char name[] = {name_lit};", - f" static const char msg[] = {msg_lit};", - f" chs_stub_set_err(err, chs_stub_make_error({deep['status']}, {deep['ch_code']}, name, sizeof(name) - 1, msg, sizeof(msg) - 1));", - f" return {deep['status']};", - " }", f" if (chs_stub_declined_sel >= 0 && chs_stub_declined_sel < {len(ids)}) {{", f" static const char *const declined_docs[] = {{{docs}}};", *_answer_doc(out, others, "declined_docs[chs_stub_declined_sel]"), @@ -1601,6 +1603,16 @@ def sig(name: str) -> str: "}", ] ) + if any(f.name == "chs_error_reason" for f in model.functions): # generation 2 on + s["chs_error_reason"] = "\n".join( + [ + f"{sig('chs_error_reason')} {{", + " if (error == NULL || chs_stub_check_handle((const chs_handle_common *) error, " + f'{_kind_const(ERROR_HANDLE)}, "error") != NULL) return NULL;', + " return chs_stub_make_buf(NULL, 0);", + "}", + ] + ) for name, field_ptr, field_len in ( ("chs_error_ch_name", "ch_name", "ch_name_len"), ("chs_error_message", "message", "message_len"), diff --git a/scripts/abi-v1/gen.py b/scripts/abi-v1/gen.py index c1ccb046..4edba304 100755 --- a/scripts/abi-v1/gen.py +++ b/scripts/abi-v1/gen.py @@ -1114,6 +1114,107 @@ def mutate(d: dict) -> None: "enum chs_format: the value 'CHS_FORMAT_UNKNOWN' takes the name (r3) reserves", ) + # The init class follows the function (the Preamble's Initialization), + # the schema pins the classes, and the header states each function's. + def set_init(fn_name: str, value): + def mutate(w: Path) -> None: + _json_edit(w / abi, lambda d: _fn_entry(d, fn_name).update(init=value)) + + return mutate + + plant( + "(init) a handshake call marked `required`", + set_init("chs_abi_version", "required"), + "function chs_abi_version: init class must be `any`, not `required`: it is class `handshake`", + ) + plant( + "(init) chs_initialize marked `required`", + set_init("chs_initialize", "required"), + "function chs_initialize: init class must be `any`, not `required`: it is process setup itself", + ) + plant( + "(init) a free marked `required`", + set_init("chs_schema_free", "required"), + "function chs_schema_free: init class must be `any`, not `required`: it returns no chs_status", + ) + plant( + "(init) an error accessor marked `required`", + set_init("chs_error_message", "required"), + "function chs_error_message: init class must be `any`, not `required`: it returns no chs_status", + ) + plant( + "(init) a status call marked `any` whose section does not say it is valid before chs_initialize", + set_init("chs_type_validate", "any"), + "function chs_type_validate: init class `any`, but it returns a status and its section does not say", + ) + + def said(w: Path) -> None: + _edit(w / docs, "### chs_type_validate\n\n", "### chs_type_validate\n\nIt is valid before `chs_initialize`. ") + + plant( + "(init) a `required` call whose section says it is valid before chs_initialize", + said, + "function chs_type_validate: its section says it is valid before `chs_initialize`, so its init class is `any`", + ) + + def said_and_any(w: Path) -> None: + said(w) + set_init("chs_type_validate", "any")(w) + + plant( + "(init) the positive control: a status call marked `any` whose section says it is valid before " + "chs_initialize generates and checks", + said_and_any, + None, + (("--major", "2", "--write"), ("--major", "2", "--check")), + ) + plant( + "(init) a `required` call that cannot return CHS_INVALID_ARGUMENT", + lambda w: _json_edit(w / abi, lambda d: _fn_entry(d, "chs_discover_query")["may_return"].remove("CHS_INVALID_ARGUMENT")), + "function chs_discover_query: init class `required` refuses with CHS_INVALID_ARGUMENT", + ) + plant( + "(init) a function with no init class", + lambda w: _json_edit(w / abi, lambda d: _fn_entry(d, "chs_schema_create").pop("init")), + "abi.schema.json", + ) + plant("(init) an init class the schema does not define", set_init("chs_schema_create", "sometimes"), "abi.schema.json") + + def init_enum_drift(w: Path) -> None: + _json_edit(w / schema, lambda d: d["$defs"]["function"]["properties"]["init"]["enum"].append("lazy")) + + plant("(init) the schema's init enum differs from model.py's classes", init_enum_drift, "the init enum") + + def header_disagrees(w: Path) -> None: + _edit(w / "include/v2/chtypes.h", "Init: required.", "Init: any.") + + plant( + "(init) the header states another init class than the description", + header_disagrees, + "include/v2/chtypes.h: differs from the generator's output", + ) + + # A document member bound to a vocabulary (F2): the binding names a vocabulary the description + # defines, and the member's enum is drawn from it. + def batch_prop(d: dict, name: str) -> dict: + return d["documents"]["batch"]["schema"]["properties"][name] + + plant( + "(vocabulary) a member bound to an enum the description does not define", + lambda w: _json_edit(w / abi, lambda d: batch_prop(d, "outcome").update(vocabulary="batch_verdict")), + "`vocabulary` names 'batch_verdict', which is not a string vocabulary", + ) + plant( + "(vocabulary) a member's enum lists a value its vocabulary lacks", + lambda w: _json_edit(w / abi, lambda d: batch_prop(d, "outcome")["enum"].append("maybe")), + "is not drawn from the `batch_outcome` vocabulary's values", + ) + plant( + "(required) a required member no property describes", + lambda w: _json_edit(w / abi, lambda d: d["documents"]["batch"]["schema"]["required"].append("rows_lost")), + "requires 'rows_lost', which no property describes", + ) + # (r4) generation 1's published codes keep their names and numbers. def renumber(d: dict) -> None: for v in d["enums"]["chs_status"]["values"]: diff --git a/scripts/abi-v1/model.py b/scripts/abi-v1/model.py index 44a27a51..3e7a955a 100644 --- a/scripts/abi-v1/model.py +++ b/scripts/abi-v1/model.py @@ -122,7 +122,7 @@ def spec(major: int) -> Spec: STABILITIES = ("unstable", "locked") # The rules every generation-2+ docs.md states in its `## Rules` section, by # label; load() refuses a Rules section that drops one. -GENERATION_RULES = ("r1", "r2", "r3", "r4", "r5", "r6", "r7") +GENERATION_RULES = ("r1", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9") # The spelling readers reserve for the member an unlisted enum value maps to # (r3): no described value may take it. UNKNOWN = "unknown" @@ -160,6 +160,22 @@ def spec(major: int) -> Spec: "the same arguments is a no-op that is safe at any time", "process_serial": "process-level: must not overlap any other call into the same library image", } +# The init classes (generation 2 on): whether a call is valid before +# chs_initialize has succeeded. Every function names one; the emitters print +# these descriptions (no second copy), and the schema pins the same set. The +# rule tying a class to a function is checked in load(): the handshake, the +# tombstone, chs_initialize and every call that returns no chs_status are +# `any`, and every other call is `required` unless its own section says it is +# valid before chs_initialize. +INITS: dict[str, str] = { + "any": "valid before and after `chs_initialize`", + "required": "valid only once `chs_initialize` has succeeded: until then the call changes nothing and returns " + "`CHS_INVALID_ARGUMENT`, ch_code 0, and with arguments that are otherwise valid its message starts " + "`: call chs_initialize first`", +} +INITIALIZE = "chs_initialize" +# The prose that makes a status-returning call `any`: its own section says so. +INIT_ANY_SAID = re.compile(r"\b(?:valid|safe) before `chs_initialize`") # What a counted byte string carries. Every one is a byte string, never a C # string; the description says which, so a binding picks a byte-safe type for # a name and the stub and the cases know what a valid input looks like. The @@ -171,9 +187,24 @@ def spec(major: int) -> Spec: "message": "a ClickHouse message: may quote input bytes, so may carry any byte", "ascii": "guaranteed ASCII", "json_object_string_values": "a JSON object whose values are JSON strings (settings, query parameters)", + "json_object_byte_values": "a JSON object whose values are JSON strings, or the object `{\"b64\": \"\"}` for a value whose bytes are not valid UTF-8 (settings, query parameters)", "json_array_names": "a JSON array of column names, each an object carrying the name as byte_strings says", "timezone": "a time zone name, validated by ClickHouse's own DateLUT; length 0 means UTC", } +# The generations a content kind exists in, when not every one: a kind added for a later generation is in +# neither an earlier one's description nor its reference, and a kind a generation replaced is in neither of +# the later ones', so no generation's outputs change with another's (generation 2: the object form for bytes, +# public issue #424, replaced the string-only settings object). load() refuses a content outside its +# generations, and the reference lists only its own. +CONTENTS_SINCE: dict[str, int] = {"json_object_byte_values": 2} +CONTENTS_UNTIL: dict[str, int] = {"json_object_string_values": 1} + + +def content_in_generation(content: str, major: int) -> bool: + """Whether a CONTENTS kind exists in generation `major`.""" + return CONTENTS_SINCE.get(content, 1) <= major <= CONTENTS_UNTIL.get(content, major) + + # A bytes_in content `input:` names an entry of the description's # `inputs` (generation 2 on): a JSON object the library validates and whose # unknown keys it refuses (r1). Beside CONTENTS, not in it: its meaning is the @@ -233,6 +264,10 @@ def __init__(self, problems: list[str]): "oneOf", "anyOf", "allOf", + # Not a JSON Schema keyword: the description's own. A document member that carries a vocabulary's value + # names the vocabulary (`"vocabulary": "batch_outcome"`), so a binding generates its decoder from the + # structure and load() checks the member's enum against the vocabulary. validate() ignores it. + "vocabulary", } ) _TYPES = { @@ -430,6 +465,7 @@ class Function: params: tuple[Param, ...] doc: str growable: bool = False # r1, generation 2 on: takes exactly one options document + init: str | None = None # generation 2 on: `any` or `required` (INITS) @property def is_firm(self) -> bool: @@ -1307,6 +1343,67 @@ def content_check(where: str, content: str | None) -> None: named = input_name(content) if named is not None and named not in inputs: problems.append(f"{where}: content {content!r} names no entry in inputs") + if content in CONTENTS and not content_in_generation(content, major): + problems.append( + f"{where}: content {content!r} does not exist in generation {major} (CONTENTS_SINCE / CONTENTS_UNTIL)" + ) + + # One copy of a vocabulary's values: wherever a schema property is named after a string vocabulary and + # carries an enum, the enum is drawn from it, equal to it in an input document (the writer may send any + # value) and a subset of it in an output document (the library writes only those; an absent member can + # mean a value, as describe's `posture` does for `create`). + def vocab_enum_problems(schema, where: str, exact: bool) -> list[str]: + out: list[str] = [] + + def walk(node, at: str) -> None: + if not isinstance(node, dict): + return + for key in ("properties", "$defs"): + for name, sub in (node.get(key) or {}).items(): + bound = sub.get("vocabulary") if isinstance(sub, dict) else None + named = key == "properties" and name in enums and enums[name].repr == "string" + if key == "properties" and isinstance(sub, dict) and (bound is not None or (named and "enum" in sub)): + vname = bound if bound is not None else name + if vname not in enums or enums[vname].repr != "string": + out.append( + f"{where}{at}/{key}/{name}: `vocabulary` names {vname!r}, which is not a string " + "vocabulary the description defines" + ) + elif "enum" not in sub: + out.append( + f"{where}{at}/{key}/{name}: is bound to the `{vname}` vocabulary but lists no enum " + "of the values the library writes" + ) + else: + want = [v.value for v in enums[vname].values] + have = sub["enum"] + if (sorted(have) != sorted(want)) if exact else not set(have) <= set(want): + out.append( + f"{where}{at}/{key}/{name}: enum {have} is not " + + ("the" if exact else "drawn from the") + + f" `{vname}` vocabulary's values {want}" + ) + walk(sub, f"{at}/{key}/{name}") + # Every member a schema requires is one it describes: a reader generates its decoder from + # `properties`, so a required member no property describes is one no binding can read. + if node.get("type") == "object" and "required" in node and "properties" in node: + for r in node["required"]: + if r not in node["properties"]: + out.append(f"{where}{at}: requires {r!r}, which no property describes") + for key in ("items", "additionalProperties"): + walk(node.get(key), f"{at}/{key}") + for key in ("oneOf", "anyOf", "allOf"): + for i, sub in enumerate(node.get(key) or []): + walk(sub, f"{at}/{key}/{i}") + + walk(schema, "") + return out + + for name, inp in inputs.items(): + problems += vocab_enum_problems(inp.schema, f"input {name}: #", exact=True) + for name, document in documents.items(): + if document.schema is not None: + problems += vocab_enum_problems(document.schema, f"document {name}: #", exact=False) # ---- build_info bi = raw["build_info"] @@ -1336,6 +1433,15 @@ def content_check(where: str, content: str | None) -> None: f"{sp.abi_schema}: the thread enum {sorted(schema_threads)} differs from the classes model.py " f"describes {sorted(THREADS)}" ) + # ---- the init vocabulary (generation 2 on), which the schema pins too + if major >= 2: + schema_inits = (_read_json(root, sp.abi_schema, problems) or {}).get("$defs", {}).get("function", {}) + schema_inits = schema_inits.get("properties", {}).get("init", {}).get("enum", []) + if set(schema_inits) != set(INITS): + problems.append( + f"{sp.abi_schema}: the init enum {sorted(schema_inits)} differs from the classes model.py " + f"describes {sorted(INITS)}" + ) # ---- functions functions: list[Function] = [] @@ -1362,6 +1468,7 @@ def content_check(where: str, content: str | None) -> None: params=params, doc=doc(name), growable=bool(f.get("growable", False)), + init=f.get("init"), ) functions.append(fn) @@ -1508,6 +1615,50 @@ def content_check(where: str, content: str | None) -> None: elif not reads and fn.thread in ("shared", "handle_serial"): problems.append(f"{where}: thread class `{fn.thread}` applies only to a call that takes a handle") + # ---- init classes follow the function (generation 2 on; the Preamble's + # Initialization). A call that returns no chs_status has no status to refuse + # with, so it is `any`: the error and buffer accessors read a refusal itself, + # and a free before chs_initialize is given NULL (nothing makes its handle). + # The handshake, the tombstone and chs_initialize are `any` by what they are. + # Every other call is `required` unless its own section says it is valid + # before chs_initialize, and a `required` call can refuse. + for fn in functions: + where = f"function {fn.name}" + if major < 2: + if fn.init is not None: + problems.append(f"{where}: `init` is generation 2 on; generation {major} names no init class") + continue + if fn.init not in INITS: + problems.append(f"{where}: init class {fn.init!r} is not one of {sorted(INITS)}") + continue + if fn.cls in ("handshake", "tombstone"): + forced = f"it is class `{fn.cls}`" + elif fn.name == INITIALIZE: + forced = "it is process setup itself" + elif fn.returns.kind != "status": + forced = "it returns no chs_status, so it has no status to refuse with" + else: + forced = None + said = bool(INIT_ANY_SAID.search(fn.doc)) + if forced is not None: + if fn.init != "any": + problems.append(f"{where}: init class must be `any`, not `{fn.init}`: {forced}") + elif fn.init == "any" and not said: + problems.append( + f"{where}: init class `any`, but it returns a status and its section does not say it is valid " + "before `chs_initialize`; such a call is `required`" + ) + elif fn.init == "required" and said: + problems.append( + f"{where}: its section says it is valid before `chs_initialize`, so its init class is `any`, " + "not `required`" + ) + elif fn.init == "required" and "CHS_INVALID_ARGUMENT" not in fn.may_return: + problems.append( + f"{where}: init class `required` refuses with CHS_INVALID_ARGUMENT before `chs_initialize`, " + "so may_return must list it" + ) + # ---- docs coverage: exactly one section per symbol required = set(handles) | set(enums) | set(fmap) | {f"{INPUT_PREFIX}{n}" for n in inputs} optional = set(constants) | {f"document:{d}" for d in documents} diff --git a/spec/abi-v2/abi.json b/spec/abi-v2/abi.json index 4a94c3f8..a67427f1 100644 --- a/spec/abi-v2/abi.json +++ b/spec/abi-v2/abi.json @@ -267,6 +267,7 @@ "fields": { "is_stored": "boolean" }, + "fallback": "skipped", "values": [ { "value": "input", @@ -311,6 +312,14 @@ { "value": "default_pending", "is_stored": false + }, + { + "value": "default_deferred", + "is_stored": false + }, + { + "value": "default_override", + "is_stored": true } ] }, @@ -393,6 +402,23 @@ } ] }, + "refusal_site": { + "repr": "string", + "values": [ + { + "value": "call" + }, + { + "value": "body_read" + }, + { + "value": "row" + }, + { + "value": "body_step" + } + ] + }, "discover_query_param": { "repr": "string", "fields": { @@ -429,6 +455,23 @@ } ] }, + "default_support": { + "repr": "string", + "values": [ + { + "value": "computed" + }, + { + "value": "generated" + }, + { + "value": "deferred" + }, + { + "value": "declined" + } + ] + }, "declined_tier": { "repr": "string", "values": [ @@ -440,6 +483,9 @@ }, { "value": "result-content" + }, + { + "value": "server-refuses" } ] }, @@ -456,6 +502,17 @@ "value": "schema" } ] + }, + "posture": { + "repr": "string", + "values": [ + { + "value": "create" + }, + { + "value": "load" + } + ] } }, "constants": { @@ -467,6 +524,10 @@ "type": "int32", "value": -1 }, + "CHS_CODE_DECLINED": { + "type": "int32", + "value": -2 + }, "CHS_DOC_VALUES": { "type": "uint32", "value": 1 @@ -527,6 +588,7 @@ "columns[].name", "columns[].type", "columns[].default_expression", + "columns[].reads[].name", "filter_declined_settings[].name", "replicated.zookeeper_path", "replicated.replica_name" @@ -548,10 +610,21 @@ "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, "tier": { - "type": "string" + "vocabulary": "declined_tier", + "enum": [ + "predicate", + "predicate-unflipped", + "result-content", + "server-refuses" + ] }, "layer": { - "type": "string" + "vocabulary": "declined_layer", + "enum": [ + "defaults", + "server", + "schema" + ] } }, "allOf": [ @@ -575,6 +648,7 @@ "type": "object", "properties": { "default_kind": { + "vocabulary": "default_kind", "enum": [ "", "DEFAULT", @@ -606,6 +680,25 @@ "default_expression_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "default_support": { + "vocabulary": "default_support", + "enum": [ + "computed", + "generated", + "deferred", + "declined" + ] + }, + "default_support_reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "reads": { + "type": "array", + "items": { + "$ref": "#/$defs/named" + } } }, "allOf": [ @@ -628,24 +721,12 @@ { "required": [ "type" - ], - "properties": { - "type_b64": false - } + ] }, { "required": [ "type_b64" - ], - "properties": { - "type": false - } - }, - { - "properties": { - "type": false, - "type_b64": false - } + ] } ] }, @@ -675,6 +756,49 @@ } ] } + ], + "required": [ + "default_kind" + ] + }, + "bytes_value": { + "type": "object", + "required": [ + "b64" + ], + "properties": { + "b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + } + }, + "named": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [ + { + "oneOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "name_b64" + ] + } + ] + } ] } }, @@ -709,7 +833,14 @@ "settings": { "type": "object", "additionalProperties": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/bytes_value" + } + ] } }, "macros": { @@ -768,6 +899,12 @@ ] } ] + }, + "posture": { + "vocabulary": "posture", + "enum": [ + "load" + ] } } } @@ -779,7 +916,6 @@ "verdict_err", "partition_id", "unknown_fields[].name", - "unsupported_settings[].name", "cols[].name", "cols[].type", "cols[].base", @@ -850,12 +986,46 @@ "col": { "type": "object", "required": [ - "null" + "null", + "src" ], "properties": { "null": { "type": "boolean" }, + "src": { + "vocabulary": "value_src", + "enum": [ + "input", + "default", + "default_substituted", + "default_generated", + "absent", + "materialized_input", + "skipped", + "ephemeral_input", + "default_expr_unsupported", + "default_volatile_unresolved", + "default_pending", + "default_deferred", + "default_override" + ] + }, + "nullable": { + "type": "boolean" + }, + "poison": { + "type": "boolean" + }, + "null_input": { + "type": "boolean" + }, + "dup_dropped": { + "type": "boolean" + }, + "ref_unclassified": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -1126,6 +1296,9 @@ "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, + "kind": { + "type": "string" + }, "stored": { "type": "string" }, @@ -1179,6 +1352,9 @@ } ] } + ], + "required": [ + "kind" ] }, "transform": { @@ -1189,6 +1365,38 @@ "minimum": 0, "maximum": 9007199254740991 }, + "reason": { + "vocabulary": "transform_reason", + "enum": [ + "overflow_wrap", + "null_to_default", + "null_loss", + "decimal_truncate", + "date_clamp", + "datetime_wrap", + "date_shift", + "uuid_mangle", + "ip_mangle", + "float_precision", + "lossy_numeric", + "fixedstring_pad", + "emptied", + "element_changed", + "enum_coerce", + "value_changed", + "poisoned", + "duplicate_key_dropped", + "reformat", + "default_filled", + "zero_filled", + "default_materialized", + "ttl_expired", + "ttl_column_expired" + ] + }, + "lossy": { + "type": "boolean" + }, "column": { "type": "string" }, @@ -1278,14 +1486,51 @@ } ] } + ], + "required": [ + "reason", + "lossy" ] } }, "type": "object", "required": [ + "outcome", + "code", "input_span" ], "properties": { + "outcome": { + "vocabulary": "row_outcome", + "enum": [ + "accepted", + "accepted_poisoned", + "rejected", + "skipped", + "unsupported" + ] + }, + "code": { + "type": "integer", + "minimum": -2 + }, + "verdict": { + "vocabulary": "filter_verdict", + "enum": [ + "t", + "f", + "e", + "d" + ] + }, + "verdict_code": { + "type": "integer", + "minimum": -2 + }, + "verdict_reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, "input_span": { "$ref": "#/$defs/span" }, @@ -1295,12 +1540,6 @@ "$ref": "#/$defs/named" } }, - "unsupported_settings": { - "type": "array", - "items": { - "$ref": "#/$defs/named" - } - }, "cols": { "type": "array", "items": { @@ -1339,6 +1578,24 @@ "partition_id_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "refused_at": { + "vocabulary": "refusal_site", + "enum": [ + "call", + "body_read", + "row", + "body_step" + ] + }, + "refused_row": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "allOf": [ @@ -1432,7 +1689,6 @@ "rows[].verdict_err", "rows[].partition_id", "rows[].unknown_fields[].name", - "rows[].unsupported_settings[].name", "rows[].cols[].name", "rows[].cols[].type", "rows[].cols[].base", @@ -1456,6 +1712,7 @@ "transformed[].input", "transformed[].stored", "unsupported_settings[].name", + "export_columns[].name", "framing.header.names[].name" ], "value_entries": [ @@ -1516,12 +1773,46 @@ "col": { "type": "object", "required": [ - "null" + "null", + "src" ], "properties": { "null": { "type": "boolean" }, + "src": { + "vocabulary": "value_src", + "enum": [ + "input", + "default", + "default_substituted", + "default_generated", + "absent", + "materialized_input", + "skipped", + "ephemeral_input", + "default_expr_unsupported", + "default_volatile_unresolved", + "default_pending", + "default_deferred", + "default_override" + ] + }, + "nullable": { + "type": "boolean" + }, + "poison": { + "type": "boolean" + }, + "null_input": { + "type": "boolean" + }, + "dup_dropped": { + "type": "boolean" + }, + "ref_unclassified": { + "type": "boolean" + }, "name": { "type": "string" }, @@ -1792,6 +2083,9 @@ "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, + "kind": { + "type": "string" + }, "stored": { "type": "string" }, @@ -1845,6 +2139,9 @@ } ] } + ], + "required": [ + "kind" ] }, "transform": { @@ -1855,6 +2152,38 @@ "minimum": 0, "maximum": 9007199254740991 }, + "reason": { + "vocabulary": "transform_reason", + "enum": [ + "overflow_wrap", + "null_to_default", + "null_loss", + "decimal_truncate", + "date_clamp", + "datetime_wrap", + "date_shift", + "uuid_mangle", + "ip_mangle", + "float_precision", + "lossy_numeric", + "fixedstring_pad", + "emptied", + "element_changed", + "enum_coerce", + "value_changed", + "poisoned", + "duplicate_key_dropped", + "reformat", + "default_filled", + "zero_filled", + "default_materialized", + "ttl_expired", + "ttl_column_expired" + ] + }, + "lossy": { + "type": "boolean" + }, "column": { "type": "string" }, @@ -1944,14 +2273,51 @@ } ] } + ], + "required": [ + "reason", + "lossy" ] }, "row": { "type": "object", "required": [ + "outcome", + "code", "input_span" ], "properties": { + "outcome": { + "vocabulary": "row_outcome", + "enum": [ + "accepted", + "accepted_poisoned", + "rejected", + "skipped", + "unsupported" + ] + }, + "code": { + "type": "integer", + "minimum": -2 + }, + "verdict": { + "vocabulary": "filter_verdict", + "enum": [ + "t", + "f", + "e", + "d" + ] + }, + "verdict_code": { + "type": "integer", + "minimum": -2 + }, + "verdict_reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, "input_span": { "$ref": "#/$defs/span" }, @@ -1961,12 +2327,6 @@ "$ref": "#/$defs/named" } }, - "unsupported_settings": { - "type": "array", - "items": { - "$ref": "#/$defs/named" - } - }, "cols": { "type": "array", "items": { @@ -2005,6 +2365,24 @@ "partition_id_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "refused_at": { + "vocabulary": "refusal_site", + "enum": [ + "call", + "body_read", + "row", + "body_step" + ] + }, + "refused_row": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "allOf": [ @@ -2232,17 +2610,21 @@ "at_merge": { "type": "object", "required": [ - "row", + "part_row", "reason" ], "properties": { - "row": { + "part_row": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "reason": { - "type": "string" + "vocabulary": "merge_reason", + "enum": [ + "ttl_delete", + "ttl_column_reset" + ] }, "column": { "type": "string" @@ -2325,11 +2707,29 @@ }, "type": "object", "required": [ + "outcome", + "code", "rows", + "rows_read", + "rows_skipped", + "unsupported_settings", "unconsumed", "framing" ], "properties": { + "outcome": { + "vocabulary": "batch_outcome", + "enum": [ + "accepted", + "accepted_poisoned", + "rejected", + "unsupported" + ] + }, + "code": { + "type": "integer", + "minimum": -2 + }, "rows": { "type": "array", "items": { @@ -2435,12 +2835,78 @@ "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" }, + "rows_read": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rows_skipped": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rows_passed": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "rows_cut": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "partition_count": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "row_spans": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/$defs/span" + }, + { + "type": "null" + } + ] + } + }, + "export_columns": { + "type": "array", + "items": { + "$ref": "#/$defs/named" + } + }, + "reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "refused_at": { + "vocabulary": "refusal_site", + "enum": [ + "call", + "body_read", + "row", + "body_step" + ] + }, + "refused_row": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, "export_declined": { "type": "string" }, "export_declined_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "export_declined_reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" } }, "allOf": [ @@ -2544,6 +3010,14 @@ "minimum": 0, "maximum": 9007199254740991 }, + "code": { + "type": "integer", + "minimum": -2 + }, + "reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, "err": { "type": "string" }, @@ -2579,11 +3053,35 @@ } ] } + ], + "required": [ + "row", + "code" ] } }, "type": "object", "properties": { + "outcome": { + "vocabulary": "filter_outcome", + "enum": [ + "ok", + "rejected", + "unsupported" + ] + }, + "code": { + "type": "integer", + "minimum": -2 + }, + "rows_read": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 + }, + "verdicts": { + "type": "string" + }, "unsupported_settings": { "type": "array", "items": { @@ -2602,6 +3100,24 @@ "err_b64": { "type": "string", "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "reason": { + "type": "string", + "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$" + }, + "refused_at": { + "vocabulary": "refusal_site", + "enum": [ + "call", + "body_read", + "row", + "body_step" + ] + }, + "refused_row": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991 } }, "allOf": [ @@ -2631,6 +3147,14 @@ } ] } + ], + "required": [ + "outcome", + "code", + "rows_read", + "unsupported_settings", + "verdicts", + "errors" ] } }, @@ -2870,7 +3394,14 @@ "settings": { "type": "object", "additionalProperties": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/$defs/bytes_value" + } + ] } }, "macros": { @@ -2882,6 +3413,21 @@ "type": "string" } } + }, + "$defs": { + "bytes_value": { + "type": "object", + "additionalProperties": false, + "required": [ + "b64" + ], + "properties": { + "b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + } + } } } }, @@ -2896,7 +3442,144 @@ "options": true, "schema": { "type": "object", - "additionalProperties": false + "additionalProperties": false, + "properties": { + "posture": { + "enum": [ + "create", + "load" + ] + } + } + } + }, + "insert_options": { + "options": true, + "schema": { + "type": "object", + "additionalProperties": false, + "properties": { + "defer_defaults": { + "enum": [ + "none", + "server" + ] + }, + "insertable_columns": { + "type": "array", + "items": { + "$ref": "#/$defs/named" + } + }, + "column_defaults": { + "type": "array", + "items": { + "$ref": "#/$defs/column_default" + } + }, + "verdict_basis": { + "enum": [ + "input", + "export" + ] + } + }, + "$defs": { + "named": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [ + { + "oneOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "name_b64" + ] + } + ] + } + ] + }, + "column_default": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "name_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "value": { + "type": "string" + }, + "value_b64": { + "type": "string", + "pattern": "^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + } + }, + "allOf": [ + { + "oneOf": [ + { + "required": [ + "name" + ] + }, + { + "required": [ + "name_b64" + ] + } + ] + }, + { + "oneOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "value_b64" + ] + } + ] + } + ] + } + } + } + }, + "filter_options": { + "options": true, + "schema": { + "type": "object", + "additionalProperties": false, + "properties": { + "param_encoding": { + "enum": [ + "escaped", + "raw" + ] + } + } } } }, @@ -3013,6 +3696,7 @@ "name": "chs_abi_version", "class": "handshake", "thread": "any", + "init": "any", "returns": { "kind": "scalar", "type": "int32" @@ -3023,6 +3707,7 @@ "name": "chs_build_info", "class": "handshake", "thread": "any", + "init": "any", "returns": { "kind": "scalar", "type": "cstr_static" @@ -3033,6 +3718,7 @@ "name": "chs_clickhouse_version", "class": "handshake", "thread": "any", + "init": "any", "returns": { "kind": "scalar", "type": "cstr_static" @@ -3043,6 +3729,7 @@ "name": "chs_abi_revision", "class": "tombstone", "thread": "any", + "init": "any", "returns": { "kind": "scalar", "type": "int", @@ -3054,6 +3741,7 @@ "name": "chs_buf_data", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "scalar", "type": "u8ptr_const", @@ -3072,6 +3760,7 @@ "name": "chs_buf_len", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "scalar", "type": "size" @@ -3089,6 +3778,7 @@ "name": "chs_buf_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3105,6 +3795,7 @@ "name": "chs_error_status", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "enum", "type": "chs_status" @@ -3122,6 +3813,7 @@ "name": "chs_error_ch_code", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "scalar", "type": "int32" @@ -3139,6 +3831,7 @@ "name": "chs_error_ch_name", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "handle", "type": "chs_buf", @@ -3159,6 +3852,7 @@ "name": "chs_error_message", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "handle", "type": "chs_buf", @@ -3179,6 +3873,7 @@ "name": "chs_error_column", "class": "api", "thread": "shared", + "init": "any", "returns": { "kind": "handle", "type": "chs_buf", @@ -3195,10 +3890,32 @@ } ] }, + { + "name": "chs_error_reason", + "class": "api", + "thread": "shared", + "init": "any", + "returns": { + "kind": "handle", + "type": "chs_buf", + "owned": true, + "nullable": true, + "content": "ascii" + }, + "params": [ + { + "name": "error", + "kind": "handle", + "type": "chs_error", + "const": true + } + ] + }, { "name": "chs_error_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3215,6 +3932,7 @@ "name": "chs_live_handles", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3240,6 +3958,7 @@ "name": "chs_error_codes", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3265,6 +3984,7 @@ "name": "chs_registered_families", "class": "tooling", "thread": "process_serial", + "init": "required", "returns": { "kind": "status" }, @@ -3290,6 +4010,7 @@ "name": "chs_function_flags", "class": "tooling", "thread": "process_serial", + "init": "required", "returns": { "kind": "status" }, @@ -3315,6 +4036,7 @@ "name": "chs_reference_type", "class": "tooling", "thread": "process_serial", + "init": "required", "returns": { "kind": "status" }, @@ -3347,6 +4069,7 @@ "name": "chs_initialize", "class": "api", "thread": "process_once", + "init": "any", "returns": { "kind": "status" }, @@ -3372,6 +4095,7 @@ "name": "chs_set_defaults", "class": "api", "thread": "process_serial", + "init": "required", "returns": { "kind": "status" }, @@ -3379,7 +4103,7 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "err", @@ -3398,6 +4122,7 @@ "name": "chs_shutdown", "class": "api", "thread": "process_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3407,6 +4132,7 @@ "name": "chs_type_validate", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3439,6 +4165,7 @@ "name": "chs_back_quote", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3469,6 +4196,7 @@ "name": "chs_back_quote_if_needed", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3499,6 +4227,7 @@ "name": "chs_quote_string", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -3529,6 +4258,7 @@ "name": "chs_server_create", "class": "api", "thread": "any", + "init": "required", "growable": true, "returns": { "kind": "status" @@ -3566,6 +4296,7 @@ "name": "chs_server_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3582,6 +4313,7 @@ "name": "chs_schema_create", "class": "api", "thread": "shared", + "init": "required", "growable": true, "returns": { "kind": "status" @@ -3602,7 +4334,7 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "options", @@ -3631,6 +4363,7 @@ "name": "chs_schema_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3647,6 +4380,7 @@ "name": "chs_schema_describe", "class": "api", "thread": "shared", + "init": "required", "returns": { "kind": "status" }, @@ -3670,7 +4404,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -3679,6 +4412,8 @@ "name": "chs_preview_row", "class": "api", "thread": "shared", + "init": "required", + "growable": true, "returns": { "kind": "status" }, @@ -3702,13 +4437,18 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "columns", "kind": "bytes_in", "content": "json_array_names" }, + { + "name": "options", + "kind": "bytes_in", + "content": "input:insert_options" + }, { "name": "out", "kind": "out_handle", @@ -3722,8 +4462,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -3732,6 +4470,8 @@ "name": "chs_preview_batch", "class": "api", "thread": "shared", + "init": "required", + "growable": true, "returns": { "kind": "status" }, @@ -3755,7 +4495,7 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "columns", @@ -3779,6 +4519,11 @@ "kind": "scalar", "type": "uint32" }, + { + "name": "options", + "kind": "bytes_in", + "content": "input:insert_options" + }, { "name": "out", "kind": "out_handle", @@ -3799,8 +4544,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -3809,6 +4552,8 @@ "name": "chs_filter_create", "class": "api", "thread": "shared", + "init": "required", + "growable": true, "returns": { "kind": "status" }, @@ -3827,12 +4572,17 @@ { "name": "query_params", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" + }, + { + "name": "options", + "kind": "bytes_in", + "content": "input:filter_options" }, { "name": "out", @@ -3856,6 +4606,7 @@ "name": "chs_filter_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3872,6 +4623,8 @@ "name": "chs_filter_eval_body", "class": "api", "thread": "shared", + "init": "required", + "growable": true, "returns": { "kind": "status" }, @@ -3895,7 +4648,17 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" + }, + { + "name": "columns", + "kind": "bytes_in", + "content": "json_array_names" + }, + { + "name": "options", + "kind": "bytes_in", + "content": "input:insert_options" }, { "name": "out", @@ -3910,8 +4673,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -3920,6 +4681,8 @@ "name": "chs_block_create", "class": "api", "thread": "shared", + "init": "required", + "growable": true, "returns": { "kind": "status" }, @@ -3943,13 +4706,18 @@ { "name": "settings", "kind": "bytes_in", - "content": "json_object_string_values" + "content": "json_object_byte_values" }, { "name": "columns", "kind": "bytes_in", "content": "json_array_names" }, + { + "name": "options", + "kind": "bytes_in", + "content": "input:insert_options" + }, { "name": "out", "kind": "out_handle", @@ -3962,8 +4730,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -3972,6 +4738,7 @@ "name": "chs_block_free", "class": "api", "thread": "handle_serial", + "init": "any", "returns": { "kind": "void" }, @@ -3988,6 +4755,7 @@ "name": "chs_filter_eval_block", "class": "api", "thread": "shared", + "init": "required", "returns": { "kind": "status" }, @@ -4017,8 +4785,6 @@ ], "may_return": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ] @@ -4027,6 +4793,7 @@ "name": "chs_discover_query", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, @@ -4052,6 +4819,7 @@ "name": "chs_discover_columns", "class": "api", "thread": "any", + "init": "required", "returns": { "kind": "status" }, diff --git a/spec/abi-v2/docs.md b/spec/abi-v2/docs.md index b8584c32..f4a29447 100644 --- a/spec/abi-v2/docs.md +++ b/spec/abi-v2/docs.md @@ -2,15 +2,15 @@ This file holds the prose for `spec/abi-v2/abi.json`, one `###` section per handle, enum, function and (optionally) constant or document. `scripts/abi-v1/gen.py --major 2` copies each section into the comment above its declaration in `include/v2/chtypes.h` and into the generated block of `docs/reference/abi-v2.md`, and `gen.py --major 2 --check` fails when a symbol has no section or a section names no symbol. The `## Rules` section below is copied into that block too. -Prose is deliberately outside the fingerprint: editing this file never changes `CHS_ABI_FINGERPRINT`, so it never invalidates a built library. The rule that follows is that anything a binding or the artifact producer must act on is structured in `abi.json` (a nullability, an ownership, a status a call may return, a thread class, a vocabulary), never only stated here. +Prose is deliberately outside the fingerprint: editing this file never changes `CHS_ABI_FINGERPRINT`, so it never invalidates a built library. The rule that follows is that anything a binding or the artifact producer must act on is structured in `abi.json` (a nullability, an ownership, a status a call may return, a thread class, an init class, a vocabulary), never only stated here. -The WHERE-affecting settings lists for each supported ClickHouse line are in `spec/abi-v2/where-settings/` (with a README); they are documentation, outside the fingerprint. The consumer rule (v2.0 and later) is in that README: refuse a tenant if `filter_declined_settings` is non-empty, or a `result-content` setting is changed. +The WHERE-affecting settings lists for each supported ClickHouse line are in `spec/abi-v2/where-settings/` (with a README); they are documentation, outside the fingerprint. The consumer rule (v2.0 and later) is one check, stated in `document:schema_description` and in that README: refuse a tenant whose `filter_declined_settings` is not empty. The catalog of every reason the library declines or refuses a filter, per supported line, is in `spec/abi-v2/declines/` (with a README); it is documentation, outside the fingerprint. A consumer maps a decline it receives to its entry there; the verdict model is `filter_verdict` below. Each section is copied into a C comment, so it may not contain a comment opener or closer, or two question marks in a row. -**UNSTABLE.** `spec/abi-v2/abi.json` declares `stability` `unstable`: generation 2 is being designed, and its fingerprint moves with every change to the description until the lock. It was seeded as generation 1's surface at generation 2; the additions land one pull request at a time. The first is the server profile: the `chs_server` handle, `chs_server_create` and `chs_server_free`, a server and an options document on `chs_schema_create`, and the `server` and `replicated` members of `schema_description`. The second is the batch document's `at_merge` list, with its `merge_reason` vocabulary, and a top-level `unsupported_settings` (public issue #544). The third is the settings a server profile sets that this build's filters do not honor in a WHERE: `schema_description`'s `server.filter_declined_settings`, with its `declined_tier` vocabulary, `chs_filter_create`'s decline on a schema whose server lists any, and the declined names from the other settings layers at evaluation. The fourth makes that one list over every layer known at schema compile (public issue #588): `schema_description`'s top-level `filter_declined_settings`, each entry with its `declined_layer` (`defaults`, `server` or `schema`), replaces `server.filter_declined_settings`, which is removed. **Changed by the fourth:** a declined setting from the `chs_set_defaults` or the schema layer, which the third named at evaluation, now declines `chs_filter_create`, at create time; a filter's own settings and an evaluation's stay evaluation-time. It also lists `CHS_REJECTED` (306, a deep input) in `chs_schema_describe`'s statuses. What generation 2 adds, and what the lock needs, are in public issue #511. +**UNSTABLE.** `spec/abi-v2/abi.json` declares `stability` `unstable`: generation 2 is being designed, and its fingerprint moves with every change to the description until the lock. It was seeded as generation 1's surface at generation 2; the additions land one pull request at a time. The first is the server profile: the `chs_server` handle, `chs_server_create` and `chs_server_free`, a server and an options document on `chs_schema_create`, and the `server` and `replicated` members of `schema_description`. The second is the batch document's `at_merge` list, with its `merge_reason` vocabulary, and a top-level `unsupported_settings` (public issue #544). The third is the settings a server profile sets that this build's filters do not honor in a WHERE: `schema_description`'s `server.filter_declined_settings`, with its `declined_tier` vocabulary, `chs_filter_create`'s decline on a schema whose server lists any, and the declined names from the other settings layers at evaluation. The fourth makes that one list over every layer known at schema compile (public issue #588): `schema_description`'s top-level `filter_declined_settings`, each entry with its `declined_layer` (`defaults`, `server` or `schema`), replaces `server.filter_declined_settings`, which is removed. **Changed by the fourth:** a declined setting from the `chs_set_defaults` or the schema layer, which the third named at evaluation, now declines `chs_filter_create`, at create time; a filter's own settings and an evaluation's stay evaluation-time. It also lists `CHS_REJECTED` (306, a deep input) in `chs_schema_describe`'s statuses. The fifth is one interface batch: `schema_options`' `posture`, with its `posture` vocabulary and `schema_description`'s `posture` (public issue #477); the per-call options document `insert_options`, taken by `chs_preview_row`, `chs_preview_batch`, `chs_block_create` and `chs_filter_eval_body`, whose member `defer_defaults` (public issue #479) is defined and declined until implemented, with the batch document's `deferred` and `export_columns` and `value_src`'s `default_deferred`; the object form `{"b64": …}` for a settings or query-parameter value whose bytes are not UTF-8 (public issue #424), as the content `json_object_byte_values`, and in `schema_description`'s server `settings`; `value_src`'s `fallback` (rule r3); the batch document's `unsupported_settings`, now required; and each function's init class, `any` or `required`, with a `required` call's refusal before `chs_initialize` (the Preamble's Initialization). The sixth is the second interface batch, from WaveHouse's sign-off review: every member the library writes is described and the load-bearing ones required, each vocabulary member bound to its enum (`vocabulary`); `chs_error_reason`, `CHS_CODE_DECLINED` and the `reason` members, so a decline is machine-keyed by its catalog id; the `refusal_site` vocabulary (`call`, `body_read`, `row`, `body_step`) with `refused_at` and `refused_row`, the members present under a condition stated as MUSTs, each with a conformance case, and the five body calls answering every verdict about a call in its document (their `may_return` loses `CHS_REJECTED` and `CHS_DECLINED`); `input:filter_options` on `chs_filter_create`, with `param_encoding` reserved; the column list `columns` on `chs_filter_eval_body`, as on the other calls that model an INSERT with one; `insert_options`' `insertable_columns`, `column_defaults` and `verdict_basis`, and `defer_defaults` with a mixed body declined (`defer-defaults-mixed-body`); `row_spans` one entry per row, and `export_columns` on every export as its one column-list member; `export_declined_reason`, an export decline's catalog id; `at_merge`'s `part_row`; the row document's `unsupported_settings` removed; `chs_schema_describe` guaranteed for a compiled schema; the `server-refuses` tier; `default_support` and `reads` per column, reserved; and rule r9 (reserve and decline), with rule r8 (crashes: the known-crash list and the breaker). What generation 2 adds, and what the lock needs, are in public issue #511. **Platform requirement.** A caller runs the library on the same operating system and CPU architecture as the ClickHouse server it models: every answer is guaranteed relative to a server on the same platform. On 26.7 and later, every platform agrees. On 26.3, ClickHouse's own parse of Float values from text input formats differs across platforms (macOS, Linux amd64, Linux arm64; mostly 1 ULP), so stored Float values, and comparisons involving them, can differ across architectures (`docs/limitations.md`). @@ -25,12 +25,12 @@ These rules bind generation 2 from its first draft: the library, every binding a **(r3) Every enum has an `unknown(n)` member.** Every enum the description defines, an `int32` enum and a string vocabulary alike (`chs_status`, `chs_format`, `value_src`, `default_kind`, the three outcomes, `filter_verdict`, `transform_reason`, every one), MUST have in every binding an explicit member `unknown(n)` carrying the raw value `n`: the integer of an `int32` enum, the exact string of a vocabulary. A reader MUST map a value its description does not list to that member, for that field alone, and go on decoding. An unlisted value never fails the document, the row or the batch that carries it. Checked by `gen.py`: no described value is spelled `unknown`, `CHS_UNKNOWN` or `CHS__UNKNOWN`, the name readers reserve for that member. - **A `fallback` changes meaning.** It no longer names the value an unlisted one is read as: the reader keeps `unknown(n)`. It names whose per-value facts `unknown(n)` reports (`lossy` for `transform_reason`, `answered` for `filter_verdict`) and how a question about the value is answered, so the fail-closed reading stays: an unknown outcome is never accepted, and an unknown verdict is never answered. -- **Facts without a fallback.** An enum whose values carry facts and which names no `fallback` MUST say what `unknown(n)` reports before the description is locked. Today that is `value_src`, whose `is_stored` has no answer for an unknown source. +- **Facts need a fallback.** An enum whose values carry facts MUST name a `fallback`, so that what `unknown(n)` reports is decided before the description is locked. `value_src` names `skipped`: an unknown source reports `is_stored` false, which is how every dev binding already read it. - **A call status.** A status outside the closed set reads as `unknown(n)` too. The call still fails, as an internal error naming `n` (the `unknown` entry of `errors.status` in `spec/abi-v2/sdk.json`), because a status a binding cannot read is never a success. - **Schemas constrain the writer.** An `enum` inside a document schema (`default_kind` in `schema_description`, `framing.container` in `batch`) constrains what the library writes, never what a reader accepts. - **Why.** Measured on the released 1.0.4 bindings, all four on 12 CI legs ([`v1-abi` run 37506595472](https://github.com/Wave-RF/chtypes/actions/runs/37506595472), the conformance legs of public pull request #509): a reader survives an unlisted value only where its vocabulary has a fallback (an outcome reads as `unsupported`, a verdict as `d`, a transform reason stays raw and reads as lossy), and an unknown call status is an internal error. An unlisted `value_src` fails the WHOLE document with an internal error, so one row's unknown source loses the entire batch; an unlisted `default_kind` fails `schema_description` the same way. -**(r4) Error codes are stable once published.** An error code a caller can match on keeps its name and its number from the first release that carries it, a `2.0.0-dev.N` pre-release included, in every later release of every generation, and is never reused for another meaning. That covers the `chs_status` values, the `CHTYPES_*` codes (`errors.codes` and `errors.fetch_codes` in `sdk.json`, and the fetch layer's table in `docs/guides/fetch-v1.md` §8) and the exit status each code maps to. A new failure gets a new code, with an exit status no published code has. Checked by `gen.py`: every `chs_status` value generation 1 published keeps its name and number, and every `errors.codes` entry of `spec/abi-v1/sdk.json` is kept and none is reused under another key. The fetch layer's table is checked against `spec/fetch-v1/constants.json`, the file §8 is generated from: `errors.fetch_codes` lists every code in it with the same exit status and no code it lacks, every `errors.codes` entry names a listed code, and every code in `spec/abi-v2/published-fetch-codes.json` (what releases have published) is kept under its exit status in both files, and a new code takes a status none of them has. The baseline is a committed file the check reads, never git history. +**(r4) Error codes are stable once published.** An error code a caller can match on keeps its name and its number from the first release that carries it, a `2.0.0-dev.N` pre-release included, in every later release of every generation, and is never reused for another meaning. That covers the `chs_status` values, the `CHTYPES_*` codes (`errors.codes` and `errors.fetch_codes` in `sdk.json`, and the fetch layer's table in `docs/guides/fetch-v1.md` §8) and the exit status each code maps to. A new failure gets a new code, with an exit status no published code has. Checked by `gen.py`: every `chs_status` value generation 1 published keeps its name and number, and every `errors.codes` entry of `spec/abi-v1/sdk.json` is kept and none is reused under another key. The fetch layer's table is checked against `spec/fetch-v1/constants.json`, the file §8 is generated from: `errors.fetch_codes` lists every code in it with the same exit status and no code it lacks, every `errors.codes` entry names a listed code, and every code in `spec/abi-v2/published-fetch-codes.json` (what releases have published) is kept under its exit status in both files, and a new code takes a status none of them has. The baseline is a committed file the check reads, never git history. The same holds for the decline catalog's reason ids (`spec/abi-v2/declines/declines.json`'s `id`, which `chs_error_reason` and every document's `reason` and `verdict_reason` carry) and for the sentinel `CHS_CODE_DECLINED`: an id keeps its meaning from the first release that carries it, a retired entry stays in the catalog under its id with a `retirement` naming its successors, and an id is never reused for another meaning. A new cause gets a new id. `scripts/abi-v2/check-declines.py` is where the check belongs, against the published catalog. **(r5) The cache.** A generation-2 binding's cache MUST be one no released 1.x reader ever reads. The normative fetch specification for generation 2 is [`docs/guides/fetch-v1.md` §"Generation 2 after the lock"](../../docs/guides/fetch-v1.md#generation-2-after-the-lock), and r5 and r6 point at it rather than restate it. @@ -58,11 +58,15 @@ These rules bind generation 2 from its first draft: the library, every binding a Checked by two shared conformance cases (`tests/fixtures/abi-v2/setup-cases.json`, generated by `scripts/abi-v1/emit/setup_cases.py`), which every binding that can express them runs through its public API over the stub library: `server.a_closed_server_is_never_passed_as_null` and `filter.a_closed_filter_is_never_passed_as_null`. Each compiles or previews with an open handle (the stub sees a non-NULL handle), with no handle (the stub sees NULL), and with a closed handle (the binding's usage error, where a NULL passed in its place would have succeeded). The server case reads the stub's live-handle counts; a batch call mints no handle, so the filter case runs on the stub variant `filter-observable`, which answers `rows_passed` 1 when a filter reached it and 0 when it received NULL. Rust declares both cases not expressible, with the reason, in the case itself. `gen.py` checks only that this rule is stated (`GENERATION_RULES`), since which parameters give NULL a meaning is prose, not structure. +**(r8) Crashes.** The library runs ClickHouse's own code on the caller's input, unmodified. A crash inside that code is a defect of security severity, like an over-accept, and is fixed or declined with a catalog entry in a later build; it is never documented as acceptable. Until then this generation does **not** promise that no input can terminate the process. What it promises is the following. (1) **The known-crash list.** Every build's release notes carry the producer's known-crash list: each entry names the line, the entry point, the input shape and the issue, and the producer's release gate counts those entries and fails on any crash not on the list. A list that is not empty is stated in the notes of every build that ships the line. (2) **The breaker.** A compile-class call (`chs_schema_create`, `chs_filter_create`, `chs_server_create`, `chs_set_defaults`, `chs_type_validate`, `chs_reference_type`, `chs_discover_columns`) records its input before it runs and clears the record after; a process that dies during the call leaves the record, and the next process of the same build on the same host declines that exact input with `CHS_DECLINED` and the reason id `vendored-crash-previously`, so a crashing input terminates a process at most once per host where the record can be written. Where it cannot (a read-only cache, no writable directory), the breaker is off and the library says so once at `chs_initialize`; `chs_build_info.capabilities.features` lists `crash_breaker`. An input whose shape is on the build's own known-crash list never reaches the code that crashes: `chs_schema_create` and `chs_filter_create` decline it at create, `CHS_DECLINED` with the reason id `known-crash-declined` and a message naming the list's rule id, the line, the clause and what to change. (3) **No wrong answer.** A breaker decline, like every decline, is never an accept and never a server's own refusal; an input the breaker does not know is answered exactly as without it. (4) **The exceptions** r8 does not cover in any generation: misuse the ABI forbids, exhaustion of the process's memory outside the admission budgets, and signals from outside the process. The full guarantee, no input terminates the process through any `chs_*` call, is generation 3's. `gen.py` checks only that this rule is stated. + +**(r9) The description evolves by reserve and decline.** After the lock a new input member, document member or enum value moves the fingerprint, so a feature that may follow the lock is reserved in the description before it: its input member is defined and a build that does not implement it declines it at any value but its default (Input documents, in the Preamble), and its output members are defined and absent until a build writes them. Implementing a reserved member is a behavior change, never a fingerprint change, and a build advertises each implemented reserved member in `chs_build_info`'s `capabilities.features`, spelled as the member's path (`insert_options.defer_defaults`). A reader treats an absent reserved output member as not reported. `gen.py` checks only that this rule is stated. + At the lock (public issue #511 says when), the description's `stability` becomes `locked`. That moves the fingerprint one last time, so a released 2.0.0 SDK never accepts a dev build and a dev SDK never accepts a 2.0.0 library. SDK 2.0.0 then ships from the production `chtypes/v2` repository, signed with the release key. ## Preamble -This header is generated from `spec/abi-v2/abi.json` by `scripts/abi-v1/gen.py --major 2`. Edit the description and regenerate; never edit this file. `docs/reference/abi-v2.md` is the normative reference, and the rules generation 2 is written under (r1 to r7) are in it. +This header is generated from `spec/abi-v2/abi.json` by `scripts/abi-v1/gen.py --major 2`. Edit the description and regenerate; never edit this file. `docs/reference/abi-v2.md` is the normative reference, and the rules generation 2 is written under (r1 to r9) are in it. Stability. Generation 2 is UNSTABLE: its description is still being designed, so `CHS_ABI_FINGERPRINT` moves with every change until the description is locked. Nothing built from an unstable description is a release. Only pre-release SDKs (`2.0.0-dev.N`) speak it, each pinned to one fingerprint, and they fetch only from the staging dev channel. ABI v1 (`include/chtypes.h`) is frozen and unaffected. @@ -70,15 +74,27 @@ Identity. `CHS_ABI_VERSION` is the generation and `CHS_ABI_FINGERPRINT` is sha25 Ownership. Every input string, statement, document and body is counted bytes, `(const uint8_t *p, size_t n)`: length 0 is none and the pointer may then be NULL, and a NULL pointer with a nonzero length is `CHS_INVALID_ARGUMENT`. Every output is an owned, opaque `chs_buf`, read with `chs_buf_data` and `chs_buf_len` and released with `chs_buf_free`. Bytes pass through unmodified, NUL and invalid UTF-8 included: a column name may contain either (measured on live servers), so no name, statement or message is ever a C string. The only static strings are the handshake's. Handles are opaque, reference-counted and tagged with their kind and the image that made them; every entry point checks both, a child holds a strong reference to what it needs, and each free drops only the caller's reference, so any free order is safe and freeing NULL does nothing. -Errors. A fallible call returns a `chs_status` and, through an optional last `chs_error **err`, an owned error carrying the status, ClickHouse's own code and name, the message and the column. The status alone carries the verdict, so `err` may be NULL. +Errors. A fallible call returns a `chs_status` and, through an optional last `chs_error **err`, an owned error carrying the status, ClickHouse's own code and name, the message, the column and, for a decline or a library-side refusal, the reason: the catalog id (`chs_error_reason`, `spec/abi-v2/declines/declines.json`) a caller keys on instead of matching message text. The status alone carries the verdict, so `err` may be NULL. + +Codes. One code space serves the error and every document's `code` and `verdict_code`: a positive value is ClickHouse's own error code, as the server's table names it; `CHS_CODE_DECLINED` (-2) is this build's decline; and 0 is written only on a success (`accepted`, `accepted_poisoned`, `ok`, a `t` or `f` verdict). A refused or declined row, outcome or verdict never carries 0, and no other negative or library-private number is ever written. Every decline and library-side refusal also names its reason, the catalog id, in the error and in the document (`reason`, `verdict_reason`), and ids and the sentinel are stable once published (rule r4). A message is informative: once a reason is structured no message text is normative, the pinned texts of generation 2's earlier drafts included. + +Conditional members. A document's schema states the members it always carries (`required`). These members are present under a condition the schema cannot see, and each condition is a MUST on the library, held by a conformance case in the SDK's suites (named in brackets) that drives it both ways; a binding MAY check them and, when it does, fails closed on a violation. + +- **The export.** When `chs_preview_batch` is asked for an export (`export_format` is not `CHS_EXPORT_NONE`), exactly one of `export_declined` and `row_spans` is present; with `export_declined`, `export_declined_reason`, its catalog id; and with `row_spans`, `export_columns`; with no export asked for, none of them is [`batch.export_members_xor`]. +- **The filter.** A row's `verdict`, `verdict_code` and `verdict_err`, and the batch's `rows_passed` and `rows_cut`, are present exactly when the call carries a filter (`chs_preview_batch` with a non-NULL `filter`) [`batch.verdict_iff_filter`]. +- **The reason.** `reason` is present exactly where its `code` is `CHS_CODE_DECLINED` (a row, a batch, a filter result, an `errors[]` entry), and `verdict_reason` exactly where its `verdict_code` is [`doc.reason_iff_declined`]. +- **The site.** `refused_at` is present exactly when the outcome of a batch, a filter result or `chs_preview_row`'s row is a refusal or a decline (`rejected` or `unsupported`), and never for `accepted`, `accepted_poisoned`, `skipped` or `ok`; `refused_row` is present exactly when `refused_at` is `row`, and on a batch or a filter result it is less than `rows_read`. The rows of a batch carry neither: the batch's own say where [`doc.refused_at_iff_refused`]. +- **The verdicts.** A filter result's `verdicts` has exactly `rows_read` characters [`filter.verdicts_len_is_rows_read`]. Threads. Every function has a thread class, listed in the reference. A handle is immutable once made, so every call that only reads handles is `shared`: any number of threads may call it at once, on the same handles too. The one thing a caller must never overlap with a call is a free of a handle that call uses. Process setup (`chs_initialize`, then `chs_set_defaults`) runs before the traffic it configures. +Initialization. Every function has an init class, listed in the reference and in its own comment: `any`, valid before and after `chs_initialize`; or `required`, valid only once `chs_initialize` has succeeded. Until then a `required` call changes nothing and returns `CHS_INVALID_ARGUMENT`, ch_code 0, and when its arguments are otherwise valid its message starts `: call chs_initialize first`; it never crashes. The `any` calls are the handshake and the tombstone, `chs_initialize`, `chs_shutdown`, the error and buffer accessors, and every free. Each of them but `chs_initialize` returns no `chs_status`, so it has nothing to refuse with. The error and buffer accessors read the refusal itself (`chs_error_message` returns a `chs_buf`), and a free given NULL does nothing, which is all a server, schema, filter or block free can be given before `chs_initialize`, because nothing makes one. Every other call is `required`, unless its own section says it is valid before `chs_initialize`. + Stacks. A call's answer doesn't depend on the calling thread's stack. Every call that returns a `chs_status` runs with an 8 MiB stack's worth of room: on the calling thread when half its stack, less what it has used, is between 3.75 MiB and 4 MiB (a default 8 MiB thread: glibc's, a Go cgo call, a Python thread, a main thread); otherwise on a library-owned thread with an 8 MiB stack, one per calling thread, made at its first such call and ended with it. The caller blocks until the call returns. The library's threads block every asynchronous signal. After `fork()`, a child's first such call makes its own thread. If the library can't make its thread, the call returns `CHS_INTERNAL`. The frees, the buffer and error accessors, `chs_shutdown` and the handshake always run on the calling thread. Time zones. A library image has one zone of its own, set once by `chs_initialize`: the image zone. A schema has a home zone, its server's `timezone` (`chs_server_create`), or else the image zone. The schema's types bind its home zone, as a server's table binds the server's zone after a restart, and so do the functions its DEFAULT, MATERIALIZED, ALIAS and CHECK expressions apply to its columns, its PARTITION BY expressions, and a Date-valued TTL under a merge. Each call runs in a zone too, the first of these that is set: the call's own `session_timezone` (a key in its settings, which ClickHouse applies through its own query context); the `session_timezone` in the schema's own settings; the one in its server's `settings`; the one set by `chs_set_defaults`; its server's `timezone`; the image zone. An empty `session_timezone` at any of these means the server's zone, the last two. The call's zone governs parsing and rendering a zone-less `DateTime` or `DateTime64`, export, a DEFAULT literal's fold, `toDateTimeOrZero` and a filter's literals. A `Date` partition id is the civil date in every zone. Every zone name is validated by ClickHouse's own `DateLUT`, which reads the library host's time zone database, on every platform. On darwin the file system is case-insensitive, so `DateLUT` there accepts a spelling such as `utc` that a Linux server refuses; that difference is documented, not patched. -Settings. Every settings input is a JSON object whose values are JSON strings, and a binding never rewrites one. A call's settings are layered, highest first: the call's own `settings`; the schema's own (`chs_schema_create`); its server's `settings` (`chs_server_create`); the defaults set by `chs_set_defaults`; the build's own. A filter's WHERE takes the same layers, lowest first: the build's own, `chs_set_defaults`, the server profile, the schema's own, and the filter's own (`chs_filter_create`), with `session_timezone` at every layer staying on the zone path (Time zones, above). One exception: a type gate in the schema's own settings binds the CREATE only, and a WHERE's CAST validates under the session layers, every layer but the schema's own. Each setting that reaches a WHERE is honored (applied to it), passed through (accepted and ignored), or declined (`chs_filter_create` says how). A server resource limit (an `execution`-tier setting, e.g. a small `max_columns_to_read`) can make the server refuse a query the library answers; that is resource policy, not row access, and the library does not model it. A server's settings are the ones its profile applies to every query, and they never change how a schema compiles. No call reads a server's version or its changed settings itself: that discovery is a recorded gap in this generation, so a caller that wants a server's settings passes them explicitly, in a server profile, through `chs_set_defaults` or in each call's settings. +Settings. Every settings input is a JSON object whose values are JSON strings, or the object form for bytes (Input values, below), and a binding never rewrites one. A call's settings are layered, highest first: the call's own `settings`; the schema's own (`chs_schema_create`); its server's `settings` (`chs_server_create`); the defaults set by `chs_set_defaults`; the build's own. A filter's WHERE takes the same layers, lowest first: the build's own, `chs_set_defaults`, the server profile, the schema's own, and the filter's own (`chs_filter_create`), with `session_timezone` at every layer staying on the zone path (Time zones, above). One exception: a type gate in the schema's own settings binds the CREATE only, and a WHERE's CAST validates under the session layers, every layer but the schema's own. Each setting that reaches a WHERE is honored (applied to it), passed through (accepted and ignored), or declined (`chs_filter_create` says how). A server resource limit (an `execution`-tier setting, e.g. a small `max_columns_to_read`) can make the server refuse a query the library answers; that is resource policy, not row access, and the library does not model it. A server's settings are the ones its profile applies to every query, and they never change how a schema compiles. No call reads a server's version or its changed settings itself: that discovery is a recorded gap in this generation, so a caller that wants a server's settings passes them explicitly, in a server profile, through `chs_set_defaults` or in each call's settings. Documents. Every output document is valid JSON that a stock parser reads: every ClickHouse rendering (a stored value, an input value, an engine's row) is a JSON string, a number written bare is within plus or minus 2^53 (anything larger is a string), and there is never a bare `inf`, `-inf` or `nan`. Every column entry of a row carries `null`, the library's verdict that the stored value is NULL, poisoned cells included. @@ -86,12 +102,14 @@ Byte strings in JSON. Every data-derived string a JSON document carries is a byt String values. Every entry that reports a stored value (a row's `cols`, `computed`, an engine row's cells, `storage_transforms`) carries `stored` or `stored_b64`, which is always ClickHouse's rendering of the value. For a scalar `String` or `FixedString` value, including one inside `Nullable` or `LowCardinality`, the entry also carries `value_b64`, the value's raw bytes in standard base64, whenever the value is not NULL, whether or not the bytes are valid UTF-8. A value nested in another type (`Array(String)`, `Map`, `Tuple`, and the like) carries no `value_b64`: that is a recorded gap in this generation, and its rendering is the only form. -Input values. A JSON input (`settings`, `query_params`, an input document) cannot carry a value that is not valid UTF-8 in this generation. The workaround is to inline the value in the SQL as `unhex('')`, built from hex digits only, which keeps it injection-safe because hex digits cannot break out of the quoted literal. A later version would add an object form, `{"": {"b64": ...}}`, never a `_b64` member, which could collide with a real key. +Input values. A value in a settings or query-parameters input (every `settings` and `query_params`, and a server profile's `settings`: the content `json_object_byte_values`) is a JSON string when its bytes are valid UTF-8, and otherwise the object `{"b64": ""}`, which carries the raw bytes: `{"format_csv_delimiter": {"b64": "/w=="}}` sets the one byte 0xFF. It is never a `_b64` member, which could collide with a real key. The two forms are the same value when the bytes are valid UTF-8, so a binding may send either, and it MUST send the object form for any other value. Any other object, and a value of any other JSON type, is `CHS_INVALID_ARGUMENT`, naming the key. A key never takes the object form: setting and query-parameter names are ClickHouse identifiers. Every JSON input is valid UTF-8 JSON (RFC 8259), so a raw byte that is not UTF-8 inside a JSON string is `CHS_INVALID_ARGUMENT`, and the object form is the one way to carry such a value. The other inputs take no object form: an options document's values are closed vocabularies, a server profile's `timezone` and `macros` are names a server's configuration spells, and the input column list carries a name that is not UTF-8 as `name_b64`, by the rule for byte strings in JSON. -Input documents. A counted input whose content is `input:` is a JSON object that the library validates against the document of that name (rule r1). A key the document does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a document that is not a JSON object or a member of the wrong JSON type. Length 0 is `{}`, and a repeated member keeps its last value. The library never ignores a key, so a caller that sets an option a build does not implement is told so instead of silently getting the old behavior. +Input documents. A counted input whose content is `input:` is a JSON object that the library validates against the document of that name (rule r1). A key the document does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a document that is not a JSON object or a member of the wrong JSON type. Length 0 is `{}`, and a repeated member keeps its last value. The library never ignores a key, so a caller that sets an option a build does not implement is told so instead of silently getting the old behavior. The whole document is validated before any member is acted on, so a document with a key it does not define is `CHS_INVALID_ARGUMENT` even when it also sets a member this build declines. A member a document defines but this build does not implement, set to any value but its default, is `CHS_DECLINED` with ch_code 0 and exactly the message `: sets to "", which this build does not implement; declined rather than answered` (for example `chs_preview_batch: insert_options sets verdict_basis to "export", which this build does not implement; declined rather than answered`), reason `option-not-implemented`, which is never a wrong answer. At a call that answers with a document (`chs_preview_row`, `chs_preview_batch`, `chs_filter_eval_body`, `chs_block_create` and `chs_filter_eval_block`), that decline is the document's, not a status: outcome `unsupported`, `code` `CHS_CODE_DECLINED`, `refused_at` `call` and `reason` `option-not-implemented`, with the status `CHS_OK` (`refusal_site`). Teardown. `chs_shutdown` stops what `chs_initialize` started. A loader never unloads a library: unloading or initializing an image again in one process is unsupported. Before `chs_initialize`, every call that returns a `chs_status`, other than `chs_initialize` itself, refuses with `CHS_INVALID_ARGUMENT` and ClickHouse code 0, naming the call, and leaves its out-parameters NULL. After `chs_shutdown` has run, every such call refuses the same way, its message saying that `chs_shutdown` has run. The handshake, `chs_abi_revision`, `chs_shutdown`, the buffer and error accessors and the frees answer at any time. +Images. (Pending the confirmation runs with the time budget at 0, and the remaining arm64 cells at the default budget and at 0.) Any number of library images, one per ClickHouse line, may be loaded and used concurrently in one process, from any number of threads, each initialized once with its own zone, and every answer is the one a single image on a single thread gives. The one exception is the admission time budget (`chs_set_defaults`): it measures the evaluating thread's CPU time, and an evaluation that costs more than about half of the budget on an idle core may be declined (`CHS_CODE_DECLINED`, reason `admission-time-budget`) when the process's load raises its CPU time, never answered differently: above the budget on an idle core it always declines, below about half of it never. Its catalog entry states this load dependence. A conformance case drives two lines, then four, concurrently in one process. + ### chs_buf An owned, immutable byte buffer: every output of the library. Read it with `chs_buf_data` and `chs_buf_len`; the bytes are exactly what the library produced, with no terminator and no encoding promise beyond what the producing parameter's content says. @@ -121,8 +139,8 @@ A body parsed once under a schema, for evaluating many filters without parsing a The verdict of every fallible call, in five values whose numbering is frozen. - `CHS_OK`: the call succeeded and its outputs are set. -- `CHS_REJECTED`: ClickHouse's own refusal, with its own code, name and message, which a server would also give. `CHS_REJECTED` with ch_code 306, TOO_DEEP_RECURSION, never `CHS_INTERNAL`, is returned for a deep argument the call compiles (an expression, a type, or a settings, columns or options JSON), at `chs_set_defaults`, `chs_server_create`, `chs_schema_create`, `chs_schema_describe`, `chs_preview_row`, `chs_preview_batch`, `chs_filter_create`, `chs_filter_eval_body`, `chs_block_create`, `chs_filter_eval_block`, `chs_type_validate`, `chs_reference_type` and `chs_discover_columns`. A body's depth refusal is reported in its document (`CHS_OK`, with code 306), as every refused body is. -- `CHS_DECLINED`: this build will not answer; a server might accept. Never scored as agreement. +- `CHS_REJECTED`: ClickHouse's own refusal, with its own code, name and message, which a server would also give. `CHS_REJECTED` with ch_code 306, TOO_DEEP_RECURSION, never `CHS_INTERNAL`, is returned for a deep argument the call compiles (an expression, a type, or a settings, columns or options JSON), at `chs_set_defaults`, `chs_server_create`, `chs_schema_create`, `chs_filter_create`, `chs_type_validate`, `chs_reference_type` and `chs_discover_columns`. A call that answers with a document (`chs_preview_row`, `chs_preview_batch`, `chs_filter_eval_body`, `chs_block_create` and `chs_filter_eval_block`) returns a status other than `CHS_OK` only for caller misuse (`CHS_INVALID_ARGUMENT`) or a library bug (`CHS_INTERNAL`): every refusal or decline of the call itself, of its settings, its column list or its options, a deep argument (306) included, is reported in its document with `refused_at` `call`, and a body's refusal with `refused_at` `row` or `body_step`, as every refused body is. `chs_schema_describe` never refuses a schema `chs_schema_create` returned: the statement's depth is judged at compile. +- `CHS_DECLINED`: this build will not answer; a server might accept. Never scored as agreement. The error names the reason (`chs_error_reason`). A call that answers with a document never returns it: its decline is the document's `unsupported` with `refused_at` `call`. - `CHS_INVALID_ARGUMENT`: caller misuse, such as a NULL pointer with a nonzero length, a wrong-kind, freed or cross-library handle, or a NULL required out-parameter. - `CHS_INTERNAL`: a guarded exception inside the library, which is a library bug. @@ -144,24 +162,40 @@ What a merge would do to one row of the part an `INSERT` writes, as the batch do A column's default kind, as ClickHouse names it: `DEFAULT`, `MATERIALIZED`, `ALIAS` or `EPHEMERAL`, or the empty string for a column with none. The schema description carries one per column. +### default_support + +What this build does with a column's DEFAULT, as `schema_description` reports it per column (reserved: a build that does not report it writes no `default_support`): `computed`, the library evaluates the expression itself (a literal, a deterministic expression of the row, an admitted clock read); `generated`, the library draws the value with an admitted generator (`default_generated`); `deferred`, the library cannot compute it and `insert_options`' `defer_defaults` `server` would defer it; `declined`, the library cannot compute it, so a body that omits the column is declined, with the catalog id in `default_support_reason`. The honor set behind it is published per line as data beside `where-settings/.json`. A value added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + ### declined_tier -The tier of a setting a settings layer sets and this build's filters do not honor in a WHERE, as `schema_description`'s `filter_declined_settings` reports it, spelled as this build line's WHERE-settings list (`spec/abi-v2/where-settings/.json`) spells it: `predicate` (measured to change a verdict), `predicate-unflipped` (read in the WHERE's scope, though no value the flip test tried changed a verdict) or `result-content` (selects rows by content). The lists' other tiers (`execution`, `result-truncate` and `output`) never appear here: those settings are passed through and ignored. A tier added later is a new value, which a reader reads as its `unknown(n)` (rule r3). +The tier of a setting a settings layer sets and this build's filters do not honor in a WHERE, as `schema_description`'s `filter_declined_settings` reports it, spelled as this build line's WHERE-settings list (`spec/abi-v2/where-settings/.json`) spells it: `predicate` (measured to change a verdict), `predicate-unflipped` (read in the WHERE's scope, though no value the flip test tried changed a verdict) `result-content` (selects rows by content), or `server-refuses` (a cell of `where-settings/documented-pathological.tsv`: a value under which the server refuses every query, so no row verdict can stand). The lists' other tiers (`execution`, `result-truncate` and `output`) never appear here: those settings are passed through and ignored. A tier added later is a new value, which a reader reads as its `unknown(n)` (rule r3). ### declined_layer The settings layer a `schema_description` `filter_declined_settings` entry comes from, the highest one that sets the name: `defaults` (the defaults `chs_set_defaults` set, as `chs_schema_create` captured them), `server` (the server profile's `settings`, `chs_server_create`) or `schema` (`chs_schema_create`'s own `settings`). The values are in layer order, lowest first, which is the list's order. A layer added later is a new value, which a reader reads as its `unknown(n)` (rule r3). +### posture + +How `chs_schema_create` compiles its statement, as `schema_options`' `posture` asks and `schema_description` reports: `create`, as a server runs a `CREATE TABLE` (ClickHouse's `LoadingStrictnessLevel::CREATE`), the default; `load`, as a server loads the stored definition of a table it already holds when it starts (`LoadingStrictnessLevel::FORCE_ATTACH`). `schema_description` carries the member only for `load`: absent is `create`. A posture added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + +### refusal_site + +Where a refusal or decline sits, as `refused_at` reports it on a batch document, a filter result and `chs_preview_row`'s row whenever the outcome is `rejected` or `unsupported` (Conditional members, in the Preamble): `call`, the call itself (its settings, its column list, its options, or a decline of the call such as a zone mismatch), before a byte of the body was read; `body_read`, the body outside any row and before the whole-body steps, so no record is to blame: a header the format reads first (a `WithNames` or `WithNamesAndTypes` header line, a header field naming a column the INSERT does not list under `input_format_skip_unknown_fields` 0), the framing (a JSON array's brackets, a separator or trailing content after the last record) or a byte-order mark the reader refuses; `row`, one row of the body, whose index is `refused_row`; `body_step`, a whole-body step over rows the parse accepted (the partition split, the writer's index expressions, the engine merge, the TTL, the projections, a CHECK over a merged block). `refused_row` is present exactly for `row`. A site added later is a new value, which a reader reads as its `unknown(n)` (rule r3). + ### discover_query_param The query parameters in the SQL `chs_discover_query` returns, each with its ClickHouse type. The caller binds them when it runs the query (over HTTP, as `param_database` and `param_table`), so no binding builds SQL. ### value_src -Where a column's value came from. `is_stored` says whether the row as stored carries a value for the column: a column the input named but the table never stores (an EPHEMERAL column, a skipped MATERIALIZED or ALIAS column) and a DEFAULT the library could not resolve carry none. +Where a column's value came from. `is_stored` says whether the row as stored carries a value for the column: a column the input named but the table never stores (an EPHEMERAL column, a skipped MATERIALIZED or ALIAS column) and a DEFAULT the library could not resolve carry none. A source this description does not list reports its fallback's facts, `skipped`'s: `is_stored` false (rule r3). `default_generated` marks a DEFAULT column whose expression calls a random or ID generator the build admits; a build that does this lists `default_generators` in `chs_build_info`'s `capabilities.features`. The label means: generated by chtypes, and it becomes the stored value when you insert this output. The library drew the value itself, with ClickHouse's own vendored function. So insert the library's output (the export of `chs_preview_batch`, or the stored values its documents report), not your original input: a server evaluates a DEFAULT only for a column the INSERT does not supply, and the original input, which omits the column, would make the server draw a different value. Generators are admitted only in DEFAULT expressions: a MATERIALIZED column is always computed by the server, and an EPHEMERAL column is not stored. No binding logic is involved; a binding reports the source as it reads it. +`default_override` marks a column whose value came from `insert_options`' `column_defaults`: the caller's value in a cell the server would have filled from the column's DEFAULT. It is stored, and the export carries it as a present value. A build that does not implement `column_defaults` never writes it. + +`default_deferred` marks a DEFAULT column the call deferred to the server (`insert_options`' `defer_defaults`): the library computed no value, the export omits the column from every line and from the batch document's `export_columns` (a body whose rows do not all defer it is declined: `defer_defaults`), and the server fills it on INSERT. A build that does not implement `defer_defaults` never writes it. + ### row_outcome The verdict on one row: accepted, accepted but unreadable afterwards (the insert succeeds and every later read fails), rejected, skipped under the server's error allowance with the batch continuing, or declined by this build. @@ -180,13 +214,13 @@ The batch answers the first of these that applies: A skipped row never changes the verdict. Steps 2 and 3 depart from "rejected if any row is refused" for one reason: never a refusal that is not definite. Both answer `unsupported`, which a caller never scores as agreement. -A filter's verdicts follow the batch. When the batch's outcome is not `accepted`, by any of steps 1 to 4, every row's filter verdict is `d`. A row the parse accepted carries `verdict_code` -2 and `verdict_err` `batch outcome is '': only a fully accepted batch has filter verdicts`; a refused row carries its own code and message. Nothing from such a batch is stored for a WHERE to read, exported or counted, so no verdict of it is an answer. +A filter's verdicts follow the batch. When the batch's outcome is not `accepted`, by any of steps 1 to 4, every row's filter verdict is `d`. A row the parse accepted carries `verdict_code` -2, its `verdict_reason`, and `verdict_err` `batch outcome is '': only a fully accepted batch has filter verdicts`; a refused row carries its own code and message. Nothing from such a batch is stored for a WHERE to read, exported or counted, so no verdict of it is an answer. ### filter_outcome -Whether a filter evaluation answers its body. `ok`: every row carries its own verdict, and per-row failures are verdicts, not outcomes. Any non-`ok` outcome, including `unsupported`, means: treat every verdict as `d`. A caller MUST do so whatever the verdict string holds. +Whether a filter evaluation answers its body. `ok`: every row carries its own verdict, and per-row failures are verdicts, not outcomes. Under any non-`ok` outcome, including `unsupported`, the library writes `d` for every verdict, on `chs_filter_eval_body` and `chs_filter_eval_block` alike, so the rule holds in the data; a caller MUST still treat every verdict as `d` whatever the verdict string holds. The document's `refused_at`, `refused_row` and `reason` say where the refusal sits and why. -`chs_filter_eval_body` and `chs_filter_eval_block` answer `ok` only for a body a server would accept, that is, when `chs_preview_batch`'s verdict over the same body and settings, with no filter, is `accepted`. Otherwise the outcome mirrors that verdict: `rejected` with its code and message; `unsupported` with its code and message, a call-level decline included; and `unsupported` with code -2 for `accepted_poisoned`, which this vocabulary does not hold and which is no refusal of the INSERT. A body the parse itself refuses answers `rejected` or `unsupported` with no verdicts. +`chs_filter_eval_body` and `chs_filter_eval_block` answer `ok` only for a body a server would accept, that is, when `chs_preview_batch`'s verdict over the same body, settings and column list, with no filter, is `accepted`. Otherwise the outcome mirrors that verdict: `rejected` with its code and message; `unsupported` with its code and message, a call-level decline included; and `unsupported` with code -2 for `accepted_poisoned`, which this vocabulary does not hold and which is no refusal of the INSERT. A body the parse itself refuses answers `rejected` or `unsupported`, with a `d` for each record read (`rows_read`, 0 when nothing was read). ### filter_verdict @@ -222,6 +256,10 @@ What `chs_abi_revision` always returns: 1001, outside every revision a released The `export_format` of a batch preview that exports nothing. +### CHS_CODE_DECLINED + +The code a declined row, outcome or verdict carries (Codes, in the Preamble): -2, outside ClickHouse's code table. It is a sentinel, stable under rule r4, and never a ClickHouse code. + ### CHS_DOC_VALUES A `doc_flags` bit: the per-row documents carry each column's stored value and provenance. @@ -248,11 +286,13 @@ ClickHouse's own error-code table for this build: a JSON array of `{"code": int, ### document:schema_description -A schema's columns, in declared order, with each column's canonical type, its `default_kind` (a value of the `default_kind` vocabulary) and default expression, and the facts a caller needs to build a row. The document is `{"columns": [...]}`, and each entry carries `name`, `type`, `default_kind` (a value of the `default_kind` vocabulary), `default_expression` (empty when there is none) and `default_is_literal`; the name, the type and the expression follow the rule for byte strings in JSON. +A schema's columns, in declared order, with each column's canonical type, its `default_kind` (a value of the `default_kind` vocabulary) and default expression, and the facts a caller needs to build a row. The document is `{"columns": [...]}`, and each entry carries `name`, `type` and `default_kind` (a value of the `default_kind` vocabulary), always, and `default_expression` (empty when there is none) and `default_is_literal`; the name, the type and the expression follow the rule for byte strings in JSON. `type` is present on every column: a column whose type the compile cannot settle (a column declared without a type whose own DEFAULT or ALIAS this build cannot type) is declined by `chs_schema_create` (`CHS_DECLINED`, reason `column-type-unsettled`), so no schema describes a column without its type. The decline is a limit of this build, never a claim about whether a server accepts the table. `default_support`, a `default_support` value with `default_support_reason` for `declined`, says what this build does with the column's DEFAULT; reserved, absent until a build writes it. `reads`, a list of name objects, names the columns the column's DEFAULT, MATERIALIZED, ALIAS or EPHEMERAL expression reads, as ClickHouse's own analysis of the expression's required source columns reports them, each once, in the table's declaration order: a build that writes it writes it on every column, `[]` for a column with no expression or one that reads none. Reserved (rule r9): absent until a build writes it, which that build advertises in `chs_build_info`'s `capabilities.features`; a reader treats it as not reported until then. A schema `chs_schema_create` returned always describes: the walk describing needs runs at compile, so a compiled schema is never refused here. + +`posture`, a `posture` value, is present only on a schema compiled in a posture other than `create`, so the document of a schema compiled in `create` is byte for byte what it was before postures existed. Two members report the server a schema was created on. Both are absent on a schema created with `server` NULL, so that document is byte for byte what it was before servers existed. -- `server`, on every schema created on a server, even one described by `{}`: `timezone` is the zone the schema's types bind (the profile's, or else the image zone, exactly as `chs_initialize` spelled it), so a reader never needs the image zone separately; `settings` is the server's settings as the profile gave them, `{}` when it gave none; `macros` is the server's macro set as the library holds it, present exactly when the profile carried `macros`. These are the caller's own JSON strings given back, not data-derived, so they are plain strings. The member no longer carries `filter_declined_settings`: that list is the top-level one, below, over every layer. +- `server`, on every schema created on a server, even one described by `{}`: `timezone` is the zone the schema's types bind (the profile's, or else the image zone, exactly as `chs_initialize` spelled it), so a reader never needs the image zone separately; `settings` is the server's settings as the profile gave them, `{}` when it gave none; `macros` is the server's macro set as the library holds it, present exactly when the profile carried `macros`. These are the caller's own values given back, not data-derived, so the rule for byte strings in JSON does not apply to them: the zone and the macros are plain strings, and each settings value is a JSON string when its bytes are valid UTF-8 and otherwise the object form `{"b64": …}` (Input values), so a settings value takes the object form only when its bytes are not UTF-8. The member no longer carries `filter_declined_settings`: that list is the top-level one, below, over every layer. - `replicated`, on a schema created on a server whose engine is a Replicated one: `zookeeper_path` and `replica_name`, the path and the replica name ClickHouse's own `TableZnodeInfo` resolved, fully expanded. The database and table names expanded into them are DDL bytes, so both follow the rule for byte strings in JSON. A caller can compare them with the server's own `system.replicas`. `filter_declined_settings` is required on every schema description, with or without a server: an array, `[]` when nothing is declined. It lists the settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known when `chs_schema_create` ran, and `chs_schema_create` computes it. The layers, lowest first, are the `declined_layer` values: @@ -263,28 +303,38 @@ Two members report the server a schema was created on. Both are absent on a sche Each entry is `{name, tier, layer}`: `name` follows the rule for byte strings in JSON (`name` or `name_b64`), `tier` is a `declined_tier` value, the tier this build line's WHERE-settings list gives the name, and `layer` is a `declined_layer` value. The entries are: -- every name set at any of the three layers that the line's list carries at tier `predicate`, `predicate-unflipped` or `result-content` and that is not in this build's honor set (the settings it applies to a WHERE, each with a measured case where the server's verdict moves and the library's tracks it); -- `compatibility`, at tier `predicate`, when its value at the highest layer that sets it moves a listed setting outside the honor set. +- every name set at any of the three layers that the line's list carries at tier `predicate` or `predicate-unflipped` and that is not in this build's honor set (the settings it applies to a WHERE, each with a measured case where the server's verdict moves and the library's tracks it); +- `compatibility`, at tier `predicate`, when its value at the highest layer that sets it moves a listed setting outside the honor set; +- every `result-content` name set at any of the three layers, honored or not: a setting that selects rows by content beyond the WHERE is outside a row verdict's scope whatever the library applies; +- every cell of the line's `where-settings/documented-pathological.tsv` present at any of the three layers, at tier `server-refuses`: the server refuses every query under that value, so no row verdict stands. + +So the consumer rule is one check: refuse when the list is not empty. No consumer copies the WHERE-settings lists or the pathological cells. A name the list does not carry is never an entry, and neither is a setting of the `execution`, `result-truncate` or `output` tiers: those are passed through and ignored. The test is presence, not value: any value at any layer counts, a declined name set to its default value included, because some of ClickHouse's reads are `isChanged()`, which asks only whether a setting was set. A higher layer that sets the same name never takes the entry away; it only moves the entry's `layer`. The order is deterministic. There is one entry per name, at the highest layer that sets it (`defaults`, then `server`, then `schema`). The entries are ordered by layer, `defaults` first, and within a layer by name as raw bytes, ascending; a `name_b64` entry sorts by its decoded bytes. -The list holds only the layers known at schema compile. A filter's own `settings` (`chs_filter_create`) are never in it: a declined one there is named at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`). The settings an evaluation brings (the call's, and the body's) are never in it either: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings (`chs_filter_create`). A schema whose list is not empty is one `chs_filter_create` declines. +The list holds only the layers known at schema compile. A filter's own `settings` (`chs_filter_create`) are never in it: a declined one there is named at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`), and a cell of `documented-pathological.tsv` there declines `chs_filter_create` itself (`setting-server-refuses`). The settings an evaluation brings (the call's, and the body's) are never in it either: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings (`chs_filter_create`). A schema whose list is not empty is one `chs_filter_create` declines. ### document:row -One row's verdict and, as the flags ask, its columns' stored values, provenance and transformations. Every entry of `cols` carries its name, `null` (whether the stored value is NULL, poisoned cells included), its renderings (`input`, `stored`, `ref`, `wire`) and its types (`type`, `base`, `ref_type`), all by the rule for byte strings in JSON, and `value_b64` for a String or FixedString value. `unknown_fields` and `unsupported_settings` are lists of name objects; `computed` entries carry a name, a kind and a stored value; `err`, `verdict_err` and `partition_id` follow the rule too. The transformations come from the library, never from a binding: each is the `transformed` entry the SDK's parity fixtures pin, with its `reason` from `transform_reason`. `input_span` is `{off, len}`, the bytes of the input body the reader consumed for this record, read from the vendored reader's own position. +One row's verdict and, as the flags ask, its columns' stored values, provenance and transformations. `outcome` (a `row_outcome` value), `code` (Codes, in the Preamble) and `input_span` are always present, `err` names the refusal, and `reason` the catalog id of a decline. With a filter attached (`chs_preview_batch`), `verdict` (a `filter_verdict` value), `verdict_code`, `verdict_err` and, for `d`, `verdict_reason` are present on every row; with none they are absent. On `chs_preview_row`'s document, which stands for the whole call, `refused_at` and `refused_row` say where a refusal sits (`refusal_site`); the rows of a batch carry neither. Every entry of `cols` carries its name, `null` (whether the stored value is NULL, poisoned cells included), `src` (a `value_src` value), its renderings (`input`, `stored`, `ref`, `wire`) and its types (`type`, `base`, `ref_type`), all by the rule for byte strings in JSON, `value_b64` for a String or FixedString value, and the booleans `nullable` (the column's type is Nullable), `poison` (the stored value is one the server cannot read back), `null_input` (the input field was the format's NULL literal) and `dup_dropped` (a duplicate key the reader dropped), with `ref_unclassified` present, true, when the reference type could not be classified. `unknown_fields` is a list of name objects (the call's declined settings are the batch document's `unsupported_settings` only, which every row of the batch shares); `computed` entries carry a name, a `kind` (the column's default kind, lower case) and a stored value; `err`, `verdict_err` and `partition_id` follow the rule too. The transformations come from the library, never from a binding: each is the `transformed` entry the SDK's parity fixtures pin, with its `reason` from `transform_reason` and its `lossy` fact. `input_span` is `{off, len}`, the bytes of the input body the reader consumed for this record, read from the vendored reader's own position. A `rejected` row always carries a nonzero ClickHouse code, and an `unsupported` one `CHS_CODE_DECLINED` with its `reason`. ### document:batch -A body's verdict, counts and per-row documents (each a row document, with its own `input_span`), and, when an export was asked for, where each accepted row sits in the export bytes. `engine_rows`, present when the table's engine merges rows at insert, is a list of rows, each a list of cells: `{name, stored, null}`, plus `value_b64` for a String or FixedString value. `storage_transforms` entries carry `row`, `column`, `reason` and `stored`, plus `value_b64`; `transformed` entries carry `row`, `column`, `input`, `stored`, `reason` and `lossy`. Every name, rendering, `err` and `export_declined` follows the rule for byte strings in JSON. +A body's verdict, counts and per-row documents, and, when an export ran, where each row sits in the export bytes. Always present: `outcome` (a `batch_outcome` value), `code` (Codes, in the Preamble), `err`, `rows` (each a row document, with its own `input_span`), `rows_read` (the records the reader read, skipped ones included), `rows_skipped` (the records skipped under the server's error allowance), `unsupported_settings`, `unconsumed` and `framing`. Present with a filter attached: `rows_passed` and `rows_cut`, which partition the accepted rows of an `accepted` batch by verdict (`t` against the rest) and read 0 and 0 on any other outcome. Present on a schema with a partition key: `partition_count`, the distinct partitions the stored rows span, which with each row's `partition_id` lets a caller cap the partitions one INSERT touches (`max_partitions_per_insert_block`). Present whenever the outcome is not `accepted` or `accepted_poisoned`: `refused_at` (a `refusal_site` value), `refused_row` for `row`, and `reason` for a decline; `rows_read` is 0 when nothing was read. With an export asked for, exactly one of `row_spans` and `export_declined` is present: `row_spans` when the export ran, one entry per `rows[]` entry, index-aligned, `{off, len}` into `out_export` for an exported row (its whole output line, `\n` included, so slicing the spans out of the bytes is the per-row payload and concatenating them reproduces it) and `null` for a row not exported; `export_declined`, naming the cause, with `export_declined_reason`, the cause's catalog id, when it did not (Conditional members, in the Preamble). When the export ran, `export_columns` is present too: the positional column list of every line in `out_export`, as name objects: the stored non-computed columns in declaration order, independent of the call's `columns` list, a column outside `insertable_columns` included, with the value the library drew, resolved or filled for it, less the columns every row defers (`defer_defaults`). A caller inserts the export naming `export_columns` as the column list; it never derives a list of its own. `export_columns` is the export's only column-list member, so no second list can disagree with it. -`unsupported_settings`, at the top level, is a list of name objects with the same shape as a row document's: the call's own settings that the library declined. It is empty when none was, and when it is not empty the batch's outcome is at least `unsupported` (see `batch_outcome`). +The export's truth table, by outcome, export asked for and filter verdict: with no export asked for, `out_export` is NULL and neither `row_spans` nor `export_declined` is present. With an export asked for and the outcome `accepted`, the export runs: `out_export` is the bytes (an empty buffer when no row is exported) and `row_spans` is present, with a span for every row whose outcome is `accepted` and, with a filter attached, whose verdict is `t`, and `null` for every other row (`skipped`, and a `f`, `e` or `d` row under a filter); the rows are exported in body order. With an export asked for and any other outcome (`accepted_poisoned`, `rejected`, `unsupported`), nothing is exported: `out_export` is NULL and `export_declined` names the cause (the outcome, a poisoned value), with `export_declined_reason` `export-batch-not-accepted`. With an export asked for and the outcome `accepted` but the serialization failing, or the export's format unable to carry a stored value, `out_export` is NULL, `export_declined` names the cause and `export_declined_reason` its catalog id; the batch's `outcome` stays `accepted`, with code 0 and no `reason`, so an export decline is keyed by `export_declined_reason`, never by the batch's `reason`. So a caller tells a declined export from an empty one by `export_declined`, never by the buffer, and never slices the payload by an index other than the row's own. -`at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part this `INSERT` writes would do, at the call's clock instant. It is absent when nothing would happen. `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. Each entry is `{row, reason, column, stored, input_rows}`: +A computed value the export cannot carry. A column the export omits is recomputed by the server that inserts the export. A DEFAULT that reads an EPHEMERAL value is stored, so the export carries its value and the stored row is the same. A MATERIALIZED expression that reads an EPHEMERAL value the row supplied is not: the export carries no EPHEMERAL column, so the inserting server computes the MATERIALIZED value from the EPHEMERAL's type default instead, and the stored row differs from the one the document reports. Such a body's export is declined (`export_declined`, with `export_declined_reason` `export-computed-value-not-carried`): a caller that needs that stored row inserts the body. -- `row` indexes `engine_rows`: the part's row, not the input body's. An engine's insert-time merge (Collapsing, Replacing, Summing, Aggregating) runs first, inside the `INSERT` writer, so `engine_rows` already excludes the rows it merged away, and after a Summing or Aggregating merge the part's rows do not map one to one to the input's. +Composition. A caller that concatenates accepted bodies into one INSERT (a worker's batch, async inserts) keeps every per-row verdict: a row's `outcome`, its stored values, its `partition_id` and its filter verdict are functions of the row and the schema alone. The whole-body steps are not compositional, and can refuse the concatenation after each body was accepted: the partition split (`max_partitions_per_insert_block`, code 252, bounded by the sum of the bodies' `partition_count`), an insert-time engine merge across bodies, `rowNumberInAllBlocks()` and the like, and a CHECK over a merged async block, which fails every INSERT in it. A caller that must not see those caps its INSERT by `partition_count` and keeps CHECK-bearing bodies isolated. `engine_rows`, present when the table's engine merges rows at insert, is a list of rows, each a list of cells: `{name, stored, null}`, plus `value_b64` for a String or FixedString value. `storage_transforms` entries carry `row`, `column`, `reason` and `stored`, plus `value_b64`; `transformed` entries carry `row`, `column`, `input`, `stored`, `reason` and `lossy`. Every name, rendering, `err` and `export_declined` follows the rule for byte strings in JSON. + +`unsupported_settings`, at the top level, is always present: a list of name objects, the call's own settings that the library declined, which every row of the batch shares. It is empty when none was, and when it is not empty the batch's outcome is at least `unsupported` (see `batch_outcome`). + +`at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part this `INSERT` writes would do, at the call's clock instant. It is absent when nothing would happen. `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. Each entry is `{part_row, reason, column, stored, input_rows}`: + +- `part_row` indexes `engine_rows`: the part's row, not the input body's. Every other `row` member of this document (`transformed[].row`, `storage_transforms[].row`, `refused_row`) and of `filter_result` (`errors[].row`) indexes `rows[]`, the input body's rows, and `input_rows` here does too. An engine's insert-time merge (Collapsing, Replacing, Summing, Aggregating) runs first, inside the `INSERT` writer, so `engine_rows` already excludes the rows it merged away, and after a Summing or Aggregating merge the part's rows do not map one to one to the input's. - `reason` is a `merge_reason` value: `ttl_delete` (the rows TTL removes the row) or `ttl_column_reset` (a column TTL resets the column's value). A row that is removed and also carries column resets lists only `ttl_delete`: it is removed in the same merge pass, so its column values are never observable. - `column` names the column, by the rule for byte strings in JSON, and is present only for `ttl_column_reset`. - `stored` is the column's value after the reset, by the same rule. It is present when that value is decided at the `INSERT`: the column's `DEFAULT` is a constant or a deterministic expression of the row, or the column has no `DEFAULT` (the type's default). It is absent when the `DEFAULT` reads the clock (`now()`) or a generator (`rand()`, `generateUUIDv4()`), and the entry then means that the value is reset at the merge and decided then. @@ -299,9 +349,9 @@ Two further fields come from the vendored reader's own state, never from a token ### document:filter_result -A filter evaluation: the call's outcome, one verdict character per row (`filter_verdict`), and each error or declined row itemized as `{row, code, err}`. `err`, at the top level and per row, follows the rule for byte strings in JSON, and `unsupported_settings` is a list of name objects. +A filter evaluation. Always present: `outcome` (a `filter_outcome` value), `code` (Codes, in the Preamble), `err`, `rows_read`, `verdicts` (one `filter_verdict` character per row read, so its length is `rows_read`, and every character `d` under any outcome but `ok`), `unsupported_settings` (a list of name objects) and `errors`, each error or declined row itemized as `{row, code, reason, err}` with `row` indexing the body's rows, `code` nonzero (ClickHouse's for `e`, `CHS_CODE_DECLINED` for `d`) and `reason` the catalog id of a `d`. Present whenever the outcome is not `ok`: `refused_at` (a `refusal_site` value), `refused_row` for `row`, and `reason` for a decline. `err`, at the top level and per row, follows the rule for byte strings in JSON. -`unsupported_settings` (after `rows_read`, before `verdicts`) names the settings the parse itself declined, and then, after them and without repeats, every setting of the filter's own `settings` that this build's filters do not honor in a WHERE. When it names one of those, every verdict is `d` and every row is itemized in `errors` as `{row, code, err}` with `code` -2. `outcome` follows its own rules all the same: `ok` for a body a server would accept, and the mirror of `chs_preview_batch`'s verdict otherwise (`filter_outcome`). A declined setting from the `chs_set_defaults`, server or schema layer never reaches an evaluation, because `chs_filter_create` declines first (`schema_description`'s `filter_declined_settings`). +`unsupported_settings` (after `rows_read`, before `verdicts`) names the settings the parse itself declined, and then, after them and without repeats, every setting of the filter's own `settings` that this build's filters do not honor in a WHERE. When it names one of those, every verdict is `d` and every row is itemized in `errors` as `{row, code, reason, err}` with `code` -2. `outcome` follows its own rules all the same: `ok` for a body a server would accept, and the mirror of `chs_preview_batch`'s verdict otherwise (`filter_outcome`). A declined setting from the `chs_set_defaults`, server or schema layer never reaches an evaluation, because `chs_filter_create` declines first (`schema_description`'s `filter_declined_settings`). ### document:discovery @@ -312,7 +362,7 @@ The column declarations reconstructed from a server's `system.columns` rows, for A server's profile, `{"timezone": "", "settings": {"": ""}, "macros": {"": ""}}`. Every member is optional, so `{}` describes a server with nothing known about it. The profile holds server facts only, in the shape a server's own tables report them, so a caller can fill it from the server it describes; library knobs go in `chs_server_create`'s options instead. - `timezone`: the server's own zone, the zone its tables bind. An empty string, or no member, means the image zone. The name is validated by ClickHouse's own `DateLUT`, which reads the library host's time zone database. -- `settings`: the settings the server's profile applies to every query, each value a JSON string, never rewritten. The library accepts exactly what the server's own SET check accepts. `session_timezone` here is the default session zone of the server's user. +- `settings`: the settings the server's profile applies to every query, each value a JSON string or the object form for bytes (Input values), never rewritten. A `chtypes_*` knob is never one of them (`chs_set_defaults`). The library accepts exactly what the server's own SET check accepts. `session_timezone` here is the default session zone of the server's user. - `macros`: the server's macros, as its `system.macros` lists them (`macro`, `substitution`). Absent, they are unknown; present, even as `{}`, they are the complete set (see `chs_server_create`). A macro name that ClickHouse's own macro reader refuses (for example `a.b`) is refused with `CHS_INVALID_ARGUMENT` naming it. ### input:server_options @@ -321,7 +371,29 @@ The options of `chs_server_create` (rule r1). It defines no member yet, so `{}` ### input:schema_options -The options of `chs_schema_create` (rule r1). It defines no member yet, so `{}` (or length 0) is the only valid document, and any key is `CHS_INVALID_ARGUMENT`, naming the key. A choice about how one statement is compiled lands here, as a new member: for example, compiling a server's own `create_table_query`, read back from the server, rather than a caller's CREATE. +The options of `chs_schema_create` (rule r1): how one statement is compiled. Its member is `posture`, a `posture` value, and `{}` (or length 0) compiles in `create`, as before the member existed. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a `posture` that is not one of its values, naming the value. + +- `create`: the statement is a `CREATE TABLE` a server would run now, with every check a server's CREATE runs, under the settings layers (`chs_schema_create`). +- `load`: the statement is the stored definition of a table the server already holds, as that server reports it (`system.tables.create_table_query`, `SHOW CREATE TABLE`, or the table's metadata file), compiled as the server loads it when it starts. It is one switch, and ClickHouse's own: every check on the compile's path that reads the loading posture reads `FORCE_ATTACH` instead of `CREATE`, in ClickHouse's own code, and nothing else changes. So the checks a server applies only to a new table are not applied: the type gates (`allow_suspicious_low_cardinality_types`, `allow_suspicious_fixed_string_types` and the rest, at the compile and at every call on the schema), the codec, statistics and suspicious-index checks, the DEFAULT validation of a typed column, the TTL checks, the MergeTree settings' constraint and sanity checks, and the floating-point partition-key, nullable-key and deprecated-syntax gates. `flatten_nested` and `data_type_default_nullable` are not applied either: a stored definition is already flattened and typed. Which checks these are on each supported line is the artifact producer's enumeration from the vendored source; no list is kept here. A projection that a starting server skips because it no longer parses is declined, because whether a given server still holds it depends on that server's history. A Replicated engine's ZooKeeper path and replica name never change a row verdict, so under `load` a stored definition whose engine arguments read a macro the server's `macros` cannot expand (or carry none, as a Replicated database's table does) compiles as the server loads it: `schema_description` then carries no `replicated` member, and only an answer that depends on the path (none today) is declined. + +**The caller's obligation.** `load` asserts that the server holds a table defined by exactly this statement. The library cannot check that assertion, because it never connects to the server. A statement a server would refuse at `CREATE` compiles under `load`, and every answer on the schema is the answer for INSERTs into that stored table. If no server holds such a table, every answer describes a table that does not exist, and an accepted row is evidence of nothing. A statement a server reported for a table it holds (its `create_table_query`, `SHOW CREATE TABLE`, or its metadata file) MUST be compiled in `load`, and a statement a caller wrote or edited in `create`. A server's `create_table_query` compiled in `create` can be wrong in both directions, because the server created the table under its own CREATE's settings, which the statement does not carry: an over-reject where a gate was set (a `LowCardinality(UInt8)` column) and an over-accept where a shape setting was (a `Nested` column of a table created with `flatten_nested = 0`). + +### input:insert_options + +The options of every call that models one INSERT of a body: `chs_preview_row`, `chs_preview_batch`, `chs_block_create` and `chs_filter_eval_body` (rule r1). Its members: `insertable_columns`, `column_defaults` and `verdict_basis`, `defer_defaults`. + +- `insertable_columns`: the columns an INSERT may supply, as name objects: the server's column-level INSERT grant (`GRANT INSERT(a, b) ON t`), checked as the server checks it, on the INSERT's header and never on the fields a body supplies. The header is the call's `columns` list, or with none the server's own no-list header: every ordinary and DEFAULT column, in declaration order (so a positional CSV, TSV, Values or JSONCompactEachRow body with no list supplies all of them). The check runs after the list's own refusals (an unknown or ALIAS column, 16; a duplicate, 15; a MATERIALIZED column, 44, unless `insert_allow_materialized_columns` is set, when it needs the grant like any other) and before any byte of the body is read. A header naming a column outside the set is refused 497 (ACCESS_DENIED), with `refused_at` `call`, in every format and whatever the body holds, an empty or a malformed body included. So `columns` MUST be a subset of the set, and a caller whose set does not cover every ordinary and DEFAULT column passes `columns`, at every call that takes this document, `chs_filter_eval_body` included: with a list a call follows the listed INSERT's rules, and with none the no-list rule. A body field naming a column the header omits (a JSON key, a `WithNames` header field) is an unknown field, never a grant question: skipped, or refused 117 under `input_format_skip_unknown_fields` 0 (`refused_at` `row` for a JSON key, `body_read` for a header field), and never stored. An omitted column keeps its declared kind, exactly as with no set: a generator DEFAULT draws (`default_generated`), a clock DEFAULT resolves, a literal DEFAULT fills, a MATERIALIZED column computes and an EPHEMERAL column takes its type default. The export carries every stored non-computed column, those outside the set included (`document:batch`), so the INSERT of the export needs a grant that covers its column list. A Nested column is granted per subcolumn (`n.x`). The set is a grant, not a column list: like a grant it may name an EPHEMERAL or MATERIALIZED column, or a name the table lacks. The 497's message is ClickHouse's own, naming the user `default` and the table `default.t`, since a schema carries neither a user nor a table name; it is informative. Absent is every column. Implemented in the first build that carries this description. +- `column_defaults`: per-call DEFAULT overrides, a list of `{name, value}` (each by the rule for byte strings in JSON). It is one rule: in a cell the server would fill from the column's DEFAULT, the call uses the caller's `value` instead. A supplied value wins: the override fills exactly the cells ClickHouse's own input format marks missing for that column under the call's format settings, as the vendored reader decides it, and no other: a field the record omits, a `null` for a non-Nullable column under `input_format_null_as_default` 1, and an empty CSV field under `input_format_csv_empty_as_default` 1. An empty JSON string `""` is a supplied value. A Nullable column's `null` stays `NULL` where the server keeps it (the implementing build confirms this against the server's own missing mark). Such a cell reads `default_override`, and everything that reads the column sees the value as an INSERT that supplied it would: a CHECK, the sorting, primary and partition keys, a skip index, a projection, a TTL, a MATERIALIZED or DEFAULT expression that reads it, and a filter. The export carries it as a stored, present value at the column's position in `export_columns`, so the INSERT of the export needs no override, and a column the override fills is not deferred in those rows (`defer_defaults`). An entry may name an ordinary or a DEFAULT column, and an EPHEMERAL column where the implementing build measures it equal to a supplied EPHEMERAL value; an entry that names an unknown column, a column that is not insertable, or a MATERIALIZED or ALIAS column is `CHS_INVALID_ARGUMENT` at `call`, naming `column_defaults[i]`. When `insertable_columns` is given, an entry's column MUST be in it, because the INSERT the override models names the column in its header; one outside it is the same `CHS_INVALID_ARGUMENT`. `value` is read in the body format's own field encoding, as if the record supplied this field (the implementing build confirms); a value that does not read as the column's type is `CHS_INVALID_ARGUMENT` at `call`, naming `column_defaults[i]`, before any byte of the body is read. Built before the lock: the build that implements it lists `insert_options.column_defaults` in `capabilities.features` (rule r9), and a build that does not declines it as Input documents (in the Preamble) says. +- `verdict_basis`: `input`, the default, or `export`: what row a filter verdict answers for. Under `input`, a filter that reads a value the library drew or resolved itself is `d` (`drawn-value`), because the server would draw its own. Under `export`, the verdict answers for the row the export would store, so a drawn or resolved value is answerable, and the caller MUST insert the export. Reserved: declined at `export` until implemented. +- A member whose value is a list or an object is refused `CHS_INVALID_ARGUMENT` when an entry does not match its shape, naming the key. `{}` (or length 0) is the INSERT these calls modeled before the document existed, byte for byte. Each member means the same at every call that takes the document. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a value that is not one of the member's, naming the value; a member this build does not implement declines as Input documents (in the Preamble) says. + +- `defer_defaults`: `none`, the default, or `server` (public issue #479). `server` defers to the server each DEFAULT this build would otherwise decline because its value is the server's to compute (a generator or a server function the build does not admit). Such a column reads `default_deferred` in the rows that omit it, and the export omits it from those rows' lines. What reads it there is not answered (the deferred-column rule): a filter that reads it makes the row's verdict `d` (`verdict_code` `CHS_CODE_DECLINED`, `verdict_reason` `defer-defaults-reader`); a CHECK, a sorting, primary or partition key, a skip index, a projection, a TTL, or a MATERIALIZED or other DEFAULT expression that reads it declines the row (`unsupported`, `reason` `defer-defaults-reader`); and such a row carries no `partition_id`. Admitted generators keep drawing (`default_generated`), so a drawn id stays usable for deduplication. A body is mixed when its exported rows' lines would not all omit the same columns (some rows supply a deferred column and others omit it). The export has one column list, so a mixed body is declined, and only when an export is asked for: the batch is `unsupported`, `refused_at` `body_step`, `reason` `defer-defaults-mixed-body`; with no export asked for no line omits anything, and the batch is answered. A body whose rows all defer the same columns, or none, is never mixed: its export omits those columns from every line and from `export_columns`. Declined until implemented. + +### input:filter_options + +The options of `chs_filter_create` (rule r1): how one filter is compiled. `{}` (or length 0) compiles as before the document existed. A key it does not define is `CHS_INVALID_ARGUMENT`, naming the key, and so is a value that is not one of the member's, naming the value; a member this build does not implement declines as Input documents (in the Preamble) says. + +- `param_encoding`: `escaped`, the default, or `raw`: how a query parameter's value is read. Under `escaped`, the value is decoded by ClickHouse's escaped-text reader, exactly as a server decodes an HTTP `param_` value, so a backslash, a tab or a newline must be escaped as that reader expects; the object form `{"b64": ...}` is decoded from base64 first and the bytes then read the same way. Under `raw`, the bytes are the value. Reserved: declined at `raw` until implemented (rule r9). ### chs_abi_version @@ -371,12 +443,16 @@ A new buffer holding the name this build's vendored error table gives the code, ### chs_error_message -A new buffer holding the message: ClickHouse's own text, verbatim, for `CHS_REJECTED`, and the library's explanation otherwise. A message may quote input bytes, so it is a byte string, not text. +A new buffer holding the message: ClickHouse's own text, verbatim, for `CHS_REJECTED`, and the library's explanation otherwise. A message may quote input bytes, so it is a byte string, not text. It is informative: a caller keys on the status, the code and the reason (`chs_error_reason`), never on message text. ### chs_error_column A new buffer holding the name of the column the error concerns, as raw bytes, empty when there is none. A column name is a byte string: NUL and invalid UTF-8 are legal in a name. +### chs_error_reason + +A new buffer holding the reason: the catalog id (`spec/abi-v2/declines/declines.json`, ASCII) of a decline or a library-side refusal, empty when the error carries none (ClickHouse's own refusal, an internal error). It is the member a caller keys on, never the message. Like the other text accessors, it returns NULL only for a NULL or invalid error. + ### chs_error_free Releases the caller's reference to an error. Freeing NULL does nothing. @@ -409,7 +485,9 @@ The zone is process state, not a per-call parameter, because ClickHouse's own Me ### chs_set_defaults -Seeds the settings every later call starts from, as a JSON object whose values are JSON strings (a binding never rewrites a value, a boolean included). Each successful call replaces the previous defaults whole. A setting name the server would refuse is refused here, with ClickHouse's own code and message, and nothing is committed. `session_timezone` here is the default zone for later calls; it does not change the image zone that compiled types bind. +Seeds the settings every later call starts from, as a JSON object whose values are JSON strings, or the object form for bytes (Input values), and a binding never rewrites a value, a boolean included. Each successful call replaces the previous defaults whole. A setting name the server would refuse is refused here, with ClickHouse's own code and message, and nothing is committed. `session_timezone` here is the default zone for later calls; it does not change the image zone that compiled types bind. + +The library's own knobs are `chtypes_*` settings, and this is the only channel for the process-level ones. Each is an integer or a string given as a JSON string, and no server knows one, so a server profile refuses it (115) and a call's settings carry only the per-call ones. The knobs, their layer, type, default and effect: `chtypes_now_epoch_nanos` (per call, int64 nanoseconds since the epoch, default the wall clock: pins the call's clock instant); `chtypes_clock_offset_nanos` (per call, int64, default 0: shifts the instant); `chtypes_max_clock_skew_nanos` (per call, int64, default 0: the skew budget a clock-reading expression may resolve under); `chtypes_default_eval_memory_bytes` (process, int64 bytes, default 268435456, 0 disables: the admission memory budget one DEFAULT, CHECK, TTL or filter evaluation may use); `chtypes_default_eval_cpu_nanos` (process, int64, default 1000000000, 0 disables: the admission time budget, the evaluating thread's CPU time, detected not interrupted; the earlier name `chtypes_default_eval_wall_nanos` is refused on v2 with `CHS_INVALID_ARGUMENT` naming this one); `chtypes_custom_settings_prefixes` (process, string, default none: the server's `custom_settings_prefixes`, comma-separated). A process knob in a call's settings is declined, named in `unsupported_settings` (`call-setting-process-only-key`), and a clock knob whose value is not an integer likewise (`call-setting-clock-value`). An evaluation that exceeds a budget is declined with that budget's own reason id, never answered: `admission-memory-budget` for the memory budget and `admission-time-budget` for the time budget; the earlier single id `admission-envelope` is retired, its catalog entry naming both successors (rule r4). The time budget is the evaluating thread's CPU time, not wall time. Each catalog entry states its load dependence. A memory-budget decline is deterministic (`measured`). A time-budget decline depends on load only near the budget: CPU time rose up to x1.03 under load on linux-arm64 and up to x2.0 on linux-amd64 when two busy hardware threads share a core, so a time-budget decline is retry-stable when the evaluation costs more than the budget on an idle core and never happens under about half of it; between half and one times the budget the host's load can decide. Normal evaluations cost about 1000 times less than the 1 s default. a schema whose DEFAULT exceeds a budget at admission is declined at compile. The defaults are this description's, not a build's: a build reports nothing about them. Setup only. The defaults are immutable once this image has created its first `chs_schema` (a filter or a block needs one): from then on the call changes nothing and answers `CHS_INVALID_ARGUMENT`. No call ever reads defaults that change under it, so no binding needs a lock around them. @@ -441,11 +519,11 @@ Everything is validated here, once, and the server never changes after it. A key `CHS_DECLINED` is reserved: a build that will not describe a profile declines it, which is never a wrong answer. The one decline foreseen is a setting outside the set that shapes an input, at a value other than its default, once that set is generated from the vendored source. -The macros are the server's whole `system.macros` when `macros` is present, even as `{}`: a macro a Replicated engine's arguments name that the set lacks is then the server's own refusal (`CHS_REJECTED`, 139, NO_ELEMENTS_IN_CONFIG). When `macros` is absent the server's macros are unknown, and a schema whose engine reads one is declined, as it is without a server. A build that validates a profile's macros but does not yet apply them declines such a schema in both cases. +The macros are the server's whole `system.macros` when `macros` is present, even as `{}`: a macro a Replicated engine's arguments name that the set lacks is then the server's own refusal (`CHS_REJECTED`, 139, NO_ELEMENTS_IN_CONFIG). When `macros` is absent the server's macros are unknown, and a schema whose engine reads one is declined, as it is without a server, except under `schema_options`' `posture` `load`, where a stored Replicated definition compiles without expanding them (`input:schema_options`). A build that validates a profile's macros but does not yet apply them declines such a schema in both cases. A setting this build's filters do not honor in a WHERE is no decline here, and the call succeeds. So does `chs_schema_create` on the server, because a profile never changes a schema compile: it lists the setting in the schema's `filter_declined_settings` (`schema_description`), at layer `server`. -The call reads no defaults and leaves `chs_set_defaults`' latch open. Like every call that is not the handshake, it is valid only after `chs_initialize`. +The call reads no defaults and leaves `chs_set_defaults`' latch open. Its init class is `required`, so before `chs_initialize` it refuses (Initialization, in the Preamble). ### chs_server_free @@ -453,7 +531,9 @@ Releases the caller's reference to a server. Schemas created on it keep it alive ### chs_schema_create -Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, settings and constraints) with ClickHouse's own parser and the checks a server's CREATE runs, on `server`, under a profile of settings given as a JSON object of string values. A trailing semicolon is allowed. A second statement is refused the way ClickHouse's own parser refuses one in a single query (`CHS_REJECTED`, with its code and message), and a statement of any other kind is `CHS_INVALID_ARGUMENT`. +Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, settings and constraints) with ClickHouse's own parser and the checks a server's CREATE runs, on `server`, under a profile of settings given as a JSON object of string values. A trailing semicolon is allowed. A second statement is refused the way ClickHouse's own parser refuses one in a single query (`CHS_REJECTED`, with its code and message), and a statement of any other kind is `CHS_INVALID_ARGUMENT`, except under `posture` `load`: there a `CREATE VIEW`, `CREATE MATERIALIZED VIEW` or `CREATE DICTIONARY` statement, the stored definition of an object a server lists in `system.tables` and whose columns it lists in `system.columns`, is `CHS_DECLINED` with the reason `engine-not-modeled`, so a caller that compiles a server's stored definitions falls back for it as it does for an engine this build does not model. The `ATTACH` spellings of those kinds that a server refuses with 62 stay `CHS_REJECTED` with that code, as the server answers. An `ATTACH TABLE` statement is not one of the other kinds (`options`, below). The library models a table, not its database's engine: `ATTACH TABLE` with a column list and an engine but no UUID is accepted under `posture` `load`, which a server's Atomic database refuses (80), and under `create` it is declined (`attach-statement-under-create`). + +A table whose engine this build does not model (a `Distributed` or `Buffer` table, for example) is `CHS_DECLINED` with the reason `engine-not-modeled`, which is returned only when the engine itself is the cause: a decline for anything else in the statement carries that thing's own reason. `server` is the server the table is on (`chs_server_create`), or NULL. On a server, the schema's home zone is the server's `timezone`: its zone-less `DateTime` and `DateTime64` columns bind it, as a restarted server binds its own, and every call on the schema that names no `session_timezone` runs in it. The server's `settings` layer under the schema's own, and a Replicated engine's ZooKeeper path and replica name expand the server's `macros`. With `server` NULL the schema is on the image's own server, exactly as before servers existed: the image zone, no server settings layer, and macros unknown. A freed server, a handle of another kind or a server from another library image is `CHS_INVALID_ARGUMENT`. The schema holds a counted reference to its server, so the caller may free the server at any time. @@ -461,7 +541,7 @@ Compiles exactly one `CREATE TABLE` statement (columns, engine, keys, TTL, setti A setting this build's filters do not honor in a WHERE, from the `chs_set_defaults` snapshot this call captures, the server profile or this call's own `settings`, is no decline here: the call succeeds and lists it in the schema's `filter_declined_settings` (`schema_description`), and `chs_filter_create` on the schema declines. -`options` (`input:schema_options`) defines no member yet, so any key in it is `CHS_INVALID_ARGUMENT`, naming the key. +`options` (`input:schema_options`) sets the `posture` the statement compiles in: `create`, the default, or `load` for a server's own stored definition (see `input:schema_options`, and the caller's obligation there). A `posture` this build does not know, or any other key, is `CHS_INVALID_ARGUMENT`, naming it. An `ATTACH TABLE` statement is a stored definition, never a statement of another kind: under `create` it is `CHS_DECLINED`, ch_code 0, with exactly the message `chs_schema_create: an ATTACH TABLE statement is a stored definition; compile it with schema_options {"posture":"load"}; declined rather than answered`, and under `load` it compiles as the same definition spelled `CREATE TABLE`. A build that does not yet compile on a server declines a non-NULL `server`, and a schema whose compile needs its server's zone is declined on a host thread that already runs a ClickHouse query of its own. Each is `CHS_DECLINED`, which is never a wrong answer. @@ -475,31 +555,35 @@ Releases the caller's reference to a schema. Filters and blocks made from it kee A JSON document describing the schema's columns, owned by the caller; it replaces v0's borrowed column accessors. -A deep input the server's stack check refuses is `CHS_REJECTED` with ch_code 306. Here the input is the schema's own statement, which describing walks again; it is never `CHS_INTERNAL`. +A schema compiled in `load` carries `"posture": "load"`; one compiled in `create` carries no `posture`. + +A schema `chs_schema_create` returned always describes: the walk this document needs runs at compile, so a deep statement is refused there (`CHS_REJECTED`, 306) and this call never refuses or declines a compiled schema. It is never `CHS_INTERNAL`. ### chs_preview_row Validates and coerces one row of `body` under the schema, as a server's INSERT would, and returns the row's document. `columns` is the INSERT column list, a JSON array of name objects (`{name}` or `{name_b64}`), and empty for none. `session_timezone` in `settings` is this call's zone. -`body` holds exactly one row, and the library reads all of it, with the same reader as `chs_preview_batch`. A malformed tail, or trailing content the format's reader does not itself skip, is rejected with the reader's own code, as a server's INSERT rejects it; what the reader skips, such as a trailing newline, is fine. A body holding no row, or more than one, is `CHS_INVALID_ARGUMENT`, naming the call: `chs_preview_batch` answers a body of many rows. The row's document equals the single row of `chs_preview_batch`'s document for the same body. +`body` holds exactly one row, and the library reads all of it, with the same reader as `chs_preview_batch`. A malformed tail, or trailing content the format's reader does not itself skip, is rejected with the reader's own code, as a server's INSERT rejects it; what the reader skips, such as a trailing newline, is fine. A body holding no row, or more than one, is `CHS_INVALID_ARGUMENT`, naming the call: `chs_preview_batch` answers a body of many rows. The row's document equals the single row of `chs_preview_batch`'s document for the same body, and stands for the call: a refusal or decline of the call, its settings, its column list or its options is this document's `outcome` with `refused_at` `call`, never a status. An INSERT whose input block has no column at all (every insertable column EPHEMERAL, and no column list) is declined: a server answers it (code 90, EMPTY_LIST_OF_COLUMNS_PASSED) from a statement inside its INSERT interpreter that the library cannot call, and a decline is never a wrong answer. +`options` (`input:insert_options`) shapes the INSERT the call models, and `{}` is the INSERT it modeled before the document existed. + ### chs_preview_batch -Validates and coerces a whole body, which may hold many rows, and returns the batch document, with the same declines as `chs_preview_row`. Row separation and the server's error allowance are ClickHouse's own, and the document's `unconsumed` and `framing` say what the reader skipped and how it framed the body. `filter`, when given, is evaluated over each stored row in the same parse. When that filter's WHERE would read a setting this build's filters do not honor, from the filter's own `settings`, the name joins the batch document's top-level `unsupported_settings` and the batch is `unsupported` by `batch_outcome`'s step 2: every row `unsupported`, filter rows `d`, and no export. The same call without the filter is unchanged. `export_format` is `CHS_EXPORT_NONE` or a `chs_format` the build can write; with an export that runs, `out_export` receives the accepted rows serialized once, an empty buffer when the batch holds no row. When the export is declined (the document carries `export_declined`), `out_export` is NULL, as it may be when no export is asked for, so a caller tells a declined export from an empty one by `export_declined`, never by the buffer. `doc_flags` chooses which groups the per-row documents carry. +Validates and coerces a whole body, which may hold many rows, and returns the batch document, with the same declines as `chs_preview_row`. Row separation and the server's error allowance are ClickHouse's own, and the document's `unconsumed` and `framing` say what the reader skipped and how it framed the body. `filter`, when given, is evaluated over each stored row in the same parse. When that filter's WHERE would read a setting this build's filters do not honor, from the filter's own `settings`, the name joins the batch document's top-level `unsupported_settings` and the batch is `unsupported` by `batch_outcome`'s step 2: every row `unsupported`, filter rows `d`, and no export. The same call without the filter is unchanged. `export_format` is `CHS_EXPORT_NONE` or a `chs_format` the build can write; with an export that runs, `out_export` receives the accepted rows serialized once, an empty buffer when the batch holds no row. When the export is declined (the document carries `export_declined` and `export_declined_reason`), `out_export` is NULL, as it may be when no export is asked for, so a caller tells a declined export from an empty one by `export_declined`, never by the buffer. `doc_flags` chooses which groups the per-row documents carry. `options` (`input:insert_options`) shapes the INSERT the call models, and `{}` is the INSERT it modeled before the document existed. `filter` must have been compiled on `schema`: a filter from another schema handle is `CHS_INVALID_ARGUMENT`, as it is at `chs_filter_eval_block`. The call returns a status other than `CHS_OK` only for misuse or an internal error: a decline of the call (the filter's zone differing from the batch's, `preview-batch-zone-mismatch`) and a refusal of its arguments are the document's, with `refused_at` `call` and `reason`. The export's truth table is in `document:batch`. ### chs_filter_create -Compiles a boolean SQL expression over the schema's columns, binding its query parameters (a JSON object of string values) by ClickHouse's own substitution, so a value is never SQL text. `settings` is the profile the expression is compiled under, so `session_timezone` there is the zone of its literals. Non-deterministic expressions are declined. +Compiles a boolean SQL expression over the schema's columns, binding its query parameters (a JSON object of string values) by ClickHouse's own substitution, so a value is never SQL text. A parameter's value is decoded by ClickHouse's escaped-text reader, exactly as a server decodes an HTTP `param_` value (`input:filter_options`' `param_encoding`, `escaped`): `\b` is an escape, so the parameter `a\b` reads as `a`, the byte 0x08 and `b`, and a literal backslash is written `\\`; `\N` reads as the empty value; `\x41`, `\'` and `\0` decode as that reader decodes them; a raw tab or newline is `CHS_REJECTED` with 457 and a trailing backslash with 25, as a server refuses them; and the bytes an escape produces are kept, valid UTF-8 or not. The object form `{"b64": ...}` (Input values) is decoded from base64 first, and its bytes are then read the same way. `options` (`input:filter_options`) shapes the compile, and `{}` is the compile before the document existed. `settings` is the profile the expression is compiled under, so `session_timezone` there is the zone of its literals. Non-deterministic expressions are declined. The WHERE's settings are layered, lowest first: the build's own, `chs_set_defaults`, the schema's server profile, the schema's own settings, and then this call's `settings`; `session_timezone` at every layer stays on the zone path. A type gate in the schema's own settings binds the CREATE only, so a WHERE's CAST validates under the session layers, every layer but the schema's own. -On a schema whose `filter_declined_settings` is not empty (`chs_schema_describe`), from any layer, the call is `CHS_DECLINED` with ch_code 0. It is checked after the argument checks and before the expression is parsed, because `dialect` can change the parse, and the message names each setting and its layer, in the list's order: `chs_filter_create: the settings this schema was compiled under set (), (), which this build's filters do not honor in a WHERE (listed in chs_schema_describe's filter_declined_settings); declined rather than answered`, where each `` is the entry's `declined_layer` value. The entries are joined by a comma and a space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as `b64:`; it does not occur in practice, because a setting name ClickHouse does not know is refused earlier, with 115. The server and the schema were created all the same: no settings layer changes a schema compile. +On a schema whose `filter_declined_settings` is not empty (`chs_schema_describe`), from any layer, the call is `CHS_DECLINED` with ch_code 0. It is checked after the argument checks and before the expression is parsed, because `dialect` can change the parse, and the message names each setting and its layer, in the list's order: `chs_filter_create: the settings this schema was compiled under set (), (), which this build's filters do not honor in a WHERE (listed in chs_schema_describe's filter_declined_settings); declined rather than answered`, where each `` is the entry's `declined_layer` value, with the reason `server-profile-setting`. The message is informative; the entries are joined by a comma and a space, in the list's order, and each name renders verbatim. A `name_b64` entry renders as `b64:`; it does not occur in practice, because a setting name ClickHouse does not know is refused earlier, with 115. The server and the schema were created all the same: no settings layer changes a schema compile. **Changed (public issue #588).** A declined setting from the `chs_set_defaults` or the schema layer moves from evaluation time to create time: it was named at evaluation (`filter_result`'s `unsupported_settings`), and it now declines this call. A server-profile setting already declined it. -A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here, and it is never in `schema_description`. A key in this build's honor set is applied to the WHERE; a key passed through is accepted and ignored; and a key this build's filters do not honor is reported at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`, and `chs_preview_batch`'s). The WHERE runs under this call's layers only (the defaults, the server profile, the schema's own settings and this call's), as a subscriber's SELECT in its own session would. An evaluation's `settings` (`chs_filter_eval_body`, `chs_preview_batch` with a filter, `chs_block_create`) are the INSERT session's: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings. So a filter evaluated in the INSERT's own session, a materialized view's WHERE for example, is created with that session's WHERE-tier settings in its own `settings`. Every key still meets the server's own SET check first, so a name the server does not know is `CHS_REJECTED` with ch_code 115. +A key of this call's own `settings` other than `session_timezone` is never `CHS_DECLINED` here but for one case, and it is never in `schema_description`. The case is a cell of the line's `where-settings/documented-pathological.tsv`, a value under which the server refuses every query: set in this call's own `settings`, it is `CHS_DECLINED` with ch_code 0 and the reason `setting-server-refuses`, because no verdict under it could stand. A key in this build's honor set is applied to the WHERE; a key passed through is accepted and ignored; and a key this build's filters do not honor is reported at evaluation (`filter_result`'s `unsupported_settings`, every verdict `d`, and `chs_preview_batch`'s). The WHERE runs under this call's layers only (the defaults, the server profile, the schema's own settings and this call's), as a subscriber's SELECT in its own session would. An evaluation's `settings` (`chs_filter_eval_body`, `chs_preview_batch` with a filter, `chs_block_create`) are the INSERT session's: they govern parsing the body and never reach the WHERE, so they are never named as WHERE settings. So a filter evaluated in the INSERT's own session, a materialized view's WHERE for example, is created with that session's WHERE-tier settings in its own `settings`. Every key still meets the server's own SET check first, so a name the server does not know is `CHS_REJECTED` with ch_code 115. A filter under a per-call `session_timezone` answers as the artifact producer has measured against live servers, including how the zone in an evaluation's settings combines with the zone the filter was compiled under. Until a path is measured to match a server it declines, which is never a wrong answer. The filter holds a counted reference to the schema. @@ -509,11 +593,11 @@ Releases the caller's reference to a filter. Freeing NULL does nothing. ### chs_filter_eval_body -Evaluates the filter over every row of a body, returning one verdict per row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. When a server would not accept an INSERT of the body, because `chs_preview_batch`'s verdict over the same body and settings, with no filter, is not `accepted`, every verdict is `d` and the outcome mirrors that verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts and that batch verdict together. +Evaluates the filter over every row of a body, returning one verdict per row. `settings` governs parsing the body, and `session_timezone` there is this call's zone. `columns` is the INSERT column list, a JSON array of name objects (`{name}` or `{name_b64}`), and empty for none, read exactly as `chs_preview_batch` reads it: with a list the call models the INSERT that names it, and with none the INSERT that names none. When a server would not accept an INSERT of the body, because `chs_preview_batch`'s verdict over the same body, settings and column list, with no filter, is not `accepted`, every verdict is `d` and the outcome mirrors that verdict (`filter_outcome`). The call reads an accepted body once, for its verdicts and that batch verdict together, and under any outcome but `ok` writes `d` for every verdict, answering a refusal or decline of the call itself in the document (`refused_at` `call`), never as a status. `options` (`input:insert_options`) is that INSERT's, read as `chs_preview_batch` reads it, so the verdict mirrored is `chs_preview_batch`'s over the same body, settings, columns and options. ### chs_block_create -Parses a body once under the schema, for evaluating many filters over it. `settings` governs the parse, `session_timezone` there included, and `columns` is read as `chs_preview_row` reads it. The block holds a counted reference to the schema. It also records `chs_preview_batch`'s verdict over the same body, settings and columns, with no filter, which `chs_filter_eval_block` answers by. An accepted body is read once, for the block and that verdict together. +Parses a body once under the schema, for evaluating many filters over it. `settings` governs the parse, `session_timezone` there included, and `columns` is read as `chs_preview_row` reads it. The block holds a counted reference to the schema. It also records `chs_preview_batch`'s verdict over the same body, settings, columns and options, with no filter, which `chs_filter_eval_block` answers by; a refusal or decline of the call or the body is recorded there, so this call returns a status other than `CHS_OK` only for misuse or an internal error. An accepted body is read once, for the block and that verdict together. `options` (`input:insert_options`) is read as `chs_preview_batch` reads it. ### chs_block_free @@ -521,7 +605,7 @@ Releases the caller's reference to a block. Freeing NULL does nothing. ### chs_filter_eval_block -Evaluates the filter over a parsed block. Over a body a server would accept, it gives the same answers `chs_filter_eval_body` gives for that body. Over one it would not (the verdict `chs_block_create` recorded is not `accepted`), the outcome mirrors that verdict as for `chs_filter_eval_body`, and every verdict from the first refused row on is `d`. The first refused row is the first row of that batch whose outcome is neither `accepted` nor `skipped`. When the refusal belongs to no row (a whole-body step, a poisoned value, the call's own settings), it is the first row of the block. The rows before it keep their evaluated verdicts. A caller treats those as `d` too: any non-`ok` outcome, including `unsupported`, means every verdict is `d`. It takes no settings: the filter brings the settings it was compiled under and the block those it was parsed under. The filter and the block must come from the same schema; a pair from two schemas is `CHS_INVALID_ARGUMENT`. +Evaluates the filter over a parsed block. Over a body a server would accept, it gives the same answers `chs_filter_eval_body` gives for that body. Over one it would not (the verdict `chs_block_create` recorded is not `accepted`), the outcome mirrors that verdict as for `chs_filter_eval_body`, and every verdict is written `d`, the rows before the refused one included: the two calls write the same document for the same body, and the rule that any non-`ok` outcome means every verdict is `d` holds in the data. `refused_at` and `refused_row` say where the refusal sat (`row` for a refused row, `body_step` for a whole-body step or a poisoned value, `call` for the block's own settings). It takes no settings: the filter brings the settings it was compiled under and the block those it was parsed under. The filter and the block must come from the same schema; a pair from two schemas is `CHS_INVALID_ARGUMENT`. ### chs_discover_query diff --git a/spec/abi-v2/generated/exports.txt b/spec/abi-v2/generated/exports.txt index 68a713d4..53555595 100644 --- a/spec/abi-v2/generated/exports.txt +++ b/spec/abi-v2/generated/exports.txt @@ -1,4 +1,4 @@ -# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT +# GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT # Every symbol a v2 library exports, one per line, sorted. Nothing else is exported. chs_abi_revision chs_abi_version @@ -19,6 +19,7 @@ chs_error_codes chs_error_column chs_error_free chs_error_message +chs_error_reason chs_error_status chs_filter_create chs_filter_eval_block diff --git a/spec/abi-v2/schema/abi.schema.json b/spec/abi-v2/schema/abi.schema.json index 0076f1df..82ed79df 100644 --- a/spec/abi-v2/schema/abi.schema.json +++ b/spec/abi-v2/schema/abi.schema.json @@ -95,7 +95,7 @@ "content": { "description": "What a counted byte string carries. Every one is a byte string, never a C string: NUL and invalid UTF-8 pass through. A document: content names an entry in documents, and an input: content an entry in inputs.", "type": "string", - "pattern": "^(bytes|name|sql|message|ascii|json_object_string_values|json_array_names|timezone|document:[a-z][a-z0-9_]*|input:[a-z][a-z0-9_]*)$" + "pattern": "^(bytes|name|sql|message|ascii|json_object_string_values|json_object_byte_values|json_array_names|timezone|document:[a-z][a-z0-9_]*|input:[a-z][a-z0-9_]*)$" }, "handle": { "type": "object", @@ -208,11 +208,12 @@ "function": { "type": "object", "additionalProperties": false, - "required": ["name", "class", "thread", "returns", "params"], + "required": ["name", "class", "thread", "init", "returns", "params"], "properties": { "name": {"$ref": "#/$defs/c_type_name"}, "class": {"enum": ["handshake", "tombstone", "api", "tooling"]}, "thread": {"enum": ["any", "shared", "handle_serial", "process_once", "process_serial"], "description": "The thread class; scripts/abi-v1/model.py THREADS describes each, and load() ties it to the function's shape: `shared` exactly when the call reads a caller handle and is not a free, `handle_serial` exactly for a free."}, + "init": {"enum": ["any", "required"], "description": "The init class: whether the call is valid before `chs_initialize` has succeeded (the Preamble's Initialization in docs.md). scripts/abi-v1/model.py INITS describes each, and load() ties it to the function: `any` for the handshake, the tombstone, `chs_initialize` and every call that returns no `chs_status` (the buffer and error accessors, the frees, `chs_shutdown`); `required` for every other call, unless its own docs.md section says it is valid before `chs_initialize`."}, "provisional": {"$ref": "#/$defs/markers"}, "growable": {"type": "boolean", "description": "Rule r1 in spec/abi-v2/docs.md: true marks a function that may gain inputs after it first ships. It takes exactly one options document (a bytes_in parameter whose content names an inputs entry with options true), and a function not marked growable takes no input document at all; scripts/abi-v1/model.py checks both."}, "returns": {"$ref": "#/$defs/return"}, diff --git a/spec/abi-v2/sdk.json b/spec/abi-v2/sdk.json index 37124af2..db6e6b9f 100644 --- a/spec/abi-v2/sdk.json +++ b/spec/abi-v2/sdk.json @@ -3,7 +3,7 @@ "reuse_v0_names": true, "provisional_markers": {}, "errors": { - "fields": ["status", "ch_code", "ch_name", "message", "column"], + "fields": ["status", "ch_code", "ch_name", "message", "column", "reason"], "codes": { "artifact_incompatible": "CHTYPES_ARTIFACT_INCOMPATIBLE", "artifact_corrupt": "CHTYPES_ARTIFACT_CORRUPT" diff --git a/spec/abi-v2/where-settings/README.md b/spec/abi-v2/where-settings/README.md index f5b345bc..aa423a90 100644 --- a/spec/abi-v2/where-settings/README.md +++ b/spec/abi-v2/where-settings/README.md @@ -30,9 +30,9 @@ Output-format settings that reach a WHERE (through `formatRow`, `toJSONString` o This is part of the v2 contract. -A consumer passes the server's changed settings in the server profile (`settings`, see `input:server_profile` in `spec/abi-v2/docs.md`). Then it must **refuse a tenant if `filter_declined_settings` is non-empty, or a `result-content` setting is changed**. +A consumer passes the server's changed settings in the server profile (`settings`, see `input:server_profile` in `spec/abi-v2/docs.md`). Then it must **refuse a tenant if `filter_declined_settings` is non-empty**. That is the whole rule: the list already carries every changed `result-content` setting, honored or not, and every cell of `documented-pathological.tsv` the schema's layers set (at tier `server-refuses`), so a consumer reads neither these lists nor that file to decide. -`filter_declined_settings` is the top-level member of `chs_schema_describe` on the tenant's schema, one check at bind time. It lists every setting that this build's filters do not honor in a WHERE, over every layer known when the schema was compiled: the `chs_set_defaults` snapshot, the server profile and the schema's own settings. Each entry carries its tier from these lists and its layer. On such a schema, `chs_filter_create` declines (`CHS_DECLINED`): this is the backstop for a consumer that did not read the list. A filter's own settings are not in it; a declined one is named at evaluation, with every verdict `d`. An evaluation's settings govern parsing the body only: they never reach the WHERE, so they are never named as WHERE settings, and a filter evaluated in the INSERT's own session is compiled with that session's WHERE-tier settings in the filter's own settings. +`filter_declined_settings` is the top-level member of `chs_schema_describe` on the tenant's schema, one check at bind time. It lists every setting that this build's filters do not honor in a WHERE, over every layer known when the schema was compiled: the `chs_set_defaults` snapshot, the server profile and the schema's own settings. Each entry carries its tier from these lists (or `server-refuses`, for a pathological cell) and its layer. On such a schema, `chs_filter_create` declines (`CHS_DECLINED`): this is the backstop for a consumer that did not read the list. A filter's own settings are not in it; a declined one is named at evaluation, with every verdict `d`. An evaluation's settings govern parsing the body only: they never reach the WHERE, so they are never named as WHERE settings, and a filter evaluated in the INSERT's own session is compiled with that session's WHERE-tier settings in the filter's own settings. A `result-truncate` setting never makes a consumer refuse: it is outside a row verdict's scope, since a stream can deliver rows that a paged SELECT would cut off. @@ -54,7 +54,7 @@ About 63 to 81 settings per line are free strings or enums for which the flip te ## Settings the server may refuse outright -Under some settings values a ClickHouse server refuses **every** filter query, so no row is ever returned, while the library answers the rows. A server configured with one of these values is outside the filter guarantee: **a consumer refuses such a profile or call.** Neither the library nor the filter's row verdicts will tell you the server would have refused. +Under some settings values a ClickHouse server refuses **every** filter query, so no row is ever returned, while the library answers the rows. A server configured with one of these values is outside the filter guarantee, and **the library says so**: for a cell set at the `chs_set_defaults`, server-profile or schema layer, `filter_declined_settings` lists it at tier `server-refuses`, so the consumer rule above refuses it with no lookup here. A filter's own settings are not in that list, and need no lookup either: a cell set there declines `chs_filter_create` (`CHS_DECLINED`, reason `setting-server-refuses`). The exact list is the data file `documented-pathological.tsv`, copied byte for byte from the artifact producer. A leading block of `#` comment lines explains it, the last comment line names the columns, and each following row is one exact cell, with these tab-separated columns: @@ -70,7 +70,7 @@ The exact list is the data file `documented-pathological.tsv`, copied byte for b Two examples from the data: `max_expanded_ast_elements=1` is refused with server code 36 on every supported line, and `page=1` is refused with code 36 on the 26.8 and 26.9 lines. -The producer's differential gate counts these cells under their own heading and does not block on them. Nothing else is excused: the list grows only by sign-off, and a listed cell that stops failing must have its row removed. Treat a value that is not on the list as unknown, not as safe. +The producer's differential gate counts these cells under their own heading and does not block on them. Nothing else is excused: the list grows only by sign-off, and a listed cell that stops failing must have its row removed. A value not on the list is one no measurement has found the server refusing; the producer's gate is what finds a new cell, and a consumer never extends the list itself. ## Not covered diff --git a/tests/fixtures/abi-v2/cases.json b/tests/fixtures/abi-v2/cases.json index c6a4df70..befa04cc 100644 --- a/tests/fixtures/abi-v2/cases.json +++ b/tests/fixtures/abi-v2/cases.json @@ -1,5 +1,5 @@ { - "_generated": "GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) \u2014 DO NOT EDIT", + "_generated": "GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) \u2014 DO NOT EDIT", "cases": [ { "expect": { @@ -946,6 +946,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -970,6 +973,11 @@ "head_hex": "5b5d", "len": 2, "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" } ], "fn": "chs_preview_row", @@ -1007,107 +1015,16 @@ "int": 0 }, { - "bytes_hex": "21533a4348535f52454a45435445443a37373a544553545f434f44453a6368735f707265766965775f726f7720666f72636564204348535f52454a4543544544" - }, - { - "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_preview_row forced CHS_REJECTED" - }, - "status": "CHS_REJECTED" - }, - "fn": "chs_preview_row", - "id": "chs_preview_row.status.CHS_REJECTED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f4445434c494e45443a37373a544553545f434f44453a6368735f707265766965775f726f7720666f72636564204348535f4445434c494e4544" + "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f707265766965775f726f7720666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" }, { "bytes_hex": "5b5d" - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_preview_row forced CHS_DECLINED" - }, - "status": "CHS_DECLINED" - }, - "fn": "chs_preview_row", - "id": "chs_preview_row.status.CHS_DECLINED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f707265766965775f726f7720666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" } ], "expect": { @@ -1155,6 +1072,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -1211,6 +1131,9 @@ }, { "int": 0 + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -1238,7 +1161,12 @@ }, null, -1, - 0 + 0, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } ], "fn": "chs_preview_batch", "out": "out" @@ -1266,7 +1194,12 @@ }, null, -1, - 0 + 0, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } ], "fn": "chs_preview_batch", "out": "out_export" @@ -1303,63 +1236,7 @@ "int": 0 }, { - "bytes_hex": "21533a4348535f52454a45435445443a37373a544553545f434f44453a6368735f707265766965775f626174636820666f72636564204348535f52454a4543544544" - }, - { - "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" - }, - { - "null_handle": true - }, - { - "int": -1 - }, - { - "int": 0 - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_preview_batch forced CHS_REJECTED" - }, - "status": "CHS_REJECTED" - }, - "fn": "chs_preview_batch", - "id": "chs_preview_batch.status.CHS_REJECTED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f4445434c494e45443a37373a544553545f434f44453a6368735f707265766965775f626174636820666f72636564204348535f4445434c494e4544" + "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f707265766965775f626174636820666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" @@ -1375,62 +1252,9 @@ }, { "int": 0 - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_preview_batch forced CHS_DECLINED" - }, - "status": "CHS_DECLINED" - }, - "fn": "chs_preview_batch", - "id": "chs_preview_batch.status.CHS_DECLINED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f707265766965775f626174636820666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" - }, - { - "null_handle": true - }, - { - "int": -1 - }, - { - "int": 0 } ], "expect": { @@ -1487,6 +1311,9 @@ }, { "int": 0 + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -1529,6 +1356,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -1567,6 +1397,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -1611,6 +1444,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -1655,6 +1491,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -1699,6 +1538,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -1746,6 +1588,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -1759,6 +1604,12 @@ { "bytes_hex": "6100ff7a" }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, { "bytes_hex": "7b7d" } @@ -1776,6 +1627,16 @@ "len": 4, "sha256": "baf9342f98f4ff9b59662fa6afac189dfcda55079f9298c54819bbd2b43bf7b9" }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + { + "head_hex": "5b5d", + "len": 2, + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, { "head_hex": "7b7d", "len": 2, @@ -1824,63 +1685,6 @@ }, { "bytes_hex": "" - } - ], - "fn": "chs_filter_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f52454a45435445443a37373a544553545f434f44453a6368735f66696c7465725f6576616c5f626f647920666f72636564204348535f52454a4543544544" - }, - { - "bytes_hex": "7b7d" - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_filter_eval_body forced CHS_REJECTED" - }, - "status": "CHS_REJECTED" - }, - "fn": "chs_filter_eval_body", - "id": "chs_filter_eval_body.status.CHS_REJECTED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "bytes_hex": "78203d2031" - }, - { - "bytes_hex": "7b7d" }, { "bytes_hex": "" @@ -1893,67 +1697,13 @@ "int": 0 }, { - "bytes_hex": "21533a4348535f4445434c494e45443a37373a544553545f434f44453a6368735f66696c7465725f6576616c5f626f647920666f72636564204348535f4445434c494e4544" + "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f66696c7465725f6576616c5f626f647920666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_filter_eval_body forced CHS_DECLINED" - }, - "status": "CHS_DECLINED" - }, - "fn": "chs_filter_eval_body", - "id": "chs_filter_eval_body.status.CHS_DECLINED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "bytes_hex": "78203d2031" - }, - { - "bytes_hex": "7b7d" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_filter_create" - } - }, - { - "int": 0 }, { - "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f66696c7465725f6576616c5f626f647920666f72636564204348535f494e56414c49445f415247554d454e54" + "bytes_hex": "5b5d" }, { "bytes_hex": "7b7d" @@ -2002,6 +1752,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -2015,6 +1768,12 @@ { "bytes_hex": "21533a4348535f494e5445524e414c3a37373a544553545f434f44453a6368735f66696c7465725f6576616c5f626f647920666f72636564204348535f494e5445524e414c" }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, { "bytes_hex": "7b7d" } @@ -2064,61 +1823,17 @@ }, { "bytes_hex": "5b5d" - } - ], - "expect": { - "status": "CHS_OK" - }, - "fn": "chs_block_create", - "id": "chs_block_create.echo", - "kind": "echo" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f52454a45435445443a37373a544553545f434f44453a6368735f626c6f636b5f63726561746520666f72636564204348535f52454a4543544544" }, { "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" } ], "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_block_create forced CHS_REJECTED" - }, - "status": "CHS_REJECTED" + "status": "CHS_OK" }, "fn": "chs_block_create", - "id": "chs_block_create.status.CHS_REJECTED", - "kind": "status" + "id": "chs_block_create.echo", + "kind": "echo" }, { "args": [ @@ -2145,60 +1860,16 @@ "int": 0 }, { - "bytes_hex": "21533a4348535f4445434c494e45443a37373a544553545f434f44453a6368735f626c6f636b5f63726561746520666f72636564204348535f4445434c494e4544" + "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f626c6f636b5f63726561746520666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" }, { "bytes_hex": "5b5d" - } - ], - "expect": { - "error": { - "ch_code": 77, - "ch_name": "TEST_CODE", - "column": "", - "message": "chs_block_create forced CHS_DECLINED" - }, - "status": "CHS_DECLINED" - }, - "fn": "chs_block_create", - "id": "chs_block_create.status.CHS_DECLINED", - "kind": "status" - }, - { - "args": [ - { - "handle": { - "args": [ - { - "null_handle": true - }, - { - "bytes_hex": "435245415445205441424c45207420287820496e74333229" - }, - { - "bytes_hex": "" - }, - { - "bytes_hex": "" - } - ], - "fn": "chs_schema_create" - } - }, - { - "int": 0 - }, - { - "bytes_hex": "21533a4348535f494e56414c49445f415247554d454e543a37373a544553545f434f44453a6368735f626c6f636b5f63726561746520666f72636564204348535f494e56414c49445f415247554d454e54" }, { "bytes_hex": "7b7d" - }, - { - "bytes_hex": "5b5d" } ], "expect": { @@ -2246,6 +1917,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -2291,6 +1965,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -2329,6 +2006,9 @@ { "bytes_hex": "" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -2558,6 +2238,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -2582,6 +2265,11 @@ "head_hex": "5b5d", "len": 2, "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" } ], "fn": "chs_preview_row", @@ -2624,6 +2312,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -2639,6 +2330,12 @@ }, { "bytes_hex": "7b2273657373696f6e5f74696d657a6f6e65223a22416d65726963612f4c6f735f416e67656c6573227d" + }, + { + "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { @@ -2658,6 +2355,16 @@ "head_hex": "7b2273657373696f6e5f74696d657a6f", "len": 42, "sha256": "57abb215d57cf42b71c0fa6c2d6d0c74a7a285f2c0584caad7cbc860d5aaf8aa" + }, + { + "head_hex": "5b5d", + "len": 2, + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" } ], "fn": "chs_filter_eval_body", @@ -2736,7 +2443,419 @@ "bytes_hex": "5b5d" }, { - "bytes_hex": "7b7d" + "bytes_hex": "7b7d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "profile: the input document server_profile is not a JSON object" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_server_create", + "id": "input_document.chs_server_create.profile.not_an_object_refused", + "kind": "status" + }, + { + "args": [ + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "7b22785f756e6b6e6f776e223a2231227d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: unknown key \"x_unknown\" in the input document server_options" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_server_create", + "id": "input_document.chs_server_create.options.unknown_key_refused", + "kind": "status" + }, + { + "args": [ + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: the input document server_options is not a JSON object" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_server_create", + "id": "input_document.chs_server_create.options.not_an_object_refused", + "kind": "status" + }, + { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "7b22706f7374757265223a22637265617465222c22785f756e6b6e6f776e223a2231227d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: unknown key \"x_unknown\" in the input document schema_options" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_schema_create", + "id": "input_document.chs_schema_create.options.unknown_key_refused", + "kind": "status" + }, + { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: the input document schema_options is not a JSON object" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_schema_create", + "id": "input_document.chs_schema_create.options.not_an_object_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b2264656665725f64656661756c7473223a226e6f6e65222c22696e7365727461626c655f636f6c756d6e73223a5b5d2c22636f6c756d6e5f64656661756c7473223a5b5d2c22766572646963745f6261736973223a22696e707574222c22785f756e6b6e6f776e223a2231227d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: unknown key \"x_unknown\" in the input document insert_options" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_preview_row", + "id": "input_document.chs_preview_row.options.unknown_key_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, + { + "bytes_hex": "5b5d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: the input document insert_options is not a JSON object" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_preview_row", + "id": "input_document.chs_preview_row.options.not_an_object_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, + { + "null_handle": true + }, + { + "int": -1 + }, + { + "int": 0 + }, + { + "bytes_hex": "7b2264656665725f64656661756c7473223a226e6f6e65222c22696e7365727461626c655f636f6c756d6e73223a5b5d2c22636f6c756d6e5f64656661756c7473223a5b5d2c22766572646963745f6261736973223a22696e707574222c22785f756e6b6e6f776e223a2231227d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: unknown key \"x_unknown\" in the input document insert_options" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_preview_batch", + "id": "input_document.chs_preview_batch.options.unknown_key_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, + { + "null_handle": true + }, + { + "int": -1 + }, + { + "int": 0 + }, + { + "bytes_hex": "5b5d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: the input document insert_options is not a JSON object" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_preview_batch", + "id": "input_document.chs_preview_batch.options.not_an_object_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "7b22706172616d5f656e636f64696e67223a2265736361706564222c22785f756e6b6e6f776e223a2231227d" + } + ], + "expect": { + "error": { + "ch_code": 0, + "ch_name": "", + "column": "", + "message": "options: unknown key \"x_unknown\" in the input document filter_options" + }, + "status": "CHS_INVALID_ARGUMENT" + }, + "fn": "chs_filter_create", + "id": "input_document.chs_filter_create.options.unknown_key_refused", + "kind": "status" + }, + { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "bytes_hex": "6100ff7a" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" } ], "expect": { @@ -2744,21 +2863,68 @@ "ch_code": 0, "ch_name": "", "column": "", - "message": "profile: the input document server_profile is not a JSON object" + "message": "options: the input document filter_options is not a JSON object" }, "status": "CHS_INVALID_ARGUMENT" }, - "fn": "chs_server_create", - "id": "input_document.chs_server_create.profile.not_an_object_refused", + "fn": "chs_filter_create", + "id": "input_document.chs_filter_create.options.not_an_object_refused", "kind": "status" }, { "args": [ + { + "handle": { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "bytes_hex": "78203d2031" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_filter_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, { "bytes_hex": "7b7d" }, { - "bytes_hex": "7b22785f756e6b6e6f776e223a2231227d" + "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b2264656665725f64656661756c7473223a226e6f6e65222c22696e7365727461626c655f636f6c756d6e73223a5b5d2c22636f6c756d6e5f64656661756c7473223a5b5d2c22766572646963745f6261736973223a22696e707574222c22785f756e6b6e6f776e223a2231227d" } ], "expect": { @@ -2766,19 +2932,66 @@ "ch_code": 0, "ch_name": "", "column": "", - "message": "options: unknown key \"x_unknown\" in the input document server_options" + "message": "options: unknown key \"x_unknown\" in the input document insert_options" }, "status": "CHS_INVALID_ARGUMENT" }, - "fn": "chs_server_create", - "id": "input_document.chs_server_create.options.unknown_key_refused", + "fn": "chs_filter_eval_body", + "id": "input_document.chs_filter_eval_body.options.unknown_key_refused", "kind": "status" }, { "args": [ + { + "handle": { + "args": [ + { + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "bytes_hex": "78203d2031" + }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_filter_create" + } + }, + { + "int": 0 + }, + { + "bytes_hex": "6100ff7a" + }, { "bytes_hex": "7b7d" }, + { + "bytes_hex": "5b5d" + }, { "bytes_hex": "5b5d" } @@ -2788,18 +3001,37 @@ "ch_code": 0, "ch_name": "", "column": "", - "message": "options: the input document server_options is not a JSON object" + "message": "options: the input document insert_options is not a JSON object" }, "status": "CHS_INVALID_ARGUMENT" }, - "fn": "chs_server_create", - "id": "input_document.chs_server_create.options.not_an_object_refused", + "fn": "chs_filter_eval_body", + "id": "input_document.chs_filter_eval_body.options.not_an_object_refused", "kind": "status" }, { "args": [ { - "null_handle": true + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 }, { "bytes_hex": "6100ff7a" @@ -2808,7 +3040,10 @@ "bytes_hex": "7b7d" }, { - "bytes_hex": "7b22785f756e6b6e6f776e223a2231227d" + "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b2264656665725f64656661756c7473223a226e6f6e65222c22696e7365727461626c655f636f6c756d6e73223a5b5d2c22636f6c756d6e5f64656661756c7473223a5b5d2c22766572646963745f6261736973223a22696e707574222c22785f756e6b6e6f776e223a2231227d" } ], "expect": { @@ -2816,18 +3051,37 @@ "ch_code": 0, "ch_name": "", "column": "", - "message": "options: unknown key \"x_unknown\" in the input document schema_options" + "message": "options: unknown key \"x_unknown\" in the input document insert_options" }, "status": "CHS_INVALID_ARGUMENT" }, - "fn": "chs_schema_create", - "id": "input_document.chs_schema_create.options.unknown_key_refused", + "fn": "chs_block_create", + "id": "input_document.chs_block_create.options.unknown_key_refused", "kind": "status" }, { "args": [ { - "null_handle": true + "handle": { + "args": [ + { + "null_handle": true + }, + { + "bytes_hex": "435245415445205441424c45207420287820496e74333229" + }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + } + ], + "fn": "chs_schema_create" + } + }, + { + "int": 0 }, { "bytes_hex": "6100ff7a" @@ -2835,6 +3089,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "5b5d" + }, { "bytes_hex": "5b5d" } @@ -2844,12 +3101,12 @@ "ch_code": 0, "ch_name": "", "column": "", - "message": "options: the input document schema_options is not a JSON object" + "message": "options: the input document insert_options is not a JSON object" }, "status": "CHS_INVALID_ARGUMENT" }, - "fn": "chs_schema_create", - "id": "input_document.chs_schema_create.options.not_an_object_refused", + "fn": "chs_block_create", + "id": "input_document.chs_block_create.options.not_an_object_refused", "kind": "status" }, { @@ -2964,11 +3221,15 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { "documents": { "out": { + "code": 0, "cols": [ { "input_b64": "/wCA", @@ -2986,8 +3247,7 @@ }, "outcome": "accepted", "transformed": [], - "unknown_fields": [], - "unsupported_settings": [] + "unknown_fields": [] } }, "status": "CHS_OK" @@ -3028,11 +3288,15 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "expect": { "documents": { "out": { + "code": 0, "cols": [ { "input": "a\u0000b", @@ -3050,8 +3314,7 @@ }, "outcome": "accepted", "transformed": [], - "unknown_fields": [], - "unsupported_settings": [] + "unknown_fields": [] } }, "status": "CHS_OK" @@ -3123,6 +3386,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3166,6 +3432,12 @@ { "bytes_hex": "7b2278223a317d" }, + { + "bytes_hex": "" + }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3185,6 +3457,16 @@ "len": 7, "sha256": "5041bf1f713df204784353e82f6a4a535931cb64f1f4b4a5aeaffcb720918b22" }, + { + "head_hex": "", + "len": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "head_hex": "", + "len": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, { "head_hex": "", "len": 0, @@ -3249,6 +3531,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3272,6 +3557,9 @@ { "bytes_hex": "" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3549,6 +3837,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "calls": 16, @@ -3574,6 +3865,11 @@ "head_hex": "5b5d", "len": 2, "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" } ], "fn": "chs_preview_row", @@ -3628,6 +3924,9 @@ }, { "int": 0 + }, + { + "bytes_hex": "7b7d" } ], "calls": 16, @@ -3656,7 +3955,12 @@ }, null, -1, - 0 + 0, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } ], "fn": "chs_preview_batch", "out": "out" @@ -3684,7 +3988,12 @@ }, null, -1, - 0 + 0, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } ], "fn": "chs_preview_batch", "out": "out_export" @@ -3724,6 +4033,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "7b7d" + }, { "bytes_hex": "7b7d" } @@ -3767,6 +4079,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3780,6 +4095,12 @@ { "bytes_hex": "6100ff7a" }, + { + "bytes_hex": "7b7d" + }, + { + "bytes_hex": "5b5d" + }, { "bytes_hex": "7b7d" } @@ -3798,6 +4119,16 @@ "len": 4, "sha256": "baf9342f98f4ff9b59662fa6afac189dfcda55079f9298c54819bbd2b43bf7b9" }, + { + "head_hex": "7b7d", + "len": 2, + "sha256": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + { + "head_hex": "5b5d", + "len": 2, + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, { "head_hex": "7b7d", "len": 2, @@ -3847,6 +4178,9 @@ }, { "bytes_hex": "5b5d" + }, + { + "bytes_hex": "7b7d" } ], "calls": 16, @@ -3888,6 +4222,9 @@ { "bytes_hex": "7b7d" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -3926,6 +4263,9 @@ { "bytes_hex": "" }, + { + "bytes_hex": "" + }, { "bytes_hex": "" } @@ -4247,6 +4587,14 @@ "kind": "loader", "variant": "missing-chs_error_message" }, + { + "expect": { + "reason": "missing_symbol:chs_error_reason" + }, + "id": "loader.missing-chs_error_reason", + "kind": "loader", + "variant": "missing-chs_error_reason" + }, { "expect": { "reason": "missing_symbol:chs_error_status" diff --git a/tests/fixtures/abi-v2/setup-cases.json b/tests/fixtures/abi-v2/setup-cases.json index cc41ba00..460125e2 100644 --- a/tests/fixtures/abi-v2/setup-cases.json +++ b/tests/fixtures/abi-v2/setup-cases.json @@ -1,5 +1,5 @@ { - "_generated": "GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) \u2014 DO NOT EDIT", + "_generated": "GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) \u2014 DO NOT EDIT", "cases": [ { "id": "setup.latches_only_on_success", @@ -714,7 +714,7 @@ "variant": "declined-settings" }, { - "id": "schema.describe_rejects_a_deep_input_306", + "id": "schema.compile_rejects_a_deep_input_306", "steps": [ { "expect": { @@ -724,13 +724,6 @@ }, { "as": "schema", - "expect": { - "ok": true - }, - "op": "compile", - "statement": "!F:deep" - }, - { "expect": { "ch_code": 306, "ch_name": "TOO_DEEP_RECURSION", @@ -738,8 +731,8 @@ "message": "Stack size too large (the stub's stand-in for the server's stack check on a deep input)", "status": "CHS_REJECTED" }, - "op": "describe", - "schema": "schema" + "op": "compile", + "statement": "!F:deep" }, { "as": "server", diff --git a/ts/src/abi2/calls.gen.ts b/ts/src/abi2/calls.gen.ts index c70a3e89..c2228371 100644 --- a/ts/src/abi2/calls.gen.ts +++ b/ts/src/abi2/calls.gen.ts @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * One typed, copy-then-free method per described call, named without the `chs_` prefix. * Each marshals through the generic `rawCall`, throws the error class sdk.json's status @@ -89,14 +89,14 @@ export class Calls { } /** The generated call wrapper for chs_preview_row. */ - previewRow(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array): Buffer { - const outs = checkStatus(rawCall(this.raw, SYMBOL.PREVIEW_ROW, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns)])); + previewRow(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array, options: Uint8Array): Buffer { + const outs = checkStatus(rawCall(this.raw, SYMBOL.PREVIEW_ROW, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns), asBuffer(options)])); return outs["out"] as Buffer; } /** The generated call wrapper for chs_preview_batch. */ - previewBatch(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array, filter: FilterHandle | null, exportFormat: number, docFlags: number): { readonly out: Buffer; readonly outExport: Buffer } { - const outs = checkStatus(rawCall(this.raw, SYMBOL.PREVIEW_BATCH, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns), filter === null ? NULL_EXTERNAL : filter.ptr, exportFormat, docFlags])); + previewBatch(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array, filter: FilterHandle | null, exportFormat: number, docFlags: number, options: Uint8Array): { readonly out: Buffer; readonly outExport: Buffer } { + const outs = checkStatus(rawCall(this.raw, SYMBOL.PREVIEW_BATCH, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns), filter === null ? NULL_EXTERNAL : filter.ptr, exportFormat, docFlags, asBuffer(options)])); return { out: outs["out"] as Buffer, outExport: outs["out_export"] as Buffer, @@ -104,20 +104,20 @@ export class Calls { } /** The generated call wrapper for chs_filter_create. */ - filterCreate(schema: SchemaHandle, expr: Uint8Array, queryParams: Uint8Array, settings: Uint8Array): FilterHandle { - const outs = checkStatus(rawCall(this.raw, SYMBOL.FILTER_CREATE, [schema.ptr, asBuffer(expr), asBuffer(queryParams), asBuffer(settings)])); + filterCreate(schema: SchemaHandle, expr: Uint8Array, queryParams: Uint8Array, settings: Uint8Array, options: Uint8Array): FilterHandle { + const outs = checkStatus(rawCall(this.raw, SYMBOL.FILTER_CREATE, [schema.ptr, asBuffer(expr), asBuffer(queryParams), asBuffer(settings), asBuffer(options)])); return wrapHandle(this.raw, outs["out"] as HandleRef) as FilterHandle; } /** The generated call wrapper for chs_filter_eval_body. */ - filterEvalBody(filter: FilterHandle, format: number, body: Uint8Array, settings: Uint8Array): Buffer { - const outs = checkStatus(rawCall(this.raw, SYMBOL.FILTER_EVAL_BODY, [filter.ptr, format, asBuffer(body), asBuffer(settings)])); + filterEvalBody(filter: FilterHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array, options: Uint8Array): Buffer { + const outs = checkStatus(rawCall(this.raw, SYMBOL.FILTER_EVAL_BODY, [filter.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns), asBuffer(options)])); return outs["out"] as Buffer; } /** The generated call wrapper for chs_block_create. */ - blockCreate(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array): BlockHandle { - const outs = checkStatus(rawCall(this.raw, SYMBOL.BLOCK_CREATE, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns)])); + blockCreate(schema: SchemaHandle, format: number, body: Uint8Array, settings: Uint8Array, columns: Uint8Array, options: Uint8Array): BlockHandle { + const outs = checkStatus(rawCall(this.raw, SYMBOL.BLOCK_CREATE, [schema.ptr, format, asBuffer(body), asBuffer(settings), asBuffer(columns), asBuffer(options)])); return wrapHandle(this.raw, outs["out"] as HandleRef) as BlockHandle; } diff --git a/ts/src/abi2/decls.gen.ts b/ts/src/abi2/decls.gen.ts index 98691cfe..64893fa0 100644 --- a/ts/src/abi2/decls.gen.ts +++ b/ts/src/abi2/decls.gen.ts @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * The ABI description as DATA: every described symbol's name, its parameter and return * shape, and the chs_status enum, both directions. See this emitter's module docstring @@ -158,8 +158,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -186,7 +184,7 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, @@ -199,6 +197,13 @@ export const FUNCTION_SPECS: Readonly> = { "nullable": false, "type": null }, + { + "content": "input:insert_options", + "kind": "bytes_in", + "name": "options", + "nullable": false, + "type": null + }, { "content": null, "kind": "out_handle", @@ -573,6 +578,29 @@ export const FUNCTION_SPECS: Readonly> = { }, "thread": "shared" }, + "chs_error_reason": { + "cls": "api", + "mayReturn": [], + "params": [ + { + "content": null, + "kind": "handle", + "name": "error", + "nullable": false, + "type": "chs_error" + } + ], + "returns": { + "borrows": null, + "constant": null, + "content": "ascii", + "kind": "handle", + "nullable": true, + "owned": true, + "type": "chs_buf" + }, + "thread": "shared" + }, "chs_error_status": { "cls": "api", "mayReturn": [], @@ -621,19 +649,26 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "query_params", "nullable": false, "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, "type": null }, + { + "content": "input:filter_options", + "kind": "bytes_in", + "name": "options", + "nullable": false, + "type": null + }, { "content": null, "kind": "out_handle", @@ -664,8 +699,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -714,8 +747,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -742,12 +773,26 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, "type": null }, + { + "content": "json_array_names", + "kind": "bytes_in", + "name": "columns", + "nullable": false, + "type": null + }, + { + "content": "input:insert_options", + "kind": "bytes_in", + "name": "options", + "nullable": false, + "type": null + }, { "content": "document:filter_result", "kind": "out_handle", @@ -904,8 +949,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -932,7 +975,7 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, @@ -966,6 +1009,13 @@ export const FUNCTION_SPECS: Readonly> = { "nullable": false, "type": "uint32" }, + { + "content": "input:insert_options", + "kind": "bytes_in", + "name": "options", + "nullable": false, + "type": null + }, { "content": "document:batch", "kind": "out_handle", @@ -1003,8 +1053,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", - "CHS_DECLINED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -1031,7 +1079,7 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, @@ -1044,6 +1092,13 @@ export const FUNCTION_SPECS: Readonly> = { "nullable": false, "type": null }, + { + "content": "input:insert_options", + "kind": "bytes_in", + "name": "options", + "nullable": false, + "type": null + }, { "content": "document:row", "kind": "out_handle", @@ -1213,7 +1268,7 @@ export const FUNCTION_SPECS: Readonly> = { "type": null }, { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, @@ -1256,7 +1311,6 @@ export const FUNCTION_SPECS: Readonly> = { "cls": "api", "mayReturn": [ "CHS_OK", - "CHS_REJECTED", "CHS_INVALID_ARGUMENT", "CHS_INTERNAL" ], @@ -1401,7 +1455,7 @@ export const FUNCTION_SPECS: Readonly> = { ], "params": [ { - "content": "json_object_string_values", + "content": "json_object_byte_values", "kind": "bytes_in", "name": "settings", "nullable": false, @@ -1507,6 +1561,7 @@ export const CAMEL_NAMES: Readonly> = { "chs_error_column": "errorColumn", "chs_error_free": "errorFree", "chs_error_message": "errorMessage", + "chs_error_reason": "errorReason", "chs_error_status": "errorStatus", "chs_filter_create": "filterCreate", "chs_filter_eval_block": "filterEvalBlock", @@ -1561,6 +1616,7 @@ export const SYMBOL = { "ERROR_COLUMN": "chs_error_column", "ERROR_FREE": "chs_error_free", "ERROR_MESSAGE": "chs_error_message", + "ERROR_REASON": "chs_error_reason", "ERROR_STATUS": "chs_error_status", "FILTER_CREATE": "chs_filter_create", "FILTER_EVAL_BLOCK": "chs_filter_eval_block", @@ -1667,6 +1723,7 @@ export const DESCRIBED_SYMBOLS: readonly string[] = [ "chs_error_column", "chs_error_free", "chs_error_message", + "chs_error_reason", "chs_error_status", "chs_filter_create", "chs_filter_eval_block", @@ -1705,7 +1762,7 @@ export const BUF_HANDLE: 'chs_buf' = 'chs_buf'; * chs_build_info().abi_fingerprint at loader step 4). */ export const ABI_VERSION = 2; -export const ABI_FINGERPRINT = "sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4"; +export const ABI_FINGERPRINT = "sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2"; /** * The description's stability: "unstable" while this generation is being designed, so * ABI_FINGERPRINT moves with every change, and "locked" after (spec/abi-v2/docs.md, rule diff --git a/ts/src/abi2/errmap.gen.ts b/ts/src/abi2/errmap.gen.ts index 405db5b5..b549908c 100644 --- a/ts/src/abi2/errmap.gen.ts +++ b/ts/src/abi2/errmap.gen.ts @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * spec/abi-v2/sdk.json's two error tables, as code: a CALL's chs_status -> one of * SchemaError/UnsupportedError/UsageError/InternalError (errors.status), and a LOADER diff --git a/ts/src/abi2/libc.gen.ts b/ts/src/abi2/libc.gen.ts index 8084caaa..ba17e2d1 100644 --- a/ts/src/abi2/libc.gen.ts +++ b/ts/src/abi2/libc.gen.ts @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * The libc primitives ts/src/abi2/libc.ts needs for plan §3.3's TS-trap fix: dlopen, * dlsym, dlerror, gnu_get_libc_version and strlen, declared through ffi-rs's define() diff --git a/ts/src/abi2/vocab.gen.ts b/ts/src/abi2/vocab.gen.ts index 2ae64244..7e75b5dc 100644 --- a/ts/src/abi2/vocab.gen.ts +++ b/ts/src/abi2/vocab.gen.ts @@ -1,4 +1,4 @@ -/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d60a681ea0e249ab9e2c83b50640ad9cef040261ddf61805106c93e2c68774e4) — DO NOT EDIT */ +/* GENERATED by scripts/abi-v1/gen.py --major 2 from spec/abi-v2/abi.json (CHS_ABI_FINGERPRINT sha256:d53239933966e15a3c363e631a94fbba1231d84c4300337115d3ae5377bb41c2) — DO NOT EDIT */ /* * Every vocabulary the description defines, with its numbers, spellings, facts and * fallbacks. A fact (a reason's `lossy`, a source's `isStored`, a verdict's `answered`) @@ -216,6 +216,8 @@ export const Source = { DefaultExprUnsupported: "default_expr_unsupported", DefaultVolatileUnresolved: "default_volatile_unresolved", DefaultPending: "default_pending", + DefaultDeferred: "default_deferred", + DefaultOverride: "default_override", } as const; /** value_src: a listed value, or its unknown(n) member, any other string, kept verbatim (rule r3); `sourceKnown` tells them apart. */ export type Source = (typeof Source)[keyof typeof Source] | Unknown; @@ -232,9 +234,11 @@ const SOURCE_ENTRIES: Readonly; + +const REFUSAL_SITE_ENTRIES: Readonly>>> = { + "call": {}, + "body_read": {}, + "row": {}, + "body_step": {}, +}; +/** Whose per-value facts an unknown(n) value reports (rule r3), or null when the description names none. Never the value an unlisted one is read as. */ +export const REFUSAL_SITE_FALLBACK: RefusalSite | null = null; + +/** Whether the description lists `value`: false for exactly the vocabulary's unknown(n) members (rule r3). */ +export function refusalSiteKnown(value: string): boolean { + return Object.hasOwn(REFUSAL_SITE_ENTRIES, value); +} + +/** The value as this vocabulary's: a listed value, or its unknown(n) member carrying the raw string (rule r3). Never a substitute. */ +export function refusalSiteOf(value: string): RefusalSite { + return value; +} + /** discover_query_param: the description's own values. */ export const DiscoverQueryParam = { Database: "database", @@ -439,11 +472,41 @@ export function defaultKindOf(value: string): DefaultKind { return value; } +/** default_support: the description's own values. */ +export const DefaultSupport = { + Computed: "computed", + Generated: "generated", + Deferred: "deferred", + Declined: "declined", +} as const; +/** default_support: a listed value, or its unknown(n) member, any other string, kept verbatim (rule r3); `defaultSupportKnown` tells them apart. */ +export type DefaultSupport = (typeof DefaultSupport)[keyof typeof DefaultSupport] | Unknown; + +const DEFAULT_SUPPORT_ENTRIES: Readonly>>> = { + "computed": {}, + "generated": {}, + "deferred": {}, + "declined": {}, +}; +/** Whose per-value facts an unknown(n) value reports (rule r3), or null when the description names none. Never the value an unlisted one is read as. */ +export const DEFAULT_SUPPORT_FALLBACK: DefaultSupport | null = null; + +/** Whether the description lists `value`: false for exactly the vocabulary's unknown(n) members (rule r3). */ +export function defaultSupportKnown(value: string): boolean { + return Object.hasOwn(DEFAULT_SUPPORT_ENTRIES, value); +} + +/** The value as this vocabulary's: a listed value, or its unknown(n) member carrying the raw string (rule r3). Never a substitute. */ +export function defaultSupportOf(value: string): DefaultSupport { + return value; +} + /** declined_tier: the description's own values. */ export const DeclinedTier = { Predicate: "predicate", PredicateUnflipped: "predicate-unflipped", ResultContent: "result-content", + ServerRefuses: "server-refuses", } as const; /** declined_tier: a listed value, or its unknown(n) member, any other string, kept verbatim (rule r3); `declinedTierKnown` tells them apart. */ export type DeclinedTier = (typeof DeclinedTier)[keyof typeof DeclinedTier] | Unknown; @@ -452,6 +515,7 @@ const DECLINED_TIER_ENTRIES: Readonly; + +const POSTURE_ENTRIES: Readonly>>> = { + "create": {}, + "load": {}, +}; +/** Whose per-value facts an unknown(n) value reports (rule r3), or null when the description names none. Never the value an unlisted one is read as. */ +export const POSTURE_FALLBACK: Posture | null = null; + +/** Whether the description lists `value`: false for exactly the vocabulary's unknown(n) members (rule r3). */ +export function postureKnown(value: string): boolean { + return Object.hasOwn(POSTURE_ENTRIES, value); +} + +/** The value as this vocabulary's: a listed value, or its unknown(n) member carrying the raw string (rule r3). Never a substitute. */ +export function postureOf(value: string): Posture { + return value; +} + /** The document groups a batch call asks for (`CHS_DOC_*`), as bit flags. */ export const DocFlags = { Values: 1, @@ -516,13 +605,16 @@ export const DESCRIBED_VOCABULARIES: Readonly Date: Mon, 19 Oct 2026 15:47:56 -0400 Subject: [PATCH 02/24] go (spec batch 2): the options documents, the b64 value form, the new call arguments WithInsertableColumns, WithColumnDefaults, WithVerdictBasis, WithDeferDefaults and WithInsertOption build insert_options for Row, Rows, ParseBlock and Filter.Rows; WithParamEncoding and WithFilterOption build filter_options; WithPosture and WithSchemaOption build schema_options. WithColumns now applies to Filter.Rows. A settings, query-parameter, defaults or server-settings value that is not UTF-8 is the {"b64": ...} object form instead of a refusal. Co-Authored-By: Claude Sonnet 5.5 --- go/chtypes/capture_batch_test.go | 6 +- go/chtypes/goldens_v1_runner_test.go | 8 +- go/chtypes/library.go | 9 +- go/chtypes/options.go | 210 +++++++++++++++++- go/chtypes/rows_pool_test.go | 2 +- go/chtypes/schema.go | 38 ++-- go/chtypes/server.go | 16 +- go/chtypes/settings.go | 135 ++++++++--- go/chtypes/settings_test.go | 96 +++++++- .../abi2/conformance_lifecycle_test.go | 20 +- 10 files changed, 458 insertions(+), 82 deletions(-) diff --git a/go/chtypes/capture_batch_test.go b/go/chtypes/capture_batch_test.go index f5ccd738..0e313623 100644 --- a/go/chtypes/capture_batch_test.go +++ b/go/chtypes/capture_batch_test.go @@ -52,7 +52,7 @@ func TestCaptureBatchDocuments(t *testing.T) { defer sch.Close() body := captureBody() for _, flags := range []DocFlags{0, DocAll} { - docBuf, exportBuf, cerr := sch.lib.tbl.PreviewBatch(sch.h, int32(JSONEachRow), body, nil, nil, nil, int32(ExportNone), uint32(flags)) + docBuf, exportBuf, cerr := sch.lib.tbl.PreviewBatch(sch.h, int32(JSONEachRow), body, nil, nil, nil, int32(ExportNone), uint32(flags), nil) if cerr != nil { t.Fatal(callError(cerr)) } @@ -67,12 +67,12 @@ func TestCaptureBatchDocuments(t *testing.T) { // The third document is the shape a filtering caller gets: a filter // attached, so the document also carries rows_passed, rows_cut and one // row_spans entry per row, and an export asked for. - flt, cerr := sch.lib.tbl.FilterCreate(sch.h, []byte("small < {m:Int8}"), []byte(`{"m":"50"}`), nil) + flt, cerr := sch.lib.tbl.FilterCreate(sch.h, []byte("small < {m:Int8}"), []byte(`{"m":"50"}`), nil, nil) if cerr != nil { t.Fatal(callError(cerr)) } defer flt.Close() - docBuf, exportBuf, cerr := sch.lib.tbl.PreviewBatch(sch.h, int32(JSONEachRow), body, nil, nil, flt, int32(JSONCompactEachRow), 0) + docBuf, exportBuf, cerr := sch.lib.tbl.PreviewBatch(sch.h, int32(JSONEachRow), body, nil, nil, flt, int32(JSONCompactEachRow), 0, nil) if cerr != nil { t.Fatal(callError(cerr)) } diff --git a/go/chtypes/goldens_v1_runner_test.go b/go/chtypes/goldens_v1_runner_test.go index 766a76d2..43f4eb18 100644 --- a/go/chtypes/goldens_v1_runner_test.go +++ b/go/chtypes/goldens_v1_runner_test.go @@ -428,7 +428,7 @@ func runGoldensCase(t *testing.T, lib *Library, c goldensCase, rec map[string]an } doc = tbl.Take(buf) case "preview_row": - buf, ce := tbl.PreviewRow(sch, c.Body.Format, c.Body.Body, c.Body.Settings, c.Body.Columns) + buf, ce := tbl.PreviewRow(sch, c.Body.Format, c.Body.Body, c.Body.Settings, c.Body.Columns, nil) if ce != nil { stopAt(rec, "call", ce) return @@ -442,7 +442,7 @@ func runGoldensCase(t *testing.T, lib *Library, c goldensCase, rec map[string]an if c.Body.DocFlags != nil { df = *c.Body.DocFlags } - dbuf, ebuf, ce := tbl.PreviewBatch(sch, c.Body.Format, c.Body.Body, c.Body.Settings, c.Body.Columns, nil, ef, uint32(df)) + dbuf, ebuf, ce := tbl.PreviewBatch(sch, c.Body.Format, c.Body.Body, c.Body.Settings, c.Body.Columns, nil, ef, uint32(df), nil) if ce != nil { stopAt(rec, "call", ce) return @@ -455,13 +455,13 @@ func runGoldensCase(t *testing.T, lib *Library, c goldensCase, rec map[string]an } } case "filter_eval_body": - f, ce := tbl.FilterCreate(sch, c.Filter.Expr, c.Filter.QueryParams, c.Filter.Settings) + f, ce := tbl.FilterCreate(sch, c.Filter.Expr, c.Filter.QueryParams, c.Filter.Settings, nil) if ce != nil { stopAt(rec, "filter_create", ce) return } defer f.Close() - buf, ce := tbl.FilterEvalBody(f, c.Body.Format, c.Body.Body, c.Body.Settings) + buf, ce := tbl.FilterEvalBody(f, c.Body.Format, c.Body.Body, c.Body.Settings, c.Body.Columns, nil) if ce != nil { stopAt(rec, "call", ce) return diff --git a/go/chtypes/library.go b/go/chtypes/library.go index 21bbae1d..ff78a32a 100644 --- a/go/chtypes/library.go +++ b/go/chtypes/library.go @@ -225,8 +225,13 @@ func (l *Library) CompileTable(createTable string, opts ...CompileOption) (*Sche defer c.server.g.leave() server = c.server.h } - // No schema option is defined yet: length 0, which the library reads as {}. - h, cerr := l.tbl.SchemaCreate(server, []byte(createTable), settings, nil) + // The options document (posture, and any member passed through), or length + // 0, which the library reads as {}. + options, err := c.schemaOpts.json("schema options") + if err != nil { + return nil, err + } + h, cerr := l.tbl.SchemaCreate(server, []byte(createTable), settings, options) if cerr != nil { return nil, callError(cerr) } diff --git a/go/chtypes/options.go b/go/chtypes/options.go index 7c4ddb91..e6d32e24 100644 --- a/go/chtypes/options.go +++ b/go/chtypes/options.go @@ -1,5 +1,10 @@ package chtypes +import ( + "encoding/base64" + "unicode/utf8" +) + // options.go — the call options (bindings-v1.md section 2, "The call options"). // One entry point per operation, with its optional inputs as Go functional // options. Each operation takes its own option interface, and an option @@ -16,6 +21,8 @@ type callConfig struct { docFlags DocFlags params map[string]string server *Server + // The three options documents (rule r1), built member by member. + schemaOpts, insertOpts, filterOpts optionDoc } func newCallConfig() *callConfig { @@ -51,12 +58,23 @@ type SettingsOption interface { EvalOption } -// ColumnsOption is an option that applies to the three calls that parse an -// INSERT body. +// ColumnsOption is an option that applies to the four calls that model an +// INSERT of a body: Row, Rows, ParseBlock and Filter.Rows. type ColumnsOption interface { RowOption RowsOption BlockOption + EvalOption +} + +// InsertOption is an option of the insert_options document: it applies to +// the four calls that model an INSERT of a body, Row, Rows, ParseBlock and +// Filter.Rows, and means the same at each. +type InsertOption interface { + RowOption + RowsOption + BlockOption + EvalOption } // The options are small named types, one per option, rather than closures: a @@ -90,12 +108,11 @@ func (o timezoneOpt) applyEval(c *callConfig) { o.apply(c) } type columnsOpt []string -func (o columnsOpt) apply(c *callConfig) { c.columns, c.haveColumns = o, true } -func (o columnsOpt) applyRow(c *callConfig) { o.apply(c) } -func (o columnsOpt) applyRows(c *callConfig) { o.apply(c) } -func (o columnsOpt) applyBlock(c *callConfig) { - o.apply(c) -} +func (o columnsOpt) apply(c *callConfig) { c.columns, c.haveColumns = o, true } +func (o columnsOpt) applyRow(c *callConfig) { o.apply(c) } +func (o columnsOpt) applyRows(c *callConfig) { o.apply(c) } +func (o columnsOpt) applyBlock(c *callConfig) { o.apply(c) } +func (o columnsOpt) applyEval(c *callConfig) { o.apply(c) } type rowFilterOpt struct{ f *Filter } @@ -133,7 +150,8 @@ func WithSessionTimezone(name string) SettingsOption { return timezoneOpt(name) } -// WithColumns sets the INSERT column list. +// WithColumns sets the INSERT column list. It applies to every call that models +// an INSERT of a body, Filter.Rows included. func WithColumns(names []string) ColumnsOption { return columnsOpt(names) } @@ -233,3 +251,177 @@ func evalConfig(opts []EvalOption) *callConfig { } return c } + +// ---- the options documents (rule r1). Each member is its own option, and a +// key the binding has no name for goes through WithInsertOption, WithFilterOption +// or WithSchemaOption as raw JSON. The binding builds the document from the +// members the caller set, sends nothing when none was set, and never validates a +// member: the library refuses a key it does not know, or a value that is not one +// of the member's, with CHS_INVALID_ARGUMENT, naming it. + +// VerdictBasis is insert_options' verdict_basis: what row a filter verdict answers +// for. +type VerdictBasis string + +const ( + // VerdictBasisInput is the default: the verdict answers for the input row. + VerdictBasisInput VerdictBasis = "input" + // VerdictBasisExport answers for the row the export would store. A build + // that does not implement it declines it. + VerdictBasisExport VerdictBasis = "export" +) + +// DeferDefaults is insert_options' defer_defaults. +type DeferDefaults string + +const ( + // DeferDefaultsNone is the default: no DEFAULT is deferred. + DeferDefaultsNone DeferDefaults = "none" + // DeferDefaultsServer defers to the server each DEFAULT the build would + // otherwise decline. A build that does not implement it declines it. + DeferDefaultsServer DeferDefaults = "server" +) + +// ParamEncoding is filter_options' param_encoding: how a query parameter's value +// is read. +type ParamEncoding string + +const ( + // ParamEncodingEscaped is the default: ClickHouse's escaped-text reader. + ParamEncodingEscaped ParamEncoding = "escaped" + // ParamEncodingRaw takes the bytes as the value. A build that does not + // implement it declines it. + ParamEncodingRaw ParamEncoding = "raw" +) + +// ColumnDefault is one entry of insert_options' column_defaults: the value the +// call uses in a cell the server would fill from the column's DEFAULT. Both +// fields are byte strings: a Name or Value that is not valid UTF-8 is sent in +// its _b64 form. +type ColumnDefault struct { + Name string + Value string +} + +type insertableColumnsOpt []string + +func (o insertableColumnsOpt) apply(c *callConfig) { + c.insertOpts.set("insertable_columns", nameObjects(o)) +} +func (o insertableColumnsOpt) applyRow(c *callConfig) { o.apply(c) } +func (o insertableColumnsOpt) applyRows(c *callConfig) { o.apply(c) } +func (o insertableColumnsOpt) applyBlock(c *callConfig) { o.apply(c) } +func (o insertableColumnsOpt) applyEval(c *callConfig) { o.apply(c) } + +type columnDefaultsOpt []ColumnDefault + +func (o columnDefaultsOpt) apply(c *callConfig) { + list := make([]map[string]string, 0, len(o)) + for _, d := range o { + e := nameObjects([]string{d.Name})[0] + if utf8.ValidString(d.Value) { + e["value"] = d.Value + } else { + e["value_b64"] = base64.StdEncoding.EncodeToString([]byte(d.Value)) + } + list = append(list, e) + } + c.insertOpts.set("column_defaults", list) +} +func (o columnDefaultsOpt) applyRow(c *callConfig) { o.apply(c) } +func (o columnDefaultsOpt) applyRows(c *callConfig) { o.apply(c) } +func (o columnDefaultsOpt) applyBlock(c *callConfig) { o.apply(c) } +func (o columnDefaultsOpt) applyEval(c *callConfig) { o.apply(c) } + +// insertMember is any other member of insert_options: a key and its value. +type insertMember struct { + key string + v any +} + +func (o insertMember) apply(c *callConfig) { + if o.key != "" { // the zero insertMember is an option that sets nothing + c.insertOpts.set(o.key, o.v) + } +} +func (o insertMember) applyRow(c *callConfig) { o.apply(c) } +func (o insertMember) applyRows(c *callConfig) { o.apply(c) } +func (o insertMember) applyBlock(c *callConfig) { o.apply(c) } +func (o insertMember) applyEval(c *callConfig) { o.apply(c) } + +type filterMember struct { + key string + v any +} + +func (o filterMember) applyFilter(c *callConfig) { c.filterOpts.set(o.key, o.v) } + +type schemaMember struct { + key string + v any +} + +func (o schemaMember) applyCompile(c *callConfig) { c.schemaOpts.set(o.key, o.v) } + +// WithInsertableColumns sets insert_options' insertable_columns: the columns an +// INSERT may supply (the server's column-level INSERT grant), checked on the +// INSERT's header before any byte of the body is read. nil leaves the member out +// (every column); a non-nil empty slice is sent as [] (no column). +func WithInsertableColumns(names []string) InsertOption { + if names == nil { + return insertMember{} + } + return insertableColumnsOpt(names) +} + +// WithColumnDefaults sets insert_options' column_defaults: per-call DEFAULT +// overrides. nil leaves the member out; a non-nil empty slice is sent as []. +func WithColumnDefaults(defaults []ColumnDefault) InsertOption { + if defaults == nil { + return insertMember{} + } + return columnDefaultsOpt(defaults) +} + +// WithVerdictBasis sets insert_options' verdict_basis. +func WithVerdictBasis(b VerdictBasis) InsertOption { + return insertMember{"verdict_basis", string(b)} +} + +// WithDeferDefaults sets insert_options' defer_defaults. +func WithDeferDefaults(d DeferDefaults) InsertOption { + return insertMember{"defer_defaults", string(d)} +} + +// WithInsertOption sets a member of insert_options this binding has no name +// for, with its value as JSON text, passed through as given. The library +// refuses a key it does not know (CHS_INVALID_ARGUMENT, naming it); a value that +// is not valid JSON is a *UsageError raised before any call. +func WithInsertOption(key string, valueJSON []byte) InsertOption { + return insertMember{key, rawMember(valueJSON)} +} + +// WithParamEncoding sets filter_options' param_encoding. +func WithParamEncoding(e ParamEncoding) FilterOption { + return filterMember{"param_encoding", string(e)} +} + +// WithFilterOption sets a member of filter_options this binding has no name +// for, as WithInsertOption does for insert_options. +func WithFilterOption(key string, valueJSON []byte) FilterOption { + return filterMember{key, rawMember(valueJSON)} +} + +// WithPosture sets schema_options' posture: how the statement is compiled, +// PostureCreate (the default) as a server's CREATE TABLE, or PostureLoad as a +// server loads the stored definition of a table it holds. The caller's +// obligation under PostureLoad is spec/abi-v2/docs.md's (input:schema_options). +func WithPosture(p Posture) CompileOption { + return schemaMember{"posture", string(p)} +} + +// WithSchemaOption sets a member of schema_options this binding has no name +// for, as WithInsertOption does for insert_options. +func WithSchemaOption(key string, valueJSON []byte) CompileOption { + return schemaMember{key, rawMember(valueJSON)} +} diff --git a/go/chtypes/rows_pool_test.go b/go/chtypes/rows_pool_test.go index 17ccddc5..7049d668 100644 --- a/go/chtypes/rows_pool_test.go +++ b/go/chtypes/rows_pool_test.go @@ -112,7 +112,7 @@ func TestStubTakeIntoMatchesTake(t *testing.T) { } defer schema.Close() call := func() *abi2.Buf { - doc, exp, cerr := lib.tbl.PreviewBatch(schema.h, int32(JSONEachRow), []byte(`{"x":1}`), nil, nil, nil, int32(ExportNone), 0) + doc, exp, cerr := lib.tbl.PreviewBatch(schema.h, int32(JSONEachRow), []byte(`{"x":1}`), nil, nil, nil, int32(ExportNone), 0, nil) if cerr != nil { t.Fatal(cerr) } diff --git a/go/chtypes/schema.go b/go/chtypes/schema.go index 50fef02e..6ddd40b3 100644 --- a/go/chtypes/schema.go +++ b/go/chtypes/schema.go @@ -95,7 +95,7 @@ func (s *Schema) Describe() (SchemaDescription, error) { // Row previews one row (chs_preview_row). func (s *Schema) Row(format Format, body []byte, opts ...RowOption) (RowResult, error) { c := rowConfig(opts) - settings, columns, err := callInputs(c) + settings, columns, options, err := callInputs(c) if err != nil { return RowResult{}, err } @@ -104,7 +104,7 @@ func (s *Schema) Row(format Format, body []byte, opts ...RowOption) (RowResult, } defer s.g.leave() raw, err := s.lib.document(func() (*abi2.Buf, *abi2.CallError) { - return s.lib.tbl.PreviewRow(s.h, int32(format), body, settings, columns) + return s.lib.tbl.PreviewRow(s.h, int32(format), body, settings, columns, options) }) if err != nil { return RowResult{}, err @@ -116,7 +116,7 @@ func (s *Schema) Row(format Format, body []byte, opts ...RowOption) (RowResult, // attached and an export of the accepted rows. func (s *Schema) Rows(format Format, body []byte, opts ...RowsOption) (BatchResult, error) { c := rowsConfig(opts) - settings, columns, err := callInputs(c) + settings, columns, options, err := callInputs(c) if err != nil { return BatchResult{}, err } @@ -132,7 +132,7 @@ func (s *Schema) Rows(format Format, body []byte, opts ...RowsOption) (BatchResu defer c.filter.g.leave() fh = c.filter.h } - docBuf, exportBuf, cerr := s.lib.tbl.PreviewBatch(s.h, int32(format), body, settings, columns, fh, int32(c.export), uint32(c.docFlags)) + docBuf, exportBuf, cerr := s.lib.tbl.PreviewBatch(s.h, int32(format), body, settings, columns, fh, int32(c.export), uint32(c.docFlags), options) if cerr != nil { return BatchResult{}, callError(cerr) } @@ -175,11 +175,15 @@ func (s *Schema) CompileFilter(expr string, opts ...FilterOption) (*Filter, erro if err != nil { return nil, err } + options, err := c.filterOpts.json("filter options") + if err != nil { + return nil, err + } if err := s.enter(); err != nil { return nil, err } defer s.g.leave() - h, cerr := s.lib.tbl.FilterCreate(s.h, []byte(expr), params, settings) + h, cerr := s.lib.tbl.FilterCreate(s.h, []byte(expr), params, settings, options) if cerr != nil { return nil, callError(cerr) } @@ -190,7 +194,7 @@ func (s *Schema) CompileFilter(expr string, opts ...FilterOption) (*Filter, erro // evaluated over it with no re-parse. func (s *Schema) ParseBlock(format Format, body []byte, opts ...BlockOption) (*Block, error) { c := blockConfig(opts) - settings, columns, err := callInputs(c) + settings, columns, options, err := callInputs(c) if err != nil { return nil, err } @@ -198,23 +202,26 @@ func (s *Schema) ParseBlock(format Format, body []byte, opts ...BlockOption) (*B return nil, err } defer s.g.leave() - h, cerr := s.lib.tbl.BlockCreate(s.h, int32(format), body, settings, columns) + h, cerr := s.lib.tbl.BlockCreate(s.h, int32(format), body, settings, columns, options) if cerr != nil { return nil, callError(cerr) } return newBlock(s.lib, h), nil } -// callInputs serializes the settings (the per-call zone written into them) and -// the column list of a call. -func callInputs(c *callConfig) (settings, columns []byte, err error) { +// callInputs serializes the settings (the per-call zone written into them), the +// column list and the insert options document of a call that models an INSERT. +func callInputs(c *callConfig) (settings, columns, options []byte, err error) { if settings, err = callSettings(c.settings, c.timezone); err != nil { - return nil, nil, err + return nil, nil, nil, err } if columns, err = columnsJSON(c.columns, c.haveColumns); err != nil { - return nil, nil, err + return nil, nil, nil, err } - return settings, columns, nil + if options, err = c.insertOpts.json("insert options"); err != nil { + return nil, nil, nil, err + } + return settings, columns, options, nil } // Filter is a compiled filter. It holds its schema inside the library. @@ -252,9 +259,10 @@ func (f *Filter) enter() error { // Rows evaluates the filter over a body (chs_filter_eval_body). The settings // are the body's parse settings only; the filter's own zone governs its WHERE. +// WithColumns and the insert options model the same INSERT as on Schema.Rows. func (f *Filter) Rows(format Format, body []byte, opts ...EvalOption) (FilterResult, error) { c := evalConfig(opts) - settings, err := callSettings(c.settings, c.timezone) + settings, columns, options, err := callInputs(c) if err != nil { return FilterResult{}, err } @@ -263,7 +271,7 @@ func (f *Filter) Rows(format Format, body []byte, opts ...EvalOption) (FilterRes } defer f.g.leave() raw, err := f.lib.document(func() (*abi2.Buf, *abi2.CallError) { - return f.lib.tbl.FilterEvalBody(f.h, int32(format), body, settings) + return f.lib.tbl.FilterEvalBody(f.h, int32(format), body, settings, columns, options) }) if err != nil { return FilterResult{}, err diff --git a/go/chtypes/server.go b/go/chtypes/server.go index 593daf30..d133dad8 100644 --- a/go/chtypes/server.go +++ b/go/chtypes/server.go @@ -42,25 +42,31 @@ type ServerProfile struct { // makes normative for macros ("absent = unknown, present even {} = complete"). type serverProfileDoc struct { Timezone string `json:"timezone,omitzero"` - Settings map[string]string `json:"settings,omitzero"` + Settings map[string]any `json:"settings,omitzero"` Macros map[string]string `json:"macros,omitzero"` } // serverProfileJSON is the profile document, with the stock encoder settings // use (keys in sorted order, no HTML escaping, no value rewritten). func serverProfileJSON(p ServerProfile) ([]byte, error) { - // A non-UTF-8 string would be replaced with U+FFFD by the encoder, and the + // A non-UTF-8 zone would be replaced with U+FFFD by the encoder, and the // library would see a different profile: refuse it instead. if !utf8.ValidString(p.Timezone) { return nil, usageError("server profile: the timezone is not valid UTF-8, and %s", nonUTF8Hint) } - if err := checkUTF8Map("server profile settings", p.Settings); err != nil { - return nil, err + // A setting's value that is not valid UTF-8 is the object form {"b64": ...}; + // a setting's name, and a macro's name or value, cannot be carried. + var settings map[string]any + if p.Settings != nil { + var err error + if settings, err = byteValues("server profile settings", p.Settings); err != nil { + return nil, err + } } if err := checkUTF8Map("server profile macros", p.Macros); err != nil { return nil, err } - return marshalNoEscape(serverProfileDoc(p)) + return marshalNoEscape(serverProfileDoc{Timezone: p.Timezone, Settings: settings, Macros: p.Macros}) } // Server is one ClickHouse server (chs_server), made by Library.NewServer and diff --git a/go/chtypes/settings.go b/go/chtypes/settings.go index 5adad9fd..b2cde551 100644 --- a/go/chtypes/settings.go +++ b/go/chtypes/settings.go @@ -18,28 +18,75 @@ import ( // sessionTimezoneKey is the settings key that carries the per-call zone. const sessionTimezoneKey = "session_timezone" -// stringMapJSON serializes a map of strings as a JSON object, or returns nil -// for an empty map (the ABI reads length 0 as "none"). A key or value that is -// not valid UTF-8 is a UsageError naming what (the input) and the key: stock -// encoding/json would replace it with U+FFFD, and the library would then see a -// different value with no error raised. +// stringMapJSON serializes a map of strings as a JSON object of byte values +// (the content json_object_byte_values), or returns nil for an empty map (the +// ABI reads length 0 as "none"). A value that is valid UTF-8 is a JSON string, +// and any other value is the object {"b64": ""} (spec/abi-v2/docs.md, +// "Input values"), so no value is ever rewritten. A key that is not valid UTF-8 +// is a UsageError naming what (the input) and the key: a JSON object cannot +// carry a key's bytes, and stock encoding/json would replace it with U+FFFD, so +// the library would see a different key with no error raised. func stringMapJSON(what string, m map[string]string) ([]byte, error) { if len(m) == 0 { return nil, nil } - if err := checkUTF8Map(what, m); err != nil { + doc, err := byteValues(what, m) + if err != nil { return nil, err } - return marshalNoEscape(m) + return marshalNoEscape(doc) } -// nonUTF8Hint is the tail of every refusal: a JSON input cannot carry the -// bytes in this ABI generation (spec/abi-v2/docs.md, "Input values"). -const nonUTF8Hint = "this ABI generation cannot carry a non-UTF-8 value in a JSON input; write the bytes as unhex('') in the expression instead" +// b64Value is the object form of a value whose bytes are not valid UTF-8. +type b64Value struct { + B64 string `json:"b64"` +} + +// byteValue is one value of json_object_byte_values: the string itself when it is +// valid UTF-8, else the object form. +func byteValue(v string) any { + if utf8.ValidString(v) { + return v + } + return b64Value{B64: base64.StdEncoding.EncodeToString([]byte(v))} +} + +// byteValues is the document of a map of byte values; it refuses a key that is +// not valid UTF-8, the first in sorted order so the message does not depend on +// map iteration. +func byteValues(what string, m map[string]string) (map[string]any, error) { + if err := checkUTF8Keys(what, m); err != nil { + return nil, err + } + out := make(map[string]any, len(m)) + for k, v := range m { + out[k] = byteValue(v) + } + return out, nil +} + +// nonUTF8Hint is the tail of every refusal: a JSON input cannot carry these +// bytes (spec/abi-v2/docs.md, "Input values"). +const nonUTF8Hint = "a JSON input cannot carry these bytes here (only a settings or query-parameter value has the {\"b64\": ...} object form)" -// checkUTF8Map refuses the first key (in sorted order, so the message does not -// depend on map iteration) or value that is not valid UTF-8. The key appears -// in the message escaped, never as raw bytes. +// checkUTF8Keys refuses the first key (in sorted order) that is not valid +// UTF-8. The key appears in the message escaped, never as raw bytes. +func checkUTF8Keys(what string, m map[string]string) error { + var bad []string + for k := range m { + if !utf8.ValidString(k) { + bad = append(bad, k) + } + } + if len(bad) == 0 { + return nil + } + sort.Strings(bad) + return usageError("%s: the key %s is not valid UTF-8, and %s", what, strconv.QuoteToASCII(bad[0]), nonUTF8Hint) +} + +// checkUTF8Map refuses the first key or value (in sorted order) that is not +// valid UTF-8, for an input with no object form for a value (a server's macros). func checkUTF8Map(what string, m map[string]string) error { var bad []string for k, v := range m { @@ -96,12 +143,9 @@ func callSettings(settings map[string]string, timezone *string) ([]byte, error) if _, dup := settings[sessionTimezoneKey]; dup { return nil, usageError("the zone is given twice: a session timezone option and a %s settings key", sessionTimezoneKey) } - if err := checkUTF8Map("settings", settings); err != nil { + if err := checkUTF8Keys("settings", settings); err != nil { return nil, err } - if !utf8.ValidString(*timezone) { - return nil, usageError("the zone: the value is not valid UTF-8, and %s", nonUTF8Hint) - } merged := make(map[string]string, len(settings)+1) for k, v := range settings { merged[k] = v @@ -110,13 +154,9 @@ func callSettings(settings map[string]string, timezone *string) ([]byte, error) return stringMapJSON("settings", merged) } -// columnsJSON serializes the INSERT column list as a JSON array of name -// objects: "name" for a valid UTF-8 name, "name_b64" (standard base64, with -// padding) for any other bytes. -func columnsJSON(names []string, have bool) ([]byte, error) { - if !have { - return nil, nil - } +// nameObjects is a list of names as name objects: "name" for a valid UTF-8 +// name, "name_b64" (standard base64, with padding) for any other bytes. +func nameObjects(names []string) []map[string]string { list := make([]map[string]string, 0, len(names)) for _, n := range names { if utf8.ValidString(n) { @@ -125,5 +165,50 @@ func columnsJSON(names []string, have bool) ([]byte, error) { list = append(list, map[string]string{"name_b64": base64.StdEncoding.EncodeToString([]byte(n))}) } } - return marshalNoEscape(list) + return list +} + +// columnsJSON serializes the INSERT column list as a JSON array of name +// objects (nameObjects). +func columnsJSON(names []string, have bool) ([]byte, error) { + if !have { + return nil, nil + } + return marshalNoEscape(nameObjects(names)) +} + +// optionDoc is an options document under construction: the members a caller +// set, by key. An option the binding has no name for is set with its raw JSON +// value and passed through as given (rule r1: the library refuses a key it does +// not know, naming it; the binding never drops or rewrites one). Setting a key +// twice keeps the last value. +type optionDoc map[string]any + +func (d *optionDoc) set(key string, v any) { + if *d == nil { + *d = optionDoc{} + } + (*d)[key] = v +} + +// json is the document, or nil when no member was set (length 0 is "none", the +// document the call took before it had one). +func (d optionDoc) json(what string) ([]byte, error) { + if len(d) == 0 { + return nil, nil + } + for k, v := range d { + if raw, ok := v.(rawMember); ok && !json.Valid(raw) { + return nil, usageError("%s: the value for key %s is not valid JSON", what, strconv.QuoteToASCII(k)) + } + if !utf8.ValidString(k) { + return nil, usageError("%s: the key %s is not valid UTF-8, and %s", what, strconv.QuoteToASCII(k), nonUTF8Hint) + } + } + return marshalNoEscape(map[string]any(d)) } + +// rawMember is a member's value given as JSON text, passed through verbatim. +type rawMember json.RawMessage + +func (r rawMember) MarshalJSON() ([]byte, error) { return []byte(r), nil } diff --git a/go/chtypes/settings_test.go b/go/chtypes/settings_test.go index b81b9757..780dbbc6 100644 --- a/go/chtypes/settings_test.go +++ b/go/chtypes/settings_test.go @@ -41,8 +41,9 @@ func TestCallSettings(t *testing.T) { } } -// A non-UTF-8 key or value is refused before any call, naming the input and the -// key in escaped form; stock encoding/json would have replaced it with U+FFFD. +// A non-UTF-8 key, and a non-UTF-8 value where the input has no object form for +// one, is refused before any call, naming the input and the key in escaped form; +// stock encoding/json would have replaced it with U+FFFD. func TestNonUTF8InputIsRefused(t *testing.T) { z := "UTC" bad := "\xff" @@ -51,13 +52,10 @@ func TestNonUTF8InputIsRefused(t *testing.T) { call func() ([]byte, error) want []string }{ - {"settings value", func() ([]byte, error) { return callSettings(map[string]string{"k": bad}, nil) }, []string{"settings", `the value for key "k"`}}, {"settings key", func() ([]byte, error) { return callSettings(map[string]string{"k" + bad: "v"}, nil) }, []string{"settings", `the key "k\xff"`}}, - {"settings value beside a zone", func() ([]byte, error) { return callSettings(map[string]string{"k": bad}, &z) }, []string{"settings", `the value for key "k"`}}, - {"zone", func() ([]byte, error) { return callSettings(nil, &bad) }, []string{"the zone"}}, - {"query parameters value", func() ([]byte, error) { return stringMapJSON("query parameters", map[string]string{"p": bad}) }, []string{"query parameters", `the value for key "p"`}}, + {"settings key beside a zone", func() ([]byte, error) { return callSettings(map[string]string{"k" + bad: "v"}, &z) }, []string{"settings", `the key "k\xff"`}}, {"query parameters key", func() ([]byte, error) { return stringMapJSON("query parameters", map[string]string{bad: "v"}) }, []string{"query parameters", `the key "\xff"`}}, - {"default settings", func() ([]byte, error) { return stringMapJSON("default settings", map[string]string{"d": bad}) }, []string{"default settings", `"d"`}}, + {"default settings key", func() ([]byte, error) { return stringMapJSON("default settings", map[string]string{"d" + bad: "1"}) }, []string{"default settings", `"d\xff"`}}, {"server profile macro", func() ([]byte, error) { return serverProfileJSON(ServerProfile{Macros: map[string]string{"m": bad}}) }, []string{"server profile macros", `"m"`}}, {"server profile settings key", func() ([]byte, error) { return serverProfileJSON(ServerProfile{Settings: map[string]string{bad: "1"}}) }, []string{"server profile settings", "key"}}, {"server profile zone", func() ([]byte, error) { return serverProfileJSON(ServerProfile{Timezone: bad}) }, []string{"timezone"}}, @@ -70,7 +68,7 @@ func TestNonUTF8InputIsRefused(t *testing.T) { continue } msg := err.Error() - for _, w := range append(c.want, "not valid UTF-8", "unhex('')") { + for _, w := range append(c.want, "not valid UTF-8", "cannot carry") { if !strings.Contains(msg, w) { t.Errorf("%s: message %q lacks %q", c.name, msg, w) } @@ -81,6 +79,31 @@ func TestNonUTF8InputIsRefused(t *testing.T) { } } +// A settings or query-parameter value that is not valid UTF-8 is the object form +// {"b64": ...} (content json_object_byte_values), standard base64 with padding; +// a valid one stays a JSON string. The per-call zone is a value too. +func TestNonUTF8ValueIsTheB64Form(t *testing.T) { + z := "\xffZ" + for _, c := range []struct { + name string + got func() ([]byte, error) + want string + }{ + {"settings", func() ([]byte, error) { return callSettings(map[string]string{"k": "\xff\xfe", "a": "ok"}, nil) }, `{"a":"ok","k":{"b64":"//4="}}`}, + {"the zone", func() ([]byte, error) { return callSettings(nil, &z) }, `{"session_timezone":{"b64":"/1o="}}`}, + {"query parameters", func() ([]byte, error) { return stringMapJSON("query parameters", map[string]string{"p": "\xff"}) }, `{"p":{"b64":"/w=="}}`}, + {"defaults", func() ([]byte, error) { return stringMapJSON("default settings", map[string]string{"d": "\x00\xff"}) }, `{"d":{"b64":"AP8="}}`}, + {"server profile settings", func() ([]byte, error) { + return serverProfileJSON(ServerProfile{Settings: map[string]string{"s": "\xff"}}) + }, `{"settings":{"s":{"b64":"/w=="}}}`}, + } { + b, err := c.got() + if err != nil || string(b) != c.want { + t.Errorf("%s = %s, %v, want %s", c.name, b, err, c.want) + } + } +} + // Valid UTF-8 serializes byte for byte as before: non-ASCII is written as // itself, in keys and values, and U+FFFD that the caller really wrote stays. func TestValidNonASCIIInputIsUnchanged(t *testing.T) { @@ -272,3 +295,60 @@ func TestAFailedOpenUnlocksOnlyItsOwnRecordAndNeverALatchedOne(t *testing.T) { t.Errorf("the latched setup again = %v, want a no-op", err) } } + +func TestInsertOptionsDocument(t *testing.T) { + // No member set: no document (length 0). + if b, err := rowConfig(nil).insertOpts.json("insert options"); err != nil || b != nil { + t.Errorf("empty = %q, %v", b, err) + } + // A nil list sets nothing; a non-nil empty list is sent. + if b, _ := rowConfig([]RowOption{WithInsertableColumns(nil), WithColumnDefaults(nil)}).insertOpts.json("x"); b != nil { + t.Errorf("nil lists set a member: %s", b) + } + c := rowsConfig([]RowsOption{ + WithInsertableColumns([]string{"a", "\xff"}), + WithColumnDefaults([]ColumnDefault{{Name: "n", Value: "v"}, {Name: "\xfe", Value: "\xff"}}), + WithVerdictBasis(VerdictBasisExport), + WithDeferDefaults(DeferDefaultsServer), + WithInsertOption("future_member", []byte(`{"x":[1,2]}`)), + }) + b, err := c.insertOpts.json("insert options") + want := `{"column_defaults":[{"name":"n","value":"v"},{"name_b64":"/g==","value_b64":"/w=="}],"defer_defaults":"server","future_member":{"x":[1,2]},"insertable_columns":[{"name":"a"},{"name_b64":"/w=="}],"verdict_basis":"export"}` + if err != nil || string(b) != want { + t.Errorf("insert options = %s, %v, want %s", b, err, want) + } + // The same option applies to all four calls that model an INSERT. + for name, cfg := range map[string]*callConfig{ + "row": rowConfig([]RowOption{WithDeferDefaults(DeferDefaultsNone)}), + "block": blockConfig([]BlockOption{WithDeferDefaults(DeferDefaultsNone)}), + "eval": evalConfig([]EvalOption{WithDeferDefaults(DeferDefaultsNone), WithColumns([]string{"a"})}), + } { + if b, _ := cfg.insertOpts.json("x"); string(b) != `{"defer_defaults":"none"}` { + t.Errorf("%s: insert options = %s", name, b) + } + } + if c := evalConfig([]EvalOption{WithColumns([]string{"a"})}); !c.haveColumns || len(c.columns) != 1 { + t.Errorf("an eval does not take the column list: %+v", c) + } + // A value that is not JSON is misuse, raised before any call. + bad := rowConfig([]RowOption{WithInsertOption("k", []byte(`{`))}) + if _, err := bad.insertOpts.json("insert options"); err == nil { + t.Error("invalid JSON passed through") + } else if _, ok := err.(*UsageError); !ok { + t.Errorf("invalid JSON = %T, want *UsageError", err) + } +} + +func TestFilterAndSchemaOptionsDocuments(t *testing.T) { + f := filterConfig([]FilterOption{WithParamEncoding(ParamEncodingRaw), WithFilterOption("k", []byte(`1`))}) + if b, err := f.filterOpts.json("filter options"); err != nil || string(b) != `{"k":1,"param_encoding":"raw"}` { + t.Errorf("filter options = %s, %v", b, err) + } + c := compileConfig([]CompileOption{WithPosture(PostureLoad), WithSchemaOption("z", []byte(`"v"`))}) + if b, err := c.schemaOpts.json("schema options"); err != nil || string(b) != `{"posture":"load","z":"v"}` { + t.Errorf("schema options = %s, %v", b, err) + } + if b, _ := compileConfig(nil).schemaOpts.json("x"); b != nil { + t.Errorf("an empty schema options document = %s", b) + } +} diff --git a/go/internal/abi2/conformance_lifecycle_test.go b/go/internal/abi2/conformance_lifecycle_test.go index 267605ee..cf7fe93e 100644 --- a/go/internal/abi2/conformance_lifecycle_test.go +++ b/go/internal/abi2/conformance_lifecycle_test.go @@ -157,38 +157,38 @@ func callResolved(t *Table, fn string, a []resolved) (*InvokeResult, error) { o, ce := t.SchemaDescribe(a[0].schema()) return finishResult(t, fn, ce, map[string]*Buf{"out": o}) case "chs_preview_row": - if err := need(5); err != nil { + if err := need(6); err != nil { return nil, err } - o, ce := t.PreviewRow(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b()) + o, ce := t.PreviewRow(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b(), a[5].b()) return finishResult(t, fn, ce, map[string]*Buf{"out": o}) case "chs_preview_batch": - if err := need(8); err != nil { + if err := need(9); err != nil { return nil, err } - o, x, ce := t.PreviewBatch(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b(), a[5].filter(), int32(a[6].i()), uint32(a[7].i())) + o, x, ce := t.PreviewBatch(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b(), a[5].filter(), int32(a[6].i()), uint32(a[7].i()), a[8].b()) return finishResult(t, fn, ce, map[string]*Buf{"out": o, "out_export": x}) case "chs_filter_create": - if err := need(4); err != nil { + if err := need(5); err != nil { return nil, err } - h, ce := t.FilterCreate(a[0].schema(), a[1].b(), a[2].b(), a[3].b()) + h, ce := t.FilterCreate(a[0].schema(), a[1].b(), a[2].b(), a[3].b(), a[4].b()) r, err := finishResult(t, fn, ce, nil) if r != nil { r.Handle = h } return r, err case "chs_filter_eval_body": - if err := need(4); err != nil { + if err := need(6); err != nil { return nil, err } - o, ce := t.FilterEvalBody(a[0].filter(), int32(a[1].i()), a[2].b(), a[3].b()) + o, ce := t.FilterEvalBody(a[0].filter(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b(), a[5].b()) return finishResult(t, fn, ce, map[string]*Buf{"out": o}) case "chs_block_create": - if err := need(5); err != nil { + if err := need(6); err != nil { return nil, err } - h, ce := t.BlockCreate(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b()) + h, ce := t.BlockCreate(a[0].schema(), int32(a[1].i()), a[2].b(), a[3].b(), a[4].b(), a[5].b()) r, err := finishResult(t, fn, ce, nil) if r != nil { r.Handle = h From b787100bac2325776cb763f92c6b81a21f120320 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 15:50:48 -0400 Subject: [PATCH 03/24] go (spec batch 2): the decoders for the refusal, export and schema members RowResult, BatchResult, FilterResult and FilterRowError carry the reason, refused_at and refused_row members; the batch carries ExportColumns, ExportDeclinedReason and a Spans slice aligned with Rows (a nil entry for a row not exported); AtMergeEntry.Row is PartRow; the row-level UnsupportedSettings is removed; Column carries DefaultSupport, DefaultSupportReason and Reads; SchemaDescription carries Posture; a server settings value in the b64 form decodes to its bytes. Co-Authored-By: Claude Sonnet 5.5 --- go/chtypes/decode.go | 118 +++++++++++++++++------ go/chtypes/decode_batch.go | 129 ++++++++++++++++++++----- go/chtypes/decode_equivalence_test.go | 13 ++- go/chtypes/decode_reference_test.go | 29 +++--- go/chtypes/decode_test.go | 133 ++++++++++++++++++++++---- go/chtypes/reader_rules_test.go | 2 +- go/chtypes/results.go | 106 +++++++++++++++----- 7 files changed, 422 insertions(+), 108 deletions(-) diff --git a/go/chtypes/decode.go b/go/chtypes/decode.go index 0e3fc011..bb7bc6a0 100644 --- a/go/chtypes/decode.go +++ b/go/chtypes/decode.go @@ -162,12 +162,26 @@ func (r *reader) stringMap(m map[string]any, key, path string) map[string]string } out := make(map[string]string, len(om)) for k, e := range om { - s, ok := e.(string) - if !ok { - r.fail(path+"."+key+"."+k, "want a JSON string") + switch s := e.(type) { + case string: + out[k] = s + case map[string]any: + // The object form of a value whose bytes are not UTF-8: {"b64": ...}. + enc, ok := s["b64"].(string) + if !ok || len(s) != 1 { + r.fail(path+"."+key+"."+k, `want a JSON string or {"b64": "..."}`) + return nil + } + raw, err := base64.StdEncoding.DecodeString(enc) + if err != nil { + r.fail(path+"."+key+"."+k, "b64 is not standard base64: %v", err) + return nil + } + out[k] = string(raw) + default: + r.fail(path+"."+key+"."+k, `want a JSON string or {"b64": "..."}`) return nil } - out[k] = s } return out } @@ -259,6 +273,16 @@ func (r *reader) u64Of(v any, path string) uint64 { return u } +// optU64 reads an unsigned 64-bit integer that may be absent: nil when absent. +func (r *reader) optU64(m map[string]any, key, path string) *uint64 { + v := r.field(m, key) + if v == nil { + return nil + } + n := r.u64Of(v, path+"."+key) + return &n +} + func (r *reader) i32(m map[string]any, key, path string) int32 { v := r.field(m, key) if v == nil { @@ -361,6 +385,24 @@ func (r *reader) span(v any, path string) *Span { return &Span{Off: r.u64(m, "off", path), Len: r.u64(m, "len", path)} } +// rowSpans reads row_spans: one entry per row, a span for an exported row and +// JSON null for a row not exported (nil here). +func (r *reader) rowSpans(m map[string]any, key, path string) []*Span { + arr := r.array(r.field(m, key), path+"."+key) + if arr == nil { + return nil + } + out := make([]*Span, 0, len(arr)) + for i, e := range arr { + if e == nil { + out = append(out, nil) + continue + } + out = append(out, r.span(e, fmt.Sprintf("%s.%s[%d]", path, key, i))) + } + return out +} + func (r *reader) spans(m map[string]any, key, path string) []Span { arr := r.array(r.field(m, key), path+"."+key) if arr == nil { @@ -417,7 +459,7 @@ func (r *reader) atMerge(m map[string]any, key, path string) []AtMergeEntry { return nil } out = append(out, AtMergeEntry{ - Row: r.int(em, "row", p), + PartRow: r.int(em, "part_row", p), Reason: MergeReason(r.text(em, "reason", p)), Column: r.optBytes(em, "column", p), Stored: r.optBytes(em, "stored", p), @@ -456,14 +498,17 @@ func (r *reader) declinedSettings(m map[string]any, key, path string) []Declined // batch's rows). func (r *reader) row(m map[string]any, path string) RowResult { res := RowResult{ - Outcome: parseRowOutcome(r.text(m, "outcome", path)), - ErrCode: r.i32(m, "code", path), - ErrMsg: r.bytes(m, "err", path), - Transformed: r.transforms(m, "transformed", path), - UnsupportedSettings: r.nameList(m, "unsupported_settings", path), - UnknownFields: r.nameList(m, "unknown_fields", path), - VerdictCode: r.i32(m, "verdict_code", path), - VerdictErr: r.bytes(m, "verdict_err", path), + Outcome: parseRowOutcome(r.text(m, "outcome", path)), + ErrCode: r.i32(m, "code", path), + ErrMsg: r.bytes(m, "err", path), + Reason: r.text(m, "reason", path), + RefusedAt: RefusalSite(r.text(m, "refused_at", path)), + RefusedRow: r.optU64(m, "refused_row", path), + Transformed: r.transforms(m, "transformed", path), + UnknownFields: r.nameList(m, "unknown_fields", path), + VerdictCode: r.i32(m, "verdict_code", path), + VerdictReason: r.text(m, "verdict_reason", path), + VerdictErr: r.bytes(m, "verdict_err", path), } if arr := r.array(r.field(m, "cols"), path+".cols"); arr != nil { for i, e := range arr { @@ -568,18 +613,23 @@ func decodeBatchGeneric(raw []byte, payload []byte) (BatchResult, error) { r := &reader{doc: "batch"} m := r.object(v, "$") res := BatchResult{ - Outcome: parseBatchOutcome(r.text(m, "outcome", "$")), - ErrCode: r.i32(m, "code", "$"), - ErrMsg: r.bytes(m, "err", "$"), - RowsRead: r.u64(m, "rows_read", "$"), - RowsSkipped: r.u64(m, "rows_skipped", "$"), - Transformed: r.transforms(m, "transformed", "$"), - Payload: payload, - Spans: r.spans(m, "row_spans", "$"), - ExportDeclined: r.bytes(m, "export_declined", "$"), - RowsPassed: r.u64(m, "rows_passed", "$"), - RowsCut: r.u64(m, "rows_cut", "$"), - Unconsumed: r.spans(m, "unconsumed", "$"), + Outcome: parseBatchOutcome(r.text(m, "outcome", "$")), + ErrCode: r.i32(m, "code", "$"), + ErrMsg: r.bytes(m, "err", "$"), + Reason: r.text(m, "reason", "$"), + RefusedAt: RefusalSite(r.text(m, "refused_at", "$")), + RefusedRow: r.optU64(m, "refused_row", "$"), + RowsRead: r.u64(m, "rows_read", "$"), + RowsSkipped: r.u64(m, "rows_skipped", "$"), + Transformed: r.transforms(m, "transformed", "$"), + Payload: payload, + Spans: r.rowSpans(m, "row_spans", "$"), + ExportColumns: r.nameList(m, "export_columns", "$"), + ExportDeclined: r.bytes(m, "export_declined", "$"), + ExportDeclinedReason: r.text(m, "export_declined_reason", "$"), + RowsPassed: r.u64(m, "rows_passed", "$"), + RowsCut: r.u64(m, "rows_cut", "$"), + Unconsumed: r.spans(m, "unconsumed", "$"), // The call's own declined settings, and what a merge of the part would // do (ABI v2): both absent on a document that predates them. UnsupportedSettings: r.nameList(m, "unsupported_settings", "$"), @@ -660,6 +710,9 @@ func decodeFilterResult(raw []byte) (FilterResult, error) { Outcome: parseFilterOutcome(r.text(m, "outcome", "$")), ErrCode: r.i32(m, "code", "$"), ErrMsg: r.bytes(m, "err", "$"), + Reason: r.text(m, "reason", "$"), + RefusedAt: RefusalSite(r.text(m, "refused_at", "$")), + RefusedRow: r.optU64(m, "refused_row", "$"), RowsRead: r.u64(m, "rows_read", "$"), UnsupportedSettings: r.nameList(m, "unsupported_settings", "$"), } @@ -674,9 +727,10 @@ func decodeFilterResult(raw []byte) (FilterResult, error) { break } res.Errors = append(res.Errors, FilterRowError{ - Row: r.int(em, "row", p), - Code: r.i32(em, "code", p), - Msg: r.bytes(em, "err", p), + Row: r.int(em, "row", p), + Code: r.i32(em, "code", p), + Reason: r.text(em, "reason", p), + Msg: r.bytes(em, "err", p), }) } } @@ -705,9 +759,17 @@ func decodeSchemaDescription(raw []byte) (SchemaDescription, error) { Type: r.bytes(cm, "type", p), DefaultKind: kind, DefaultExpr: r.bytes(cm, "default_expression", p), + // Reserved members: absent until a build reports them. An unlisted + // default_support is its unknown(n), kept (r3). + DefaultSupport: DefaultSupport(r.text(cm, "default_support", p)), + DefaultSupportReason: r.text(cm, "default_support_reason", p), + Reads: r.nameList(cm, "reads", p), }) } } + // Present only for a schema compiled in the load posture; an unlisted + // posture is its unknown(n), kept (r3). + res.Posture = Posture(r.text(m, "posture", "$")) // Over every layer known at schema compile, with or without a server. res.FilterDeclinedSettings = r.declinedSettings(m, "filter_declined_settings", "$") // Both are absent on a schema compiled without a server, so that document diff --git a/go/chtypes/decode_batch.go b/go/chtypes/decode_batch.go index d75b7a9b..4c113a81 100644 --- a/go/chtypes/decode_batch.go +++ b/go/chtypes/decode_batch.go @@ -505,6 +505,7 @@ type colDoc struct { RefTypeB64 fIgnore `json:"ref_type_b64"` Wire fIgnore `json:"wire"` WireB64 fIgnore `json:"wire_b64"` + RefUnclass fIgnore `json:"ref_unclassified"` Nullable fIgnore `json:"nullable"` Poison fIgnore `json:"poison"` NullInput fIgnore `json:"null_input"` @@ -532,7 +533,7 @@ type cellDoc struct { // atMergeDoc is one entry of at_merge (ABI v2). type atMergeDoc struct { - Row fInt `json:"row"` + PartRow fInt `json:"part_row"` Reason fWord `json:"reason"` Column fWord `json:"column"` ColumnB64 fB64 `json:"column_b64"` @@ -577,19 +578,26 @@ type framingDoc struct { // batchRowDoc is one entry of rows. The lists keep their backing arrays between // rows (reset), the scalars are cleared. type batchRowDoc struct { - Outcome fWord `json:"outcome"` - Code fI32 `json:"code"` - Err fString `json:"err"` - ErrB64 fB64 `json:"err_b64"` - Verdict fString `json:"verdict"` - VerdictCode fI32 `json:"verdict_code"` - VerdictErr fString `json:"verdict_err"` - VerdictErrB64 fB64 `json:"verdict_err_b64"` - PartitionID fString `json:"partition_id"` - PartitionIDB64 fB64 `json:"partition_id_b64"` - InputSpan fObj[spanDoc] `json:"input_span"` - UnknownFields fList[nameDoc] `json:"unknown_fields"` - UnsupportedSetting fList[nameDoc] `json:"unsupported_settings"` + Outcome fWord `json:"outcome"` + Code fI32 `json:"code"` + Err fString `json:"err"` + ErrB64 fB64 `json:"err_b64"` + Reason fWord `json:"reason"` + RefusedAt fWord `json:"refused_at"` + RefusedRow fU64 `json:"refused_row"` + Verdict fString `json:"verdict"` + VerdictCode fI32 `json:"verdict_code"` + VerdictReason fWord `json:"verdict_reason"` + VerdictErr fString `json:"verdict_err"` + VerdictErrB64 fB64 `json:"verdict_err_b64"` + PartitionID fString `json:"partition_id"` + PartitionIDB64 fB64 `json:"partition_id_b64"` + InputSpan fObj[spanDoc] `json:"input_span"` + UnknownFields fList[nameDoc] `json:"unknown_fields"` + // unsupported_settings was removed from the row document (spec batch 2); a + // library that still writes it is read as the fast path always has, the + // member ignored (rule r2), rather than sent to the slower generic reader. + UnsupportedSetting fIgnore `json:"unsupported_settings"` Computed fList[computedDoc] `json:"computed"` Cols fList[colDoc] `json:"cols"` Transformed fList[transformDoc] `json:"transformed"` @@ -600,7 +608,6 @@ func newBatchRowDoc() *batchRowDoc { r := &batchRowDoc{} r.InputSpan.sub = sub r.UnknownFields.sub = sub - r.UnsupportedSetting.sub = sub r.Computed.sub = sub r.Cols.sub = sub r.Transformed.sub = sub @@ -611,12 +618,13 @@ func newBatchRowDoc() *batchRowDoc { // emptied with its backing array kept. func (d *batchRowDoc) reset() { d.Outcome, d.Code, d.Err, d.ErrB64 = fWord{}, fI32{}, fString{}, fB64{} - d.Verdict, d.VerdictCode = fString{}, fI32{} + d.Reason, d.RefusedAt, d.RefusedRow = fWord{}, fWord{}, fU64{} + d.Verdict, d.VerdictCode, d.VerdictReason = fString{}, fI32{}, fWord{} d.VerdictErr, d.VerdictErrB64 = fString{}, fB64{} d.PartitionID, d.PartitionIDB64 = fString{}, fB64{} d.InputSpan.seen, d.InputSpan.null, d.InputSpan.v = false, false, spanDoc{} + d.UnsupportedSetting = fIgnore{} d.UnknownFields.reset() - d.UnsupportedSetting.reset() d.Computed.reset() d.Cols.reset() d.Transformed.reset() @@ -733,7 +741,7 @@ func atMergeOf(list []atMergeDoc, present bool) ([]AtMergeEntry, bool) { out := make([]AtMergeEntry, 0, len(list)) for i := range list { a := &list[i] - e := AtMergeEntry{Row: a.Row.v, Reason: MergeReason(a.Reason.s)} + e := AtMergeEntry{PartRow: a.PartRow.v, Reason: MergeReason(a.Reason.s)} column, present, ok := byteOf(&a.Column.fString, &a.ColumnB64) if !ok { return nil, false @@ -793,6 +801,55 @@ func decodeSpanList(dec *json.Decoder, hint int) ([]Span, error) { return list, nil } +// decodeRowSpans reads row_spans as decodeSpanList reads a list of spans, but +// a null element is a row not exported (a nil entry). The entries share chunks +// of Span rather than allocating one each. +func decodeRowSpans(dec *json.Decoder, hint int) ([]*Span, error) { + t, err := dec.Token() + if err != nil { + return nil, errDeclined + } + if t == nil { + return nil, nil + } + if t != json.Delim('[') { + return nil, errDeclined + } + list := make([]*Span, 0, hint) + var chunk []Span + slot := new(spanSlot) + for dec.More() { + *slot = spanSlot{} + if err := dec.Decode(slot); err != nil { + return nil, errDeclined + } + if slot.null { + list = append(list, nil) + continue + } + if len(chunk) == 0 { + chunk = make([]Span, slabChunk) + } + p := &chunk[0] + chunk = chunk[1:] + *p = Span{Off: slot.v.Off.v, Len: slot.v.Len.v} + list = append(list, p) + } + if t, err := dec.Token(); err != nil || t != json.Delim(']') { + return nil, errDeclined + } + return list, nil +} + +// optU64Of is reader.optU64: nil when the member is absent or null. +func optU64Of(f *fU64) *uint64 { + if !f.seen || f.null { + return nil + } + n := f.v + return &n +} + // spanHint is the count a document has already announced (one span per row), // or 0 when none was or it is not credible for a document of docLen bytes. func spanHint(n fU64, docLen int) int { @@ -813,9 +870,9 @@ func (d *batchRowDoc) result(slabs *rowSlabs) (RowResult, bool) { if res.Transformed, ok = transformsOf(d.Transformed.v, d.Transformed.present()); !ok { return res, false } - if res.UnsupportedSettings, ok = namesOf(&d.UnsupportedSetting); !ok { - return res, false - } + res.Reason, res.RefusedAt = d.Reason.s, RefusalSite(d.RefusedAt.s) + res.RefusedRow = optU64Of(&d.RefusedRow) + res.VerdictReason = d.VerdictReason.s if res.UnknownFields, ok = namesOf(&d.UnknownFields); !ok { return res, false } @@ -963,6 +1020,11 @@ const ( kStorageTransforms kUnsupportedSettings kAtMerge + kReason + kRefusedAt + kRefusedRow + kExportDeclinedReason + kExportColumns ) // decodeBatchFast reads a batch document; ok is false when the generic reader @@ -988,7 +1050,12 @@ func decodeBatchFast(raw, payload []byte) (res BatchResult, ok bool) { exportDeclinedB64 fB64 partitionCount fU64 transformed []transformDoc - rowSpans, unconsumed []Span + rowSpans []*Span + unconsumed []Span + reason, refusedAt fWord + refusedRow fU64 + exportDeclinedReason fWord + exportColumns fList[nameDoc] engineRows [][]cellDoc framing *framingDoc storageTransforms fIgnore @@ -1038,7 +1105,7 @@ func decodeBatchFast(raw, payload []byte) (res BatchResult, ok bool) { case "row_spans": // One span per row, so rows_read, when it came first, sizes the list. bit = kRowSpans - top.rowSpans, err = decodeSpanList(dec, spanHint(top.rowsRead, len(raw))) + top.rowSpans, err = decodeRowSpans(dec, spanHint(top.rowsRead, len(raw))) case "unconsumed": bit = kUnconsumed top.unconsumed, err = decodeSpanList(dec, 0) @@ -1052,6 +1119,16 @@ func decodeBatchFast(raw, payload []byte) (res BatchResult, ok bool) { bit, err = kUnsupportedSettings, dec.Decode(&top.unsupportedSettings) case "at_merge": bit, err = kAtMerge, dec.Decode(&top.atMerge) + case "reason": + bit, err = kReason, dec.Decode(&top.reason) + case "refused_at": + bit, err = kRefusedAt, dec.Decode(&top.refusedAt) + case "refused_row": + bit, err = kRefusedRow, dec.Decode(&top.refusedRow) + case "export_declined_reason": + bit, err = kExportDeclinedReason, dec.Decode(&top.exportDeclinedReason) + case "export_columns": + bit, err = kExportColumns, dec.Decode(&top.exportColumns) case "rows": bit = kRows hint := 0 @@ -1080,6 +1157,9 @@ func decodeBatchFast(raw, payload []byte) (res BatchResult, ok bool) { if res.ErrMsg, good = bytesOf(&top.errText, &top.errB64); !good { return res, false } + res.Reason, res.RefusedAt = top.reason.s, RefusalSite(top.refusedAt.s) + res.RefusedRow = optU64Of(&top.refusedRow) + res.ExportDeclinedReason = top.exportDeclinedReason.s res.RowsRead, res.RowsSkipped = top.rowsRead.v, top.rowsSkipped.v if res.Transformed, good = transformsOf(top.transformed, top.transformed != nil); !good { return res, false @@ -1095,6 +1175,9 @@ func decodeBatchFast(raw, payload []byte) (res BatchResult, ok bool) { res.Payload = payload res.Spans = top.rowSpans } + if res.ExportColumns, good = namesOf(&top.exportColumns); !good { + return res, false + } res.RowsPassed, res.RowsCut = top.rowsPassed.v, top.rowsCut.v res.Unconsumed = top.unconsumed if res.UnsupportedSettings, good = namesOf(&top.unsupportedSettings); !good { diff --git a/go/chtypes/decode_equivalence_test.go b/go/chtypes/decode_equivalence_test.go index 6eeb554c..550751bf 100644 --- a/go/chtypes/decode_equivalence_test.go +++ b/go/chtypes/decode_equivalence_test.go @@ -67,9 +67,15 @@ var batchFixtures = map[string]string{ "byte-safe batch fields": `{"outcome":"rejected","code":27,"err_b64":"/w==","export_declined_b64":"/gA=","rows_read":"7"}`, "storage transforms": `{"outcome":"accepted","storage_transforms":[{"row":0,"column":"a","stored":"1"}]}`, "at_merge, every shape": atMergeBatchDoc, - "at_merge, nulls": `{"outcome":"accepted","at_merge":[{"row":null,"reason":null,"column":null,"stored":null,"input_rows":null}],"unsupported_settings":null}`, + "at_merge, nulls": `{"outcome":"accepted","at_merge":[{"part_row":null,"reason":null,"column":null,"stored":null,"input_rows":null}],"unsupported_settings":null}`, "at_merge, empty": `{"outcome":"accepted","at_merge":[],"unsupported_settings":[]}`, "at_merge, null list": `{"outcome":"accepted","at_merge":null}`, + // Spec batch 2: the refusal members, the export's column list and reason, and + // a row_spans with a null entry for a row not exported. + "refusal members": `{"outcome":"unsupported","code":-2,"reason":"preview-batch-zone-mismatch","refused_at":"body_read","refused_row":3,"rows_read":0,"rows":[]}`, + "refusal, unlisted site": `{"outcome":"rejected","code":27,"refused_at":"a_site_nobody_listed","refused_row":"9007199254740991"}`, + "export, spans with nulls": `{"outcome":"accepted","rows_read":3,"rows":[],"row_spans":[{"off":0,"len":5},null,{"off":5,"len":6}],"export_columns":[{"name":"a"},{"name_b64":"/w=="}]}`, + "export declined with a reason": `{"outcome":"unsupported","export_declined":"x","export_declined_reason":"export-batch-not-accepted","export_columns":[]}`, } // rowFixtures are documents of one row, as decodeRow reads them; each also @@ -77,8 +83,9 @@ var batchFixtures = map[string]string{ var rowFixtures = map[string]string{ "row, every field": rowDoc, "row, minimal": `{"outcome":"accepted"}`, - "row, empty lists": `{"outcome":"accepted","cols":[],"computed":[],"transformed":[],"unknown_fields":[],"unsupported_settings":[]}`, - "row, null lists": `{"outcome":"accepted","cols":null,"computed":null,"transformed":null,"unknown_fields":null,"unsupported_settings":null,"input_span":null,"verdict":null,"partition_id":null}`, + "row, declined": `{"outcome":"unsupported","code":-2,"reason":"drawn-value","refused_at":"row","refused_row":0,"verdict":"d","verdict_code":-2,"verdict_reason":"defer-defaults-reader"}`, + "row, empty lists": `{"outcome":"accepted","cols":[],"computed":[],"transformed":[],"unknown_fields":[]}`, + "row, null lists": `{"outcome":"accepted","cols":null,"computed":null,"transformed":null,"unknown_fields":null,"input_span":null,"verdict":null,"partition_id":null}`, "row, partition id bytes": `{"outcome":"accepted","partition_id_b64":"/w==","err_b64":"/gA=","verdict_err_b64":"/w==","verdict_code":3}`, "row, string value": `{"outcome":"accepted","cols":[{"name":"s","src":"input","null":false,"stored":"\"a\\\"b\"","value_b64":"YSJi","type":"String","ref":"x"}]}`, "row, unknown outcome": `{"outcome":"something_new"}`, diff --git a/go/chtypes/decode_reference_test.go b/go/chtypes/decode_reference_test.go index 255649b5..9416f80f 100644 --- a/go/chtypes/decode_reference_test.go +++ b/go/chtypes/decode_reference_test.go @@ -17,18 +17,23 @@ func decodeBatchReference(raw []byte, payload []byte) (BatchResult, error) { r := &reader{doc: "batch"} m := r.object(v, "$") res := BatchResult{ - Outcome: parseBatchOutcome(r.text(m, "outcome", "$")), - ErrCode: r.i32(m, "code", "$"), - ErrMsg: r.bytes(m, "err", "$"), - RowsRead: r.u64(m, "rows_read", "$"), - RowsSkipped: r.u64(m, "rows_skipped", "$"), - Transformed: r.transforms(m, "transformed", "$"), - Payload: payload, - Spans: r.spans(m, "row_spans", "$"), - ExportDeclined: r.bytes(m, "export_declined", "$"), - RowsPassed: r.u64(m, "rows_passed", "$"), - RowsCut: r.u64(m, "rows_cut", "$"), - Unconsumed: r.spans(m, "unconsumed", "$"), + Outcome: parseBatchOutcome(r.text(m, "outcome", "$")), + ErrCode: r.i32(m, "code", "$"), + ErrMsg: r.bytes(m, "err", "$"), + Reason: r.text(m, "reason", "$"), + RefusedAt: RefusalSite(r.text(m, "refused_at", "$")), + RefusedRow: r.optU64(m, "refused_row", "$"), + RowsRead: r.u64(m, "rows_read", "$"), + RowsSkipped: r.u64(m, "rows_skipped", "$"), + Transformed: r.transforms(m, "transformed", "$"), + Payload: payload, + Spans: r.rowSpans(m, "row_spans", "$"), + ExportColumns: r.nameList(m, "export_columns", "$"), + ExportDeclinedReason: r.text(m, "export_declined_reason", "$"), + ExportDeclined: r.bytes(m, "export_declined", "$"), + RowsPassed: r.u64(m, "rows_passed", "$"), + RowsCut: r.u64(m, "rows_cut", "$"), + Unconsumed: r.spans(m, "unconsumed", "$"), // Added with ABI v2's at_merge (public issue #544), through the shared // reader helpers, so the comparison covers the two new members too. UnsupportedSettings: r.nameList(m, "unsupported_settings", "$"), diff --git a/go/chtypes/decode_test.go b/go/chtypes/decode_test.go index a337a4df..718d803f 100644 --- a/go/chtypes/decode_test.go +++ b/go/chtypes/decode_test.go @@ -212,13 +212,13 @@ const atMergeBatchDoc = `{ "outcome": "accepted", "unsupported_settings": [{"name": "session_timezone"}, {"name_b64": "/w=="}], "at_merge": [ - {"row": 0, "reason": "ttl_delete"}, - {"row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, - {"row": 2, "reason": "ttl_column_reset", "column": "t"}, - {"row": 3, "reason": "ttl_column_reset", "column_b64": "/wA=", "stored_b64": "/w=="}, - {"row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, - {"row": 5, "reason": "a_reason_nobody_listed"}, - {"row": 6, "reason": "ttl_column_reset", "column": "e", "stored": "", "input_rows": []} + {"part_row": 0, "reason": "ttl_delete"}, + {"part_row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, + {"part_row": 2, "reason": "ttl_column_reset", "column": "t"}, + {"part_row": 3, "reason": "ttl_column_reset", "column_b64": "/wA=", "stored_b64": "/w=="}, + {"part_row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, + {"part_row": 5, "reason": "a_reason_nobody_listed"}, + {"part_row": 6, "reason": "ttl_column_reset", "column": "e", "stored": "", "input_rows": []} ] }` @@ -258,9 +258,9 @@ func TestDecodeBatchAtMerge(t *testing.T) { {MergeTTLColumnReset, "e", "", []int{}, false, true, true}, // an empty stored is not an absent one } { e := b.AtMerge[i] - if e.Row != i || e.Reason != tc.reason || e.Reason.Known() != tc.known || str(e.Column) != tc.column || str(e.Stored) != tc.stored { - t.Errorf("entry %d = {Row %d Reason %q Known %v Column %q Stored %q}, want {%d %q %v %q %q}", - i, e.Row, e.Reason, e.Reason.Known(), str(e.Column), str(e.Stored), i, tc.reason, tc.known, tc.column, tc.stored) + if e.PartRow != i || e.Reason != tc.reason || e.Reason.Known() != tc.known || str(e.Column) != tc.column || str(e.Stored) != tc.stored { + t.Errorf("entry %d = {PartRow %d Reason %q Known %v Column %q Stored %q}, want {%d %q %v %q %q}", + i, e.PartRow, e.Reason, e.Reason.Known(), str(e.Column), str(e.Stored), i, tc.reason, tc.known, tc.column, tc.stored) } if (e.Column != nil) != tc.columnPresent { t.Errorf("entry %d: Column present = %v, want %v", i, e.Column != nil, tc.columnPresent) @@ -279,7 +279,7 @@ func TestDecodeBatchAtMerge(t *testing.T) { } // An unknown member, in an entry and beside the list, is ignored (r2). - b, err = decodeBatch([]byte(`{"outcome":"accepted","at_merge":[{"row":0,"reason":"ttl_delete","x_future":{"a":[1]},"x_future_b64":"/w=="}],"x_future_list":[1]}`), nil) + b, err = decodeBatch([]byte(`{"outcome":"accepted","at_merge":[{"part_row":0,"reason":"ttl_delete","x_future":{"a":[1]},"x_future_b64":"/w=="}],"x_future_list":[1]}`), nil) if err != nil || len(b.AtMerge) != 1 || b.AtMerge[0].Reason != MergeTTLDelete || b.AtMerge[0].Column != nil { t.Errorf("at_merge with unknown members = %+v, %v", b.AtMerge, err) } @@ -298,11 +298,11 @@ func TestDecodeBatchAtMerge(t *testing.T) { // A document that breaks its schema is an InternalError naming the key. for _, bad := range []string{ - `{"at_merge":[{"row":0,"reason":"ttl_column_reset","column":"c","column_b64":"Yw=="}]}`, - `{"at_merge":[{"row":0,"reason":"ttl_column_reset","column":"c","stored_b64":"!!"}]}`, - `{"at_merge":[{"row":0,"reason":"ttl_delete","input_rows":["x"]}]}`, - `{"at_merge":[{"row":0,"reason":"ttl_delete","input_rows":[-1]}]}`, - `{"at_merge":[{"row":0,"reason":7}]}`, + `{"at_merge":[{"part_row":0,"reason":"ttl_column_reset","column":"c","column_b64":"Yw=="}]}`, + `{"at_merge":[{"part_row":0,"reason":"ttl_column_reset","column":"c","stored_b64":"!!"}]}`, + `{"at_merge":[{"part_row":0,"reason":"ttl_delete","input_rows":["x"]}]}`, + `{"at_merge":[{"part_row":0,"reason":"ttl_delete","input_rows":[-1]}]}`, + `{"at_merge":[{"part_row":0,"reason":7}]}`, `{"at_merge":{"row":0}}`, `{"unsupported_settings":["session_timezone"]}`, } { @@ -494,3 +494,104 @@ func TestDeclinedExportPayloadIsAbsent(t *testing.T) { }) } } + +// Spec batch 2: a refusal or decline names its code (CodeDeclined is -2), its +// catalog id and where it sits; a batch fast path and the generic reader agree. +func TestDecodeBatchRefusalMembers(t *testing.T) { + doc := `{"outcome":"unsupported","code":-2,"reason":"preview-batch-zone-mismatch","refused_at":"body_read","refused_row":3,"rows_read":0,"rows":[]}` + for name, dec := range map[string]func([]byte, []byte) (BatchResult, error){"production": decodeBatch, "generic": decodeBatchGeneric} { + b, err := dec([]byte(doc), nil) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if b.ErrCode != CodeDeclined || b.Reason != "preview-batch-zone-mismatch" || b.RefusedAt != RefusedAtBodyRead || !b.RefusedAt.Known() || + b.RefusedRow == nil || *b.RefusedRow != 3 { + t.Errorf("%s: batch = %+v", name, b) + } + b, err = dec([]byte(`{"outcome":"accepted","code":0,"rows_read":1}`), nil) + if err != nil || b.Reason != "" || b.RefusedAt != "" || b.RefusedRow != nil { + t.Errorf("%s: an accepted batch carries no refusal members: %+v, %v", name, b, err) + } + // An unlisted site is its unknown(n): kept, never a failure (r3). + b, err = dec([]byte(`{"outcome":"rejected","refused_at":"a_site_nobody_listed"}`), nil) + if err != nil || b.RefusedAt != "a_site_nobody_listed" || b.RefusedAt.Known() { + t.Errorf("%s: unlisted site = %+v, %v", name, b.RefusedAt, err) + } + } +} + +func TestDecodeBatchExportMembers(t *testing.T) { + doc := `{"outcome":"accepted","rows_read":3,"row_spans":[{"off":0,"len":5},null,{"off":5,"len":6}],"export_columns":[{"name":"a"},{"name_b64":"/w=="}]}` + for name, dec := range map[string]func([]byte, []byte) (BatchResult, error){"production": decodeBatch, "generic": decodeBatchGeneric} { + b, err := dec([]byte(doc), []byte("0123456789a")) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if len(b.Spans) != 3 || b.Spans[0] == nil || b.Spans[1] != nil || b.Spans[2] == nil || b.Spans[2].Off != 5 || b.Spans[2].Len != 6 { + t.Errorf("%s: spans = %v: a row not exported is a nil entry, aligned with Rows", name, b.Spans) + } + if len(b.ExportColumns) != 2 || b.ExportColumns[0] != "a" || b.ExportColumns[1] != "\xff" { + t.Errorf("%s: export_columns = %q", name, b.ExportColumns) + } + d, err := dec([]byte(`{"outcome":"unsupported","export_declined":"nope","export_declined_reason":"export-batch-not-accepted","row_spans":[null]}`), []byte("x")) + if err != nil || d.ExportDeclined != "nope" || d.ExportDeclinedReason != "export-batch-not-accepted" || d.Payload != nil || d.Spans != nil || d.ExportColumns != nil { + t.Errorf("%s: declined export = %+v, %v", name, d, err) + } + } +} + +func TestDecodeRowRefusalMembers(t *testing.T) { + r, err := decodeRow([]byte(`{"outcome":"unsupported","code":-2,"reason":"drawn-value","refused_at":"row","refused_row":0,"verdict":"d","verdict_code":-2,"verdict_reason":"defer-defaults-reader"}`)) + if err != nil { + t.Fatal(err) + } + if r.ErrCode != CodeDeclined || r.Reason != "drawn-value" || r.RefusedAt != RefusedAtRow || r.RefusedRow == nil || *r.RefusedRow != 0 || + r.Verdict == nil || *r.Verdict != VerdictDecline || r.VerdictCode != CodeDeclined || r.VerdictReason != "defer-defaults-reader" { + t.Errorf("row = %+v", r) + } +} + +func TestDecodeFilterResultRefusalMembers(t *testing.T) { + f, err := decodeFilterResult([]byte(`{"outcome":"unsupported","code":-2,"reason":"filter-zone","refused_at":"call","rows_read":0,"verdicts":"","errors":[{"row":0,"code":-2,"reason":"drawn-value","err":"x"}],"unsupported_settings":[]}`)) + if err != nil { + t.Fatal(err) + } + if f.ErrCode != CodeDeclined || f.Reason != "filter-zone" || f.RefusedAt != RefusedAtCall || f.RefusedRow != nil || + len(f.Errors) != 1 || f.Errors[0].Code != CodeDeclined || f.Errors[0].Reason != "drawn-value" { + t.Errorf("filter result = %+v", f) + } +} + +func TestDecodeSchemaDescriptionBatch2(t *testing.T) { + d, err := decodeSchemaDescription([]byte(`{"columns":[ + {"name":"a","type":"Int32","default_kind":"","default_support":"deferred","default_support_reason":"r-x","reads":[{"name":"b"},{"name_b64":"/w=="}]}, + {"name":"b","type":"Int32","default_kind":"DEFAULT","default_support":"a_support_nobody_listed"}], + "filter_declined_settings":[],"posture":"load", + "server":{"timezone":"UTC","settings":{"s":"v","t":{"b64":"/w=="}}}}`)) + if err != nil { + t.Fatal(err) + } + if d.Posture != PostureLoad || len(d.Columns) != 2 { + t.Fatalf("description = %+v", d) + } + if c := d.Columns[0]; c.DefaultSupport != DefaultSupportDeferred || c.DefaultSupportReason != "r-x" || len(c.Reads) != 2 || c.Reads[1] != "\xff" { + t.Errorf("column 0 = %+v", c) + } + if c := d.Columns[1]; c.DefaultSupport != "a_support_nobody_listed" || c.DefaultSupport.Known() || c.Reads != nil { + t.Errorf("column 1 = %+v: an unlisted support is its unknown(n), an absent reads is nil", c) + } + if d.Server == nil || d.Server.Settings["s"] != "v" || d.Server.Settings["t"] != "\xff" { + t.Errorf("server = %+v: a settings value in the b64 form is its bytes", d.Server) + } + // Absent members: a create-posture schema, and a build that reports no default_support. + d, err = decodeSchemaDescription([]byte(`{"columns":[{"name":"a","type":"Int32","default_kind":""}],"filter_declined_settings":[]}`)) + if err != nil || d.Posture != "" || d.Columns[0].DefaultSupport != "" || d.Columns[0].Reads != nil { + t.Errorf("absent members = %+v, %v", d, err) + } + // A b64 settings value that is not the one-member object form is an InternalError. + _, err = decodeSchemaDescription([]byte(`{"columns":[],"server":{"settings":{"t":{"b64":"/w==","x":1}}}}`)) + var ie *InternalError + if !errors.As(err, &ie) { + t.Errorf("a malformed b64 value = %v, want an *InternalError", err) + } +} diff --git a/go/chtypes/reader_rules_test.go b/go/chtypes/reader_rules_test.go index f2e38ae7..d1c99a10 100644 --- a/go/chtypes/reader_rules_test.go +++ b/go/chtypes/reader_rules_test.go @@ -57,7 +57,7 @@ func TestR2UnknownMembersAreIgnored(t *testing.T) { if r, err := schema.Row(JSONEachRow, []byte(`{"x":1}`)); err != nil { t.Errorf("row with unknown members: %v", err) } else if r.Outcome != Accepted || len(r.Columns) != 1 || r.Columns[0].Text != "abc" || r.InputSpan == nil || r.InputSpan.Len != 3 || - len(r.Computed) != 1 || len(r.Transformed) != 1 || len(r.UnknownFields) != 1 || len(r.UnsupportedSettings) != 1 { + len(r.Computed) != 1 || len(r.Transformed) != 1 || len(r.UnknownFields) != 1 { t.Errorf("row = %+v", r) } else if tr := r.Transformed[0]; tr.Column != "s" || tr.Input != "1" || tr.Stored != "2" || tr.Reason != ReasonValueChanged || !tr.Lossy || tr.Row != 0 { diff --git a/go/chtypes/results.go b/go/chtypes/results.go index d722d18d..5520d5c2 100644 --- a/go/chtypes/results.go +++ b/go/chtypes/results.go @@ -69,8 +69,9 @@ type EngineCell struct { // the batch's at_merge list. EngineRows stays exactly what the INSERT writer // produces, so the two never contradict each other. type AtMergeEntry struct { - // Row indexes EngineRows: the part's row, not the input body's. - Row int + // PartRow indexes EngineRows: the part's row, not the input body's (the + // document's part_row). + PartRow int // Reason is a merge_reason value. One the description does not list is its // unknown(n), kept as it came (rule r3). Reason MergeReason @@ -91,20 +92,35 @@ type AtMergeEntry struct { type RowResult struct { // Outcome is final: the library applies every promotion. Outcome Outcome + // ErrCode is the document's code: ClickHouse's own error code, 0 only on a + // success, and CodeDeclined (-2) where this build declined the row. ErrCode int32 ErrMsg string + // Reason is the decline's catalog id (spec/abi-v2/declines/declines.json), + // from reason; "" when the document carries none, which it does exactly where + // ErrCode is CodeDeclined. Key on it, never on ErrMsg. + Reason string + // RefusedAt is where a single-row result's refusal sits, from refused_at: "" + // for an accepted or skipped row. The rows of a batch carry neither this nor + // RefusedRow: the batch's own say where. + RefusedAt RefusalSite + // RefusedRow is the row index of a RefusedAtRow refusal, from refused_row; nil + // when the document carries none. + RefusedRow *uint64 // Columns is every entry of cols, in document order; Values is the subset // whose IsStored is true. - Columns []Value - Values []Value - Transformed []Transform - UnknownFields []string - UnsupportedSettings []string - Computed []Computed + Columns []Value + Values []Value + Transformed []Transform + UnknownFields []string + Computed []Computed // Verdict is present only with an attached filter. Verdict *Verdict VerdictCode int32 - VerdictErr string + // VerdictReason is the catalog id of a declined verdict (VerdictCode is + // CodeDeclined), from verdict_reason; "" when the document carries none. + VerdictReason string + VerdictErr string // PartitionID is nil when the document carries none; it is bytes. PartitionID *string // InputSpan is the bytes the reader consumed for this record. @@ -130,9 +146,18 @@ type Framing struct { // BatchResult is a body's verdict, counts and per-row documents. type BatchResult struct { - Outcome Outcome - ErrCode int32 - ErrMsg string + Outcome Outcome + // ErrCode is the document's code, as RowResult.ErrCode. + ErrCode int32 + ErrMsg string + // Reason is the catalog id of a declined batch (ErrCode is CodeDeclined), + // from reason; "" when the document carries none. + Reason string + // RefusedAt is where a refusal or decline sits, from refused_at: "" for an + // accepted batch. A RefusedAtRow one names its row in RefusedRow (below + // RowsRead); nil otherwise. + RefusedAt RefusalSite + RefusedRow *uint64 Rows []RowResult RowsRead uint64 RowsSkipped uint64 @@ -156,13 +181,23 @@ type BatchResult struct { // 20261011.105019 returned a zero-length buffer), non-nil and empty for an // accepted batch with zero rows. Payload []byte - // Spans is each exported row's place in Payload; nil when absent, and - // always nil when the export was declined. - Spans []Span - ExportDeclined string - RowsPassed uint64 - RowsCut uint64 - PartitionCount *uint64 + // Spans is each row's place in Payload, from row_spans: one entry per + // Rows entry, index-aligned, nil for a row not exported (a skipped row, or a + // row a filter cut). The slice itself is nil when absent, and always nil when + // the export was declined. + Spans []*Span + // ExportColumns is the export's one column list, in the order every line + // carries its columns, from export_columns; nil when the document carries + // none (no export ran). + ExportColumns []string + // ExportDeclined names the cause of a declined export, and ExportDeclinedReason + // its catalog id, from export_declined_reason; both "" when the export ran or + // none was asked for. + ExportDeclined string + ExportDeclinedReason string + RowsPassed uint64 + RowsCut uint64 + PartitionCount *uint64 // Unconsumed is the byte ranges the reader's error recovery skipped. It // does not account for every record: a skipped row's InputSpan can cover // more than one input record, so verdicts can be fewer than records while @@ -177,14 +212,23 @@ type BatchResult struct { type FilterRowError struct { Row int Code int32 - Msg string + // Reason is the catalog id where Code is CodeDeclined, from reason; "" when + // the entry carries none. + Reason string + Msg string } // FilterResult is a filter evaluation. type FilterResult struct { - Outcome FilterOutcome - ErrCode int32 - ErrMsg string + Outcome FilterOutcome + // ErrCode is the document's code, as RowResult.ErrCode. + ErrCode int32 + ErrMsg string + // Reason, RefusedAt and RefusedRow are the batch's (see BatchResult): where a + // refused or declined evaluation sits, and its catalog id. + Reason string + RefusedAt RefusalSite + RefusedRow *uint64 RowsRead uint64 UnsupportedSettings []string // Verdicts is one verdict per row. A verdict counts only when Outcome is @@ -203,12 +247,23 @@ type Column struct { Type string DefaultKind DefaultKind DefaultExpr string + // DefaultSupport is what this build does with the column's DEFAULT, from + // default_support (reserved: "" when the build reports none), and + // DefaultSupportReason the catalog id behind a DefaultSupportDeclined one. + DefaultSupport DefaultSupport + DefaultSupportReason string + // Reads is the columns the column's expression reads, from reads (reserved: + // nil when the build reports none). + Reads []string } // SchemaDescription is a schema's columns, in declared order, the settings // its filters decline, and the server it was compiled on. type SchemaDescription struct { Columns []Column + // Posture is how the schema's statement was compiled, from posture, which the + // document carries only for PostureLoad: "" is PostureCreate. + Posture Posture // FilterDeclinedSettings is the settings the schema compiles under that // this build's filters do not honor in a WHERE, over every layer known at // schema compile (the defaults Setup set, the server profile and the @@ -234,8 +289,9 @@ type SchemaServer struct { // Timezone is the zone the schema's types bind: the profile's, or else the // image zone (Setup's), exactly as chs_initialize spelled it. Timezone string - // Settings is the server's settings as the profile gave them; the library - // reports {} (an empty map) when it gave none. + // Settings is the server's settings as the profile gave them (a value that + // was not UTF-8 comes back as its bytes); the library reports {} (an empty + // map) when it gave none. Settings map[string]string // Macros is the server's macro set: nil exactly when the profile carried // no macros (unknown), and non-nil, even empty, when it did (the complete From 01e958f93bf84b84c2998a97763b295ff5ae6143 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 15:56:17 -0400 Subject: [PATCH 04/24] go (spec batch 2): the stub goldens document states the documents' code member Co-Authored-By: Claude Sonnet 5.5 --- go/chtypes/testdata/goldens-v1-stub.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/go/chtypes/testdata/goldens-v1-stub.json b/go/chtypes/testdata/goldens-v1-stub.json index 57f9082b..016fbb97 100644 --- a/go/chtypes/testdata/goldens-v1-stub.json +++ b/go/chtypes/testdata/goldens-v1-stub.json @@ -68,12 +68,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 6 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -112,12 +112,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 6 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -164,12 +164,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 6 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", From f5ab978a08f615a34fd280ba625db1ccf1f200f8 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 15:58:20 -0400 Subject: [PATCH 05/24] go (spec batch 2): bindings-v1.md rows and the parity allowlist for the new names Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/bindings-v1.md | 51 +++++++++++++++++++++++-------- scripts/parity-surface-allow.json | 5 ++- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 22976d62..59c0217d 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -95,7 +95,7 @@ One row per public operation. `request` is a version spelling under [`fetch-v1.m A `ServerProfile` carries: `timezone` (a zone name), `settings` (settings) and `macros` (settings), each optional (input:server_profile, [`abi-v2.md`](abi-v2.md)). -**The server profile (ABI v2).** `new_server` describes one ClickHouse server with `chs_server_create`, and the compile option `server` compiles a table on it (`chs_schema_create`'s `server`): the schema's home zone is the server's `timezone`, the server's `settings` layer under the schema's own, and a Replicated engine's ZooKeeper path and replica name expand its `macros`. On a server schema, `timezone()` answers the server's zone when no session zone is in force; a session zone that would change the answer is declined (named in `unsupported_settings`). That applies equally to a profile whose `settings` carry `session_timezone`. The binding serializes the profile with the stock encoder and validates nothing in it: a zone `DateLUT` cannot load, or a setting the server's `SET` check refuses, is the library's own refusal (`SchemaError`), a malformed macro set is its `UsageError`, and a build that does not compile on a server yet declines (`UnsupportedError`). An empty zone and an absent map are omitted from the document; a present map is sent as given, `{}` when empty. **Absent `macros` is not empty `macros`:** absent means the server's macros are unknown, and present, even `{}`, is the server's complete set (Go nil against a non-nil map, Python and TypeScript absent against `{}`, Rust `None` against `Some`). No server option and no schema option is defined yet, so each options document is empty. The server is immutable; a schema holds a counted reference to it inside the library, so the two close in any order, and a closed server is a `UsageError` raised before any call, never NULL in its place (rule r7: NULL means the image's own server, so the library cannot tell a freed server from none); Rust meets the rule by construction, since a `Server` is freed only on its last drop and a compile borrows it. A server from another library is that library's own `CHS_INVALID_ARGUMENT`. +**The server profile (ABI v2).** `new_server` describes one ClickHouse server with `chs_server_create`, and the compile option `server` compiles a table on it (`chs_schema_create`'s `server`): the schema's home zone is the server's `timezone`, the server's `settings` layer under the schema's own, and a Replicated engine's ZooKeeper path and replica name expand its `macros`. On a server schema, `timezone()` answers the server's zone when no session zone is in force; a session zone that would change the answer is declined (named in `unsupported_settings`). That applies equally to a profile whose `settings` carry `session_timezone`. The binding serializes the profile with the stock encoder and validates nothing in it: a zone `DateLUT` cannot load, or a setting the server's `SET` check refuses, is the library's own refusal (`SchemaError`), a malformed macro set is its `UsageError`, and a build that does not compile on a server yet declines (`UnsupportedError`). An empty zone and an absent map are omitted from the document; a present map is sent as given, `{}` when empty. **Absent `macros` is not empty `macros`:** absent means the server's macros are unknown, and present, even `{}`, is the server's complete set (Go nil against a non-nil map, Python and TypeScript absent against `{}`, Rust `None` against `Some`). No server option is defined yet, so its options document is empty; `schema_options` has `posture` (§2, the call options). The server is immutable; a schema holds a counted reference to it inside the library, so the two close in any order, and a closed server is a `UsageError` raised before any call, never NULL in its place (rule r7: NULL means the image's own server, so the library cannot tell a freed server from none); Rust meets the rule by construction, since a `Server` is freed only on its last drop and a compile borrows it. A server from another library is that library's own `CHS_INVALID_ARGUMENT`. ### A schema @@ -125,14 +125,25 @@ One entry point per operation, with its optional inputs spelled in the language' | ---------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------- | ------------------------------------------------ | | settings | `settings` | compile (the profile), row, rows, parse a block, filter compile (the filter's own zone and profile), filter rows (the body's parse settings) | `WithSettings(map[string]string)`: one function whose value implements `CompileOption`, `RowOption`, `RowsOption`, `BlockOption`, `FilterOption` and `EvalOption` | `settings=` | `settings` | `settings: Vec<(String, String)>` | | the per-call zone | the `session_timezone` key inside `settings` | the same six calls; on a filter compile it is the filter's zone, on a filter evaluation the body's parse zone | `WithSessionTimezone(name string)`, implementing the same six option interfaces | `session_timezone=` | `sessionTimezone` | `session_timezone: Option` | -| the INSERT column list | `columns` | row, rows, parse a block | `WithColumns(names []string)`: a `RowOption`, `RowsOption` and `BlockOption` | `columns=` | `columns` | `columns: Vec>` | +| the INSERT column list | `columns` | row, rows, parse a block, filter rows (ABI v2) | `WithColumns(names []string)`: a `RowOption`, `RowsOption`, `BlockOption` and `EvalOption` | `columns=` | `columns` | `columns: Vec>` | | an attached filter | `filter` | rows | `WithRowFilter(*Filter)` | `row_filter=` | `rowFilter` | `filter: Option` | | an export | `export_format`, `out_export` | rows | `WithExport(Format)`; absent means `CHS_EXPORT_NONE` | `export=` | `exportFormat` | `export: Option` | | document groups | `doc_flags` | rows | `WithDocFlags(DocFlags)`; absent means `DocAll` | `doc_flags=` | `docFlags` | `doc_flags: Option`, `None` means all | | query parameters | `query_params` | filter compile | `WithFilterParams(map[string]string)` | `params=` | `params` | `params: Vec<(String, String)>` | | the server | `server` | compile (ABI v2) | `OnServer(*Server)`; `OnServer(nil)` is no server | `server=` | `server` | `server: Option`, `None` means no server | +| insertable columns | `options`: `insert_options.insertable_columns` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertableColumns(names []string)`: an `InsertOption`; nil leaves the member out | `insertable_columns=` | `insertableColumns` | `insertable_columns: Option>>` | +| column defaults | `options`: `insert_options.column_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithColumnDefaults(defaults []ColumnDefault)`: an `InsertOption`; `ColumnDefault{Name, Value string}` | `column_defaults=` | `columnDefaults` | `column_defaults: Option>` | +| verdict basis | `options`: `insert_options.verdict_basis` | row, rows, parse a block, filter rows (ABI v2) | `WithVerdictBasis(VerdictBasis)`: `VerdictBasisInput`, `VerdictBasisExport` | `verdict_basis=` | `verdictBasis` | `verdict_basis: Option` | +| defer defaults | `options`: `insert_options.defer_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithDeferDefaults(DeferDefaults)`: `DeferDefaultsNone`, `DeferDefaultsServer` | `defer_defaults=` | `deferDefaults` | `defer_defaults: Option` | +| an insert option with no name here | `options`: `insert_options.` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertOption(key string, valueJSON []byte)`: a member passed through as raw JSON | `insert_options_extra=` | `insertOptionsExtra` | `insert_options_extra: Vec<(String, String)>` | +| parameter encoding | `options`: `filter_options.param_encoding` | filter compile (ABI v2) | `WithParamEncoding(ParamEncoding)`: `ParamEncodingEscaped`, `ParamEncodingRaw` | `param_encoding=` | `paramEncoding` | `param_encoding: Option` | +| a filter option with no name here | `options`: `filter_options.` | filter compile (ABI v2) | `WithFilterOption(key string, valueJSON []byte)`: passed through as raw JSON | `filter_options_extra=` | `filterOptionsExtra` | `filter_options_extra: Vec<(String, String)>` | +| posture | `options`: `schema_options.posture` | compile (ABI v2) | `WithPosture(Posture)`: `PostureCreate`, `PostureLoad` | `posture=` | `posture` | `posture: Option` | +| a schema option with no name here | `options`: `schema_options.` | compile (ABI v2) | `WithSchemaOption(key string, valueJSON []byte)`: passed through as raw JSON | `schema_options_extra=` | `schemaOptionsExtra` | `schema_options_extra: Vec<(String, String)>` | -The TypeScript option types are `CompileOptions { settings?, sessionTimezone?, server? }`, `RowOptions { settings?, sessionTimezone?, columns? }`, `RowsOptions extends RowOptions { rowFilter?, exportFormat?, docFlags? }`, `FilterOptions { params?, settings?, sessionTimezone? }` and `EvalOptions { settings?, sessionTimezone? }`. The Rust ones are `CompileOptions { settings, session_timezone, server }`, `RowOptions { settings, session_timezone, columns }`, `RowsOptions { settings, session_timezone, columns, filter, export, doc_flags }`, `FilterOptions { params, settings, session_timezone }` and `EvalOptions { settings, session_timezone }`, each with public fields and `Default`, built as `RowOptions { session_timezone: Some(z), ..Default::default() }`. They are deliberately not `#[non_exhaustive]`, which would forbid that construction outside the crate; the result types are. +The TypeScript option types are `CompileOptions { settings?, sessionTimezone?, server?, posture?, schemaOptionsExtra? }`, `RowOptions { settings?, sessionTimezone?, columns?, insertableColumns?, columnDefaults?, verdictBasis?, deferDefaults?, insertOptionsExtra? }`, `RowsOptions extends RowOptions { rowFilter?, exportFormat?, docFlags? }`, `FilterOptions { params?, settings?, sessionTimezone?, paramEncoding?, filterOptionsExtra? }` and `EvalOptions { settings?, sessionTimezone?, columns?, insertableColumns?, columnDefaults?, verdictBasis?, deferDefaults?, insertOptionsExtra? }`. The Rust ones are `CompileOptions { settings, session_timezone, server, posture, schema_options_extra }`, `RowOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra }`, `RowsOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra, filter, export, doc_flags }`, `FilterOptions { params, settings, session_timezone, param_encoding, filter_options_extra }` and `EvalOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra }`, each with public fields and `Default`, built as `RowOptions { session_timezone: Some(z), ..Default::default() }`. They are deliberately not `#[non_exhaustive]`, which would forbid that construction outside the crate; the result types are. + +**The options documents (ABI v2).** Three calls take an options document (rule r1): `insert_options` on the four calls that model an INSERT of a body (`chs_preview_row`, `chs_preview_batch`, `chs_block_create`, `chs_filter_eval_body`), `filter_options` on `chs_filter_create`, and `schema_options` on `chs_schema_create`. Each member is its own option in the table above, and the binding builds the document from the members the caller set: a document with no member set is not sent (length 0, which the library reads as `{}`, the call before the document existed). The binding validates no member and no value: the library refuses a key it does not know, or a value that is not one of the member's, with `CHS_INVALID_ARGUMENT`, naming it, and a member a build does not implement yet (`defer_defaults` `server`, `verdict_basis` `export`, `param_encoding` `raw`, and `column_defaults`, until a build lists it in `capabilities.features`) declines. A member the binding has no name for is passed through as JSON text under its key, so a caller can set an option the library implements before the binding names it; a key set twice keeps the last value, and text that is not valid JSON is a `UsageError` before any call. `insertable_columns` and the `columns` list are name lists: a name that is not valid UTF-8 is its `name_b64` form, as for `columns`. A `column_defaults` entry is `{name, value}`, each a byte string in the same two forms (`name_b64`, `value_b64`). An absent list is the member left out (every column; no override), and a present empty list is sent as `[]`. **The per-call zone is a settings key, and nothing more.** The ABI has no zone parameter on any call: the per-call zone is `"session_timezone": ""` inside the call's settings object. The option exists so the zone is visible in every signature it affects; the binding writes its value into the settings object as that one key, verbatim, and does nothing else with it: no default, no validation, no canonicalization. The library validates it with `DateLUT` and answers a bad name with ClickHouse's own refusal. Passing the option and a `session_timezone` key in `settings` together is a `UsageError`, raised before the call whether or not the two agree, so a call never has two spellings of its zone. In a compile profile it is a default for later calls on that schema; a compiled type always takes the image zone, never the profile's. @@ -188,7 +199,7 @@ The two zones, spelled in each binding: - **A Go `string` is a byte string.** Go has no separate byte-string type, and a `string` holds any bytes, NUL included. The rule is only that it is built from copied bytes (`string(b)`), never through `C.GoString`, which stops at the first NUL. - **A `str` or `string` input is encoded as UTF-8**, because that is the only way a text value becomes bytes. That is not an assumption about the payload: a caller with non-UTF-8 bytes passes them as `bytes`, `Uint8Array` or `&[u8]`. A body never accepts the text type, as in v0. - **Rust keeps v0's `RawText`** (the bytes, `as_bytes()`, `as_str() -> Option<&str>` and a lossy `Display`) for every byte output, so it is the one byte-out type in that binding. -- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object of string values with the language's stock encoder and passes it verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. A value whose bytes are not valid UTF-8 cannot be passed as a setting or a query parameter in 1.0; the workaround is to inline it in the SQL as `unhex('')`, built from hex digits only, which cannot break out of the quoted literal. The bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the zone) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different value. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. +- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object with the language's stock encoder and passes each value verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. **A value whose bytes are not valid UTF-8 is the object form `{"b64": ""}`** (ABI v2, `json_object_byte_values`: a settings value, a query-parameter value, a `chs_set_defaults` default, and a value of a server profile's `settings`), written by the binding in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a JSON string. A key that is not valid UTF-8 cannot be carried (a JSON object has no byte form for a key, and a macro name or value and a zone have none either): the bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the profile) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different key. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. A per-call zone is a settings value, so a zone that is not valid UTF-8 is written in the object form too, and the library validates it. - **A zone name is text**, because ClickHouse's zone names are, and a name `DateLUT` cannot load is refused by the library whatever its bytes. It crosses the ABI as counted bytes (`chs_initialize`) or as a JSON string (`session_timezone`), UTF-8 encoded. - **A list of column names** (the `columns` option) is serialized as a JSON array of names under the name encoding of §5. @@ -252,11 +263,15 @@ Every vocabulary is generated from the description into all four bindings, with | filter outcomes | `filter_outcome` | `FilterOutcome`: `FilterOK`, `FilterRejected`, `FilterUnsupported` | `FilterOutcome` | `FilterOutcome` | `FilterOutcome` | FIRM. Unknown reads as `unsupported` | | filter verdicts | `filter_verdict` | `Verdict`: `VerdictTrue`, `VerdictFalse`, `VerdictError`, `VerdictDecline`, and `Answered()` | `Verdict`, `.answered` | `Verdict`, `answered` | `Verdict`, `answered()` | FIRM. Unknown reads as `d`. `answered` is the description's own fact, never a list kept here | | transform reasons | `transform_reason` | `ReasonOverflowWrap` … | `Reason.OVERFLOW_WRAP` … | `Reason.OverflowWrap` … | `reason::OVERFLOW_WRAP` … | FIRM. The spellings are v0's. Each carries its `lossy` fact, which `Transform.lossy` reads | -| value sources | `value_src` | `SourceInput` … | `Source.INPUT` … | `Source.Input` … | `source::INPUT` … | FIRM. Every value now, not v0's two. Each carries its `is_stored` fact, which `Value.is_stored` reads | +| value sources | `value_src` | `SourceInput` …, `SourceDefaultDeferred`, `SourceDefaultOverride` | `Source.INPUT` … | `Source.Input` … | `source::INPUT` … | FIRM. Every value now, not v0's two. Each carries its `is_stored` fact, which `Value.is_stored` reads; an unlisted value reads the description's fallback (`skipped`), not stored. `default_deferred` (not stored) and `default_override` (stored) are ABI v2 | | `default_generated` | a `value_src` value | `SourceDefaultGenerated` | `Source.DEFAULT_GENERATED` | `Source.DefaultGenerated` | `source::DEFAULT_GENERATED` | FIRM. `is_stored` is true: the value is stored once the caller inserts the library's output (§5, Generated defaults). Generated from the description like every other value, never hand-added | | merge reasons | `merge_reason` | `MergeReason`: `MergeTTLDelete`, `MergeTTLColumnReset` | `MergeReason` | `MergeReason` | `MergeReason`, `#[non_exhaustive]` | ABI v2 dev. The reason of a batch's `at_merge` entry (§5). The spellings are the description's | -| declined tiers | `declined_tier` | `DeclinedTier`: `TierPredicate`, `TierPredicateUnflipped`, `TierResultContent` | `DeclinedTier` | `DeclinedTier` | `DeclinedTier`, `#[non_exhaustive]` | ABI v2 dev. The tier of an entry of a schema description's `filter_declined_settings` (§5). The spellings are the description's; each binding reads a hyphen as a word boundary in its identifiers | +| declined tiers | `declined_tier` | `DeclinedTier`: `TierPredicate`, `TierPredicateUnflipped`, `TierResultContent`, `TierServerRefuses` | `DeclinedTier` | `DeclinedTier` | `DeclinedTier`, `#[non_exhaustive]` | ABI v2 dev. The tier of an entry of a schema description's `filter_declined_settings` (§5). The spellings are the description's; each binding reads a hyphen as a word boundary in its identifiers | | declined layers | `declined_layer` | `DeclinedLayer`: `LayerDefaults`, `LayerServer`, `LayerSchema` | `DeclinedLayer` | `DeclinedLayer` | `DeclinedLayer`, `#[non_exhaustive]` | ABI v2 dev. The settings layer of an entry of a schema description's `filter_declined_settings` (§5), the highest that sets the name: `defaults`, `server` or `schema`, lowest first. The spellings are the description's | +| refusal sites | `refusal_site` | `RefusalSite`: `RefusedAtCall`, `RefusedAtBodyRead`, `RefusedAtRow`, `RefusedAtBodyStep` | `RefusalSite` | `RefusalSite` | `RefusalSite`, `#[non_exhaustive]` | ABI v2 dev. Where a batch, a row or a filter result's refusal sits (`refused_at`, §5). The spellings are the description's; each binding reads an underscore as a word boundary in its identifiers | +| postures | `posture` | `Posture`: `PostureCreate`, `PostureLoad` | `Posture` | `Posture` | `Posture`, `#[non_exhaustive]` | ABI v2 dev. How a schema's statement was compiled (`schema_options`' `posture`, and a schema description's `posture`). The spellings are the description's | +| default support | `default_support` | `DefaultSupport`: `DefaultSupportComputed`, `DefaultSupportGenerated`, `DefaultSupportDeferred`, `DefaultSupportDeclined` | `DefaultSupport` | `DefaultSupport` | `DefaultSupport`, `#[non_exhaustive]` | ABI v2 dev, reserved. What a build does with a column's DEFAULT (`default_support`, §5). The spellings are the description's | +| the decline code | `CHS_CODE_DECLINED` | `CodeDeclined` (`int32`, -2) | `CODE_DECLINED` | `CODE_DECLINED` | `code::DECLINED` (`i32`) | ABI v2 dev. The `code` of a declined row, outcome or verdict, outside ClickHouse's code table; 0 is written only on a success | | discover query parameters | `discover_query_param` | `DiscoverQueryParam`: `QueryParamDatabase`, `QueryParamTable` | `DiscoverQueryParam` | `DiscoverQueryParam` | `DiscoverQueryParam`, `#[non_exhaustive]` | ABI v2 dev. The discovery query's two parameters (§2, the discovery query), which a caller binds when it runs the query against its server. The spellings are the description's | | default kinds | `default_kind` | `DefaultKind`: `KindNone`, `KindDefault`, `KindMaterialized`, `KindAlias`, `KindEphemeral` | `DefaultKind` | `DefaultKind` | `DefaultKind` | FIRM; joins the description in the regeneration that freezes the fingerprint (§8 question 3). The spellings are v0's | | document groups | `CHS_DOC_*` | `DocFlags`: `DocValues`, `DocTransforms`, `DocDefaults`, `DocAll` | `DocFlags` | `DocFlags` | `DocFlags` | FIRM | @@ -370,16 +385,19 @@ Every table below lists a document's fields as the regenerated description draws | field | from | type | | ---------------------- | ------------------------------------------------------------------ | ------------------- | | `outcome` | `outcome`, already final: the library applies every promotion (D6) | `Outcome` | -| `err_code` | `code` | int32 | +| `err_code` | `code`: ClickHouse's own code, 0 only on a success, `CHS_CODE_DECLINED` (-2) for a decline | int32 | | `err_msg` | `err` (rule 5) | bytes | +| `reason` | `reason`: the decline's catalog id, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at` | `refused_at`: where a single-row result's refusal sits; the rows of a batch carry neither this nor `refused_row` | optional `RefusalSite` | +| `refused_row` | `refused_row`, present exactly when `refused_at` is `row` | optional unsigned 64-bit | | `columns` | `cols`, every entry, in document order | list of `Value` | | `values` | the entries of `cols` whose `src` is stored, in order | list of `Value` | | `transformed` | `transformed`, computed by the library (D6) | list of `Transform` | | `unknown_fields` | `unknown_fields`, name objects (rule 5) | list of bytes | -| `unsupported_settings` | `unsupported_settings`, name objects (rule 5) | list of bytes | | `computed` | `computed` | list of `Computed` | | `verdict` | `verdict`, present only with an attached filter | optional `Verdict` | | `verdict_code` | `verdict_code` | int32 | +| `verdict_reason` | `verdict_reason`: the catalog id, present exactly where `verdict_code` is `CHS_CODE_DECLINED` | optional ASCII text | | `verdict_err` | `verdict_err` (rule 5) | bytes | | `partition_id` | `partition_id` (rule 5) | optional bytes | | `input_span` | `input_span`: the bytes the reader consumed for this record | optional `Span` | @@ -402,6 +420,8 @@ The transform entry keeps v0's shape (`column`, `input`, `stored`, `reason`, `ro | ------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------- | | `outcome` | `outcome`, final | `Outcome` | | `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | +| `reason` | `reason`: the catalog id of a declined batch, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at`, `refused_row` | `refused_at` (a `refusal_site` value) and `refused_row`: where a refusal or decline sits, present whenever the outcome is not `accepted` or `accepted_poisoned`; `refused_row` for `row`, below `rows_read` | optional `RefusalSite`, optional unsigned 64-bit | | `rows` | `rows`, each decoded as a `RowResult` | list of `RowResult` | | `rows_read` | `rows_read` | unsigned 64-bit | | `rows_skipped` | `rows_skipped` | unsigned 64-bit | @@ -410,8 +430,10 @@ The transform entry keeps v0's shape (`column`, `input`, `stored`, `reason`, `ro | `engine_rows` | `engine_rows`: each stored row after the engine's insert-time merge, a list of cells | optional list of rows, each a list of `EngineCell` | | `at_merge` | `at_merge`: what an `OPTIMIZE TABLE ... FINAL` of the part would do to its rows (ABI v2) | list of `AtMergeEntry` | | `payload` | the `out_export` buffer, present only for an export that ran (absent when `export_declined` is non-empty) | optional payload | -| `spans` | `row_spans`: each exported row's place in `payload` | optional list of `Span` | +| `spans` | `row_spans`: one entry per `rows` entry, index-aligned, each exported row's place in `payload` and null for a row not exported | optional list of optional `Span` | +| `export_columns` | `export_columns`: the export's one column list, name objects (rule 5) | optional list of bytes | | `export_declined` | `export_declined` (rule 5) | bytes | +| `export_declined_reason` | `export_declined_reason`: the catalog id of a declined export | optional ASCII text | | `rows_passed`, `rows_cut` | `rows_passed`, `rows_cut`, with an attached filter | unsigned 64-bit | | `partition_count` | `partition_count` | optional unsigned 64-bit | | `unconsumed` | `unconsumed`: the byte ranges the reader's error recovery skipped | list of `Span` | @@ -419,9 +441,9 @@ The transform entry keeps v0's shape (`column`, `input`, `stored`, `reason`, `ro | type | fields, each from the same-named key unless noted | | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `AtMergeEntry` | `row` (int: an index into `engine_rows`, the part's row), `reason` (`MergeReason`, a `merge_reason` value), `column` (optional bytes, from `column` or `column_b64`: rule 5), `stored` (optional bytes, from `stored` or `stored_b64`: rule 5), `input_rows` (optional list of int) | +| `AtMergeEntry` | `part_row` (int: an index into `engine_rows`, the part's row), `reason` (`MergeReason`, a `merge_reason` value), `column` (optional bytes, from `column` or `column_b64`: rule 5), `stored` (optional bytes, from `stored` or `stored_b64`: rule 5), `input_rows` (optional list of int) | -**`at_merge` and the batch's own `unsupported_settings` (ABI v2).** `at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part the `INSERT` writes would do to its rows, at the call's clock instant: `ttl_delete` (the rows TTL removes the row) or `ttl_column_reset` (a column TTL resets the column's value). `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. An entry's `row` indexes `engine_rows`, not the input body, and `input_rows`, when present, names the input rows, each by its index in the body, that formed that row after an engine's insert-time merge. A row both removed and reset lists only `ttl_delete`. `column` is present only for `ttl_column_reset`, and `stored` is absent when the column's `DEFAULT` reads the clock or a generator, so the value is decided at the merge. Absent is never empty, in any binding: Go `nil`, Python `None`, TypeScript `undefined`, Rust `None`. The list is "at least these", not "only these": a later background merge can remove more rows as more of them expire. The batch's own `unsupported_settings` names the call's settings the library declined. Both lists are empty when the document carries none. [`spec/abi-v2/docs.md`](../../spec/abi-v2/docs.md) is normative for both. +**`at_merge` and the batch's own `unsupported_settings` (ABI v2).** `at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part the `INSERT` writes would do to its rows, at the call's clock instant: `ttl_delete` (the rows TTL removes the row) or `ttl_column_reset` (a column TTL resets the column's value). `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. An entry's `part_row` indexes `engine_rows`, not the input body, and `input_rows`, when present, names the input rows, each by its index in the body, that formed that row after an engine's insert-time merge. A row both removed and reset lists only `ttl_delete`. `column` is present only for `ttl_column_reset`, and `stored` is absent when the column's `DEFAULT` reads the clock or a generator, so the value is decided at the merge. Absent is never empty, in any binding: Go `nil`, Python `None`, TypeScript `undefined`, Rust `None`. The list is "at least these", not "only these": a later background merge can remove more rows as more of them expire. The batch's own `unsupported_settings` names the call's settings the library declined. Both lists are empty when the document carries none. [`spec/abi-v2/docs.md`](../../spec/abi-v2/docs.md) is normative for both. `Framing` is `bom_skipped`, `container` and `header`, each the vendored reader's own state: @@ -451,10 +473,12 @@ A column whose DEFAULT calls one of the random or UUID generators this build adm | ---------------------- | -------------------------------------------------------------------------- | ------------------------ | | `outcome` | `outcome` | `FilterOutcome` | | `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | +| `reason` | `reason`: the catalog id of a declined evaluation, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at`, `refused_row` | `refused_at` and `refused_row`, as the batch's: where a refusal or decline sits | optional `RefusalSite`, optional unsigned 64-bit | | `rows_read` | `rows_read` | unsigned 64-bit | | `unsupported_settings` | `unsupported_settings`, name objects (rule 5) | list of bytes | | `verdicts` | `verdicts`: one character per row, each mapped through the generated table | list of `Verdict` | -| `errors` | `errors`: `row` (int), `code` (int32), `msg` (bytes, from `err`, rule 5) | list of `FilterRowError` | +| `errors` | `errors`: `row` (int), `code` (int32), `reason` (optional ASCII text, present where `code` is `CHS_CODE_DECLINED`), `msg` (bytes, from `err`, rule 5) | list of `FilterRowError` | `e` and `d` are never answers, and a caller enforcing visibility fails closed on both; `Verdict.answered` says which. A non-ok outcome's verdicts are whatever the document holds: v0's rule that a binding empties them is the library's now. @@ -462,7 +486,8 @@ A column whose DEFAULT calls one of the random or UUID generators this build adm | field | from | type | | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| `columns` | `columns`, in declared order. Each `Column`: `name` (bytes, rule 5), `type` (bytes, rule 5: the canonical type), `default_kind` (`DefaultKind`), `default_expr` (bytes, rule 5, from `default_expression`) | list of `Column` | +| `columns` | `columns`, in declared order. Each `Column`: `name` (bytes, rule 5), `type` (bytes, rule 5: the canonical type), `default_kind` (`DefaultKind`), `default_expr` (bytes, rule 5, from `default_expression`), `default_support` (optional `DefaultSupport`) and `default_support_reason` (optional ASCII text), reserved, and `reads` (optional list of bytes, name objects, rule 5), reserved | list of `Column` | +| `posture` | `posture` (a `posture` value): how the statement was compiled, present only for `load`; absent is `create` (ABI v2) | optional `Posture` | | `filter_declined_settings` | `filter_declined_settings`: the settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known at schema compile, as the library wrote them (one entry per name, at the highest layer that sets it, by layer and then by name bytes), empty when it lists none (ABI v2) | list of `DeclinedSetting` | | `server` | `SchemaServer`: `timezone` (text: the zone the schema's types bind, the profile's or else the image zone), `settings` (settings, `{}` when the profile gave none) and `macros` (settings, absent exactly when the profile carried none). From `server`, which the document carries exactly when the schema was compiled on a server (ABI v2) | optional `SchemaServer`: absent exactly when the schema was compiled without a server | | `replicated` | `SchemaReplicated`: `zookeeper_path` (bytes, rule 5) and `replica_name` (bytes, rule 5), fully expanded. From `replicated`: what ClickHouse's own `TableZnodeInfo` resolved for a Replicated engine on a server, to compare with the server's `system.replicas` (ABI v2) | optional `SchemaReplicated` | diff --git a/scripts/parity-surface-allow.json b/scripts/parity-surface-allow.json index 82ca767a..47e4a021 100644 --- a/scripts/parity-surface-allow.json +++ b/scripts/parity-surface-allow.json @@ -341,7 +341,10 @@ "MergeReason.Known", "DeclinedTier.Known", "DeclinedLayer.Known", - "DiscoverQueryParam.Known" + "DiscoverQueryParam.Known", + "RefusalSite.Known", + "Posture.Known", + "DefaultSupport.Known" ], "issue": 511, "reason": "The ABI v2 dev binding's rule r3 (spec/abi-v2/docs.md): every vocabulary has an unknown(n) member, an unlisted value kept per field, which Go spells as the raw value plus Known() false. docs/reference/bindings-v1.md is the 1.x surface and names none of them; the v2 surface document lands with the other three dev bindings (public issue #511), and this entry goes with it." From 7e9141688e0b5b9eeee394a4f6ef1f243a03e8ec Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 15:59:39 -0400 Subject: [PATCH 06/24] go (spec batch 2): CHANGELOG Co-Authored-By: Claude Sonnet 5.5 --- go/CHANGELOG.md | 9 +++++++++ go/chtypes/options.go | 3 ++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index 0ec6d201..4791bb51 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -23,6 +23,10 @@ The four bindings in this repository are released together and give one answer, - The v2 spec's decline catalog adds `server-dependent-function` (a column's DEFAULT, MATERIALIZED, ALIAS or EPHEMERAL expression, or a CHECK, that calls a function whose resolver needs a server is declined at `chs_schema_create`), `function-name-as-lambda` (a bare function name written where a higher-order function takes a lambda, such as `arrayExists(isNull, arr)`, is declined at `chs_filter_create` on 26.7, 26.8 and 26.9) and `skipped-key-in-squashed-block` (a batch whose answer depends on how the INSERT's squashing step groups its chunks is unsupported), and `call-setting-session-timezone` names only a per-call `session_timezone` the call cannot attach (55 reasons). - A `410 Gone` is a retired repository, and permanent (#571). Any request the fetch layer makes (a tag, an alias, a digest, a blob, `tags/list`, the referrers API or its fallback tag, after a redirect too) that answers `410` is now the new code `CHTYPES_SOURCE_RETIRED` (exit status 10), never retried and never sent to the next base, nor from the dev channel's alias to its tag. It was `CHTYPES_SOURCE_UNREACHABLE` (exit 3) with the registry's message dropped, and Go, Python and Rust moved on to the next base, so a mirror's 404 could hide the retirement. The error names the URL that answered and carries the registry's own message, with control and bidirectional-formatting characters removed and cut to 256 code points: Go `CodeSourceRetired` (`errors.Is(err, ErrSourceRetired)`), Python and TypeScript `SourceRetiredError`, Rust `Error::SourceRetired`. The rule is `docs/guides/fetch-v1.md` section 2, "A retired repository", and the `retired-410-*` and `list-tags-retired-410` conformance cases and one shared message table (`tests/fixtures/retired-message/cases.json`) hold all four bindings to it. - A 2.0.0-dev SDK prefers the newest build of its own fingerprint (#511). A version request resolves `--fp-` first, the alias the staging dev registry serves beside every dev tag, with the binding's own generated ABI fingerprint, and the tag itself only when every base answers the alias 404; any other failure of the alias request is that failure, with no fallback, and an alias whose index lacks the platform is `CHTYPES_ARTIFACT_UNPUBLISHED`. The dev channel's cache lookups (the installed-build lookup a registry open makes first, the offline answer, and the monotonic rule of a fetch) now see only records whose signed `abi_fingerprint` is the SDK's own, so a newer build of another fingerprint, installed by another dev SDK in the shared cache, is never returned or kept over this one's own, and is never removed or rewritten. So a dev build of a newer fingerprint no longer strands an older dev SDK until its consumers upgrade. Trust is unchanged (the signed predicate and the load checks decide), a listing never shows an alias, nothing records one, `list` and `where` are unchanged, and there is no option: both steps are the dev channel's alone, and the v1 contract and every production channel never take them. The rule is `docs/guides/fetch-v1.md` section 3, and the `dev-alias-*`, `resolve-installed-dev-alias-*` and `list-tags-dev-alias` conformance cases hold all four bindings to it. +- Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): `WithInsertableColumns`, `WithColumnDefaults` (with `ColumnDefault`), `WithVerdictBasis` (`VerdictBasisInput`, `VerdictBasisExport`) and `WithDeferDefaults` (`DeferDefaultsNone`, `DeferDefaultsServer`) build `insert_options` for `Schema.Row`, `Schema.Rows`, `Schema.ParseBlock` and `Filter.Rows`; `WithParamEncoding` (`ParamEncodingEscaped`, `ParamEncodingRaw`) builds `filter_options` for `CompileFilter`; `WithPosture` (`PostureCreate`, `PostureLoad`) builds `schema_options` for `CompileTable`. `WithInsertOption`, `WithFilterOption` and `WithSchemaOption` pass a member the binding has no name for through as JSON text, so the library, which refuses an unknown key (rule r1), is the only judge. A document with no member set is not sent. `WithColumns` now applies to `Filter.Rows` too (`chs_filter_eval_body` takes the column list). +- Spec batch 2, a value that is not UTF-8 (ABI v2): a settings value, a query-parameter value, a `Setup` default and a server profile's settings value is sent as `{"b64": ...}` (the content `json_object_byte_values`), where the binding used to refuse it; and `SchemaServer.Settings` decodes the same form back to its bytes. +- Spec batch 2, the decline and refusal members (ABI v2): `RowResult`, `BatchResult` and `FilterResult` carry `Reason` (the catalog id, present where the code is `CodeDeclined`, -2), `RefusedAt` (a `RefusalSite`: `RefusedAtCall`, `RefusedAtBodyRead`, `RefusedAtRow`, `RefusedAtBodyStep`) and `RefusedRow`; `RowResult.VerdictReason`; `FilterRowError.Reason`; `BatchResult.ExportColumns` and `BatchResult.ExportDeclinedReason`. +- Spec batch 2, the schema description (ABI v2): `SchemaDescription.Posture`, and `Column.DefaultSupport` (a `DefaultSupport`), `Column.DefaultSupportReason` and `Column.Reads`, reserved members a build reports when it implements them. `SourceDefaultDeferred`, `SourceDefaultOverride` (stored) and `TierServerRefuses` join the vocabularies. ### Changed @@ -67,9 +71,14 @@ The four bindings in this repository are released together and give one answer, - **Enforcement gate lifted** for `26.8` on `darwin-arm64`. - **Enforcement gate lifted** for `26.9` on `darwin-arm64`. - `docs/limitations.md` is re-checked against artifact build `20261012.142240` (v2-dev.13), the library half measured on that build: functions that need a server, in a column expression, and a body the server may read serially or in parallel under a non-zero error budget are listed under "Declined by design"; the known divergence for a column expression that calls a function needing a server is retired, since that build declines those schemas (on 26.8 and 26.9 the `AI*` and `ai*` functions compile, as the page says); and the `lossy` raw-NUL TSV gap is marked fixed on that build and still present in the 1.0 libraries. `docs/guides/filters.md` gains a section on subcolumn filters and a JSON value typed by its batch. +- **Breaking (ABI v2 dev): `BatchResult.Spans` is `[]*Span`**, one entry per `Rows` entry and index-aligned with it: `row_spans` carries `null` for a row the export did not carry (a skipped row, or a row a filter cut), which the entry reads as nil. It was `[]Span` of the exported rows only. +- **Breaking (ABI v2 dev): `AtMergeEntry.Row` is `AtMergeEntry.PartRow`**, the document's `part_row`. +- A document's `code` can be `CodeDeclined` (-2) for a row, batch, filter result or verdict this build declined, and every decline names its `Reason`; a `Source` the description does not list reads the description's fallback (`skipped`), not stored. ### Removed +- **Breaking (ABI v2 dev): `RowResult.UnsupportedSettings`**: the row document no longer carries `unsupported_settings`; the batch's and the filter result's lists stay. +- The refusal of a settings, query-parameter, default or server-settings value that is not valid UTF-8 (it is the `{"b64": ...}` form now); a key that is not valid UTF-8 is still refused. - `SchemaServer.FilterDeclinedSettings` (a breaking change inside v2 dev, #588): the server member no longer carries the list. Read `SchemaDescription.FilterDeclinedSettings`, whose `server`-layer entries are the profile's. ### Fixed diff --git a/go/chtypes/options.go b/go/chtypes/options.go index e6d32e24..ef3564e0 100644 --- a/go/chtypes/options.go +++ b/go/chtypes/options.go @@ -135,7 +135,8 @@ type serverOpt struct{ s *Server } func (o serverOpt) applyCompile(c *callConfig) { c.server = o.s } // WithSettings sets the call's settings: a map of strings to strings, -// serialized as a JSON object and passed verbatim. No value is ever rewritten. +// serialized as a JSON object and passed verbatim. No value is ever rewritten: +// a value that is not valid UTF-8 is sent as {"b64": ...}, the object form. // On a compile it is the profile; on a filter compile the filter's own profile; // on the calls that parse a body, the body's parse settings. func WithSettings(settings map[string]string) SettingsOption { From 1e94950d4cac6a1673496cca5a2a147c10a5076f Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:06:05 -0400 Subject: [PATCH 07/24] v2 generator: call errors carry chs_error_reason as a reason field Go's CallError, Python's ErrorInfo and the call error classes, and Rust's RawCallError read chs_error_reason when the description declares it. Output for ABI v1 is unchanged; the v2 fingerprint is unchanged. Co-Authored-By: Claude Sonnet 5.5 --- go/internal/abi2/abi_gen.go | 4 +++- python/src/chtypes/_abi2/_decls.py | 4 ++++ python/src/chtypes/errors.py | 3 +++ rust/src/abi2/calls_gen.rs | 9 +++++++++ rust/src/error.rs | 1 + scripts/abi-v1/emit/go.py | 17 ++++++++++++++++- scripts/abi-v1/emit/python.py | 19 +++++++++++++++---- scripts/abi-v1/emit/rust.py | 16 +++++++++++++++- 8 files changed, 66 insertions(+), 7 deletions(-) diff --git a/go/internal/abi2/abi_gen.go b/go/internal/abi2/abi_gen.go index 1e50745d..f67fd027 100644 --- a/go/internal/abi2/abi_gen.go +++ b/go/internal/abi2/abi_gen.go @@ -292,7 +292,7 @@ func (t *Table) ResolveAll() string { } // CallError is D3's per-call error shape, read verbatim off a non-OK status: -// the chs_status NAME (never the bare int), ch_code, ch_name, message and column. +// the chs_status NAME (never the bare int), ch_code, ch_name, message and column and, from chs_error_reason, reason. // It carries no class of its own -- StatusClass(e.Status) is sdk.json's table, // generated into errmap_gen.go, and the public API (wave C) is what turns this // into an idiomatic *SchemaError/*UsageError/... @@ -302,6 +302,7 @@ type CallError struct { ChName string Message string Column string + Reason string } func (e *CallError) Error() string { @@ -379,6 +380,7 @@ func makeCallError(t *Table, status int32, err *ErrorHandle) *CallError { ce.ChName = string(readAndFreeBuf(t, t.ErrorChName(err))) ce.Message = string(readAndFreeBuf(t, t.ErrorMessage(err))) ce.Column = string(readAndFreeBuf(t, t.ErrorColumn(err))) + ce.Reason = string(readAndFreeBuf(t, t.ErrorReason(err))) err.Close() } return ce diff --git a/python/src/chtypes/_abi2/_decls.py b/python/src/chtypes/_abi2/_decls.py index 1b146878..33bf02a7 100644 --- a/python/src/chtypes/_abi2/_decls.py +++ b/python/src/chtypes/_abi2/_decls.py @@ -571,6 +571,7 @@ def _check(self, status: int, err: ctypes.c_void_p | None) -> None: info.ch_name.decode("ascii", "replace") if info else "", message, info.column if info else b"", + reason=info.reason.decode("ascii", "replace") if info else "", ) def live_handles(self) -> bytes: @@ -1027,6 +1028,7 @@ class ErrorInfo: ch_name: bytes message: bytes column: bytes + reason: bytes = b"" @dataclass @@ -1088,6 +1090,7 @@ def _decode_error(api: Api, err_ptr: int) -> ErrorInfo: ch_name = _read_buf(api, api._raw["chs_error_ch_name"](err_ptr)) message = _read_buf(api, api._raw["chs_error_message"](err_ptr)) column = _read_buf(api, api._raw["chs_error_column"](err_ptr)) + reason = _read_buf(api, api._raw["chs_error_reason"](err_ptr)) api._raw["chs_error_free"](err_ptr) return ErrorInfo( status=STATUS_BY_VALUE.get(status_val, f"UNKNOWN:{status_val}"), @@ -1095,6 +1098,7 @@ def _decode_error(api: Api, err_ptr: int) -> ErrorInfo: ch_name=ch_name, message=message, column=column, + reason=reason, ) diff --git a/python/src/chtypes/errors.py b/python/src/chtypes/errors.py index d3f2a392..838f859d 100644 --- a/python/src/chtypes/errors.py +++ b/python/src/chtypes/errors.py @@ -96,6 +96,7 @@ class CallError(ChtypesError): message: ClickHouse's own message for a refusal, the library's otherwise; bytes. column: the column concerned, empty if none; bytes. + reason: the catalog id (`chs_error_reason`), ASCII; empty if none. """ def __init__( @@ -105,6 +106,7 @@ def __init__( ch_name: str = "", message: bytes = b"", column: bytes = b"", + reason: str = "", ) -> None: super().__init__(message.decode("utf-8", "replace")) self.status = status @@ -112,6 +114,7 @@ def __init__( self.ch_name = ch_name self.message = message self.column = column + self.reason = reason class SchemaError(CallError): diff --git a/rust/src/abi2/calls_gen.rs b/rust/src/abi2/calls_gen.rs index a2016b89..b0760d2a 100644 --- a/rust/src/abi2/calls_gen.rs +++ b/rust/src/abi2/calls_gen.rs @@ -22,6 +22,7 @@ pub(crate) struct RawCallError { pub(crate) ch_name: String, pub(crate) message: Vec, pub(crate) column: Vec, + pub(crate) reason: String, } /// Copy one owned `chs_buf *` into a `Vec` and free it. NULL reads as @@ -63,6 +64,7 @@ unsafe fn read_free_error(api: &Api, status: i32, err: *mut ChsError) -> RawCall ch_name: String::new(), message: b"the library returned a non-OK status with no error object".to_vec(), column: Vec::new(), + reason: String::new(), }; } // SAFETY: the caller's `# Safety` clause; every accessor reads the live @@ -76,6 +78,11 @@ unsafe fn read_free_error(api: &Api, status: i32, err: *mut ChsError) -> RawCall .into_owned(); let message = copy_free_buf(api, (api.chs_error_message)(err as *const ChsError)); let column = copy_free_buf(api, (api.chs_error_column)(err as *const ChsError)); + let reason = String::from_utf8_lossy(©_free_buf( + api, + (api.chs_error_reason)(err as *const ChsError), + )) + .into_owned(); (api.chs_error_free)(err); RawCallError { status, @@ -83,6 +90,7 @@ unsafe fn read_free_error(api: &Api, status: i32, err: *mut ChsError) -> RawCall ch_name, message, column, + reason, } } } @@ -97,6 +105,7 @@ fn missing_output(status: i32, what: &str) -> RawCallError { ch_name: String::new(), message: format!("the library answered OK and returned no {what}").into_bytes(), column: Vec::new(), + reason: String::new(), } } diff --git a/rust/src/error.rs b/rust/src/error.rs index 47337884..4697709d 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -326,6 +326,7 @@ mod tests { ch_name: "CANNOT_PARSE_INPUT_ASSERTION_FAILED".to_string(), message: b"boom".to_vec(), column: b"c".to_vec(), + reason: String::new(), } } diff --git a/scripts/abi-v1/emit/go.py b/scripts/abi-v1/emit/go.py index bf04db70..00f1e109 100644 --- a/scripts/abi-v1/emit/go.py +++ b/scripts/abi-v1/emit/go.py @@ -579,14 +579,24 @@ def render_abi_gen(model) -> str: return text +def _has_error_reason(model) -> bool: + """True when the description declares chs_error_reason (ABI v2): every call error then carries it.""" + return any(fn.name == "chs_error_reason" for fn in model.functions) + + def render_call_error(model) -> str: fields = [("Status", "string"), ("ChCode", "int32"), ("ChName", "string"), ("Message", "string"), ("Column", "string")] + has_reason = _has_error_reason(model) + if has_reason: + fields.append(("Reason", "string")) s = _struct( "CallError", fields, doc=( "CallError is D3's per-call error shape, read verbatim off a non-OK status:\n" - "the chs_status NAME (never the bare int), ch_code, ch_name, message and column.\n" + "the chs_status NAME (never the bare int), ch_code, ch_name, message and column" + + (" and, from chs_error_reason, reason" if has_reason else "") + + ".\n" "It carries no class of its own -- StatusClass(e.Status) is sdk.json's table,\n" "generated into errmap_gen.go, and the public API (wave C) is what turns this\n" "into an idiomatic *SchemaError/*UsageError/..." @@ -672,6 +682,11 @@ def render_helpers(model) -> str: f"{TAB}{TAB}ce.ChName = string(readAndFreeBuf(t, t.ErrorChName(err)))", f"{TAB}{TAB}ce.Message = string(readAndFreeBuf(t, t.ErrorMessage(err)))", f"{TAB}{TAB}ce.Column = string(readAndFreeBuf(t, t.ErrorColumn(err)))", + *( + [f"{TAB}{TAB}ce.Reason = string(readAndFreeBuf(t, t.ErrorReason(err)))"] + if _has_error_reason(model) + else [] + ), f"{TAB}{TAB}err.Close()", f"{TAB}}}", f"{TAB}return ce", diff --git a/scripts/abi-v1/emit/python.py b/scripts/abi-v1/emit/python.py index 1366c162..e5b8d390 100644 --- a/scripts/abi-v1/emit/python.py +++ b/scripts/abi-v1/emit/python.py @@ -384,6 +384,17 @@ def render_decls(model: Model) -> str: typed_wrappers = _typed_wrappers(model) handle_class_dict = _dict_block([(_lit(h), cls) for h, cls in handle_classes.items()]) cross_check = _cross_check_literal(model) + # A description that declares chs_error_reason (ABI v2) has every call error + # carry it; one that does not (ABI v1) emits the layer unchanged. + has_reason = any(fn.name == "chs_error_reason" for fn in model.functions) + info_reason_field = "\n reason: bytes = b\"\"" if has_reason else "" + info_reason_read = ( + '\n reason = _read_buf(api, api._raw["chs_error_reason"](err_ptr))' if has_reason else "" + ) + info_reason_arg = "\n reason=reason," if has_reason else "" + raise_reason_arg = ( + '\n reason=info.reason.decode("ascii", "replace") if info else "",' if has_reason else "" + ) handle_class_defs = "\n\n\n".join( f'class {cls}(Handle):\n """A `{h}` the library owns the free function for."""' @@ -639,7 +650,7 @@ def _check(self, status: int, err: ctypes.c_void_p | None) -> None: info.ch_code if info else 0, info.ch_name.decode("ascii", "replace") if info else "", message, - info.column if info else b"", + info.column if info else b"",{raise_reason_arg} ) {typed_wrappers} @@ -675,7 +686,7 @@ class ErrorInfo: ch_code: int ch_name: bytes message: bytes - column: bytes + column: bytes{info_reason_field} @dataclass @@ -736,14 +747,14 @@ def _decode_error(api: Api, err_ptr: int) -> ErrorInfo: ch_code = api._raw["chs_error_ch_code"](err_ptr) ch_name = _read_buf(api, api._raw["chs_error_ch_name"](err_ptr)) message = _read_buf(api, api._raw["chs_error_message"](err_ptr)) - column = _read_buf(api, api._raw["chs_error_column"](err_ptr)) + column = _read_buf(api, api._raw["chs_error_column"](err_ptr)){info_reason_read} api._raw["chs_error_free"](err_ptr) return ErrorInfo( status=STATUS_BY_VALUE.get(status_val, f"UNKNOWN:{{status_val}}"), ch_code=ch_code, ch_name=ch_name, message=message, - column=column, + column=column,{info_reason_arg} ) diff --git a/scripts/abi-v1/emit/rust.py b/scripts/abi-v1/emit/rust.py index fede99b2..a1c88572 100644 --- a/scripts/abi-v1/emit/rust.py +++ b/scripts/abi-v1/emit/rust.py @@ -842,6 +842,7 @@ def _method_name(fn, model) -> str: def render_calls(model) -> str: + has_reason = any(fn.name == "chs_error_reason" for fn in model.functions) out: list[str] = [ f"// {banner(model)}", "//", @@ -872,6 +873,7 @@ def render_calls(model) -> str: " pub(crate) ch_name: String,", " pub(crate) message: Vec,", " pub(crate) column: Vec,", + *([" pub(crate) reason: String,"] if has_reason else []), "}", "", "/// Copy one owned `chs_buf *` into a `Vec` and free it. NULL reads as", @@ -913,6 +915,7 @@ def render_calls(model) -> str: " ch_name: String::new(),", " message: b\"the library returned a non-OK status with no error object\".to_vec(),", " column: Vec::new(),", + *([" reason: String::new(),"] if has_reason else []), " };", " }", " // SAFETY: the caller's `# Safety` clause; every accessor reads the live", @@ -923,8 +926,18 @@ def render_calls(model) -> str: " .into_owned();", " let message = copy_free_buf(api, (api.chs_error_message)(err as *const ChsError));", " let column = copy_free_buf(api, (api.chs_error_column)(err as *const ChsError));", + *( + [ + " let reason = String::from_utf8_lossy(©_free_buf(api, (api.chs_error_reason)(err as *const ChsError)))", + " .into_owned();", + ] + if has_reason + else [] + ), " (api.chs_error_free)(err);", - " RawCallError { status, ch_code, ch_name, message, column }", + " RawCallError { status, ch_code, ch_name, message, column" + + (", reason" if has_reason else "") + + " }", " }", "}", "", @@ -938,6 +951,7 @@ def render_calls(model) -> str: " ch_name: String::new(),", " message: format!(\"the library answered OK and returned no {what}\").into_bytes(),", " column: Vec::new(),", + *([" reason: String::new(),"] if has_reason else []), " }", "}", "", From 79db1a7cdccd66d032acf4e773da8373305a1a97 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:07:54 -0400 Subject: [PATCH 08/24] stubtest: call the v2 signatures of preview_row, filter_create and filter_eval_body Co-Authored-By: Claude Sonnet 5.5 --- scripts/abi-v1/stubtest.c | 58 ++++++++++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/scripts/abi-v1/stubtest.c b/scripts/abi-v1/stubtest.c index 68ad7aeb..8de5a16b 100644 --- a/scripts/abi-v1/stubtest.c +++ b/scripts/abi-v1/stubtest.c @@ -73,15 +73,47 @@ typedef chs_status (*fn_schema_create)(const uint8_t *, size_t, const uint8_t *, (l)->schema_create((sql), (n), (settings), (settings_n), (out), (err)) #endif typedef void (*fn_schema_free)(chs_schema *); +#if CHS_ABI_VERSION >= 2 +/* Generation 2: chs_filter_create takes an options document after the settings, + chs_preview_row one after the columns, and chs_filter_eval_body a columns list + and an options document after the settings. Every call below passes none, + which is generation 1's behavior. */ +typedef chs_status (*fn_filter_create)( + const chs_schema *, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, + size_t, chs_filter **, chs_error **); +#define FILTER_CREATE(l, schema, expr, n, params, params_n, settings, settings_n, out, err) \ + (l)->filter_create((schema), (expr), (n), (params), (params_n), (settings), (settings_n), NULL, 0, (out), (err)) +#else typedef chs_status (*fn_filter_create)( const chs_schema *, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, size_t, chs_filter **, chs_error **); +#define FILTER_CREATE(l, schema, expr, n, params, params_n, settings, settings_n, out, err) \ + (l)->filter_create((schema), (expr), (n), (params), (params_n), (settings), (settings_n), (out), (err)) +#endif typedef void (*fn_filter_free)(chs_filter *); +#if CHS_ABI_VERSION >= 2 +typedef chs_status (*fn_filter_eval_body)( + const chs_filter *, chs_format, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, size_t, + const uint8_t *, size_t, chs_buf **, chs_error **); +typedef chs_status (*fn_preview_row)( + const chs_schema *, chs_format, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, size_t, + const uint8_t *, size_t, chs_buf **, chs_error **); +#define FILTER_EVAL_BODY(l, f, format, body, n, settings, settings_n, out, err) \ + (l)->filter_eval_body((f), (format), (body), (n), (settings), (settings_n), NULL, 0, NULL, 0, (out), (err)) +#define PREVIEW_ROW(l, schema, format, body, n, settings, settings_n, columns, columns_n, out, err) \ + (l)->preview_row((schema), (format), (body), (n), (settings), (settings_n), (columns), (columns_n), NULL, 0, \ + (out), (err)) +#else typedef chs_status (*fn_filter_eval_body)( const chs_filter *, chs_format, const uint8_t *, size_t, const uint8_t *, size_t, chs_buf **, chs_error **); typedef chs_status (*fn_preview_row)( const chs_schema *, chs_format, const uint8_t *, size_t, const uint8_t *, size_t, const uint8_t *, size_t, chs_buf **, chs_error **); +#define FILTER_EVAL_BODY(l, f, format, body, n, settings, settings_n, out, err) \ + (l)->filter_eval_body((f), (format), (body), (n), (settings), (settings_n), (out), (err)) +#define PREVIEW_ROW(l, schema, format, body, n, settings, settings_n, columns, columns_n, out, err) \ + (l)->preview_row((schema), (format), (body), (n), (settings), (settings_n), (columns), (columns_n), (out), (err)) +#endif typedef struct { void *handle; @@ -245,8 +277,8 @@ static void test_long_echo_not_truncated(lib_t *l) { const char *columns = "[\"col_a\",\"col_b\",\"col_c\",\"col_d\",\"col_e\"]"; chs_buf *out = NULL; - chs_status st = l->preview_row( - schema, CHS_JSON_EACH_ROW, (const uint8_t *) body, strlen(body), (const uint8_t *) settings, + chs_status st = PREVIEW_ROW( + l, schema, CHS_JSON_EACH_ROW, (const uint8_t *) body, strlen(body), (const uint8_t *) settings, strlen(settings), (const uint8_t *) columns, strlen(columns), &out, &err); CHECK(st == CHS_OK, "chs_preview_row for the long-echo test: status %d", (int) st); if (out != NULL) { @@ -348,7 +380,7 @@ static void test_holds_and_live(lib_t *l) { CHECK(live_count(l, "chs_schema") == schema_before + 1, "live chs_schema did not increase by one on create"); chs_filter *filter = NULL; - CHECK(l->filter_create(schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err) == CHS_OK, "chs_filter_create"); + CHECK(FILTER_CREATE(l, schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err) == CHS_OK, "chs_filter_create"); CHECK(live_count(l, "chs_filter") == filter_before + 1, "live chs_filter did not increase by one on create"); l->schema_free(schema); /* the caller's own reference; the filter still holds one */ @@ -356,7 +388,7 @@ static void test_holds_and_live(lib_t *l) { "live chs_schema dropped after freeing the schema while a filter still holds it"); chs_buf *out = NULL; - CHECK(l->filter_eval_body(filter, CHS_JSON_EACH_ROW, (const uint8_t *) "{}", 2, NULL, 0, &out, &err) == CHS_OK, + CHECK(FILTER_EVAL_BODY(l, filter, CHS_JSON_EACH_ROW, (const uint8_t *) "{}", 2, NULL, 0, &out, &err) == CHS_OK, "chs_filter_eval_body after its schema was freed by the caller"); if (out != NULL) l->buf_free(out); @@ -407,20 +439,20 @@ static void *conc_worker(void *p) { for (int i = 0; i < CONC_CALLS; i++) { chs_buf *out = NULL; chs_error *err = NULL; - if (a->l->preview_row(a->schema, CHS_JSON_EACH_ROW, (const uint8_t *) body, sizeof body - 1, - (const uint8_t *) settings, sizeof settings - 1, NULL, 0, &out, &err) != CHS_OK) { + if (PREVIEW_ROW(a->l, a->schema, CHS_JSON_EACH_ROW, (const uint8_t *) body, sizeof body - 1, + (const uint8_t *) settings, sizeof settings - 1, NULL, 0, &out, &err) != CHS_OK) { a->failures++; } if (out != NULL) a->l->buf_free(out); chs_filter *f = NULL; - if (a->l->filter_create(a->schema, (const uint8_t *) "x = 1", 5, NULL, 0, (const uint8_t *) settings, - sizeof settings - 1, &f, &err) != CHS_OK) { + if (FILTER_CREATE(a->l, a->schema, (const uint8_t *) "x = 1", 5, NULL, 0, (const uint8_t *) settings, + sizeof settings - 1, &f, &err) != CHS_OK) { a->failures++; } if (f != NULL) a->l->filter_free(f); out = NULL; - if (a->l->filter_eval_body(a->filter, CHS_JSON_EACH_ROW, (const uint8_t *) body, sizeof body - 1, NULL, 0, - &out, &err) != CHS_OK) { + if (FILTER_EVAL_BODY(a->l, a->filter, CHS_JSON_EACH_ROW, (const uint8_t *) body, sizeof body - 1, NULL, 0, + &out, &err) != CHS_OK) { a->failures++; } if (out != NULL) a->l->buf_free(out); @@ -437,7 +469,7 @@ static void test_concurrent_shared(lib_t *l) { chs_error *err = NULL; CHECK(SCHEMA_CREATE(l, (const uint8_t *) "x", 1, NULL, 0, &schema, &err) == CHS_OK, "chs_schema_create"); chs_filter *filter = NULL; - CHECK(l->filter_create(schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err) == CHS_OK, + CHECK(FILTER_CREATE(l, schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err) == CHS_OK, "chs_filter_create"); pthread_t threads[CONC_THREADS]; @@ -498,7 +530,7 @@ static void test_document_mode(lib_t *l) { memcpy(body, "!D:", 3); memcpy(body + 3, cases[i].value, cases[i].len); chs_buf *out = NULL; - chs_status st = l->preview_row(schema, CHS_JSON_EACH_ROW, body, 3 + cases[i].len, NULL, 0, NULL, 0, &out, &err); + chs_status st = PREVIEW_ROW(l, schema, CHS_JSON_EACH_ROW, body, 3 + cases[i].len, NULL, 0, NULL, 0, &out, &err); CHECK(st == CHS_OK && out != NULL, "document mode, %s: status %d", cases[i].what, (int) st); if (out == NULL) continue; const char *doc = (const char *) l->buf_data(out); @@ -529,7 +561,7 @@ static void test_cross_image(lib_t *a, lib_t *b) { chs_schema *schema = NULL; CHECK(SCHEMA_CREATE(a, (const uint8_t *) "x", 1, NULL, 0, &schema, &err) == CHS_OK, "chs_schema_create on lib a"); chs_filter *filter = NULL; - chs_status st = b->filter_create(schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err); + chs_status st = FILTER_CREATE(b, schema, (const uint8_t *) "1", 1, NULL, 0, NULL, 0, &filter, &err); CHECK(st == CHS_INVALID_ARGUMENT, "lib b's chs_filter_create over lib a's schema gave %d, want CHS_INVALID_ARGUMENT", (int) st); if (err != NULL) b->error_free(err); From 628eedcefd1312914821a83f1324987b9e17e9e1 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:11:23 -0400 Subject: [PATCH 09/24] go: CallError.Reason from the generated reason field Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/bindings-v1.md | 3 ++- docs/reference/go.md | 2 +- go/CHANGELOG.md | 1 + go/README.md | 2 +- go/chtypes/doc.go | 2 +- go/chtypes/errors.go | 9 ++++++--- go/chtypes/errors_test.go | 6 +++--- 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 59c0217d..7a84f68d 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -300,7 +300,7 @@ The class for each status and each loader refusal is [`spec/abi-v1/sdk.json`](.. ### The fields -**Every call error carries the five fields of `chs_error`, verbatim, and nothing synthesized:** +**Every call error carries the six fields of `chs_error`, verbatim, and nothing synthesized:** | field | meaning | Go | Python | TypeScript | Rust | | --------- | ------------------------------------------------------------------------- | ---------------- | ---------------- | ---------------------- | ------------------ | @@ -309,6 +309,7 @@ The class for each status and each loader refusal is [`spec/abi-v1/sdk.json`](.. | `ch_name` | the name this build's vendored table gives the code; ASCII; empty if none | `ChName string` | `ch_name: str` | `chName: string` | `ch_name: String` | | `message` | ClickHouse's own message for a refusal, the library's otherwise; bytes | `Message string` | `message: bytes` | `messageBytes: Buffer` | `message: RawText` | | `column` | the column concerned, empty if none; bytes | `Column string` | `column: bytes` | `column: Buffer` | `column: RawText` | +| `reason` | the catalog id (`chs_error_reason`); ASCII; empty if none | `Reason string` | `reason: str` | `reason: string` | `reason: String` | A misuse the binding detects itself carries `status` `CHS_INVALID_ARGUMENT`, `ch_code` 0, an empty `ch_name` and `column`, and a message naming the misuse, so a handler sees one shape whichever side caught it. diff --git a/docs/reference/go.md b/docs/reference/go.md index 7c0fe0b1..d0f1f8d9 100644 --- a/docs/reference/go.md +++ b/docs/reference/go.md @@ -31,7 +31,7 @@ Row and batch verdicts are **never** Go errors: they are `Outcome` in the result | `*InternalError` | `CHS_INTERNAL`, or a document that does not decode | | `*ArtifactError` | a fetch or load failure, with a `Code` (an `ErrorCode`, one of the generated `Code…` constants) and a sentinel per code for `errors.Is`, such as `ErrArtifactMissing` | -All four call types embed `CallError`, whose five fields (`Status`, `ChCode`, `ChName`, `Message`, `Column`) read directly; `AsCallError` reads them from any of the four. `Message` and `Column` are byte strings, and `Error()` is the one lossy display form. +All four call types embed `CallError`, whose six fields (`Status`, `ChCode`, `ChName`, `Message`, `Column`, `Reason`) read directly; `AsCallError` reads them from any of the four. `Reason` is the catalog id (ASCII). `Message` and `Column` are byte strings, and `Error()` is the one lossy display form. ## The Go spellings diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index 4791bb51..5e526113 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `CallError.Reason` (ABI v2): the error's catalog id, read through `chs_error_reason` by the generated layer and passed through verbatim (ASCII, empty if none), on all four call classes. - A fetch-only call (#492): `(*Registry).Fetch(ctx, request) (Resolved, error)` resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing: no library is loaded and the process setup is untouched, so it needs no `Setup`. It honors the registry's fetch options (offline, the cache, and under the production channel the bases, the trust and the lock), and a concurrent `Fetch` and open of one request share one fetch. The `Resolved` names the installed version, build, digests and library path; errors are the fetch codes, and a done `ctx` is `ctx.Err()`. - `chtypes resolve [--json]` (#493): what a line or version resolves to on every platform the registry offers, one `resolved ` line each, or with `--json` one JSON array of `{platform, version, build, manifest}` objects, each signed statement verified exactly as a fetch verifies it and no layer downloaded; `--offline` answers from the cache. The four CLIs print the same bytes, so one parser reads them (a CI cache key, for example). - `chtypes prune [--line ] [--keep ] [--dry-run]` (#494): removes the installed builds of each line and platform that at least n newer installed builds supersede (n defaults to 1), each with its record, its blobs and its `index.json` entry; `--dry-run` prints exactly what would go. It never removes a build a running process holds: a registry now holds every build it opens or fetches, with a shared `flock` on the entry's `verified.json` until the process exits, and prune removes an entry only after taking the exclusive lock without waiting, reporting it `in-use` otherwise. It never touches a system directory, nor, on the dev channel, another fingerprint's builds. The rules are `docs/guides/fetch-v1.md` sections 1 and 3, and the `resolve-build-*` and `prune-*` conformance cases hold all four bindings to them. diff --git a/go/README.md b/go/README.md index cee14956..c356b7d5 100644 --- a/go/README.md +++ b/go/README.md @@ -108,7 +108,7 @@ A bad **row** is a verdict, not an error: `Outcome` becomes `Rejected` with Clic - `*InternalError`: a library bug, or a document that does not decode. - `*ArtifactError`: fetch and load failures, with a `Code` and a sentinel per code for `errors.Is`. -`chtypes.AsCallError` reads the five fields every call error carries. +`chtypes.AsCallError` reads the six fields every call error carries. ## Threads diff --git a/go/chtypes/doc.go b/go/chtypes/doc.go index 6262d972..493e041b 100644 --- a/go/chtypes/doc.go +++ b/go/chtypes/doc.go @@ -56,7 +56,7 @@ // answer is a *UnsupportedError, a PEER of the first that errors.As never // confuses with it. *UsageError is misuse (a closed object, a refused // version spelling, a conflicting setup) and *InternalError a library bug. -// AsCallError reads the five fields every one of the four carries. A +// AsCallError reads the six fields every one of the four carries. A // loader or fetch failure is an *ArtifactError whose Code is the shared // vocabulary, with a sentinel per code for errors.Is. // diff --git a/go/chtypes/errors.go b/go/chtypes/errors.go index 64f38b1d..055d031e 100644 --- a/go/chtypes/errors.go +++ b/go/chtypes/errors.go @@ -1,7 +1,7 @@ package chtypes // errors.go — the error classes (bindings-v1.md section 4). Every call error -// carries the five fields of chs_error verbatim; the class of a status is +// carries the six fields of chs_error verbatim; the class of a status is // spec/abi-v2/sdk.json's table, generated into the abi2 layer. A status the // description does not list is its unknown(n) and an InternalError (rule r3). @@ -15,7 +15,7 @@ import ( "github.com/wave-rf/chtypes/go/v2/internal/ocifetch" ) -// CallError is the five fields every call error carries, verbatim and with +// CallError is the six fields every call error carries, verbatim and with // nothing synthesized. It is embedded in *SchemaError, *UnsupportedError, // *UsageError and *InternalError, so the fields read directly // (se.ChCode); AsCallError reads them from any of the four. @@ -33,6 +33,9 @@ type CallError struct { Message string // Column is the column concerned, empty if none. It is bytes. Column string + // Reason is the error's catalog id, read through chs_error_reason: ASCII, + // empty if none. + Reason string } // display is the one lossy display form: invalid UTF-8 replaced, for printing @@ -125,7 +128,7 @@ func callError(c *abi2.CallError) error { if c == nil { return nil } - ce := CallError{ChCode: c.ChCode, ChName: c.ChName, Message: c.Message, Column: c.Column} + ce := CallError{ChCode: c.ChCode, ChName: c.ChName, Message: c.Message, Column: c.Column, Reason: c.Reason} status, known := statusOfName(c.Status) if !known { raw := int64(StatusInternal) diff --git a/go/chtypes/errors_test.go b/go/chtypes/errors_test.go index 38f86227..16945224 100644 --- a/go/chtypes/errors_test.go +++ b/go/chtypes/errors_test.go @@ -22,13 +22,13 @@ func TestCallErrorClasses(t *testing.T) { {"CHS_INTERNAL", "*chtypes.InternalError", StatusInternal}, } for _, c := range cases { - err := callError(&abi2.CallError{Status: c.status, ChCode: 53, ChName: "TYPE_MISMATCH", Message: "m\xff", Column: "c\x00"}) + err := callError(&abi2.CallError{Status: c.status, ChCode: 53, ChName: "TYPE_MISMATCH", Message: "m\xff", Column: "c\x00", Reason: "option-not-implemented"}) if got := fmt.Sprintf("%T", err); got != c.want { t.Errorf("%s -> %s, want %s", c.status, got, c.want) } ce, ok := AsCallError(err) - if !ok || ce.Status != c.code || ce.ChCode != 53 || ce.ChName != "TYPE_MISMATCH" || ce.Message != "m\xff" || ce.Column != "c\x00" { - t.Errorf("%s: AsCallError = %+v, %v: the five fields are verbatim", c.status, ce, ok) + if !ok || ce.Status != c.code || ce.ChCode != 53 || ce.ChName != "TYPE_MISMATCH" || ce.Message != "m\xff" || ce.Column != "c\x00" || ce.Reason != "option-not-implemented" { + t.Errorf("%s: AsCallError = %+v, %v: the six fields are verbatim", c.status, ce, ok) } // Wrapped, still reachable. if _, ok := AsCallError(fmt.Errorf("wrapped: %w", err)); !ok { From d4732943ca547fcb88ecf92bd4a6173cfe78b385 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:19:39 -0400 Subject: [PATCH 10/24] docs: bindings-v1.md tables realigned (dprint fmt) Co-Authored-By: Claude Opus 5.5 --- docs/reference/bindings-v1.md | 200 +++++++++++++++++----------------- 1 file changed, 100 insertions(+), 100 deletions(-) diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 7a84f68d..597e19b9 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -121,25 +121,25 @@ A `ServerProfile` carries: `timezone` (a zone name), `settings` (settings) and ` One entry point per operation, with its optional inputs spelled in the language's own idiom, as the 0.x binding contract's "One compile function" already ruled: Go functional options, Python keyword-only arguments, a TypeScript options object, and a Rust options struct with `Default` (or a positional slice where the call has only that one optional input). -| option | C parameter | applies to | Go | Python | TypeScript | Rust | -| ---------------------- | -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------- | ------------------------------------------------ | -| settings | `settings` | compile (the profile), row, rows, parse a block, filter compile (the filter's own zone and profile), filter rows (the body's parse settings) | `WithSettings(map[string]string)`: one function whose value implements `CompileOption`, `RowOption`, `RowsOption`, `BlockOption`, `FilterOption` and `EvalOption` | `settings=` | `settings` | `settings: Vec<(String, String)>` | -| the per-call zone | the `session_timezone` key inside `settings` | the same six calls; on a filter compile it is the filter's zone, on a filter evaluation the body's parse zone | `WithSessionTimezone(name string)`, implementing the same six option interfaces | `session_timezone=` | `sessionTimezone` | `session_timezone: Option` | -| the INSERT column list | `columns` | row, rows, parse a block, filter rows (ABI v2) | `WithColumns(names []string)`: a `RowOption`, `RowsOption`, `BlockOption` and `EvalOption` | `columns=` | `columns` | `columns: Vec>` | -| an attached filter | `filter` | rows | `WithRowFilter(*Filter)` | `row_filter=` | `rowFilter` | `filter: Option` | -| an export | `export_format`, `out_export` | rows | `WithExport(Format)`; absent means `CHS_EXPORT_NONE` | `export=` | `exportFormat` | `export: Option` | -| document groups | `doc_flags` | rows | `WithDocFlags(DocFlags)`; absent means `DocAll` | `doc_flags=` | `docFlags` | `doc_flags: Option`, `None` means all | -| query parameters | `query_params` | filter compile | `WithFilterParams(map[string]string)` | `params=` | `params` | `params: Vec<(String, String)>` | -| the server | `server` | compile (ABI v2) | `OnServer(*Server)`; `OnServer(nil)` is no server | `server=` | `server` | `server: Option`, `None` means no server | -| insertable columns | `options`: `insert_options.insertable_columns` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertableColumns(names []string)`: an `InsertOption`; nil leaves the member out | `insertable_columns=` | `insertableColumns` | `insertable_columns: Option>>` | -| column defaults | `options`: `insert_options.column_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithColumnDefaults(defaults []ColumnDefault)`: an `InsertOption`; `ColumnDefault{Name, Value string}` | `column_defaults=` | `columnDefaults` | `column_defaults: Option>` | -| verdict basis | `options`: `insert_options.verdict_basis` | row, rows, parse a block, filter rows (ABI v2) | `WithVerdictBasis(VerdictBasis)`: `VerdictBasisInput`, `VerdictBasisExport` | `verdict_basis=` | `verdictBasis` | `verdict_basis: Option` | -| defer defaults | `options`: `insert_options.defer_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithDeferDefaults(DeferDefaults)`: `DeferDefaultsNone`, `DeferDefaultsServer` | `defer_defaults=` | `deferDefaults` | `defer_defaults: Option` | -| an insert option with no name here | `options`: `insert_options.` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertOption(key string, valueJSON []byte)`: a member passed through as raw JSON | `insert_options_extra=` | `insertOptionsExtra` | `insert_options_extra: Vec<(String, String)>` | -| parameter encoding | `options`: `filter_options.param_encoding` | filter compile (ABI v2) | `WithParamEncoding(ParamEncoding)`: `ParamEncodingEscaped`, `ParamEncodingRaw` | `param_encoding=` | `paramEncoding` | `param_encoding: Option` | -| a filter option with no name here | `options`: `filter_options.` | filter compile (ABI v2) | `WithFilterOption(key string, valueJSON []byte)`: passed through as raw JSON | `filter_options_extra=` | `filterOptionsExtra` | `filter_options_extra: Vec<(String, String)>` | -| posture | `options`: `schema_options.posture` | compile (ABI v2) | `WithPosture(Posture)`: `PostureCreate`, `PostureLoad` | `posture=` | `posture` | `posture: Option` | -| a schema option with no name here | `options`: `schema_options.` | compile (ABI v2) | `WithSchemaOption(key string, valueJSON []byte)`: passed through as raw JSON | `schema_options_extra=` | `schemaOptionsExtra` | `schema_options_extra: Vec<(String, String)>` | +| option | C parameter | applies to | Go | Python | TypeScript | Rust | +| ---------------------------------- | ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -------------------- | ------------------------------------------------ | +| settings | `settings` | compile (the profile), row, rows, parse a block, filter compile (the filter's own zone and profile), filter rows (the body's parse settings) | `WithSettings(map[string]string)`: one function whose value implements `CompileOption`, `RowOption`, `RowsOption`, `BlockOption`, `FilterOption` and `EvalOption` | `settings=` | `settings` | `settings: Vec<(String, String)>` | +| the per-call zone | the `session_timezone` key inside `settings` | the same six calls; on a filter compile it is the filter's zone, on a filter evaluation the body's parse zone | `WithSessionTimezone(name string)`, implementing the same six option interfaces | `session_timezone=` | `sessionTimezone` | `session_timezone: Option` | +| the INSERT column list | `columns` | row, rows, parse a block, filter rows (ABI v2) | `WithColumns(names []string)`: a `RowOption`, `RowsOption`, `BlockOption` and `EvalOption` | `columns=` | `columns` | `columns: Vec>` | +| an attached filter | `filter` | rows | `WithRowFilter(*Filter)` | `row_filter=` | `rowFilter` | `filter: Option` | +| an export | `export_format`, `out_export` | rows | `WithExport(Format)`; absent means `CHS_EXPORT_NONE` | `export=` | `exportFormat` | `export: Option` | +| document groups | `doc_flags` | rows | `WithDocFlags(DocFlags)`; absent means `DocAll` | `doc_flags=` | `docFlags` | `doc_flags: Option`, `None` means all | +| query parameters | `query_params` | filter compile | `WithFilterParams(map[string]string)` | `params=` | `params` | `params: Vec<(String, String)>` | +| the server | `server` | compile (ABI v2) | `OnServer(*Server)`; `OnServer(nil)` is no server | `server=` | `server` | `server: Option`, `None` means no server | +| insertable columns | `options`: `insert_options.insertable_columns` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertableColumns(names []string)`: an `InsertOption`; nil leaves the member out | `insertable_columns=` | `insertableColumns` | `insertable_columns: Option>>` | +| column defaults | `options`: `insert_options.column_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithColumnDefaults(defaults []ColumnDefault)`: an `InsertOption`; `ColumnDefault{Name, Value string}` | `column_defaults=` | `columnDefaults` | `column_defaults: Option>` | +| verdict basis | `options`: `insert_options.verdict_basis` | row, rows, parse a block, filter rows (ABI v2) | `WithVerdictBasis(VerdictBasis)`: `VerdictBasisInput`, `VerdictBasisExport` | `verdict_basis=` | `verdictBasis` | `verdict_basis: Option` | +| defer defaults | `options`: `insert_options.defer_defaults` | row, rows, parse a block, filter rows (ABI v2) | `WithDeferDefaults(DeferDefaults)`: `DeferDefaultsNone`, `DeferDefaultsServer` | `defer_defaults=` | `deferDefaults` | `defer_defaults: Option` | +| an insert option with no name here | `options`: `insert_options.` | row, rows, parse a block, filter rows (ABI v2) | `WithInsertOption(key string, valueJSON []byte)`: a member passed through as raw JSON | `insert_options_extra=` | `insertOptionsExtra` | `insert_options_extra: Vec<(String, String)>` | +| parameter encoding | `options`: `filter_options.param_encoding` | filter compile (ABI v2) | `WithParamEncoding(ParamEncoding)`: `ParamEncodingEscaped`, `ParamEncodingRaw` | `param_encoding=` | `paramEncoding` | `param_encoding: Option` | +| a filter option with no name here | `options`: `filter_options.` | filter compile (ABI v2) | `WithFilterOption(key string, valueJSON []byte)`: passed through as raw JSON | `filter_options_extra=` | `filterOptionsExtra` | `filter_options_extra: Vec<(String, String)>` | +| posture | `options`: `schema_options.posture` | compile (ABI v2) | `WithPosture(Posture)`: `PostureCreate`, `PostureLoad` | `posture=` | `posture` | `posture: Option` | +| a schema option with no name here | `options`: `schema_options.` | compile (ABI v2) | `WithSchemaOption(key string, valueJSON []byte)`: passed through as raw JSON | `schema_options_extra=` | `schemaOptionsExtra` | `schema_options_extra: Vec<(String, String)>` | The TypeScript option types are `CompileOptions { settings?, sessionTimezone?, server?, posture?, schemaOptionsExtra? }`, `RowOptions { settings?, sessionTimezone?, columns?, insertableColumns?, columnDefaults?, verdictBasis?, deferDefaults?, insertOptionsExtra? }`, `RowsOptions extends RowOptions { rowFilter?, exportFormat?, docFlags? }`, `FilterOptions { params?, settings?, sessionTimezone?, paramEncoding?, filterOptionsExtra? }` and `EvalOptions { settings?, sessionTimezone?, columns?, insertableColumns?, columnDefaults?, verdictBasis?, deferDefaults?, insertOptionsExtra? }`. The Rust ones are `CompileOptions { settings, session_timezone, server, posture, schema_options_extra }`, `RowOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra }`, `RowsOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra, filter, export, doc_flags }`, `FilterOptions { params, settings, session_timezone, param_encoding, filter_options_extra }` and `EvalOptions { settings, session_timezone, columns, insertable_columns, column_defaults, verdict_basis, defer_defaults, insert_options_extra }`, each with public fields and `Default`, built as `RowOptions { session_timezone: Some(z), ..Default::default() }`. They are deliberately not `#[non_exhaustive]`, which would forbid that construction outside the crate; the result types are. @@ -255,26 +255,26 @@ TypeScript runs every call synchronously on one thread, so within an isolate the Every vocabulary is generated from the description into all four bindings, with the description's numbers and spellings. No binding keeps a copy of its own. -| vocabulary | from | Go | Python | TypeScript | Rust | notes | -| ------------------------- | ------------------------------ | -------------------------------------------------------------------------------------------- | -------------------------- | ------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| formats | `chs_format` | `Format` (`int32`): `JSONEachRow`, `CSV`, …, `TSVWithNames` | `Format` (`IntEnum`) | `Format` | `Format` (`#[repr(i32)]`) | FIRM and frozen (D1.4). Each carries `ch_name`, ClickHouse's own name, which is how `capabilities` lists a format | -| call statuses | `chs_status` | `Status` (`int32`): `StatusOK` … `StatusInternal` | `Status` (`IntEnum`) | `Status` | `chtypes::status::{OK, …}` (`i32`) | FIRM (D3) | -| row and batch outcomes | `row_outcome`, `batch_outcome` | `Outcome`: `Accepted`, `AcceptedPoisoned`, `Rejected`, `Skipped`, `Unsupported` | `Outcome` | `Outcome` | `Outcome`, `#[non_exhaustive]` | FIRM. An unknown value reads as the description's fallback, `unsupported` | -| filter outcomes | `filter_outcome` | `FilterOutcome`: `FilterOK`, `FilterRejected`, `FilterUnsupported` | `FilterOutcome` | `FilterOutcome` | `FilterOutcome` | FIRM. Unknown reads as `unsupported` | -| filter verdicts | `filter_verdict` | `Verdict`: `VerdictTrue`, `VerdictFalse`, `VerdictError`, `VerdictDecline`, and `Answered()` | `Verdict`, `.answered` | `Verdict`, `answered` | `Verdict`, `answered()` | FIRM. Unknown reads as `d`. `answered` is the description's own fact, never a list kept here | -| transform reasons | `transform_reason` | `ReasonOverflowWrap` … | `Reason.OVERFLOW_WRAP` … | `Reason.OverflowWrap` … | `reason::OVERFLOW_WRAP` … | FIRM. The spellings are v0's. Each carries its `lossy` fact, which `Transform.lossy` reads | -| value sources | `value_src` | `SourceInput` …, `SourceDefaultDeferred`, `SourceDefaultOverride` | `Source.INPUT` … | `Source.Input` … | `source::INPUT` … | FIRM. Every value now, not v0's two. Each carries its `is_stored` fact, which `Value.is_stored` reads; an unlisted value reads the description's fallback (`skipped`), not stored. `default_deferred` (not stored) and `default_override` (stored) are ABI v2 | -| `default_generated` | a `value_src` value | `SourceDefaultGenerated` | `Source.DEFAULT_GENERATED` | `Source.DefaultGenerated` | `source::DEFAULT_GENERATED` | FIRM. `is_stored` is true: the value is stored once the caller inserts the library's output (§5, Generated defaults). Generated from the description like every other value, never hand-added | -| merge reasons | `merge_reason` | `MergeReason`: `MergeTTLDelete`, `MergeTTLColumnReset` | `MergeReason` | `MergeReason` | `MergeReason`, `#[non_exhaustive]` | ABI v2 dev. The reason of a batch's `at_merge` entry (§5). The spellings are the description's | -| declined tiers | `declined_tier` | `DeclinedTier`: `TierPredicate`, `TierPredicateUnflipped`, `TierResultContent`, `TierServerRefuses` | `DeclinedTier` | `DeclinedTier` | `DeclinedTier`, `#[non_exhaustive]` | ABI v2 dev. The tier of an entry of a schema description's `filter_declined_settings` (§5). The spellings are the description's; each binding reads a hyphen as a word boundary in its identifiers | -| declined layers | `declined_layer` | `DeclinedLayer`: `LayerDefaults`, `LayerServer`, `LayerSchema` | `DeclinedLayer` | `DeclinedLayer` | `DeclinedLayer`, `#[non_exhaustive]` | ABI v2 dev. The settings layer of an entry of a schema description's `filter_declined_settings` (§5), the highest that sets the name: `defaults`, `server` or `schema`, lowest first. The spellings are the description's | -| refusal sites | `refusal_site` | `RefusalSite`: `RefusedAtCall`, `RefusedAtBodyRead`, `RefusedAtRow`, `RefusedAtBodyStep` | `RefusalSite` | `RefusalSite` | `RefusalSite`, `#[non_exhaustive]` | ABI v2 dev. Where a batch, a row or a filter result's refusal sits (`refused_at`, §5). The spellings are the description's; each binding reads an underscore as a word boundary in its identifiers | -| postures | `posture` | `Posture`: `PostureCreate`, `PostureLoad` | `Posture` | `Posture` | `Posture`, `#[non_exhaustive]` | ABI v2 dev. How a schema's statement was compiled (`schema_options`' `posture`, and a schema description's `posture`). The spellings are the description's | -| default support | `default_support` | `DefaultSupport`: `DefaultSupportComputed`, `DefaultSupportGenerated`, `DefaultSupportDeferred`, `DefaultSupportDeclined` | `DefaultSupport` | `DefaultSupport` | `DefaultSupport`, `#[non_exhaustive]` | ABI v2 dev, reserved. What a build does with a column's DEFAULT (`default_support`, §5). The spellings are the description's | -| the decline code | `CHS_CODE_DECLINED` | `CodeDeclined` (`int32`, -2) | `CODE_DECLINED` | `CODE_DECLINED` | `code::DECLINED` (`i32`) | ABI v2 dev. The `code` of a declined row, outcome or verdict, outside ClickHouse's code table; 0 is written only on a success | -| discover query parameters | `discover_query_param` | `DiscoverQueryParam`: `QueryParamDatabase`, `QueryParamTable` | `DiscoverQueryParam` | `DiscoverQueryParam` | `DiscoverQueryParam`, `#[non_exhaustive]` | ABI v2 dev. The discovery query's two parameters (§2, the discovery query), which a caller binds when it runs the query against its server. The spellings are the description's | -| default kinds | `default_kind` | `DefaultKind`: `KindNone`, `KindDefault`, `KindMaterialized`, `KindAlias`, `KindEphemeral` | `DefaultKind` | `DefaultKind` | `DefaultKind` | FIRM; joins the description in the regeneration that freezes the fingerprint (§8 question 3). The spellings are v0's | -| document groups | `CHS_DOC_*` | `DocFlags`: `DocValues`, `DocTransforms`, `DocDefaults`, `DocAll` | `DocFlags` | `DocFlags` | `DocFlags` | FIRM | +| vocabulary | from | Go | Python | TypeScript | Rust | notes | +| ------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------- | -------------------------- | ------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| formats | `chs_format` | `Format` (`int32`): `JSONEachRow`, `CSV`, …, `TSVWithNames` | `Format` (`IntEnum`) | `Format` | `Format` (`#[repr(i32)]`) | FIRM and frozen (D1.4). Each carries `ch_name`, ClickHouse's own name, which is how `capabilities` lists a format | +| call statuses | `chs_status` | `Status` (`int32`): `StatusOK` … `StatusInternal` | `Status` (`IntEnum`) | `Status` | `chtypes::status::{OK, …}` (`i32`) | FIRM (D3) | +| row and batch outcomes | `row_outcome`, `batch_outcome` | `Outcome`: `Accepted`, `AcceptedPoisoned`, `Rejected`, `Skipped`, `Unsupported` | `Outcome` | `Outcome` | `Outcome`, `#[non_exhaustive]` | FIRM. An unknown value reads as the description's fallback, `unsupported` | +| filter outcomes | `filter_outcome` | `FilterOutcome`: `FilterOK`, `FilterRejected`, `FilterUnsupported` | `FilterOutcome` | `FilterOutcome` | `FilterOutcome` | FIRM. Unknown reads as `unsupported` | +| filter verdicts | `filter_verdict` | `Verdict`: `VerdictTrue`, `VerdictFalse`, `VerdictError`, `VerdictDecline`, and `Answered()` | `Verdict`, `.answered` | `Verdict`, `answered` | `Verdict`, `answered()` | FIRM. Unknown reads as `d`. `answered` is the description's own fact, never a list kept here | +| transform reasons | `transform_reason` | `ReasonOverflowWrap` … | `Reason.OVERFLOW_WRAP` … | `Reason.OverflowWrap` … | `reason::OVERFLOW_WRAP` … | FIRM. The spellings are v0's. Each carries its `lossy` fact, which `Transform.lossy` reads | +| value sources | `value_src` | `SourceInput` …, `SourceDefaultDeferred`, `SourceDefaultOverride` | `Source.INPUT` … | `Source.Input` … | `source::INPUT` … | FIRM. Every value now, not v0's two. Each carries its `is_stored` fact, which `Value.is_stored` reads; an unlisted value reads the description's fallback (`skipped`), not stored. `default_deferred` (not stored) and `default_override` (stored) are ABI v2 | +| `default_generated` | a `value_src` value | `SourceDefaultGenerated` | `Source.DEFAULT_GENERATED` | `Source.DefaultGenerated` | `source::DEFAULT_GENERATED` | FIRM. `is_stored` is true: the value is stored once the caller inserts the library's output (§5, Generated defaults). Generated from the description like every other value, never hand-added | +| merge reasons | `merge_reason` | `MergeReason`: `MergeTTLDelete`, `MergeTTLColumnReset` | `MergeReason` | `MergeReason` | `MergeReason`, `#[non_exhaustive]` | ABI v2 dev. The reason of a batch's `at_merge` entry (§5). The spellings are the description's | +| declined tiers | `declined_tier` | `DeclinedTier`: `TierPredicate`, `TierPredicateUnflipped`, `TierResultContent`, `TierServerRefuses` | `DeclinedTier` | `DeclinedTier` | `DeclinedTier`, `#[non_exhaustive]` | ABI v2 dev. The tier of an entry of a schema description's `filter_declined_settings` (§5). The spellings are the description's; each binding reads a hyphen as a word boundary in its identifiers | +| declined layers | `declined_layer` | `DeclinedLayer`: `LayerDefaults`, `LayerServer`, `LayerSchema` | `DeclinedLayer` | `DeclinedLayer` | `DeclinedLayer`, `#[non_exhaustive]` | ABI v2 dev. The settings layer of an entry of a schema description's `filter_declined_settings` (§5), the highest that sets the name: `defaults`, `server` or `schema`, lowest first. The spellings are the description's | +| refusal sites | `refusal_site` | `RefusalSite`: `RefusedAtCall`, `RefusedAtBodyRead`, `RefusedAtRow`, `RefusedAtBodyStep` | `RefusalSite` | `RefusalSite` | `RefusalSite`, `#[non_exhaustive]` | ABI v2 dev. Where a batch, a row or a filter result's refusal sits (`refused_at`, §5). The spellings are the description's; each binding reads an underscore as a word boundary in its identifiers | +| postures | `posture` | `Posture`: `PostureCreate`, `PostureLoad` | `Posture` | `Posture` | `Posture`, `#[non_exhaustive]` | ABI v2 dev. How a schema's statement was compiled (`schema_options`' `posture`, and a schema description's `posture`). The spellings are the description's | +| default support | `default_support` | `DefaultSupport`: `DefaultSupportComputed`, `DefaultSupportGenerated`, `DefaultSupportDeferred`, `DefaultSupportDeclined` | `DefaultSupport` | `DefaultSupport` | `DefaultSupport`, `#[non_exhaustive]` | ABI v2 dev, reserved. What a build does with a column's DEFAULT (`default_support`, §5). The spellings are the description's | +| the decline code | `CHS_CODE_DECLINED` | `CodeDeclined` (`int32`, -2) | `CODE_DECLINED` | `CODE_DECLINED` | `code::DECLINED` (`i32`) | ABI v2 dev. The `code` of a declined row, outcome or verdict, outside ClickHouse's code table; 0 is written only on a success | +| discover query parameters | `discover_query_param` | `DiscoverQueryParam`: `QueryParamDatabase`, `QueryParamTable` | `DiscoverQueryParam` | `DiscoverQueryParam` | `DiscoverQueryParam`, `#[non_exhaustive]` | ABI v2 dev. The discovery query's two parameters (§2, the discovery query), which a caller binds when it runs the query against its server. The spellings are the description's | +| default kinds | `default_kind` | `DefaultKind`: `KindNone`, `KindDefault`, `KindMaterialized`, `KindAlias`, `KindEphemeral` | `DefaultKind` | `DefaultKind` | `DefaultKind` | FIRM; joins the description in the regeneration that freezes the fingerprint (§8 question 3). The spellings are v0's | +| document groups | `CHS_DOC_*` | `DocFlags`: `DocValues`, `DocTransforms`, `DocDefaults`, `DocAll` | `DocFlags` | `DocFlags` | `DocFlags` | FIRM | **A fact in TypeScript is a free function.** TypeScript's vocabularies are const objects (`Format.JSONEachRow` is a number, not an object), which cannot carry a member, so each fact the table names (`ch_name`, `answered`, `lossy`, `is_stored`) is a free function named for its vocabulary: `formatChName(f)`, `verdictAnswered(v)`, `reasonLossy(r)` and `sourceIsStored(s)`, generated from the description like the facts of the other three. This is the language's idiom, not a convenience of its own; the parity gate carries the four as spelling entries. @@ -383,25 +383,25 @@ Every table below lists a document's fields as the regenerated description draws ### `RowResult`, from the `row` document -| field | from | type | -| ---------------------- | ------------------------------------------------------------------ | ------------------- | -| `outcome` | `outcome`, already final: the library applies every promotion (D6) | `Outcome` | -| `err_code` | `code`: ClickHouse's own code, 0 only on a success, `CHS_CODE_DECLINED` (-2) for a decline | int32 | -| `err_msg` | `err` (rule 5) | bytes | -| `reason` | `reason`: the decline's catalog id, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | -| `refused_at` | `refused_at`: where a single-row result's refusal sits; the rows of a batch carry neither this nor `refused_row` | optional `RefusalSite` | -| `refused_row` | `refused_row`, present exactly when `refused_at` is `row` | optional unsigned 64-bit | -| `columns` | `cols`, every entry, in document order | list of `Value` | -| `values` | the entries of `cols` whose `src` is stored, in order | list of `Value` | -| `transformed` | `transformed`, computed by the library (D6) | list of `Transform` | -| `unknown_fields` | `unknown_fields`, name objects (rule 5) | list of bytes | -| `computed` | `computed` | list of `Computed` | -| `verdict` | `verdict`, present only with an attached filter | optional `Verdict` | -| `verdict_code` | `verdict_code` | int32 | -| `verdict_reason` | `verdict_reason`: the catalog id, present exactly where `verdict_code` is `CHS_CODE_DECLINED` | optional ASCII text | -| `verdict_err` | `verdict_err` (rule 5) | bytes | -| `partition_id` | `partition_id` (rule 5) | optional bytes | -| `input_span` | `input_span`: the bytes the reader consumed for this record | optional `Span` | +| field | from | type | +| ---------------- | ---------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `outcome` | `outcome`, already final: the library applies every promotion (D6) | `Outcome` | +| `err_code` | `code`: ClickHouse's own code, 0 only on a success, `CHS_CODE_DECLINED` (-2) for a decline | int32 | +| `err_msg` | `err` (rule 5) | bytes | +| `reason` | `reason`: the decline's catalog id, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at` | `refused_at`: where a single-row result's refusal sits; the rows of a batch carry neither this nor `refused_row` | optional `RefusalSite` | +| `refused_row` | `refused_row`, present exactly when `refused_at` is `row` | optional unsigned 64-bit | +| `columns` | `cols`, every entry, in document order | list of `Value` | +| `values` | the entries of `cols` whose `src` is stored, in order | list of `Value` | +| `transformed` | `transformed`, computed by the library (D6) | list of `Transform` | +| `unknown_fields` | `unknown_fields`, name objects (rule 5) | list of bytes | +| `computed` | `computed` | list of `Computed` | +| `verdict` | `verdict`, present only with an attached filter | optional `Verdict` | +| `verdict_code` | `verdict_code` | int32 | +| `verdict_reason` | `verdict_reason`: the catalog id, present exactly where `verdict_code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `verdict_err` | `verdict_err` (rule 5) | bytes | +| `partition_id` | `partition_id` (rule 5) | optional bytes | +| `input_span` | `input_span`: the bytes the reader consumed for this record | optional `Span` | `values` filters on `Value.is_stored`, which is the description's own `is_stored` fact for the entry's `src`; it consults no list the binding keeps, which is the whole difference from v0's "exclude `skipped` and `ephemeral_input`" rule. `columns` is new, and it is how a caller still sees the entries that are never stored (an `ephemeral_input` column, an unresolved DEFAULT). @@ -417,31 +417,31 @@ The transform entry keeps v0's shape (`column`, `input`, `stored`, `reason`, `ro ### `BatchResult`, from the `batch` document and the export buffer -| field | from | type | -| ------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------- | -| `outcome` | `outcome`, final | `Outcome` | -| `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | -| `reason` | `reason`: the catalog id of a declined batch, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | -| `refused_at`, `refused_row` | `refused_at` (a `refusal_site` value) and `refused_row`: where a refusal or decline sits, present whenever the outcome is not `accepted` or `accepted_poisoned`; `refused_row` for `row`, below `rows_read` | optional `RefusalSite`, optional unsigned 64-bit | -| `rows` | `rows`, each decoded as a `RowResult` | list of `RowResult` | -| `rows_read` | `rows_read` | unsigned 64-bit | -| `rows_skipped` | `rows_skipped` | unsigned 64-bit | -| `transformed` | `transformed`: every transform in the batch with its `row`, storage transforms (TTL) included, as the library lists them | list of `Transform` | -| `unsupported_settings` | `unsupported_settings`, name objects (rule 5): the call's own settings the library declined (ABI v2) | list of bytes | -| `engine_rows` | `engine_rows`: each stored row after the engine's insert-time merge, a list of cells | optional list of rows, each a list of `EngineCell` | -| `at_merge` | `at_merge`: what an `OPTIMIZE TABLE ... FINAL` of the part would do to its rows (ABI v2) | list of `AtMergeEntry` | -| `payload` | the `out_export` buffer, present only for an export that ran (absent when `export_declined` is non-empty) | optional payload | -| `spans` | `row_spans`: one entry per `rows` entry, index-aligned, each exported row's place in `payload` and null for a row not exported | optional list of optional `Span` | -| `export_columns` | `export_columns`: the export's one column list, name objects (rule 5) | optional list of bytes | -| `export_declined` | `export_declined` (rule 5) | bytes | -| `export_declined_reason` | `export_declined_reason`: the catalog id of a declined export | optional ASCII text | -| `rows_passed`, `rows_cut` | `rows_passed`, `rows_cut`, with an attached filter | unsigned 64-bit | -| `partition_count` | `partition_count` | optional unsigned 64-bit | -| `unconsumed` | `unconsumed`: the byte ranges the reader's error recovery skipped | list of `Span` | -| `framing` | `framing`: what the reader decided about the body's framing | optional `Framing` | - -| type | fields, each from the same-named key unless noted | -| -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| field | from | type | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | +| `outcome` | `outcome`, final | `Outcome` | +| `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | +| `reason` | `reason`: the catalog id of a declined batch, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at`, `refused_row` | `refused_at` (a `refusal_site` value) and `refused_row`: where a refusal or decline sits, present whenever the outcome is not `accepted` or `accepted_poisoned`; `refused_row` for `row`, below `rows_read` | optional `RefusalSite`, optional unsigned 64-bit | +| `rows` | `rows`, each decoded as a `RowResult` | list of `RowResult` | +| `rows_read` | `rows_read` | unsigned 64-bit | +| `rows_skipped` | `rows_skipped` | unsigned 64-bit | +| `transformed` | `transformed`: every transform in the batch with its `row`, storage transforms (TTL) included, as the library lists them | list of `Transform` | +| `unsupported_settings` | `unsupported_settings`, name objects (rule 5): the call's own settings the library declined (ABI v2) | list of bytes | +| `engine_rows` | `engine_rows`: each stored row after the engine's insert-time merge, a list of cells | optional list of rows, each a list of `EngineCell` | +| `at_merge` | `at_merge`: what an `OPTIMIZE TABLE ... FINAL` of the part would do to its rows (ABI v2) | list of `AtMergeEntry` | +| `payload` | the `out_export` buffer, present only for an export that ran (absent when `export_declined` is non-empty) | optional payload | +| `spans` | `row_spans`: one entry per `rows` entry, index-aligned, each exported row's place in `payload` and null for a row not exported | optional list of optional `Span` | +| `export_columns` | `export_columns`: the export's one column list, name objects (rule 5) | optional list of bytes | +| `export_declined` | `export_declined` (rule 5) | bytes | +| `export_declined_reason` | `export_declined_reason`: the catalog id of a declined export | optional ASCII text | +| `rows_passed`, `rows_cut` | `rows_passed`, `rows_cut`, with an attached filter | unsigned 64-bit | +| `partition_count` | `partition_count` | optional unsigned 64-bit | +| `unconsumed` | `unconsumed`: the byte ranges the reader's error recovery skipped | list of `Span` | +| `framing` | `framing`: what the reader decided about the body's framing | optional `Framing` | + +| type | fields, each from the same-named key unless noted | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `AtMergeEntry` | `part_row` (int: an index into `engine_rows`, the part's row), `reason` (`MergeReason`, a `merge_reason` value), `column` (optional bytes, from `column` or `column_b64`: rule 5), `stored` (optional bytes, from `stored` or `stored_b64`: rule 5), `input_rows` (optional list of int) | **`at_merge` and the batch's own `unsupported_settings` (ABI v2).** `at_merge` lists what an `OPTIMIZE TABLE ... FINAL` of the part the `INSERT` writes would do to its rows, at the call's clock instant: `ttl_delete` (the rows TTL removes the row) or `ttl_column_reset` (a column TTL resets the column's value). `engine_rows` stays exactly what the `INSERT` writer produces, so the two never contradict each other. An entry's `part_row` indexes `engine_rows`, not the input body, and `input_rows`, when present, names the input rows, each by its index in the body, that formed that row after an engine's insert-time merge. A row both removed and reset lists only `ttl_delete`. `column` is present only for `ttl_column_reset`, and `stored` is absent when the column's `DEFAULT` reads the clock or a generator, so the value is decided at the merge. Absent is never empty, in any binding: Go `nil`, Python `None`, TypeScript `undefined`, Rust `None`. The list is "at least these", not "only these": a later background merge can remove more rows as more of them expire. The batch's own `unsupported_settings` names the call's settings the library declined. Both lists are empty when the document carries none. [`spec/abi-v2/docs.md`](../../spec/abi-v2/docs.md) is normative for both. @@ -470,28 +470,28 @@ A column whose DEFAULT calls one of the random or UUID generators this build adm ### `FilterResult`, from the `filter_result` document -| field | from | type | -| ---------------------- | -------------------------------------------------------------------------- | ------------------------ | -| `outcome` | `outcome` | `FilterOutcome` | -| `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | -| `reason` | `reason`: the catalog id of a declined evaluation, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | -| `refused_at`, `refused_row` | `refused_at` and `refused_row`, as the batch's: where a refusal or decline sits | optional `RefusalSite`, optional unsigned 64-bit | -| `rows_read` | `rows_read` | unsigned 64-bit | -| `unsupported_settings` | `unsupported_settings`, name objects (rule 5) | list of bytes | -| `verdicts` | `verdicts`: one character per row, each mapped through the generated table | list of `Verdict` | -| `errors` | `errors`: `row` (int), `code` (int32), `reason` (optional ASCII text, present where `code` is `CHS_CODE_DECLINED`), `msg` (bytes, from `err`, rule 5) | list of `FilterRowError` | +| field | from | type | +| --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | +| `outcome` | `outcome` | `FilterOutcome` | +| `err_code`, `err_msg` | `code`, `err` (rule 5) | int32, bytes | +| `reason` | `reason`: the catalog id of a declined evaluation, present exactly where `code` is `CHS_CODE_DECLINED` | optional ASCII text | +| `refused_at`, `refused_row` | `refused_at` and `refused_row`, as the batch's: where a refusal or decline sits | optional `RefusalSite`, optional unsigned 64-bit | +| `rows_read` | `rows_read` | unsigned 64-bit | +| `unsupported_settings` | `unsupported_settings`, name objects (rule 5) | list of bytes | +| `verdicts` | `verdicts`: one character per row, each mapped through the generated table | list of `Verdict` | +| `errors` | `errors`: `row` (int), `code` (int32), `reason` (optional ASCII text, present where `code` is `CHS_CODE_DECLINED`), `msg` (bytes, from `err`, rule 5) | list of `FilterRowError` | `e` and `d` are never answers, and a caller enforcing visibility fails closed on both; `Verdict.answered` says which. A non-ok outcome's verdicts are whatever the document holds: v0's rule that a binding empties them is the library's now. ### `SchemaDescription`, from the `schema_description` document -| field | from | type | -| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | -| `columns` | `columns`, in declared order. Each `Column`: `name` (bytes, rule 5), `type` (bytes, rule 5: the canonical type), `default_kind` (`DefaultKind`), `default_expr` (bytes, rule 5, from `default_expression`), `default_support` (optional `DefaultSupport`) and `default_support_reason` (optional ASCII text), reserved, and `reads` (optional list of bytes, name objects, rule 5), reserved | list of `Column` | -| `posture` | `posture` (a `posture` value): how the statement was compiled, present only for `load`; absent is `create` (ABI v2) | optional `Posture` | -| `filter_declined_settings` | `filter_declined_settings`: the settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known at schema compile, as the library wrote them (one entry per name, at the highest layer that sets it, by layer and then by name bytes), empty when it lists none (ABI v2) | list of `DeclinedSetting` | -| `server` | `SchemaServer`: `timezone` (text: the zone the schema's types bind, the profile's or else the image zone), `settings` (settings, `{}` when the profile gave none) and `macros` (settings, absent exactly when the profile carried none). From `server`, which the document carries exactly when the schema was compiled on a server (ABI v2) | optional `SchemaServer`: absent exactly when the schema was compiled without a server | -| `replicated` | `SchemaReplicated`: `zookeeper_path` (bytes, rule 5) and `replica_name` (bytes, rule 5), fully expanded. From `replicated`: what ClickHouse's own `TableZnodeInfo` resolved for a Replicated engine on a server, to compare with the server's `system.replicas` (ABI v2) | optional `SchemaReplicated` | +| field | from | type | +| -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------- | +| `columns` | `columns`, in declared order. Each `Column`: `name` (bytes, rule 5), `type` (bytes, rule 5: the canonical type), `default_kind` (`DefaultKind`), `default_expr` (bytes, rule 5, from `default_expression`), `default_support` (optional `DefaultSupport`) and `default_support_reason` (optional ASCII text), reserved, and `reads` (optional list of bytes, name objects, rule 5), reserved | list of `Column` | +| `posture` | `posture` (a `posture` value): how the statement was compiled, present only for `load`; absent is `create` (ABI v2) | optional `Posture` | +| `filter_declined_settings` | `filter_declined_settings`: the settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known at schema compile, as the library wrote them (one entry per name, at the highest layer that sets it, by layer and then by name bytes), empty when it lists none (ABI v2) | list of `DeclinedSetting` | +| `server` | `SchemaServer`: `timezone` (text: the zone the schema's types bind, the profile's or else the image zone), `settings` (settings, `{}` when the profile gave none) and `macros` (settings, absent exactly when the profile carried none). From `server`, which the document carries exactly when the schema was compiled on a server (ABI v2) | optional `SchemaServer`: absent exactly when the schema was compiled without a server | +| `replicated` | `SchemaReplicated`: `zookeeper_path` (bytes, rule 5) and `replica_name` (bytes, rule 5), fully expanded. From `replicated`: what ClickHouse's own `TableZnodeInfo` resolved for a Replicated engine on a server, to compare with the server's `system.replicas` (ABI v2) | optional `SchemaReplicated` | | type | fields, each from the same-named key unless noted | | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | From 7bb745f90099e06102d777720329d8562a2efc80 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:27:06 -0400 Subject: [PATCH 11/24] python: spec batch 2 hand-written surface (options, b64 values, refusal and export members, part_row) Co-Authored-By: Claude Sonnet 5.5 --- python/src/chtypes/__init__.py | 20 +- python/src/chtypes/_decode.py | 55 ++++- python/src/chtypes/_input.py | 227 ++++++++++++++++-- python/src/chtypes/library.py | 106 +++++++- python/src/chtypes/results.py | 72 +++++- python/tests/abi2/test_public_library.py | 79 +++++- python/tests/abi2/test_reader_rules.py | 2 +- python/tests/api/test_decode.py | 208 ++++++++++++++-- python/tests/api/test_input.py | 133 +++++++++- .../tests/goldens_v1/data/stub-goldens.json | 4 +- python/tests/goldens_v1/test_runner.py | 5 + 11 files changed, 830 insertions(+), 81 deletions(-) diff --git a/python/src/chtypes/__init__.py b/python/src/chtypes/__init__.py index 21d1ff08..197fc2e4 100644 --- a/python/src/chtypes/__init__.py +++ b/python/src/chtypes/__init__.py @@ -51,18 +51,29 @@ DeclinedLayer, DeclinedTier, DefaultKind, + DefaultSupport, DiscoverQueryParam, DocFlags, FilterOutcome, Format, MergeReason, Outcome, + Posture, Reason, + RefusalSite, Source, Status, Verdict, ) -from ._input import BytesIn, ServerProfile, Settings +from ._input import ( + BytesIn, + ColumnDefault, + DeferDefaults, + ParamEncoding, + ServerProfile, + Settings, + VerdictBasis, +) from ._setup import setup from .errors import ( CODE_ARTIFACT_CORRUPT, @@ -156,11 +167,14 @@ "Capabilities", "ChtypesError", "Column", + "ColumnDefault", "Computed", "DeclinedLayer", "DeclinedSetting", "DeclinedTier", + "DeferDefaults", "DefaultKind", + "DefaultSupport", "DiscoverQueryParam", "DiscoveredColumn", "Discovery", @@ -180,8 +194,11 @@ "Library", "MergeReason", "Outcome", + "ParamEncoding", + "Posture", "Reason", "Registry", + "RefusalSite", "Resolved", "RowResult", "Schema", @@ -205,6 +222,7 @@ "UsageError", "Value", "Verdict", + "VerdictBasis", "cache_root", "open_unverified", "search_dirs", diff --git a/python/src/chtypes/_decode.py b/python/src/chtypes/_decode.py index 76879430..d494dabd 100644 --- a/python/src/chtypes/_decode.py +++ b/python/src/chtypes/_decode.py @@ -32,10 +32,13 @@ DeclinedLayer, DeclinedTier, DefaultKind, + DefaultSupport, FilterOutcome, MergeReason, Outcome, + Posture, Reason, + RefusalSite, Source, Verdict, ) @@ -291,6 +294,20 @@ def _spans(obj: Mapping[str, Any], key: str, where: str) -> tuple[Span, ...]: return _objects(obj, key, where, _span) +def _row_spans(obj: Mapping[str, Any], key: str, where: str) -> tuple[Span | None, ...]: + """`row_spans`: one entry per row, a span or null for a row not exported.""" + out: list[Span | None] = [] + for i, item in enumerate(_list(obj, key, where)): + out.append(None if item is None else _span(_obj(item, where, key), f"{where}.{key}[{i}]")) + return tuple(out) + + +def _refused_at(obj: Mapping[str, Any], where: str) -> RefusalSite | None: + # An unlisted site is its unknown(n): kept, never a failure (rule r3). + value = _opt_text(obj, "refused_at", where) + return None if value is None else RefusalSite.of(value) + + def _value(obj: dict[str, Any], where: str) -> Value: # An unlisted src is its unknown(n): kept, never a failure (rule r3), and # not stored until the description says what one reports. An ABSENT src is @@ -353,7 +370,7 @@ def _int_list(obj: Mapping[str, Any], key: str, where: str) -> tuple[int, ...] | def _at_merge(obj: dict[str, Any], where: str) -> AtMergeEntry: # An unlisted reason is its unknown(n): kept, never a failure (rule r3). return AtMergeEntry( - row=_int(obj, "row", where), + part_row=_int(obj, "part_row", where), reason=MergeReason.of(_text(obj, "reason", where)), column=_bytes_or_none(obj, "column", where), stored=_bytes_or_none(obj, "stored", where), @@ -386,14 +403,17 @@ def decode_row(obj: Mapping[str, Any], where: str = "row") -> RowResult: outcome=Outcome.of(_text(obj, "outcome", where)), err_code=_int(obj, "code", where), err_msg=_bytes(obj, "err", where), + reason=_opt_text(obj, "reason", where), + refused_at=_refused_at(obj, where), + refused_row=_opt_u64(obj, "refused_row", where), columns=columns, values=tuple(v for v in columns if v.is_stored), transformed=_objects(obj, "transformed", where, _transform), unknown_fields=_name_list(obj, "unknown_fields", where), - unsupported_settings=_name_list(obj, "unsupported_settings", where), computed=_objects(obj, "computed", where, _computed), verdict=None if verdict is None else Verdict.of(verdict), verdict_code=_int(obj, "verdict_code", where), + verdict_reason=_opt_text(obj, "verdict_reason", where), verdict_err=_bytes(obj, "verdict_err", where), partition_id=partition, input_span=_opt_span(obj, "input_span", where), @@ -427,7 +447,10 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult: obj = _obj(strict_loads(raw, where), where) framing = obj.get("framing") engine_rows = None if obj.get("engine_rows") is None else _engine_rows(obj, where) - spans = None if obj.get("row_spans") is None else _spans(obj, "row_spans", where) + spans = None if obj.get("row_spans") is None else _row_spans(obj, "row_spans", where) + export_columns = ( + None if obj.get("export_columns") is None else _name_list(obj, "export_columns", where) + ) export_declined = _bytes(obj, "export_declined", where) if export_declined: # A declined export has no payload and no spans, whatever buffer the call @@ -439,6 +462,9 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult: outcome=Outcome.of(_text(obj, "outcome", where)), err_code=_int(obj, "code", where), err_msg=_bytes(obj, "err", where), + reason=_opt_text(obj, "reason", where), + refused_at=_refused_at(obj, where), + refused_row=_opt_u64(obj, "refused_row", where), rows=_objects(obj, "rows", where, decode_row), rows_read=_u64(obj, "rows_read", where), rows_skipped=_u64(obj, "rows_skipped", where), @@ -446,7 +472,9 @@ def decode_batch(raw: bytes, export: bytes | None) -> BatchResult: engine_rows=engine_rows, payload=export, spans=spans, + export_columns=export_columns, export_declined=export_declined, + export_declined_reason=_opt_text(obj, "export_declined_reason", where), rows_passed=_u64(obj, "rows_passed", where), rows_cut=_u64(obj, "rows_cut", where), partition_count=_opt_u64(obj, "partition_count", where), @@ -467,6 +495,7 @@ def _filter_row_error(obj: dict[str, Any], where: str) -> FilterRowError: return FilterRowError( row=_int(obj, "row", where), code=_int(obj, "code", where), + reason=_opt_text(obj, "reason", where), msg=_bytes(obj, "err", where), ) @@ -479,6 +508,9 @@ def decode_filter_result(raw: bytes) -> FilterResult: outcome=FilterOutcome.of(_text(obj, "outcome", where)), err_code=_int(obj, "code", where), err_msg=_bytes(obj, "err", where), + reason=_opt_text(obj, "reason", where), + refused_at=_refused_at(obj, where), + refused_row=_opt_u64(obj, "refused_row", where), rows_read=_u64(obj, "rows_read", where), unsupported_settings=_name_list(obj, "unsupported_settings", where), verdicts=tuple(Verdict.of(c) for c in verdicts), @@ -487,12 +519,17 @@ def decode_filter_result(raw: bytes) -> FilterResult: def _column(obj: dict[str, Any], where: str) -> Column: - # An unlisted default_kind is its unknown(n): kept, never a failure (rule r3). + # An unlisted default_kind or default_support is its unknown(n): kept, never a + # failure (rule r3). + support = _opt_text(obj, "default_support", where) return Column( name=_name(obj, "name", where), type=_bytes(obj, "type", where), default_kind=DefaultKind.of(_text(obj, "default_kind", where)), default_expr=_bytes(obj, "default_expression", where), + default_support=None if support is None else DefaultSupport.of(support), + default_support_reason=_opt_text(obj, "default_support_reason", where), + reads=None if obj.get("reads") is None else _name_list(obj, "reads", where), ) @@ -506,6 +543,14 @@ def _string_map(obj: Mapping[str, Any], key: str, where: str) -> dict[str, str] inner = _obj(value, where, key) out: dict[str, str] = {} for name, item in inner.items(): + if isinstance(item, dict): + # The object form of a value that was not UTF-8: its bytes, spelled + # with surrogateescape (the spelling the input side takes back). + if set(item) != {"b64"}: + raise _wrong(f"{where}.{key}", name, "a string or a b64 object", item) + item = _decode_b64(item["b64"], f"{where}.{key}", name).decode( + "utf-8", "surrogateescape" + ) if not isinstance(item, str): raise _wrong(f"{where}.{key}", name, "a string", item) out[name] = item @@ -543,6 +588,7 @@ def decode_schema_description(raw: bytes) -> SchemaDescription: # decodes exactly as it did before servers existed. server = obj.get("server") replicated = obj.get("replicated") + posture = _opt_text(obj, "posture", where) return SchemaDescription( columns=_objects(obj, "columns", where, _column), server=None @@ -557,6 +603,7 @@ def decode_schema_description(raw: bytes) -> SchemaDescription: filter_declined_settings=_objects( obj, "filter_declined_settings", where, _declined_setting ), + posture=None if posture is None else Posture.of(posture), ) diff --git a/python/src/chtypes/_input.py b/python/src/chtypes/_input.py index 5ab6f8b0..4b7a3b06 100644 --- a/python/src/chtypes/_input.py +++ b/python/src/chtypes/_input.py @@ -13,10 +13,20 @@ import json from collections.abc import Mapping, Sequence from dataclasses import dataclass +from enum import StrEnum +from ._abi2._vocab import Posture from .errors import _misuse as misuse -__all__ = ["BytesIn", "ServerProfile", "Settings"] +__all__ = [ + "BytesIn", + "ColumnDefault", + "DeferDefaults", + "ParamEncoding", + "ServerProfile", + "Settings", + "VerdictBasis", +] BytesIn = bytes | str Settings = Mapping[str, str] @@ -46,8 +56,8 @@ def body_bytes(value: bytes, what: str = "body") -> bytes: _NON_UTF8_HINT = ( - "this ABI generation cannot carry a non-UTF-8 value in a JSON input; " - "write the bytes as unhex('') in the expression instead" + "a JSON input cannot carry these bytes here " + '(only a settings or query-parameter value has the {"b64": ...} object form)' ) @@ -70,7 +80,40 @@ def _refuse_non_utf8(what: str, part: str, key: str) -> Exception: ) -def _string_map(mapping: Mapping[str, str], what: str) -> dict[str, str]: +def _byte_value(text: str, what: str, key: str) -> str | dict[str, str]: + """One value of `json_object_byte_values`: the string itself when it is valid + UTF-8, else the object form `{"b64": }` of its bytes (a `str` + holding the lone surrogates `surrogateescape` makes of undecodable bytes + carries those bytes). A surrogate that is not one of those has no bytes to + carry and is refused.""" + if _is_utf8(text): + return text + try: + raw = text.encode("utf-8", "surrogateescape") + except UnicodeEncodeError: + raise _refuse_non_utf8(what, "value", key) from None + return {"b64": base64.b64encode(raw).decode("ascii")} + + +def _string_map(mapping: Mapping[str, str], what: str) -> dict[str, str | dict[str, str]]: + out: dict[str, str | dict[str, str]] = {} + for key, value in mapping.items(): + if not isinstance(key, str) or not isinstance(value, str): + raise TypeError( + f"{what} maps strings to strings only; got {type(key).__name__} -> " + f"{type(value).__name__} for {key!r} (a value is never rewritten)" + ) + # json.dumps would write a lone surrogate as a \udXXX escape, a key the + # library does not read as the bytes the caller meant: refuse. + if not _is_utf8(key): + raise _refuse_non_utf8(what, "key", key) + out[key] = _byte_value(value, what, key) + return out + + +def _text_map(mapping: Mapping[str, str], what: str) -> dict[str, str]: + """A map whose values have no object form (a server's macros): a key or a + value that is not valid UTF-8 is refused.""" out: dict[str, str] = {} for key, value in mapping.items(): if not isinstance(key, str) or not isinstance(value, str): @@ -78,8 +121,6 @@ def _string_map(mapping: Mapping[str, str], what: str) -> dict[str, str]: f"{what} maps strings to strings only; got {type(key).__name__} -> " f"{type(value).__name__} for {key!r} (a value is never rewritten)" ) - # json.dumps would write a lone surrogate as a \udXXX escape, a value - # the library does not read as the bytes the caller meant: refuse. if not _is_utf8(key): raise _refuse_non_utf8(what, "key", key) if not _is_utf8(value): @@ -96,24 +137,25 @@ def settings_json( settings: Settings | None, session_timezone: str | None = None, ) -> bytes | None: - """The settings object, as a JSON object of string values, or None for none. + """The settings object, as a JSON object of byte values (a string, or `{"b64": ...}` + for a value that is not UTF-8), or None for none. `session_timezone` is written into it as that one key, verbatim. Giving it alongside a settings key of the same name is a `UsageError`, whether or not the two agree: a call never carries two spellings of its zone. """ - merged = {} if settings is None else _string_map(settings, "settings") + merged: dict[str, str | dict[str, str]] = ( + {} if settings is None else _string_map(settings, "settings") + ) if session_timezone is not None: if not isinstance(session_timezone, str): raise TypeError(f"session_timezone must be str, not {type(session_timezone).__name__}") - if not _is_utf8(session_timezone): - raise misuse(f"the zone: the value is not valid UTF-8, and {_NON_UTF8_HINT}") if _ZONE_KEY in merged: raise misuse( f"the {_ZONE_KEY} option and a {_ZONE_KEY} settings key were both given; " "a call carries one spelling of its zone" ) - merged[_ZONE_KEY] = session_timezone + merged[_ZONE_KEY] = _byte_value(session_timezone, "the zone", _ZONE_KEY) return _dump(merged) if merged else None @@ -124,6 +166,17 @@ def string_map_json(mapping: Mapping[str, str] | None, what: str) -> bytes | Non return _dump(_string_map(mapping, what)) +def _name_objects(names: Sequence[BytesIn]) -> list[dict[str, str]]: + out = [] + for name in names: + raw = to_bytes(name, "a column name") + try: + out.append({"name": raw.decode("utf-8")}) + except UnicodeDecodeError: + out.append({"name_b64": base64.b64encode(raw).decode("ascii")}) + return out + + def columns_json(columns: Sequence[BytesIn] | None) -> bytes | None: """The INSERT column list: a JSON array of name objects, `{"name": ...}` for a name that is valid UTF-8 and `{"name_b64": ...}` otherwise.""" @@ -133,14 +186,152 @@ def columns_json(columns: Sequence[BytesIn] | None) -> bytes | None: raise TypeError("columns must be a sequence of names, not a single name") if len(columns) == 0: return None - names = [] - for column in columns: - raw = to_bytes(column, "a column name") + return _dump(_name_objects(columns)) + + +# ---- the options documents (rule r1). Each member is its own keyword, and a key +# the binding has no name for goes through the `*_options_extra` mapping as JSON +# text. The binding builds the document from the members the caller set, sends +# nothing when none was set, and never validates a member: the library refuses a +# key it does not know, or a value that is not one of the member's, with +# CHS_INVALID_ARGUMENT, naming it. + + +class VerdictBasis(StrEnum): + """`insert_options.verdict_basis`: what row a filter verdict answers for.""" + + INPUT = "input" # the default: the input row + EXPORT = "export" # the row the export would store; a build without it declines + + +class DeferDefaults(StrEnum): + """`insert_options.defer_defaults`.""" + + NONE = "none" # the default: no DEFAULT is deferred + SERVER = "server" # defer to the server each DEFAULT the build would decline + + +class ParamEncoding(StrEnum): + """`filter_options.param_encoding`: how a query parameter's value is read.""" + + ESCAPED = "escaped" # the default: ClickHouse's escaped-text reader + RAW = "raw" # the bytes are the value; a build without it declines + + +@dataclass(frozen=True, slots=True) +class ColumnDefault: + """One entry of `insert_options.column_defaults`: the value the call uses in + a cell the server would fill from the column's DEFAULT. Both fields are byte + strings (`str` is UTF-8 encoded); one that is not valid UTF-8 is sent in its + `_b64` form.""" + + name: BytesIn + value: BytesIn + + +def _enum_text(value: object, what: str) -> str: + if isinstance(value, str): # a StrEnum member is a str: its value is the spelling + return value.value if isinstance(value, StrEnum) else value + raise TypeError(f"{what} must be a str or its enum, not {type(value).__name__}") + + +def _column_default_objects(defaults: Sequence[ColumnDefault]) -> list[dict[str, str]]: + if isinstance(defaults, str | bytes): + raise TypeError("column_defaults must be a sequence of ColumnDefault") + out = [] + for entry in defaults: + if not isinstance(entry, ColumnDefault): + raise TypeError( + f"column_defaults entries must be ColumnDefault, not {type(entry).__name__}" + ) + item = _name_objects([entry.name])[0] + raw = to_bytes(entry.value, "a column default value") try: - names.append({"name": raw.decode("utf-8")}) + item["value"] = raw.decode("utf-8") except UnicodeDecodeError: - names.append({"name_b64": base64.b64encode(raw).decode("ascii")}) - return _dump(names) + item["value_b64"] = base64.b64encode(raw).decode("ascii") + out.append(item) + return out + + +def _options_json( + what: str, members: dict[str, object], extra: Mapping[str, str] | None +) -> bytes | None: + """The document from the members set (value already JSON-encodable) and the + extra members (JSON text, spliced verbatim). A key set twice keeps the last + value, and an extra member is set after the named ones. Nothing set is None + (length 0, which the library reads as `{}`).""" + parts: dict[str, str] = {key: json.dumps(value) for key, value in members.items()} + if extra is not None: + if not isinstance(extra, Mapping): + raise TypeError(f"{what}_extra must map keys to JSON text") + for key, text in extra.items(): + if not isinstance(key, str) or not isinstance(text, str): + raise TypeError(f"{what}_extra maps strings to JSON text only") + if not _is_utf8(key): + raise _refuse_non_utf8(f"{what} options", "key", key) + try: + json.loads(text) + except ValueError: + raise misuse( + f"{what} options: the value for key {json.dumps(key)} is not valid JSON" + ) from None + parts[key] = text + if not parts: + return None + body = ",".join(f"{json.dumps(key)}:{text}" for key, text in parts.items()) + try: + return ("{" + body + "}").encode("utf-8") + except UnicodeEncodeError: + raise misuse(f"{what} options: a value is not valid UTF-8 JSON text") from None + + +def insert_options_json( + *, + insertable_columns: Sequence[BytesIn] | None = None, + column_defaults: Sequence[ColumnDefault] | None = None, + verdict_basis: VerdictBasis | str | None = None, + defer_defaults: DeferDefaults | str | None = None, + insert_options_extra: Mapping[str, str] | None = None, +) -> bytes | None: + """`insert_options`, from the members set. None leaves a member out; an empty + list is sent as `[]`.""" + members: dict[str, object] = {} + if insertable_columns is not None: + if isinstance(insertable_columns, str | bytes): + raise TypeError("insertable_columns must be a sequence of names, not a single name") + members["insertable_columns"] = _name_objects(insertable_columns) + if column_defaults is not None: + members["column_defaults"] = _column_default_objects(column_defaults) + if verdict_basis is not None: + members["verdict_basis"] = _enum_text(verdict_basis, "verdict_basis") + if defer_defaults is not None: + members["defer_defaults"] = _enum_text(defer_defaults, "defer_defaults") + return _options_json("insert", members, insert_options_extra) + + +def filter_options_json( + *, + param_encoding: ParamEncoding | str | None = None, + filter_options_extra: Mapping[str, str] | None = None, +) -> bytes | None: + """`filter_options`, from the members set.""" + members: dict[str, object] = {} + if param_encoding is not None: + members["param_encoding"] = _enum_text(param_encoding, "param_encoding") + return _options_json("filter", members, filter_options_extra) + + +def schema_options_json( + *, + posture: Posture | str | None = None, + schema_options_extra: Mapping[str, str] | None = None, +) -> bytes | None: + """`schema_options`, from the members set.""" + members: dict[str, object] = {} + if posture is not None: + members["posture"] = _enum_text(posture, "posture") + return _options_json("schema", members, schema_options_extra) @dataclass(frozen=True, slots=True) @@ -181,5 +372,5 @@ def server_profile_json(profile: ServerProfile) -> bytes: if profile.settings is not None: doc["settings"] = _string_map(profile.settings, "server profile settings") if profile.macros is not None: - doc["macros"] = _string_map(profile.macros, "server profile macros") + doc["macros"] = _text_map(profile.macros, "server profile macros") return _dump(doc) diff --git a/python/src/chtypes/library.py b/python/src/chtypes/library.py index a3a4d152..61378c8f 100644 --- a/python/src/chtypes/library.py +++ b/python/src/chtypes/library.py @@ -13,20 +13,27 @@ import contextlib import os import threading -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from pathlib import Path from types import TracebackType from . import _decode, _setup from ._abi2 import _decls, _loader -from ._abi2._vocab import EXPORT_NONE, DocFlags, Format +from ._abi2._vocab import EXPORT_NONE, DocFlags, Format, Posture from ._guard import CloseGuard from ._input import ( BytesIn, + ColumnDefault, + DeferDefaults, + ParamEncoding, ServerProfile, Settings, + VerdictBasis, body_bytes, columns_json, + filter_options_json, + insert_options_json, + schema_options_json, server_profile_json, settings_json, string_map_json, @@ -197,14 +204,30 @@ def rows( *, settings: Settings | None = None, session_timezone: str | None = None, + columns: Sequence[BytesIn] | None = None, + insertable_columns: Sequence[BytesIn] | None = None, + column_defaults: Sequence[ColumnDefault] | None = None, + verdict_basis: VerdictBasis | str | None = None, + defer_defaults: DeferDefaults | str | None = None, + insert_options_extra: Mapping[str, str] | None = None, ) -> FilterResult: """Evaluate over a body. `settings` and `session_timezone` are the body's - PARSE settings only: the filter's own zone was fixed when it compiled.""" + PARSE settings only: the filter's own zone was fixed when it compiled. + `columns` is the INSERT column list and the keywords after it the + `insert_options` members, as for `Schema.rows`.""" payload = body_bytes(body) settings_bytes = settings_json(settings, session_timezone) + columns_bytes = columns_json(columns) + options = insert_options_json( + insertable_columns=insertable_columns, + column_defaults=column_defaults, + verdict_basis=verdict_basis, + defer_defaults=defer_defaults, + insert_options_extra=insert_options_extra, + ) with _hold(self._guard): raw = self._library._api.filter_eval_body( - self._handle, int(format), payload, settings_bytes + self._handle, int(format), payload, settings_bytes, columns_bytes, options ) return _decode.decode_filter_result(raw) @@ -239,14 +262,30 @@ def row( settings: Settings | None = None, session_timezone: str | None = None, columns: Sequence[BytesIn] | None = None, + insertable_columns: Sequence[BytesIn] | None = None, + column_defaults: Sequence[ColumnDefault] | None = None, + verdict_basis: VerdictBasis | str | None = None, + defer_defaults: DeferDefaults | str | None = None, + insert_options_extra: Mapping[str, str] | None = None, ) -> RowResult: - """One row, as a server's INSERT would validate and coerce it.""" + """One row, as a server's INSERT would validate and coerce it. The keywords + after `columns` are the `insert_options` members (rule r1), passed through + unvalidated: `insertable_columns`, `column_defaults`, `verdict_basis`, + `defer_defaults`, and `insert_options_extra` (JSON text by key) for a + member this binding has no name for.""" payload = body_bytes(body) settings_bytes = settings_json(settings, session_timezone) columns_bytes = columns_json(columns) + options = insert_options_json( + insertable_columns=insertable_columns, + column_defaults=column_defaults, + verdict_basis=verdict_basis, + defer_defaults=defer_defaults, + insert_options_extra=insert_options_extra, + ) with _hold(self._guard): raw = self._library._api.preview_row( - self._handle, int(format), payload, settings_bytes, columns_bytes + self._handle, int(format), payload, settings_bytes, columns_bytes, options ) return _decode.decode_row_document(raw) @@ -258,6 +297,11 @@ def rows( settings: Settings | None = None, session_timezone: str | None = None, columns: Sequence[BytesIn] | None = None, + insertable_columns: Sequence[BytesIn] | None = None, + column_defaults: Sequence[ColumnDefault] | None = None, + verdict_basis: VerdictBasis | str | None = None, + defer_defaults: DeferDefaults | str | None = None, + insert_options_extra: Mapping[str, str] | None = None, row_filter: Filter | None = None, export: Format | int | None = None, doc_flags: DocFlags = DocFlags.ALL, @@ -268,6 +312,13 @@ def rows( payload = body_bytes(body) settings_bytes = settings_json(settings, session_timezone) columns_bytes = columns_json(columns) + options = insert_options_json( + insertable_columns=insertable_columns, + column_defaults=column_defaults, + verdict_basis=verdict_basis, + defer_defaults=defer_defaults, + insert_options_extra=insert_options_extra, + ) if row_filter is not None and not isinstance(row_filter, Filter): raise TypeError(f"row_filter must be a Filter, not {type(row_filter).__name__}") guards = [self._guard] + ([] if row_filter is None else [row_filter._guard]) @@ -281,6 +332,7 @@ def rows( None if row_filter is None else row_filter._handle, EXPORT_NONE if export is None else int(export), int(doc_flags), + options, ) return _decode.decode_batch(raw, exported if export is not None else None) @@ -291,16 +343,24 @@ def compile_filter( params: Settings | None = None, settings: Settings | None = None, session_timezone: str | None = None, + param_encoding: ParamEncoding | str | None = None, + filter_options_extra: Mapping[str, str] | None = None, ) -> Filter: """Compile a boolean expression over this schema's columns. The filter's zone is its own, fixed here: its WHERE runs in the session this call's - `session_timezone` (or `settings`) names, on every evaluation.""" + `session_timezone` (or `settings`) names, on every evaluation. + `param_encoding` and `filter_options_extra` (JSON text by key, for a + member this binding has no name for) are the `filter_options` members, + passed through unvalidated (rule r1).""" expr_bytes = to_bytes(expr, "expr") params_bytes = string_map_json(params, "query parameters") settings_bytes = settings_json(settings, session_timezone) + options = filter_options_json( + param_encoding=param_encoding, filter_options_extra=filter_options_extra + ) with _hold(self._guard): handle = self._library._api.filter_create( - self._handle, expr_bytes, params_bytes, settings_bytes + self._handle, expr_bytes, params_bytes, settings_bytes, options ) return Filter(self._library, handle) @@ -312,14 +372,27 @@ def parse_block( settings: Settings | None = None, session_timezone: str | None = None, columns: Sequence[BytesIn] | None = None, + insertable_columns: Sequence[BytesIn] | None = None, + column_defaults: Sequence[ColumnDefault] | None = None, + verdict_basis: VerdictBasis | str | None = None, + defer_defaults: DeferDefaults | str | None = None, + insert_options_extra: Mapping[str, str] | None = None, ) -> Block: - """Parse a body once, for evaluating many filters over it.""" + """Parse a body once, for evaluating many filters over it. The keywords + after `columns` are the `insert_options` members, as for `row`.""" payload = body_bytes(body) settings_bytes = settings_json(settings, session_timezone) columns_bytes = columns_json(columns) + options = insert_options_json( + insertable_columns=insertable_columns, + column_defaults=column_defaults, + verdict_basis=verdict_basis, + defer_defaults=defer_defaults, + insert_options_extra=insert_options_extra, + ) with _hold(self._guard): handle = self._library._api.block_create( - self._handle, int(format), payload, settings_bytes, columns_bytes + self._handle, int(format), payload, settings_bytes, columns_bytes, options ) return Block(self._library, handle) @@ -421,6 +494,8 @@ def compile_table( settings: Settings | None = None, session_timezone: str | None = None, server: Server | None = None, + posture: Posture | str | None = None, + schema_options_extra: Mapping[str, str] | None = None, ) -> Schema: """Compile exactly one `CREATE TABLE` statement. `settings` is the profile; a compiled type always takes the image zone, never the profile's. @@ -431,17 +506,22 @@ def compile_table( replica name expand the server's macros. A closed server is a `UsageError`, raised before any call; a server from another library is that library's own refusal, also a `UsageError`. `None` is no server: the - schema is on the image's own server.""" + schema is on the image's own server. + + `posture` (`Posture.CREATE`, the default, or `Posture.LOAD`) and + `schema_options_extra` (JSON text by key, for a member this binding has no + name for) are the `schema_options` members, passed through unvalidated + (rule r1).""" statement = to_bytes(create_table, "create_table") settings_bytes = settings_json(settings, session_timezone) if server is not None and not isinstance(server, Server): raise TypeError(f"server must be a Server, not {type(server).__name__}") - # No schema option is defined yet: none, which the library reads as `{}`. + options = schema_options_json(posture=posture, schema_options_extra=schema_options_extra) # A closed server is refused here, before the call, because its freed # handle would cross as NULL and the library would compile on the # image's server instead of refusing. with _hold(*(() if server is None else (server._guard,))): handle = self._api.schema_create( - None if server is None else server._handle, statement, settings_bytes, None + None if server is None else server._handle, statement, settings_bytes, options ) return Schema(self, handle) diff --git a/python/src/chtypes/results.py b/python/src/chtypes/results.py index c08521f6..f8628259 100644 --- a/python/src/chtypes/results.py +++ b/python/src/chtypes/results.py @@ -18,9 +18,12 @@ DeclinedLayer, DeclinedTier, DefaultKind, + DefaultSupport, FilterOutcome, MergeReason, Outcome, + Posture, + RefusalSite, Verdict, ) @@ -125,7 +128,7 @@ class AtMergeEntry: `at_merge` list (ABI v2). `engine_rows` stays exactly what the INSERT writer produces, so the two never contradict each other. - `row` indexes `engine_rows`: the part's row, not the input body's. `reason` is + `part_row` indexes `engine_rows`: the part's row, not the input body's. `reason` is a `MergeReason`; one the description does not list is its unknown(n) member (rule r3). `column` is the column a `ttl_column_reset` resets (from `column` or `column_b64`), None when the entry names none. `stored` is the column's @@ -136,7 +139,7 @@ class AtMergeEntry: the document carries none. """ - row: int + part_row: int reason: MergeReason column: bytes | None = None stored: bytes | None = None @@ -146,19 +149,32 @@ class AtMergeEntry: @dataclass(frozen=True, slots=True) class RowResult: """The `row` document. `columns` is every `cols` entry in document order; - `values` is the subset whose `is_stored` is true, in order.""" + `values` is the subset whose `is_stored` is true, in order. + + `err_code` is the document's `code`: ClickHouse's own code, 0 only on a + success, and -2 (`CHS_CODE_DECLINED`) where this build declined the row. + `reason` is the decline's catalog id (ASCII), present exactly where `err_code` + is -2, else None; key on it, never on `err_msg`. `refused_at` is where a + single-row result's refusal sits (the rows of a batch carry neither this nor + `refused_row`: the batch's own say where), None for an accepted or skipped + row. `refused_row` is the row index of a `RefusalSite.ROW` refusal, else None. + `verdict_reason` is the catalog id of a declined verdict (`verdict_code` is + -2), else None.""" outcome: Outcome err_code: int err_msg: bytes + reason: str | None + refused_at: RefusalSite | None + refused_row: int | None columns: tuple[Value, ...] values: tuple[Value, ...] transformed: tuple[Transform, ...] unknown_fields: tuple[bytes, ...] - unsupported_settings: tuple[bytes, ...] computed: tuple[Computed, ...] verdict: Verdict | None verdict_code: int + verdict_reason: str | None verdict_err: bytes partition_id: bytes | None input_span: Span | None @@ -196,6 +212,16 @@ class BatchResult: a body with any skipped row or any `unconsumed` range (docs/guides/batches.md). + `reason`, `refused_at` and `refused_row` are the row's (see `RowResult`) for the + batch: the catalog id of a declined batch, and where a refusal or decline + sits, None when the document carries none. `spans` is `row_spans`: one entry + per `rows` entry, index-aligned, None for a row not exported (a skipped row, or + a row a filter cut); the tuple itself is None when absent, and always None when + the export was declined. `export_columns` is the export's one column list, in + the order every line carries its columns (None when no export ran), and + `export_declined_reason` the catalog id of a declined export (None when the + export ran or none was asked for); `export_declined` stays the cause text. + `unsupported_settings` is the call's own settings the library declined, from the batch's top-level list (ABI v2). `at_merge` is what an `OPTIMIZE TABLE ... FINAL` of the part would do to its rows, at least: a later background merge @@ -206,14 +232,19 @@ class BatchResult: outcome: Outcome err_code: int err_msg: bytes + reason: str | None + refused_at: RefusalSite | None + refused_row: int | None rows: tuple[RowResult, ...] rows_read: int rows_skipped: int transformed: tuple[Transform, ...] engine_rows: tuple[tuple[EngineCell, ...], ...] | None payload: bytes | None - spans: tuple[Span, ...] | None + spans: tuple[Span | None, ...] | None + export_columns: tuple[bytes, ...] | None export_declined: bytes + export_declined_reason: str | None rows_passed: int rows_cut: int partition_count: int | None @@ -225,21 +256,30 @@ class BatchResult: @dataclass(frozen=True, slots=True) class FilterRowError: + """One entry of a filter result's `errors`. `reason` is the catalog id where + `code` is -2 (`CHS_CODE_DECLINED`), else None.""" + row: int code: int + reason: str | None msg: bytes @dataclass(frozen=True, slots=True) class FilterResult: """The `filter_result` document. A verdict counts only when `outcome` is `ok`; - otherwise treat every verdict as `d`, whatever the verdict string holds. Under + otherwise treat every verdict as `d`, whatever the verdict string holds. + `reason`, `refused_at` and `refused_row` are the batch's (see `BatchResult`): + where a refused or declined evaluation sits, and its catalog id. Under `ok`, `e` and `d` are never answers, and a caller enforcing visibility fails closed on both; `Verdict.answered` says which.""" outcome: FilterOutcome err_code: int err_msg: bytes + reason: str | None + refused_at: RefusalSite | None + refused_row: int | None rows_read: int unsupported_settings: tuple[bytes, ...] verdicts: tuple[Verdict, ...] @@ -250,12 +290,20 @@ class FilterResult: class Column: """One column of a compiled schema. `type` is the canonical type; it can differ between ClickHouse lines, so a cross-line hash is taken over the - caller's own statement, never over `columns`.""" + caller's own statement, never over `columns`. + + `default_support` is what this build does with the column's DEFAULT and + `default_support_reason` the catalog id behind a `DECLINED` one; `reads` is + the columns the column's expression reads. All three are reserved: None when + the build reports none.""" name: bytes type: bytes default_kind: DefaultKind default_expr: bytes + default_support: DefaultSupport | None = None + default_support_reason: str | None = None + reads: tuple[bytes, ...] | None = None @dataclass(frozen=True, slots=True) @@ -280,7 +328,9 @@ class SchemaServer: data-derived. `timezone` is the zone the schema's types bind: the profile's, or else the image zone. `settings` is the profile's settings (`{}` when it gave none). `macros` is None exactly when the profile carried no macros - (unknown), and a mapping, even an empty one, when it did (the complete set).""" + (unknown), and a mapping, even an empty one, when it did (the complete set). A + setting value that was not UTF-8 comes back as its bytes decoded with + `surrogateescape`, the spelling that goes back in unchanged.""" timezone: str settings: Mapping[str, str] @@ -308,12 +358,16 @@ class SchemaDescription: the highest layer that sets it, ordered by layer and then by name bytes, as the library wrote it. A schema that lists any is one `compile_filter` declines (`UnsupportedError`), and a consumer that streams rows refuses - such a tenant (spec/abi-v2/where-settings/README.md).""" + such a tenant (spec/abi-v2/where-settings/README.md). + + `posture` is how the statement was compiled, which the document carries only + for `Posture.LOAD`: None is `Posture.CREATE`.""" columns: tuple[Column, ...] server: SchemaServer | None = None replicated: SchemaReplicated | None = None filter_declined_settings: tuple[DeclinedSetting, ...] = () + posture: Posture | None = None @dataclass(frozen=True, slots=True) diff --git a/python/tests/abi2/test_public_library.py b/python/tests/abi2/test_public_library.py index ed556d99..b458f8e3 100644 --- a/python/tests/abi2/test_public_library.py +++ b/python/tests/abi2/test_public_library.py @@ -15,13 +15,18 @@ from chtypes import ( BatchResult, CallError, + ColumnDefault, + DeferDefaults, DocFlags, Format, InternalError, Outcome, + ParamEncoding, + Posture, SchemaError, UnsupportedError, UsageError, + VerdictBasis, ) from chtypes._abi2 import _decls @@ -149,8 +154,8 @@ def spy(*args): # lists, reads as its unknown(n) member, never as accepted (rule r3). assert result.outcome == "" and not result.outcome.known and result.payload is not None assert result.outcome is not Outcome.ACCEPTED - (_, fmt, body, settings, columns, handle, export, flags), plain = calls - assert fmt == 0 and body == b'{"a":1}\n' and settings is None + (_, fmt, body, settings, columns, handle, export, flags, options), plain = calls + assert fmt == 0 and body == b'{"a":1}\n' and settings is None and options is None assert json.loads(columns) == [{"name": "a"}, {"name_b64": "/w=="}] assert handle is not None and export == int(Format.CSV) and flags == 3 assert plain[5] is None and plain[6] == -1 and plain[7] == 7 # no export, all groups @@ -169,12 +174,12 @@ def test_filter_zone_and_parse_zone_are_distinct_arguments(lib, monkeypatch) -> monkeypatch.setattr( lib._api, "filter_create", - lambda s, e, p, st: (created.append(st), real_create(s, e, p, st))[1], + lambda s, e, p, st, o: (created.append(st), real_create(s, e, p, st, o))[1], ) monkeypatch.setattr( lib._api, "filter_eval_body", - lambda f, fmt, b, st: (evaluated.append(st), real_eval(f, fmt, b, st))[1], + lambda f, fmt, b, st, c, o: (evaluated.append(st), real_eval(f, fmt, b, st, c, o))[1], ) with lib.compile_table(CREATE) as schema: with schema.compile_filter("a > 1", params={"p": "1"}, session_timezone="Asia/Tokyo") as f: @@ -327,3 +332,69 @@ def fake(): table = lib.error_codes() assert table.name(62) == "SYNTAX_ERROR" assert lib.error_codes() is table and len(calls) == 2 + + +def _spy(lib, monkeypatch, name): + calls: list[tuple] = [] + real = getattr(lib._api, name) + + def spy(*args): + calls.append(args) + return real(*args) + + monkeypatch.setattr(lib._api, name, spy) + return calls + + +def test_insert_options_reach_the_four_insert_calls_and_nothing_else(lib, monkeypatch) -> None: + """Spec batch 2: `options` on preview_row, preview_batch, block_create and + filter_eval_body, the `columns` of filter_eval_body, and no document at all + when no member is set. What the library did with each is the stub's echo.""" + row = _spy(lib, monkeypatch, "preview_row") + batch = _spy(lib, monkeypatch, "preview_batch") + block = _spy(lib, monkeypatch, "block_create") + body = _spy(lib, monkeypatch, "filter_eval_body") + with lib.compile_table(CREATE) as schema, schema.compile_filter("a > 1") as flt: + kw = { + "insertable_columns": ["a"], + "column_defaults": [ColumnDefault("a", "1")], + "verdict_basis": VerdictBasis.EXPORT, + "defer_defaults": DeferDefaults.SERVER, + } + schema.row(Format.CSV, b"1", **kw) + schema.rows(Format.CSV, b"1", **kw) + schema.parse_block(Format.CSV, b"1", **kw).close() + flt.rows(Format.CSV, b"1", columns=["a"], **kw) + schema.row(Format.CSV, b"1") + want = { + "insertable_columns": [{"name": "a"}], + "column_defaults": [{"name": "a", "value": "1"}], + "verdict_basis": "export", + "defer_defaults": "server", + } + assert json.loads(row[0][-1]) == want and row[1][-1] is None + assert json.loads(batch[0][-1]) == want + assert json.loads(block[0][-1]) == want + assert json.loads(body[0][-1]) == want and json.loads(body[0][-2]) == [{"name": "a"}] + + +def test_filter_and_schema_options_reach_their_creates(lib, monkeypatch) -> None: + created = _spy(lib, monkeypatch, "filter_create") + schemas = _spy(lib, monkeypatch, "schema_create") + with lib.compile_table(CREATE, posture=Posture.LOAD) as schema: + schema.compile_filter("a > 1", param_encoding=ParamEncoding.RAW).close() + schema.compile_filter("a > 1").close() + assert json.loads(schemas[0][-1]) == {"posture": "load"} + assert json.loads(created[0][-1]) == {"param_encoding": "raw"} and created[1][-1] is None + + +def test_an_unnamed_member_reaches_the_library_which_refuses_it_by_name(lib) -> None: + """Rule r1: the binding never drops a key it has no name for; the library + refuses one it does not know, naming it (here the stub, which enforces r1).""" + with lib.compile_table(CREATE) as schema: + with pytest.raises(UsageError, match='unknown key "future"'): + schema.row(Format.CSV, b"1", insert_options_extra={"future": "[1]"}) + with pytest.raises(UsageError, match='unknown key "future"'): + schema.compile_filter("a > 1", filter_options_extra={"future": "1"}) + with pytest.raises(UsageError, match='unknown key "future"'): + lib.compile_table(CREATE, schema_options_extra={"future": "1"}) diff --git a/python/tests/abi2/test_reader_rules.py b/python/tests/abi2/test_reader_rules.py index 60615ab4..bd7f5a19 100644 --- a/python/tests/abi2/test_reader_rules.py +++ b/python/tests/abi2/test_reader_rules.py @@ -86,7 +86,7 @@ def test_r2_unknown_members_are_ignored(open_variant) -> None: entry0 = row.transformed[0] assert (entry0.column, entry0.input, entry0.stored) == (b"s", b"1", b"2") assert entry0.reason == Reason.VALUE_CHANGED and entry0.lossy is True and entry0.row == 0 - assert row.unknown_fields == (b"u",) and row.unsupported_settings == (b"st",) + assert row.unknown_fields == (b"u",) batch = schema.rows(JSON, b'{"x":1}', export=JSON) assert batch.outcome is Outcome.ACCEPTED and batch.rows_read == 1 diff --git a/python/tests/api/test_decode.py b/python/tests/api/test_decode.py index fd96a6ea..8c739c29 100644 --- a/python/tests/api/test_decode.py +++ b/python/tests/api/test_decode.py @@ -7,7 +7,17 @@ import pytest -from chtypes import DefaultKind, FilterOutcome, InternalError, MergeReason, Outcome, Verdict +from chtypes import ( + DefaultKind, + DefaultSupport, + FilterOutcome, + InternalError, + MergeReason, + Outcome, + Posture, + RefusalSite, + Verdict, +) from chtypes._decode import ( decode_batch, decode_discovery, @@ -42,7 +52,7 @@ def dumps(doc) -> bytes: {"column_b64": b64(b"\xff"), "input_b64": b64(b"\x80"), "stored": "x", "reason": "zzz"}, ], "unknown_fields": [{"name": "extra"}, {"name_b64": b64(b"\xff\x00")}], - "unsupported_settings": [{"name": "foo"}], + "unsupported_settings": [{"name": "removed_at_row_level"}], # skipped: not a member now "computed": [{"name": "c", "kind": "materialized", "stored": "9", "value_b64": b64(b"9")}], "verdict": "t", "verdict_code": 0, @@ -67,7 +77,9 @@ def test_a_row_decodes_one_to_one_with_names_and_values_as_bytes() -> None: assert row.transformed[1].column == b"\xff" and row.transformed[1].input == b"\x80" assert row.transformed[1].reason == "zzz" and row.transformed[1].lossy is True assert row.transformed[1].row == 0 - assert row.unknown_fields == (b"extra", b"\xff\x00") and row.unsupported_settings == (b"foo",) + assert row.unknown_fields == (b"extra", b"\xff\x00") and not hasattr( + row, "unsupported_settings" + ) assert row.computed[0].text == b"9" and row.computed[0].value == b"9" assert row.verdict is Verdict.TRUE assert row.partition_id == b"p1" and row.input_span.off == 0 and row.input_span.len == 5 @@ -80,6 +92,8 @@ def test_absent_is_the_default() -> None: assert row.outcome == "" and not row.outcome.known and row.outcome is not Outcome.ACCEPTED assert row.columns == () and row.verdict is None and row.input_span is None assert row.err_msg == b"" and row.partition_id is None + assert row.reason is None and row.refused_at is None and row.refused_row is None + assert row.verdict_reason is None @pytest.mark.parametrize( @@ -232,30 +246,38 @@ def test_a_batch_decodes_rows_payload_framing_and_spans() -> None: "outcome": "accepted", "unsupported_settings": [{"name": "session_timezone"}, {"name_b64": b64(b"\xff")}], "at_merge": [ - {"row": 0, "reason": "ttl_delete"}, - {"row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, - {"row": 2, "reason": "ttl_column_reset", "column": "t"}, + {"part_row": 0, "reason": "ttl_delete"}, + {"part_row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, + {"part_row": 2, "reason": "ttl_column_reset", "column": "t"}, { - "row": 3, + "part_row": 3, "reason": "ttl_column_reset", "column_b64": b64(b"\xff\x00"), "stored_b64": b64(b"\xff"), }, - {"row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, + {"part_row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, { - "row": 5, + "part_row": 5, "reason": "a_reason_nobody_listed", "x_future": {"a": [1]}, "x_future_b64": "/w==", }, - {"row": 6, "reason": "ttl_column_reset", "column": "e", "stored": "", "input_rows": []}, + { + "part_row": 6, + "reason": "ttl_column_reset", + "column": "e", + "stored": "", + "input_rows": [], + }, ], } def test_a_batch_decodes_at_merge_one_to_one() -> None: entries = decode_batch(dumps(AT_MERGE_BATCH), None).at_merge - got = [(e.row, e.reason, e.reason.known, e.column, e.stored, e.input_rows) for e in entries] + got = [ + (e.part_row, e.reason, e.reason.known, e.column, e.stored, e.input_rows) for e in entries + ] assert got == [ (0, MergeReason.TTL_DELETE, True, None, None, None), (1, MergeReason.TTL_COLUMN_RESET, True, b"c", b"0", None), @@ -298,14 +320,18 @@ def test_absent_at_merge_and_unsupported_settings_are_empty(doc) -> None: [ { "at_merge": [ - {"row": 0, "reason": "ttl_column_reset", "column": "c", "column_b64": "Yw=="} + {"part_row": 0, "reason": "ttl_column_reset", "column": "c", "column_b64": "Yw=="} + ] + }, + { + "at_merge": [ + {"part_row": 0, "reason": "ttl_column_reset", "column": "c", "stored_b64": "!!"} ] }, - {"at_merge": [{"row": 0, "reason": "ttl_column_reset", "column": "c", "stored_b64": "!!"}]}, - {"at_merge": [{"row": 0, "reason": "ttl_delete", "input_rows": ["x"]}]}, - {"at_merge": [{"row": 0, "reason": "ttl_delete", "input_rows": [-1]}]}, - {"at_merge": [{"row": 0, "reason": 7}]}, - {"at_merge": {"row": 0}}, + {"at_merge": [{"part_row": 0, "reason": "ttl_delete", "input_rows": ["x"]}]}, + {"at_merge": [{"part_row": 0, "reason": "ttl_delete", "input_rows": [-1]}]}, + {"at_merge": [{"part_row": 0, "reason": 7}]}, + {"at_merge": {"part_row": 0}}, {"unsupported_settings": ["session_timezone"]}, ], ) @@ -415,7 +441,6 @@ def test_error_code_table_and_live_handles() -> None: {"unknown_fields": ["bare string"]}, {"unknown_fields": [{}]}, {"unknown_fields": [{"name": "a", "name_b64": "YQ=="}]}, - {"unsupported_settings": ["x"]}, {"partition_id": "p", "partition_id_b64": "cA=="}, {"cols": [{"name": "a", "null": False, "src": "input", "value_b64": "YQ"}]}, # unpadded {"computed": [{"name": "c", "value_b64": "!!"}]}, @@ -431,3 +456,150 @@ def test_an_engine_row_must_be_a_list_of_cells_never_an_object() -> None: decode_batch(dumps({"engine_rows": [{"a": "1"}]}), None) with pytest.raises(InternalError): decode_batch(dumps({"engine_rows": [[{"stored": "1", "null": False}]]}), None) # no name + + +# ABI v2 spec batch 2: the refusal members, the export members, the null row span +# and the schema description's new members. + + +def test_a_declined_batch_names_its_code_reason_and_site() -> None: + batch = decode_batch( + dumps( + { + "outcome": "unsupported", + "code": -2, + "reason": "preview-batch-zone-mismatch", + "refused_at": "body_read", + "rows_read": 0, + "rows_skipped": 0, + "unsupported_settings": [], + } + ), + None, + ) + assert batch.err_code == -2 and batch.reason == "preview-batch-zone-mismatch" + assert batch.refused_at is RefusalSite.BODY_READ and batch.refused_at.known + assert batch.refused_row is None + + +def test_a_row_refusal_names_its_row_and_a_future_site_is_kept() -> None: + row = decode_row_document( + dumps( + { + "outcome": "unsupported", + "code": -2, + "reason": "drawn-value", + "refused_at": "row", + "refused_row": 0, + "verdict": "d", + "verdict_code": -2, + "verdict_reason": "defer-defaults-reader", + } + ) + ) + assert row.reason == "drawn-value" and row.refused_at is RefusalSite.ROW + assert row.refused_row == 0 # present zero is not absent + assert row.verdict is Verdict.DECLINE and row.verdict_code == -2 + assert row.verdict_reason == "defer-defaults-reader" + future = decode_row_document(dumps({"refused_at": "a_site_nobody_listed"})) + assert future.refused_at == "a_site_nobody_listed" and not future.refused_at.known + + +def test_a_filter_result_carries_the_batchs_refusal_members_and_error_reasons() -> None: + res = decode_filter_result( + dumps( + { + "outcome": "unsupported", + "code": -2, + "reason": "filter-zone", + "refused_at": "call", + "rows_read": 1, + "verdicts": "d", + "errors": [ + {"row": 0, "code": -2, "reason": "drawn-value"}, + {"row": 1, "code": 386}, + ], + } + ) + ) + assert res.err_code == -2 and res.reason == "filter-zone" + assert res.refused_at is RefusalSite.CALL and res.refused_row is None + assert [(e.code, e.reason) for e in res.errors] == [(-2, "drawn-value"), (386, None)] + + +def test_the_export_members_and_the_null_row_span() -> None: + batch = decode_batch( + dumps( + { + "outcome": "accepted", + "row_spans": [{"off": 0, "len": 4}, None, {"off": 4, "len": 2}], + "export_columns": [{"name": "a"}, {"name_b64": b64(b"\xff")}], + "export_declined": "", + } + ), + b"abcdef", + ) + assert batch.spans is not None + assert [s and (s.off, s.len) for s in batch.spans] == [(0, 4), None, (4, 2)] + assert batch.export_columns == (b"a", b"\xff") and batch.export_declined_reason is None + declined = decode_batch( + dumps( + { + "outcome": "rejected", + "row_spans": [None], + "export_columns": [{"name": "a"}], + "export_declined": "no export", + "export_declined_reason": "export-after-refusal", + } + ), + None, + ) + assert declined.spans is None and declined.payload is None + assert declined.export_declined == b"no export" + assert declined.export_declined_reason == "export-after-refusal" + assert decode_batch(dumps({}), None).export_columns is None + + +def test_a_schema_description_decodes_posture_default_support_reads_and_b64_settings() -> None: + desc = decode_schema_description( + dumps( + { + "posture": "load", + "columns": [ + { + "name": "a", + "type": "UInt8", + "default_kind": "DEFAULT", + "default_support": "declined", + "default_support_reason": "drawn-value", + "reads": [{"name": "b"}, {"name_b64": b64(b"\xff")}], + }, + {"name": "b", "type": "UInt8", "default_kind": ""}, + ], + "server": { + "timezone": "UTC", + "settings": {"plain": "1", "raw": {"b64": b64(b"\xff\x00")}}, + }, + } + ) + ) + assert desc.posture is Posture.LOAD + a, b = desc.columns + assert ( + a.default_support is DefaultSupport.DECLINED and a.default_support_reason == "drawn-value" + ) + assert a.reads == (b"b", b"\xff") + assert b.default_support is None and b.default_support_reason is None and b.reads is None + assert desc.server is not None + assert desc.server.settings == {"plain": "1", "raw": "\udcff\x00"} + assert desc.server.settings["raw"].encode("utf-8", "surrogateescape") == b"\xff\x00" + assert decode_schema_description(dumps({"columns": []})).posture is None + + +@pytest.mark.parametrize( + "settings", + [{"k": {"b64": "!!"}}, {"k": {"b64": "YQ==", "x": 1}}, {"k": 5}], +) +def test_a_bad_server_settings_value_is_an_internal_error(settings) -> None: + with pytest.raises(InternalError): + decode_schema_description(dumps({"server": {"settings": settings}})) diff --git a/python/tests/api/test_input.py b/python/tests/api/test_input.py index b006a850..ca85e04f 100644 --- a/python/tests/api/test_input.py +++ b/python/tests/api/test_input.py @@ -7,8 +7,18 @@ import pytest -from chtypes import UsageError -from chtypes._input import body_bytes, columns_json, settings_json, string_map_json, to_bytes +from chtypes import ColumnDefault, ParamEncoding, Posture, ServerProfile, UsageError, VerdictBasis +from chtypes._input import ( + body_bytes, + columns_json, + filter_options_json, + insert_options_json, + schema_options_json, + server_profile_json, + settings_json, + string_map_json, + to_bytes, +) def test_settings_are_a_json_object_of_string_values_written_verbatim() -> None: @@ -69,33 +79,134 @@ def test_text_inputs_are_utf8_and_bodies_never_accept_text() -> None: _LONE = "\udcff" # what surrogateescape makes of the byte 0xff +def test_a_non_utf8_value_travels_as_the_b64_object_form() -> None: + # ABI v2 json_object_byte_values: the bytes are exact; a UTF-8 value stays a string. + got = json.loads(settings_json({"k": _LONE, "ok": "é"}) or b"") + assert got == {"k": {"b64": "/w=="}, "ok": "é"} + assert json.loads(settings_json({"k": _LONE}, "UTC") or b"") == { + "k": {"b64": "/w=="}, + "session_timezone": "UTC", + } + # the zone is a settings value too + assert json.loads(settings_json(None, _LONE) or b"") == {"session_timezone": {"b64": "/w=="}} + assert json.loads(string_map_json({"p": _LONE + "a"}, "query parameters") or b"") == { + "p": {"b64": base64.b64encode(b"\xffa").decode()} + } + + @pytest.mark.parametrize( ("call", "words"), [ - (lambda: settings_json({"k": _LONE}), ("settings", 'the value for key "k"')), (lambda: settings_json({"k" + _LONE: "v"}), ("settings", 'the key "k\\udcff"')), - (lambda: settings_json({"k": _LONE}, "UTC"), ("settings", 'the value for key "k"')), - (lambda: settings_json(None, _LONE), ("the zone",)), - ( - lambda: string_map_json({"p": _LONE}, "query parameters"), - ("query parameters", 'the value for key "p"'), - ), ( lambda: string_map_json({_LONE: "v"}, "query parameters"), ("query parameters", 'the key "\\udcff"'), ), + # a surrogate that is not a surrogateescape byte has no bytes to carry + (lambda: settings_json({"k": "\ud800"}), ("settings", 'the value for key "k"')), + # a macro (name or value) and a server timezone have no object form + ( + lambda: server_profile_json(ServerProfile(macros={"m": _LONE})), + ("macros", 'the value for key "m"'), + ), + ( + lambda: server_profile_json(ServerProfile(macros={_LONE: "v"})), + ("macros", 'the key "\\udcff"'), + ), + (lambda: server_profile_json(ServerProfile(timezone=_LONE)), ("timezone",)), ], ) -def test_a_non_utf8_key_or_value_is_a_usage_error_naming_the_input_and_the_key(call, words) -> None: +def test_a_key_or_value_with_no_byte_form_is_a_usage_error_naming_the_input_and_the_key( + call, words +) -> None: with pytest.raises(UsageError) as info: call() message = info.value.message.decode("utf-8") - for word in (*words, "not valid UTF-8", "unhex('')"): + for word in (*words, "not valid UTF-8", "cannot carry these bytes"): assert word in message assert info.value.status == 3 and info.value.ch_code == 0 assert message.isascii() # the key is escaped, never raw +def test_server_profile_settings_take_the_b64_form_too() -> None: + doc = json.loads(server_profile_json(ServerProfile(settings={"k": _LONE, "j": "v"}))) + assert doc == {"settings": {"k": {"b64": "/w=="}, "j": "v"}} + + +def test_no_option_set_sends_no_document() -> None: + assert insert_options_json() is None + assert filter_options_json() is None + assert schema_options_json() is None + + +def test_insert_options_are_the_members_set_and_nothing_else() -> None: + raw = insert_options_json( + insertable_columns=["a", b"\xff"], + column_defaults=[ColumnDefault("a", "1"), ColumnDefault(b"\xfe", b"\xfd")], + verdict_basis=VerdictBasis.EXPORT, + defer_defaults="server", + ) + assert json.loads(raw or b"") == { + "insertable_columns": [{"name": "a"}, {"name_b64": "/w=="}], + "column_defaults": [ + {"name": "a", "value": "1"}, + {"name_b64": "/g==", "value_b64": "/Q=="}, + ], + "verdict_basis": "export", + "defer_defaults": "server", + } + + +def test_an_absent_list_is_left_out_and_a_present_empty_one_is_sent() -> None: + assert json.loads(insert_options_json(insertable_columns=[]) or b"") == { + "insertable_columns": [] + } + assert json.loads(insert_options_json(column_defaults=[]) or b"") == {"column_defaults": []} + assert insert_options_json(insertable_columns=None, column_defaults=None) is None + + +def test_an_unnamed_member_is_passed_through_as_json_text_verbatim() -> None: + raw = insert_options_json( + verdict_basis="input", insert_options_extra={"future": '{"a": [1, 2.50]}', "n": "7"} + ) + # the JSON text is spliced as given (no re-serialization), after the named members + assert raw == b'{"verdict_basis":"input","future":{"a": [1, 2.50]},"n":7}' + # a key set twice keeps the last value: the extra member wins + assert json.loads( + insert_options_json( + verdict_basis="input", insert_options_extra={"verdict_basis": '"export"'} + ) + or b"" + ) == {"verdict_basis": "export"} + with pytest.raises(UsageError) as info: + insert_options_json(insert_options_extra={"k": "{not json"}) + assert b"not valid JSON" in info.value.message + with pytest.raises(TypeError): + insert_options_json(insert_options_extra={"k": 1}) # type: ignore[dict-item] + + +def test_filter_and_schema_options() -> None: + assert json.loads(filter_options_json(param_encoding=ParamEncoding.RAW) or b"") == { + "param_encoding": "raw" + } + assert json.loads( + schema_options_json(posture=Posture.LOAD, schema_options_extra={"x": "true"}) or b"" + ) == {"posture": "load", "x": True} + + +@pytest.mark.parametrize("bad", [5, b"input", None.__class__]) +def test_an_enum_option_is_a_str_or_its_enum(bad) -> None: + with pytest.raises(TypeError): + insert_options_json(verdict_basis=bad) + + +def test_the_options_never_validate_a_member() -> None: + # The library refuses an unknown value, naming it; the binding sends it. + assert json.loads(insert_options_json(verdict_basis="nonsense") or b"") == { + "verdict_basis": "nonsense" + } + + def test_valid_non_ascii_serializes_byte_for_byte_as_before() -> None: # Measured before the change on the same inputs: json.dumps, ASCII escapes. assert string_map_json({"p": "é日本"}, "query parameters") == b'{"p":"\\u00e9\\u65e5\\u672c"}' diff --git a/python/tests/goldens_v1/data/stub-goldens.json b/python/tests/goldens_v1/data/stub-goldens.json index 43f46547..07810968 100644 --- a/python/tests/goldens_v1/data/stub-goldens.json +++ b/python/tests/goldens_v1/data/stub-goldens.json @@ -43,12 +43,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 8 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -120,12 +120,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 9 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", diff --git a/python/tests/goldens_v1/test_runner.py b/python/tests/goldens_v1/test_runner.py index 1959612b..4a7d531b 100644 --- a/python/tests/goldens_v1/test_runner.py +++ b/python/tests/goldens_v1/test_runner.py @@ -147,6 +147,7 @@ def decoded(call: Any) -> bool: _b64d(body["body_b64"]), _b64d(body["settings_b64"]), _b64d(body["columns_b64"]) if "columns_b64" in body else None, + None, ), None, ) @@ -162,6 +163,7 @@ def decoded(call: Any) -> bool: None, requested, body["doc_flags"], + None, ) if requested == -1: export = None @@ -174,6 +176,7 @@ def decoded(call: Any) -> bool: _b64d(f["expr_b64"]), _b64d(f["query_params_b64"]), _b64d(f["settings_b64"]), + None, ) except CallError as err: return failed("filter_create", err) @@ -183,6 +186,8 @@ def decoded(call: Any) -> bool: body["format"], _b64d(body["body_b64"]), _b64d(body["settings_b64"]), + _b64d(body["columns_b64"]) if "columns_b64" in body else None, + None, ), None, ) From 9e295332c5583859c059686cd50a9ace923f7164 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:27:20 -0400 Subject: [PATCH 12/24] rust: spec batch 2 hand-written surface (options documents, refusal and decline members, error reason, schema description) Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/bindings-v1.md | 2 +- rust/CHANGELOG.md | 9 + rust/src/decode.rs | 266 +++++++++++++++++++-- rust/src/error.rs | 21 +- rust/src/lib.rs | 7 +- rust/src/library.rs | 12 +- rust/src/options.rs | 371 ++++++++++++++++++++++++++++++ rust/src/result.rs | 72 +++++- rust/src/schema.rs | 249 +++++++++++++++++--- rust/tests/reader_rules.rs | 5 +- scripts/parity-surface-allow.json | 34 ++- 11 files changed, 965 insertions(+), 83 deletions(-) create mode 100644 rust/src/options.rs diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 597e19b9..a904ffdb 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -199,7 +199,7 @@ The two zones, spelled in each binding: - **A Go `string` is a byte string.** Go has no separate byte-string type, and a `string` holds any bytes, NUL included. The rule is only that it is built from copied bytes (`string(b)`), never through `C.GoString`, which stops at the first NUL. - **A `str` or `string` input is encoded as UTF-8**, because that is the only way a text value becomes bytes. That is not an assumption about the payload: a caller with non-UTF-8 bytes passes them as `bytes`, `Uint8Array` or `&[u8]`. A body never accepts the text type, as in v0. - **Rust keeps v0's `RawText`** (the bytes, `as_bytes()`, `as_str() -> Option<&str>` and a lossy `Display`) for every byte output, so it is the one byte-out type in that binding. -- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object with the language's stock encoder and passes each value verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. **A value whose bytes are not valid UTF-8 is the object form `{"b64": ""}`** (ABI v2, `json_object_byte_values`: a settings value, a query-parameter value, a `chs_set_defaults` default, and a value of a server profile's `settings`), written by the binding in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a JSON string. A key that is not valid UTF-8 cannot be carried (a JSON object has no byte form for a key, and a macro name or value and a zone have none either): the bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the profile) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different key. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. A per-call zone is a settings value, so a zone that is not valid UTF-8 is written in the object form too, and the library validates it. +- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object with the language's stock encoder and passes each value verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. **A value whose bytes are not valid UTF-8 is the object form `{"b64": ""}`** (ABI v2, `json_object_byte_values`: a settings value, a query-parameter value, a `chs_set_defaults` default, and a value of a server profile's `settings`), written by the binding in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a JSON string. A key that is not valid UTF-8 cannot be carried (a JSON object has no byte form for a key, and a macro name or value and a zone have none either): the bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the profile) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different key. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. What a schema description reports back is bytes, though: a server profile's settings value in the object form decodes to its bytes, so Rust's `SchemaServer::settings` holds `RawText` values. A per-call zone is a settings value, so a zone that is not valid UTF-8 is written in the object form too, and the library validates it. - **A zone name is text**, because ClickHouse's zone names are, and a name `DateLUT` cannot load is refused by the library whatever its bytes. It crosses the ABI as counted bytes (`chs_initialize`) or as a JSON string (`session_timezone`), UTF-8 encoded. - **A list of column names** (the `columns` option) is serialized as a JSON array of names under the name encoding of §5. diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 9805e13f..6f9a5668 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -24,9 +24,17 @@ The four bindings in this repository are released together and give one answer, - The v2 spec's decline catalog adds `server-dependent-function` (a column's DEFAULT, MATERIALIZED, ALIAS or EPHEMERAL expression, or a CHECK, that calls a function whose resolver needs a server is declined at `chs_schema_create`), `function-name-as-lambda` (a bare function name written where a higher-order function takes a lambda, such as `arrayExists(isNull, arr)`, is declined at `chs_filter_create` on 26.7, 26.8 and 26.9) and `skipped-key-in-squashed-block` (a batch whose answer depends on how the INSERT's squashing step groups its chunks is unsupported), and `call-setting-session-timezone` names only a per-call `session_timezone` the call cannot attach (55 reasons). - A `410 Gone` is a retired repository, and permanent (#571). Any request the fetch layer makes (a tag, an alias, a digest, a blob, `tags/list`, the referrers API or its fallback tag, after a redirect too) that answers `410` is now the new code `CHTYPES_SOURCE_RETIRED` (exit status 10), never retried and never sent to the next base, nor from the dev channel's alias to its tag. It was `CHTYPES_SOURCE_UNREACHABLE` (exit 3) with the registry's message dropped, and Go, Python and Rust moved on to the next base, so a mirror's 404 could hide the retirement. The error names the URL that answered and carries the registry's own message, with control and bidirectional-formatting characters removed and cut to 256 code points: Go `CodeSourceRetired` (`errors.Is(err, ErrSourceRetired)`), Python and TypeScript `SourceRetiredError`, Rust `Error::SourceRetired`. The rule is `docs/guides/fetch-v1.md` section 2, "A retired repository", and the `retired-410-*` and `list-tags-retired-410` conformance cases and one shared message table (`tests/fixtures/retired-message/cases.json`) hold all four bindings to it. - A 2.0.0-dev SDK prefers the newest build of its own fingerprint (#511). A version request resolves `--fp-` first, the alias the staging dev registry serves beside every dev tag, with the binding's own generated ABI fingerprint, and the tag itself only when every base answers the alias 404; any other failure of the alias request is that failure, with no fallback, and an alias whose index lacks the platform is `CHTYPES_ARTIFACT_UNPUBLISHED`. The dev channel's cache lookups (the installed-build lookup a registry open makes first, the offline answer, and the monotonic rule of a fetch) now see only records whose signed `abi_fingerprint` is the SDK's own, so a newer build of another fingerprint, installed by another dev SDK in the shared cache, is never returned or kept over this one's own, and is never removed or rewritten. So a dev build of a newer fingerprint no longer strands an older dev SDK until its consumers upgrade. Trust is unchanged (the signed predicate and the load checks decide), a listing never shows an alias, nothing records one, `list` and `where` are unchanged, and there is no option: both steps are the dev channel's alone, and the v1 contract and every production channel never take them. The rule is `docs/guides/fetch-v1.md` section 3, and the `dev-alias-*`, `resolve-installed-dev-alias-*` and `list-tags-dev-alias` conformance cases hold all four bindings to it. +- `CallError::reason` (ABI v2): the error's catalog id, read through `chs_error_reason` by the generated layer and copied into the public `CallError` (ASCII, empty if none), on all four call classes. A misuse the binding detects itself carries an empty one. +- Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): `RowOptions`, `RowsOptions` and `EvalOptions` gain `insertable_columns` (`Option>>`), `column_defaults` (`Option>`), `verdict_basis` (`Option`: `Input`, `Export`), `defer_defaults` (`Option`: `None`, `Server`) and `insert_options_extra`, which build `insert_options` for `Schema::row`, `Schema::rows`, `Schema::parse_block` and `Filter::rows`; `FilterOptions` gains `param_encoding` (`Option`: `Escaped`, `Raw`) and `filter_options_extra`; `CompileOptions` gains `posture` (`Option`: `Create`, `Load`) and `schema_options_extra`. Each `*_extra` is a list of `(key, JSON text)` passed through verbatim, so the library, which refuses an unknown key (rule r1), is the only judge; text that is not valid JSON is `Error::Usage` before any call. A document with no member set is not sent. `EvalOptions` gains `columns`, because `chs_filter_eval_body` takes the column list now. +- Spec batch 2, the decline and refusal members (ABI v2): `RowResult`, `BatchResult` and `FilterResult` carry `reason` (the catalog id, present where the code is -2, `CHS_CODE_DECLINED`), `refused_at` (an `Option`: `Call`, `BodyRead`, `Row`, `BodyStep`) and `refused_row`; `RowResult::verdict_reason`; `FilterRowError::reason`; `BatchResult::export_columns` and `BatchResult::export_declined_reason`. A document's `code` can be -2 for a row, batch, filter result or verdict this build declined. +- Spec batch 2, the schema description (ABI v2): `SchemaDescription::posture`, and `Column::default_support` (an `Option`), `Column::default_support_reason` and `Column::reads`, reserved members a build reports when it implements them. `Source::DefaultDeferred`, `Source::DefaultOverride` (stored) and `DeclinedTier::ServerRefuses` join the generated vocabularies, and `RefusalSite`, `Posture` and `DefaultSupport` are exported. +- Spec batch 2, a settings value that is not UTF-8 (ABI v2): `SchemaServer::settings` decodes the `{"b64": ...}` form back to its bytes. Its values are `RawText` now (see Changed). Rust takes `String` for every value it sends, so sending a non-UTF-8 value stays unreachable. ### Changed +- **Breaking (ABI v2 dev): `BatchResult::spans` is `Option>>`**, one entry per `rows` entry and index-aligned with it: `row_spans` carries `null` for a row the export did not carry (a skipped row, or a row a filter cut), which the entry reads as `None`. It was `Option>` of the exported rows only. +- **Breaking (ABI v2 dev): `AtMergeEntry::row` is `AtMergeEntry::part_row`**, the document's `part_row`. +- **Breaking (ABI v2 dev): `SchemaServer::settings` is `BTreeMap`** (was `BTreeMap`), because a server profile's setting value can be bytes that are not valid UTF-8. - `chtypes list` and the tag listing name the published lines only (two-part, numeric order), as Go's always did; `chtypes resolve ` prints the exact version a line resolves to (#613). - **Breaking: the public surface the parity gate carried as exceptions is resolved, before the v2 lock (#500).** A convenience exists in every binding or in none (`docs/reference/bindings-v1.md` §1, principle 2). Migration, old → new: - `Refusal.detail` → `Refusal.got`: what a one-sided refusal found (the `dlopen` error, what a malformed `build_info` lacks) is in `got`, with `want` `None`, as in the other bindings; the display still prints it. `Refusal` is `#[non_exhaustive]`, so build one only through the crate. @@ -71,6 +79,7 @@ The four bindings in this repository are released together and give one answer, ### Removed +- **Breaking (ABI v2 dev): `RowResult::unsupported_settings`**: the row document no longer carries `unsupported_settings`; the batch's and the filter result's lists stay. - `SchemaServer::filter_declined_settings` (a breaking change inside v2 dev, #588): the server member no longer carries the list. Read `SchemaDescription::filter_declined_settings`, whose `server`-layer entries are the profile's. ### Fixed diff --git a/rust/src/decode.rs b/rust/src/decode.rs index e7fcd15b..5e27cc63 100644 --- a/rust/src/decode.rs +++ b/rust/src/decode.rs @@ -26,8 +26,8 @@ use serde::de::{Deserialize, Deserializer, MapAccess, SeqAccess, Visitor}; use serde_json::{Map, Value as Json}; use crate::abi2::vocab_gen::{ - BYTES_SUFFIX, DeclinedLayer, DeclinedTier, DefaultKind, FilterOutcome, MergeReason, Outcome, - Reason, Source, VALUE_BYTES, Verdict, + BYTES_SUFFIX, DeclinedLayer, DeclinedTier, DefaultKind, DefaultSupport, FilterOutcome, + MergeReason, Outcome, Posture, Reason, RefusalSite, Source, VALUE_BYTES, Verdict, }; use crate::error::{Error, Result}; use crate::raw::RawText; @@ -273,6 +273,46 @@ impl<'a> Obj<'a> { .map(Some) } + /// An object whose values are byte values (`json_object_byte_values`): a + /// JSON string, or the object `{"b64": ""}` for bytes that + /// are not valid UTF-8. `None` when absent, `Some`, even of an empty map, + /// when present. + fn byte_value_map(&self, key: &str) -> Result>> { + let Some(o) = self.object(key)? else { + return Ok(None); + }; + let mut out = BTreeMap::new(); + for (k, v) in o.map { + let bytes = match v { + Json::String(s) => s.clone().into_bytes(), + Json::Object(m) => match (m.len(), m.get("b64")) { + (1, Some(Json::String(b))) => base64::engine::general_purpose::STANDARD + .decode(b) + .map_err(|_| { + Error::internal(format!( + "the {} document: {}.{k}.b64 is not standard base64", + o.doc, o.at + )) + })?, + _ => { + return Err(Error::internal(format!( + "the {} document: {}.{k} is not a string or a {{\"b64\": ...}} object", + o.doc, o.at + ))); + } + }, + _ => { + return Err(Error::internal(format!( + "the {} document: {}.{k} is not a string or a {{\"b64\": ...}} object", + o.doc, o.at + ))); + } + }; + out.insert(k.clone(), RawText::from(bytes)); + } + Ok(Some(out)) + } + fn strings(&self, key: &str) -> Result> { self.array(key)? .iter() @@ -369,6 +409,24 @@ fn span(o: &Obj<'_>, key: &str) -> Result> { } } +/// `row_spans`: one entry per row, `null` for a row that was not exported. +fn nullable_spans(o: &Obj<'_>, key: &str) -> Result>> { + o.array(key)? + .iter() + .enumerate() + .map(|(i, v)| { + if v.is_null() { + return Ok(None); + } + let s = Obj::of(o.doc, format!("{}[{i}]", o.child(key)), v)?; + Ok(Some(Span { + off: s.uint64("off")?, + len: s.uint64("len")?, + })) + }) + .collect() +} + fn spans(o: &Obj<'_>, key: &str) -> Result> { o.array(key)? .iter() @@ -462,14 +520,19 @@ fn row_of(o: &Obj<'_>) -> Result { outcome: Outcome::from_wire(&o.string("outcome")?), err_code: o.int32("code")?, err_msg: o.bytes("err")?, + reason: o.opt_string("reason")?, + refused_at: o + .opt_string("refused_at")? + .map(|v| RefusalSite::from_wire(&v)), + refused_row: o.opt_uint64("refused_row")?, columns, values, transformed: transforms(o, "transformed")?, unknown_fields, - unsupported_settings: o.name_objects("unsupported_settings")?, computed, verdict: o.opt_string("verdict")?.map(|v| Verdict::from_wire(&v)), verdict_code: o.int32("verdict_code")?, + verdict_reason: o.opt_string("verdict_reason")?, verdict_err: o.bytes("verdict_err")?, partition_id: o.bytes_opt("partition_id")?.map(RawText::from), input_span: span(o, "input_span")?, @@ -517,7 +580,11 @@ pub(crate) fn batch(bytes: &[u8], payload: Option>) -> Result None, - Some(_) => Some(spans(&o, "row_spans")?), + Some(_) => Some(nullable_spans(&o, "row_spans")?), + }; + let export_columns = match o.opt_array("export_columns")? { + None => None, + Some(_) => Some(o.name_objects("export_columns")?), }; let framing = match o.object("framing")? { None => None, @@ -545,6 +612,11 @@ pub(crate) fn batch(bytes: &[u8], payload: Option>) -> Result>) -> Result) -> Result { ), }; Ok(AtMergeEntry { - row: o.uint64("row")?, + part_row: o.uint64("part_row")?, reason: MergeReason::from_wire(&o.string("reason")?), column: o.bytes_opt("column")?.map(RawText::from), stored: o.bytes_opt("stored")?.map(RawText::from), @@ -622,6 +696,7 @@ pub(crate) fn filter_result(bytes: &[u8]) -> Result { errors.push(FilterRowError { row: e.uint64("row")?, code: e.int32("code")?, + reason: e.opt_string("reason")?, msg: e.bytes("err")?, }); } @@ -629,6 +704,11 @@ pub(crate) fn filter_result(bytes: &[u8]) -> Result { outcome: FilterOutcome::from_wire(&o.string("outcome")?), err_code: o.int32("code")?, err_msg: o.bytes("err")?, + reason: o.opt_string("reason")?, + refused_at: o + .opt_string("refused_at")? + .map(|v| RefusalSite::from_wire(&v)), + refused_row: o.opt_uint64("refused_row")?, rows_read: o.uint64("rows_read")?, unsupported_settings: o.name_objects("unsupported_settings")?, verdicts, @@ -668,6 +748,15 @@ pub(crate) fn schema_description(bytes: &[u8]) -> Result { r#type: c.bytes("type")?, default_kind, default_expr: c.bytes("default_expression")?, + // Reserved members: absent while the library does not write them. + default_support: c + .opt_string("default_support")? + .map(|v| DefaultSupport::from_wire(&v)), + default_support_reason: c.opt_string("default_support_reason")?, + reads: match c.opt_array("reads")? { + None => None, + Some(_) => Some(c.name_objects("reads")?), + }, }); } // Both are absent on a schema compiled without a server, so that document @@ -676,7 +765,7 @@ pub(crate) fn schema_description(bytes: &[u8]) -> Result { None => None, Some(sm) => Some(SchemaServer { timezone: sm.string("timezone")?, - settings: sm.string_map("settings")?.unwrap_or_default(), + settings: sm.byte_value_map("settings")?.unwrap_or_default(), macros: sm.string_map("macros")?, }), }; @@ -689,6 +778,8 @@ pub(crate) fn schema_description(bytes: &[u8]) -> Result { }; Ok(SchemaDescription { columns, + // Present only for `load`; absent is create. + posture: o.opt_string("posture")?.map(|v| Posture::from_wire(&v)), // Over every layer known at schema compile, with or without a server. filter_declined_settings: declined_settings(&o)?, server, @@ -1026,7 +1117,7 @@ mod tests { assert_eq!(b.rows_read, 9_007_199_254_740_993); assert_eq!(b.partition_count, None); assert_eq!(b.payload.as_deref(), Some(&b"{\"a\":1}\n"[..])); - assert_eq!(b.spans, Some(vec![Span { off: 0, len: 5 }])); + assert_eq!(b.spans, Some(vec![Some(Span { off: 0, len: 5 })])); assert_eq!( b.rows[0].partition_id.as_ref().map(RawText::as_bytes), Some(&b"202601"[..]) @@ -1095,7 +1186,11 @@ mod tests { .unwrap(); let s = d.server.expect("server"); assert_eq!(s.timezone, "Asia/Tokyo"); - assert_eq!(s.settings, map(&[("max_threads", "8")])); + assert_eq!( + s.settings.get("max_threads").map(RawText::as_bytes), + Some(&b"8"[..]) + ); + assert_eq!(s.settings.len(), 1); assert_eq!(s.macros, Some(BTreeMap::new())); // Macros and a Replicated engine's resolved path, a name in its _b64 @@ -1173,13 +1268,13 @@ mod tests { "outcome": "accepted", "unsupported_settings": [{"name": "session_timezone"}, {"name_b64": "/w=="}], "at_merge": [ - {"row": 0, "reason": "ttl_delete"}, - {"row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, - {"row": 2, "reason": "ttl_column_reset", "column": "t"}, - {"row": 3, "reason": "ttl_column_reset", "column_b64": "/wA=", "stored_b64": "/w=="}, - {"row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, - {"row": 5, "reason": "a_reason_nobody_listed", "x_future": {"a": [1]}, "x_future_b64": "/w=="}, - {"row": 6, "reason": "ttl_column_reset", "column": "e", "stored": "", "input_rows": []} + {"part_row": 0, "reason": "ttl_delete"}, + {"part_row": 1, "reason": "ttl_column_reset", "column": "c", "stored": "0"}, + {"part_row": 2, "reason": "ttl_column_reset", "column": "t"}, + {"part_row": 3, "reason": "ttl_column_reset", "column_b64": "/wA=", "stored_b64": "/w=="}, + {"part_row": 4, "reason": "ttl_delete", "input_rows": [4, 7]}, + {"part_row": 5, "reason": "a_reason_nobody_listed", "x_future": {"a": [1]}, "x_future_b64": "/w=="}, + {"part_row": 6, "reason": "ttl_column_reset", "column": "e", "stored": "", "input_rows": []} ] }"#; @@ -1192,7 +1287,7 @@ mod tests { .iter() .map(|e| { ( - e.row, + e.part_row, e.reason.clone(), e.reason.is_known(), bytes(&e.column), @@ -1291,11 +1386,11 @@ mod tests { #[test] fn an_at_merge_that_breaks_its_schema_is_internal() { let docs: [&[u8]; 7] = [ - br#"{"at_merge":[{"row":0,"reason":"ttl_column_reset","column":"c","column_b64":"Yw=="}]}"#, - br#"{"at_merge":[{"row":0,"reason":"ttl_column_reset","column":"c","stored_b64":"!!"}]}"#, - br#"{"at_merge":[{"row":0,"reason":"ttl_delete","input_rows":["x"]}]}"#, - br#"{"at_merge":[{"row":0,"reason":"ttl_delete","input_rows":[-1]}]}"#, - br#"{"at_merge":[{"row":0,"reason":7}]}"#, + br#"{"at_merge":[{"part_row":0,"reason":"ttl_column_reset","column":"c","column_b64":"Yw=="}]}"#, + br#"{"at_merge":[{"part_row":0,"reason":"ttl_column_reset","column":"c","stored_b64":"!!"}]}"#, + br#"{"at_merge":[{"part_row":0,"reason":"ttl_delete","input_rows":["x"]}]}"#, + br#"{"at_merge":[{"part_row":0,"reason":"ttl_delete","input_rows":[-1]}]}"#, + br#"{"at_merge":[{"part_row":0,"reason":7}]}"#, br#"{"at_merge":{"row":0}}"#, br#"{"unsupported_settings":["session_timezone"]}"#, ]; @@ -1306,19 +1401,142 @@ mod tests { #[test] fn unsupported_settings_are_name_objects_and_surface_as_bytes() { + // The row-level member is REMOVED (ABI v2 batch 2): a row document that + // still carries one decodes on, as an unknown member (rule r2). let r = row( br#"{"outcome":"accepted","unsupported_settings":[{"name":"a_setting"},{"name_b64":"AP8="}]}"#, ) .unwrap(); - assert_eq!(r.unsupported_settings[0].as_bytes(), b"a_setting"); - assert_eq!(r.unsupported_settings[1].as_bytes(), &[0x00, 0xff]); + assert_eq!(r.outcome, Outcome::Accepted); let f = filter_result( br#"{"outcome":"ok","verdicts":"t","unsupported_settings":[{"name":"x"}]}"#, ) .unwrap(); assert_eq!(f.unsupported_settings[0].as_bytes(), b"x"); // A bare string is not a name object. - assert!(row(br#"{"outcome":"accepted","unsupported_settings":["x"]}"#).is_err()); + assert!(filter_result(br#"{"outcome":"ok","unsupported_settings":["x"]}"#).is_err()); + } + + /// ABI v2 batch 2: the refusal members, the decline `reason`s, and the + /// export's column list and decline reason, each read off the document. + #[test] + fn a_refusal_carries_its_site_row_and_reason() { + let r = row( + br#"{"outcome":"unsupported","code":-2,"reason":"a-catalog-id","refused_at":"body_read", + "verdict":"d","verdict_code":-2,"verdict_reason":"another-id"}"#, + ) + .unwrap(); + assert_eq!(r.err_code, -2); + assert_eq!(r.reason.as_deref(), Some("a-catalog-id")); + assert_eq!(r.refused_at, Some(RefusalSite::BodyRead)); + assert_eq!(r.refused_row, None); + assert_eq!(r.verdict_code, -2); + assert_eq!(r.verdict_reason.as_deref(), Some("another-id")); + // Absent is None, never an empty string or a default site. + let r = row(br#"{"outcome":"accepted","code":0}"#).unwrap(); + assert_eq!( + (r.reason, r.refused_at, r.refused_row, r.verdict_reason), + (None, None, None, None) + ); + + let b = batch( + br#"{"outcome":"rejected","code":27,"refused_at":"row","refused_row":3,"rows_read":4, + "export_columns":[{"name":"a"},{"name_b64":"AP8="}], + "export_declined":"","export_declined_reason":null, + "row_spans":[{"off":0,"len":2},null,{"off":2,"len":2}]}"#, + Some(b"xxxx".to_vec()), + ) + .unwrap(); + assert_eq!(b.refused_at, Some(RefusalSite::Row)); + assert_eq!(b.refused_row, Some(3)); + assert_eq!(b.reason, None); + let cols: Vec<&[u8]> = b + .export_columns + .as_ref() + .unwrap() + .iter() + .map(RawText::as_bytes) + .collect(); + assert_eq!(cols, vec![&b"a"[..], &[0x00, 0xff][..]]); + // One entry per row, `null` for a row that was not exported. + assert_eq!( + b.spans, + Some(vec![ + Some(Span { off: 0, len: 2 }), + None, + Some(Span { off: 2, len: 2 }) + ]) + ); + + let b = batch( + br#"{"outcome":"unsupported","code":-2,"reason":"r-id","refused_at":"call", + "export_declined":"nope","export_declined_reason":"export-id"}"#, + None, + ) + .unwrap(); + assert_eq!(b.reason.as_deref(), Some("r-id")); + assert_eq!(b.refused_at, Some(RefusalSite::Call)); + assert_eq!(b.export_declined_reason.as_deref(), Some("export-id")); + assert_eq!(b.export_columns, None); + + let f = filter_result( + br#"{"outcome":"unsupported","code":-2,"reason":"f-id","refused_at":"body_step", + "verdicts":"d","errors":[{"row":0,"code":-2,"reason":"e-id","err":"m"}]}"#, + ) + .unwrap(); + assert_eq!(f.reason.as_deref(), Some("f-id")); + assert_eq!(f.refused_at, Some(RefusalSite::BodyStep)); + assert_eq!(f.errors[0].reason.as_deref(), Some("e-id")); + // A site the description does not list is its Unknown (rule r3). + let f = filter_result(br#"{"outcome":"rejected","refused_at":"somewhere_new"}"#).unwrap(); + assert_eq!( + f.refused_at, + Some(RefusalSite::Unknown("somewhere_new".into())) + ); + } + + #[test] + fn a_schema_description_carries_posture_support_reads_and_b64_server_settings() { + let d = schema_description( + br#"{"posture":"load", + "columns":[ + {"name":"a","type":"UInt8","default_kind":"","default_expression":""}, + {"name":"b","type":"UInt8","default_kind":"DEFAULT","default_expression":"a", + "default_support":"deferred","default_support_reason":"why-id", + "reads":[{"name":"a"},{"name_b64":"AP8="}]}], + "server":{"timezone":"UTC","settings":{"s":"v","raw":{"b64":"AP8="}}}}"#, + ) + .unwrap(); + assert_eq!(d.posture, Some(Posture::Load)); + assert_eq!(d.columns[0].default_support, None); + assert_eq!(d.columns[0].reads, None); + assert_eq!(d.columns[1].default_support, Some(DefaultSupport::Deferred)); + assert_eq!( + d.columns[1].default_support_reason.as_deref(), + Some("why-id") + ); + let reads: Vec<&[u8]> = d.columns[1] + .reads + .as_ref() + .unwrap() + .iter() + .map(RawText::as_bytes) + .collect(); + assert_eq!(reads, vec![&b"a"[..], &[0x00, 0xff][..]]); + let s = d.server.unwrap().settings; + assert_eq!(s["s"].as_bytes(), b"v"); + assert_eq!(s["raw"].as_bytes(), &[0x00, 0xff]); + // Absent posture is create, as None. + let d = schema_description(br#"{"columns":[]}"#).unwrap(); + assert_eq!(d.posture, None); + // An object that is not the b64 form breaks the document's schema. + for bad in [ + br#"{"columns":[],"server":{"settings":{"x":{"b64":"!!"}}}}"#.as_slice(), + br#"{"columns":[],"server":{"settings":{"x":{"other":"AP8="}}}}"#.as_slice(), + br#"{"columns":[],"server":{"settings":{"x":7}}}"#.as_slice(), + ] { + internal(schema_description(bad).unwrap_err()); + } } #[test] diff --git a/rust/src/error.rs b/rust/src/error.rs index 4697709d..02aa0cff 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -1,6 +1,6 @@ //! The error classes (`docs/reference/bindings-v1.md` §4). //! -//! Four **call** classes carry the five fields of the library's own error +//! Four **call** classes carry the six fields of the library's own error //! ([`CallError`]); two **artifact** classes carry a loader [`Refusal`]; the //! fetch layer's own codes are one variant each. They are one family: a caller //! matching [`Error::ArtifactCorrupt`] sees the fetch layer's corruption and @@ -46,10 +46,11 @@ impl fmt::Display for CacheFault { } } -/// The five fields of one `chs_error`, verbatim, and nothing synthesized. +/// The six fields of one `chs_error`, verbatim, and nothing synthesized. /// /// A misuse the binding detects itself carries the same shape: `status` -/// `CHS_INVALID_ARGUMENT`, `ch_code` 0, an empty `ch_name` and `column`, and a +/// `CHS_INVALID_ARGUMENT`, `ch_code` 0, an empty `ch_name`, `column` and +/// `reason`, and a /// message naming the misuse, so a handler sees one shape whichever side /// caught it. #[derive(Debug, Clone, PartialEq, Eq)] @@ -65,6 +66,8 @@ pub struct CallError { pub message: RawText, /// The column concerned, empty if none. Bytes. pub column: RawText, + /// The catalog id of the error (`chs_error_reason`). ASCII; empty if none. + pub reason: String, } impl CallError { @@ -86,6 +89,7 @@ impl CallError { ch_name: String::new(), message: RawText::from(message.into()), column: RawText::default(), + reason: String::new(), } } } @@ -98,6 +102,7 @@ impl From for CallError { ch_name: e.ch_name, message: RawText::from(e.message), column: RawText::from(e.column), + reason: e.reason, } } } @@ -183,7 +188,7 @@ impl Error { matches!(self, Error::Unsupported(_)) } - /// The five fields of the library's error, from any of the four call + /// The six fields of the library's error, from any of the four call /// classes; `None` for an artifact or fetch error. pub fn call_error(&self) -> Option<&CallError> { match self { @@ -326,12 +331,12 @@ mod tests { ch_name: "CANNOT_PARSE_INPUT_ASSERTION_FAILED".to_string(), message: b"boom".to_vec(), column: b"c".to_vec(), - reason: String::new(), + reason: "cannot-parse-input".to_string(), } } #[test] - fn each_status_maps_to_its_own_class_and_the_five_fields_survive() { + fn each_status_maps_to_its_own_class_and_the_six_fields_survive() { assert!(matches!( Error::from_call(raw(status::REJECTED)), Error::Schema(_) @@ -355,6 +360,10 @@ mod tests { assert_eq!(c.ch_name, "CANNOT_PARSE_INPUT_ASSERTION_FAILED"); assert_eq!(c.message.as_bytes(), b"boom"); assert_eq!(c.column.as_bytes(), b"c"); + // The library's catalog id, copied from the generated RawCallError. + assert_eq!(c.reason, "cannot-parse-input"); + // A misuse this binding detects itself carries no reason. + assert_eq!(CallError::usage("x").reason, ""); } #[test] diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 71652b94..3a85184c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -93,6 +93,7 @@ mod abi2; mod decode; mod error; mod library; +mod options; // The fetch layer's own documentation is its module docs; its record types are // re-exported below, undocumented field by field. #[allow(missing_docs)] @@ -117,8 +118,9 @@ mod server; mod setup; pub use abi2::vocab_gen::{ - DeclinedLayer, DeclinedTier, DefaultKind, DiscoverQueryParam, DocFlags, FilterOutcome, Format, - MergeReason, Outcome, Reason, Source, Status, Verdict, reason, source, status, + DeclinedLayer, DeclinedTier, DefaultKind, DefaultSupport, DiscoverQueryParam, DocFlags, + FilterOutcome, Format, MergeReason, Outcome, Posture, Reason, RefusalSite, Source, Status, + Verdict, reason, source, status, }; pub use error::{CacheFault, CallError, Error, Refusal, Result}; pub use library::Library; @@ -127,6 +129,7 @@ pub use library::Library; /// (`docs/reference/bindings-v1.md` §4, the codes). pub use ocifetch::constants::code; pub use ocifetch::ensure::Resolved; +pub use options::{ColumnDefault, DeferDefaults, ParamEncoding, VerdictBasis}; pub use raw::RawText; pub use registry::{FetchOptions, Registry, RegistryOptions, cache_root, search_dirs}; pub use result::{ diff --git a/rust/src/library.rs b/rust/src/library.rs index 55d0cfb8..3869d199 100644 --- a/rust/src/library.rs +++ b/rust/src/library.rs @@ -27,6 +27,7 @@ use crate::abi2::loader::{self, LoadError, LoadInput, Refusal}; use crate::decode; use crate::error::{Error, Result}; use crate::ocifetch::ensure::Resolved; +use crate::options; use crate::raw::RawText; use crate::result::{BuildInfo, Discovery, ErrorCodeTable}; use crate::schema::{CompileOptions, Schema, settings_object}; @@ -296,11 +297,14 @@ impl Library { ) -> Result { let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; // No server (NULL) puts the schema on the image's own server, exactly as - // before the server profile existed. No schema option is defined yet, - // so the options document is empty (length 0). + // before the server profile existed. The schema options document is + // empty (length 0) unless a member was set. let server = options.server.as_ref().map(Server::handle); - let handle: SchemaHandle = - self.call(|api| api.schema_create(server, create_table.as_ref(), &settings, &[]))?; + let schema_options = + options::schema_options(options.posture.as_ref(), &options.schema_options_extra)?; + let handle: SchemaHandle = self.call(|api| { + api.schema_create(server, create_table.as_ref(), &settings, &schema_options) + })?; Ok(Schema::new(Arc::clone(self), handle)) } diff --git a/rust/src/options.rs b/rust/src/options.rs new file mode 100644 index 00000000..90feef61 --- /dev/null +++ b/rust/src/options.rs @@ -0,0 +1,371 @@ +//! The options documents (`docs/reference/bindings-v1.md` §2, "The options +//! documents"; rule r1 of `spec/abi-v2/docs.md`). +//! +//! Three calls take one: `insert_options` on the four calls that model an +//! INSERT of a body, `filter_options` on `chs_filter_create` and +//! `schema_options` on `chs_schema_create`. Each member is its own field of +//! the call's options struct, and the binding builds the document from the +//! members the caller set. It validates no member and no value: the library +//! refuses a key it does not know, or a value that is not one of the member's, +//! with `CHS_INVALID_ARGUMENT`, naming it. A member the binding has no name for +//! goes through the struct's `*_extra` list as JSON text under its key and is +//! passed through verbatim; a key set twice keeps the last value (an extra +//! after the named member), and text that is not valid JSON is +//! [`Error::Usage`] before any call. A document with no member set is not sent +//! (length 0, which the library reads as `{}`). + +use std::collections::BTreeMap; + +use serde::Serialize; +use serde::de::IgnoredAny; +use serde_json::{Map, Value as Json}; + +use crate::abi2::vocab_gen::Posture; +use crate::error::{Error, Result}; + +/// `insert_options`' `verdict_basis`: what row a filter verdict answers for. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum VerdictBasis { + /// The default: the verdict answers for the input row. + Input, + /// The verdict answers for the row the export would store. A build that + /// does not implement it declines it. + Export, +} + +impl VerdictBasis { + /// The wire spelling. + pub fn as_str(self) -> &'static str { + match self { + VerdictBasis::Input => "input", + VerdictBasis::Export => "export", + } + } +} + +/// `insert_options`' `defer_defaults`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DeferDefaults { + /// The default: no DEFAULT is deferred. + None, + /// Defer to the server each DEFAULT the build would otherwise decline. A + /// build that does not implement it declines it. + Server, +} + +impl DeferDefaults { + /// The wire spelling. + pub fn as_str(self) -> &'static str { + match self { + DeferDefaults::None => "none", + DeferDefaults::Server => "server", + } + } +} + +/// `filter_options`' `param_encoding`: how a query parameter's value is read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ParamEncoding { + /// The default: ClickHouse's escaped-text reader. + Escaped, + /// The bytes are the value. A build that does not implement it declines + /// it. + Raw, +} + +impl ParamEncoding { + /// The wire spelling. + pub fn as_str(self) -> &'static str { + match self { + ParamEncoding::Escaped => "escaped", + ParamEncoding::Raw => "raw", + } + } +} + +/// One entry of `insert_options`' `column_defaults`: the value the call uses +/// in a cell the server would fill from the column's DEFAULT. Both fields are +/// byte strings: one that is not valid UTF-8 is sent in its `_b64` form. +#[derive(Debug, Clone, Default, PartialEq, Eq, Hash)] +pub struct ColumnDefault { + /// The column's name, as bytes. + pub name: Vec, + /// The value, as bytes. + pub value: Vec, +} + +/// One byte string as the object member pair it travels as: `key` for valid +/// UTF-8, `key_b64` (standard base64, padded) for any other bytes. +fn put_bytes(o: &mut Map, key: &str, bytes: &[u8]) { + use base64::Engine as _; + match std::str::from_utf8(bytes) { + Ok(s) => o.insert(key.to_string(), Json::String(s.to_string())), + Err(_) => o.insert( + format!("{key}_b64"), + Json::String(base64::engine::general_purpose::STANDARD.encode(bytes)), + ), + }; +} + +/// A name as the name object the ABI takes: `{"name"}` or `{"name_b64"}`. +pub(crate) fn name_object(name: &[u8]) -> Json { + let mut o = Map::new(); + put_bytes(&mut o, "name", name); + Json::Object(o) +} + +/// The members of an options document under construction: key to JSON text. +#[derive(Default)] +struct Doc { + members: BTreeMap>, +} + +impl Doc { + fn set(&mut self, key: &str, value: &impl Serialize) -> Result<()> { + let text = serde_json::to_vec(value) + .map_err(|e| Error::internal(format!("the option {key:?} does not encode: {e}")))?; + self.members.insert(key.to_string(), text); + Ok(()) + } + + /// A member the binding has no name for: its JSON text, passed through as + /// given. A later one replaces an earlier one under the same key. + fn extras(&mut self, what: &str, extra: &[(String, String)]) -> Result<()> { + for (key, text) in extra { + if serde_json::from_str::(text).is_err() { + return Err(Error::usage(format!( + "{what}: the value for key {key:?} is not valid JSON" + ))); + } + self.members.insert(key.clone(), text.clone().into_bytes()); + } + Ok(()) + } + + /// The document, or empty when no member was set. + fn finish(self) -> Result> { + if self.members.is_empty() { + return Ok(Vec::new()); + } + let mut out = vec![b'{']; + for (i, (key, text)) in self.members.iter().enumerate() { + if i > 0 { + out.push(b','); + } + let k = serde_json::to_vec(key) + .map_err(|e| Error::internal(format!("an option key does not encode: {e}")))?; + out.extend_from_slice(&k); + out.push(b':'); + out.extend_from_slice(text); + } + out.push(b'}'); + Ok(out) + } +} + +/// The members of `insert_options` a call's options struct carries. +pub(crate) struct InsertMembers<'a> { + pub(crate) insertable_columns: Option<&'a [Vec]>, + pub(crate) column_defaults: Option<&'a [ColumnDefault]>, + pub(crate) verdict_basis: Option, + pub(crate) defer_defaults: Option, + pub(crate) extra: &'a [(String, String)], +} + +impl InsertMembers<'_> { + /// The `insert_options` document (empty for none). + pub(crate) fn document(&self) -> Result> { + let mut d = Doc::default(); + if let Some(names) = self.insertable_columns { + let list: Vec = names.iter().map(|n| name_object(n)).collect(); + d.set("insertable_columns", &list)?; + } + if let Some(defaults) = self.column_defaults { + let list: Vec = defaults + .iter() + .map(|c| { + let mut o = Map::new(); + put_bytes(&mut o, "name", &c.name); + put_bytes(&mut o, "value", &c.value); + Json::Object(o) + }) + .collect(); + d.set("column_defaults", &list)?; + } + if let Some(b) = self.verdict_basis { + d.set("verdict_basis", &b.as_str())?; + } + if let Some(b) = self.defer_defaults { + d.set("defer_defaults", &b.as_str())?; + } + d.extras("insert options", self.extra)?; + d.finish() + } +} + +/// The `filter_options` document (empty for none). +pub(crate) fn filter_options( + param_encoding: Option, + extra: &[(String, String)], +) -> Result> { + let mut d = Doc::default(); + if let Some(e) = param_encoding { + d.set("param_encoding", &e.as_str())?; + } + d.extras("filter options", extra)?; + d.finish() +} + +/// The `schema_options` document (empty for none). +pub(crate) fn schema_options( + posture: Option<&Posture>, + extra: &[(String, String)], +) -> Result> { + let mut d = Doc::default(); + if let Some(p) = posture { + d.set("posture", &p.as_str())?; + } + d.extras("schema options", extra)?; + d.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn members<'a>() -> InsertMembers<'a> { + InsertMembers { + insertable_columns: None, + column_defaults: None, + verdict_basis: None, + defer_defaults: None, + extra: &[], + } + } + + fn text(v: Vec) -> String { + String::from_utf8(v).unwrap() + } + + #[test] + fn no_member_set_is_no_document() { + assert!(members().document().unwrap().is_empty()); + assert!(filter_options(None, &[]).unwrap().is_empty()); + assert!(schema_options(None, &[]).unwrap().is_empty()); + } + + #[test] + fn every_named_member_is_written_under_its_key() { + let names = vec![b"a".to_vec(), vec![0x00, 0xff]]; + let defaults = vec![ + ColumnDefault { + name: b"c".to_vec(), + value: b"1".to_vec(), + }, + ColumnDefault { + name: vec![0xff], + value: vec![0xfe], + }, + ]; + let doc = InsertMembers { + insertable_columns: Some(&names), + column_defaults: Some(&defaults), + verdict_basis: Some(VerdictBasis::Export), + defer_defaults: Some(DeferDefaults::Server), + extra: &[], + } + .document() + .unwrap(); + assert_eq!( + text(doc), + concat!( + r#"{"column_defaults":[{"name":"c","value":"1"},{"name_b64":"/w==","value_b64":"/g=="}],"#, + r#""defer_defaults":"server","#, + r#""insertable_columns":[{"name":"a"},{"name_b64":"AP8="}],"#, + r#""verdict_basis":"export"}"# + ) + ); + } + + #[test] + fn an_empty_list_is_sent_as_an_empty_list_and_none_is_left_out() { + let none: Vec> = Vec::new(); + let doc = InsertMembers { + insertable_columns: Some(&none), + ..members() + } + .document() + .unwrap(); + assert_eq!(text(doc), r#"{"insertable_columns":[]}"#); + } + + #[test] + fn an_extra_is_passed_through_verbatim_and_the_last_value_of_a_key_wins() { + let extra = vec![ + ( + "x_future".to_string(), + r#"{"a": [1, 2.50, 1e3]}"#.to_string(), + ), + ("verdict_basis".to_string(), r#""export""#.to_string()), + ("x_future".to_string(), "7".to_string()), + ]; + let doc = InsertMembers { + verdict_basis: Some(VerdictBasis::Input), + extra: &extra, + ..members() + } + .document() + .unwrap(); + // The extra after the named member replaces it; the repeated extra + // keeps its last value; no value is re-encoded. + assert_eq!(text(doc), r#"{"verdict_basis":"export","x_future":7}"#); + let doc = filter_options( + Some(ParamEncoding::Raw), + &[("y".to_string(), r#"{"a": [1, 2.50, 1e3]}"#.to_string())], + ) + .unwrap(); + assert_eq!( + text(doc), + r#"{"param_encoding":"raw","y":{"a": [1, 2.50, 1e3]}}"# + ); + } + + #[test] + fn text_that_is_not_json_is_misuse_before_any_call() { + for bad in ["", "{", "1 2", "nope", "'x'"] { + let extra = vec![("k".to_string(), bad.to_string())]; + let err = InsertMembers { + extra: &extra, + ..members() + } + .document() + .unwrap_err(); + assert!(matches!(err, Error::Usage(_)), "{bad:?}: {err:?}"); + assert!(matches!( + filter_options(None, &extra).unwrap_err(), + Error::Usage(_) + )); + assert!(matches!( + schema_options(None, &extra).unwrap_err(), + Error::Usage(_) + )); + } + } + + #[test] + fn posture_and_param_encoding_use_the_wire_spelling_and_keep_an_unknown() { + assert_eq!( + text(schema_options(Some(&Posture::Load), &[]).unwrap()), + r#"{"posture":"load"}"# + ); + // An unlisted posture is passed through as given: the library refuses it. + assert_eq!( + text(schema_options(Some(&Posture::Unknown("later".into())), &[]).unwrap()), + r#"{"posture":"later"}"# + ); + assert_eq!( + text(filter_options(Some(ParamEncoding::Escaped), &[]).unwrap()), + r#"{"param_encoding":"escaped"}"# + ); + } +} diff --git a/rust/src/result.rs b/rust/src/result.rs index c3debb32..db8189e4 100644 --- a/rust/src/result.rs +++ b/rust/src/result.rs @@ -20,8 +20,8 @@ use std::collections::BTreeMap; use crate::abi2::vocab_gen::{ - DeclinedLayer, DeclinedTier, DefaultKind, FilterOutcome, MergeReason, Outcome, Reason, Source, - Verdict, + DeclinedLayer, DeclinedTier, DefaultKind, DefaultSupport, FilterOutcome, MergeReason, Outcome, + Posture, Reason, RefusalSite, Source, Verdict, }; use crate::raw::RawText; @@ -117,8 +117,8 @@ pub struct EngineCell { #[non_exhaustive] pub struct AtMergeEntry { /// Indexes [`BatchResult::engine_rows`]: the part's row, not the input - /// body's. - pub row: u64, + /// body's (`part_row`). + pub part_row: u64, /// A `merge_reason` value. One the description does not list is /// [`MergeReason::Unknown`] (rule r3). pub reason: MergeReason, @@ -145,6 +145,16 @@ pub struct RowResult { pub err_code: i32, /// ClickHouse's message for this row (`err`). pub err_msg: RawText, + /// The catalog id of a declined row (`reason`), present exactly where + /// [`RowResult::err_code`] is `CHS_CODE_DECLINED` (-2). ASCII. + pub reason: Option, + /// Where a single-row result's refusal sits (`refused_at`). One the + /// description does not list is [`RefusalSite::Unknown`] (rule r3). The + /// rows of a batch carry neither this nor `refused_row`. + pub refused_at: Option, + /// The row a refusal sits at (`refused_row`), present exactly when + /// `refused_at` is [`RefusalSite::Row`]. + pub refused_row: Option, /// Every entry of `cols`, in document order. pub columns: Vec, /// The entries of `cols` whose [`Value::is_stored`] is true, in order. @@ -153,15 +163,16 @@ pub struct RowResult { pub transformed: Vec, /// Input fields that name no column (`unknown_fields`), as bytes. pub unknown_fields: Vec, - /// Settings this build does not support (`unsupported_settings`, name - /// objects), as bytes. - pub unsupported_settings: Vec, /// MATERIALIZED values (`computed`). pub computed: Vec, /// The attached filter's verdict, present only with an attached filter. pub verdict: Option, /// The filter's error code for this row. pub verdict_code: i32, + /// The catalog id of a declined verdict (`verdict_reason`), present + /// exactly where [`RowResult::verdict_code`] is `CHS_CODE_DECLINED` + /// (-2). ASCII. + pub verdict_reason: Option, /// The filter's message for this row. pub verdict_err: RawText, /// The row's partition id, when the schema has a partition key. @@ -206,6 +217,16 @@ pub struct BatchResult { pub err_code: i32, /// ClickHouse's message (`err`). pub err_msg: RawText, + /// The catalog id of a declined batch (`reason`), present exactly where + /// [`BatchResult::err_code`] is `CHS_CODE_DECLINED` (-2). ASCII. + pub reason: Option, + /// Where a refusal or decline sits (`refused_at`), present whenever the + /// outcome is not accepted. One the description does not list is + /// [`RefusalSite::Unknown`] (rule r3). + pub refused_at: Option, + /// The row a refusal sits at (`refused_row`), for [`RefusalSite::Row`]; + /// below [`BatchResult::rows_read`]. + pub refused_row: Option, /// Each row, decoded as a [`RowResult`]. pub rows: Vec, /// Rows the reader read. @@ -229,10 +250,17 @@ pub struct BatchResult { /// The export bytes: present only for an export that ran (possibly empty); /// `None` when none was asked for or it was declined ([`BatchResult::export_declined`]). pub payload: Option>, - /// Each exported row's place in [`BatchResult::payload`]. - pub spans: Option>, + /// Each exported row's place in [`BatchResult::payload`] (`row_spans`): + /// one entry per [`BatchResult::rows`] entry, index-aligned, `None` for a + /// row that was not exported. + pub spans: Option>>, + /// The export's one column list (`export_columns`, name objects); `None` + /// when the document carries none. + pub export_columns: Option>, /// Why an export was declined; empty when it was not. pub export_declined: RawText, + /// The catalog id of a declined export (`export_declined_reason`). ASCII. + pub export_declined_reason: Option, /// Rows an attached filter passed. pub rows_passed: u64, /// Rows an attached filter cut. @@ -258,6 +286,9 @@ pub struct FilterRowError { pub row: u64, /// ClickHouse's error code for the row. pub code: i32, + /// The catalog id of a declined row (`reason`), present exactly where + /// [`FilterRowError::code`] is `CHS_CODE_DECLINED` (-2). ASCII. + pub reason: Option, /// ClickHouse's message for the row (from `err`). pub msg: RawText, } @@ -272,6 +303,13 @@ pub struct FilterResult { pub err_code: i32, /// ClickHouse's message (`err`). pub err_msg: RawText, + /// The catalog id of a declined evaluation (`reason`), present exactly + /// where [`FilterResult::err_code`] is `CHS_CODE_DECLINED` (-2). ASCII. + pub reason: Option, + /// Where a refusal or decline sits (`refused_at`), as the batch's. + pub refused_at: Option, + /// The row a refusal sits at (`refused_row`), as the batch's. + pub refused_row: Option, /// Rows the evaluation read. pub rows_read: u64, /// Settings this build does not support (name objects), as bytes. @@ -299,6 +337,15 @@ pub struct Column { pub default_kind: DefaultKind, /// The DEFAULT expression; empty when there is none. pub default_expr: RawText, + /// What a build does with the column's DEFAULT (`default_support`). + /// Reserved: `None` while the library does not write it. + pub default_support: Option, + /// The catalog id behind [`Column::default_support`] + /// (`default_support_reason`). ASCII. Reserved. + pub default_support_reason: Option, + /// The columns the DEFAULT expression reads (`reads`, name objects). + /// Reserved: `None` while the library does not write it. + pub reads: Option>, } /// A schema's columns, from `chs_schema_describe`. @@ -307,6 +354,9 @@ pub struct Column { pub struct SchemaDescription { /// The columns, in declared order. pub columns: Vec, + /// How the statement was compiled (`posture`): present only for + /// [`Posture::Load`]; `None` is create (ABI v2). + pub posture: Option, /// The settings the schema compiles under that this build's filters do /// not honor in a WHERE, over every layer known at schema compile (the /// defaults `setup` set, the server profile and the schema's own @@ -335,7 +385,9 @@ pub struct SchemaServer { /// (`setup`'s), exactly as `chs_initialize` spelled it. pub timezone: String, /// The server's settings as the profile gave them; empty when it gave none. - pub settings: BTreeMap, + /// A value is bytes: one that is not valid UTF-8 comes back from its + /// `{"b64": ...}` form as exactly those bytes. + pub settings: BTreeMap, /// The server's macro set: `None` exactly when the profile carried no /// macros (unknown), and `Some`, even of an empty map, when it did (the /// complete set). diff --git a/rust/src/schema.rs b/rust/src/schema.rs index bafaa350..74410137 100644 --- a/rust/src/schema.rs +++ b/rust/src/schema.rs @@ -28,10 +28,14 @@ use std::sync::Arc; use serde_json::{Map, Value as Json}; use crate::abi2::calls_gen::{BlockHandle, FilterHandle, SchemaHandle}; -use crate::abi2::vocab_gen::{DocFlags, EXPORT_NONE, Format}; +use crate::abi2::vocab_gen::{DocFlags, EXPORT_NONE, Format, Posture}; use crate::decode; use crate::error::{Error, Result}; use crate::library::Library; +use crate::options::{ + ColumnDefault, DeferDefaults, InsertMembers, ParamEncoding, VerdictBasis, filter_options, + name_object, +}; use crate::result::{BatchResult, FilterResult, RowResult, SchemaDescription}; /// The settings key a call's zone travels under. @@ -77,24 +81,10 @@ fn string_object( /// each name is `{"name": ""}`, or `{"name_b64": ""}` when its /// bytes are not valid UTF-8. Empty means none. fn columns_array(columns: &[Vec]) -> Result> { - use base64::Engine as _; if columns.is_empty() { return Ok(Vec::new()); } - let items: Vec = columns - .iter() - .map(|name| { - let mut o = Map::new(); - match std::str::from_utf8(name) { - Ok(s) => o.insert("name".to_string(), Json::String(s.to_string())), - Err(_) => o.insert( - "name_b64".to_string(), - Json::String(base64::engine::general_purpose::STANDARD.encode(name)), - ), - }; - Json::Object(o) - }) - .collect(); + let items: Vec = columns.iter().map(|name| name_object(name)).collect(); serde_json::to_vec(&Json::Array(items)) .map_err(|e| Error::internal(format!("the column list does not encode: {e}"))) } @@ -117,6 +107,35 @@ pub struct CompileOptions { /// server, exactly as before the server profile existed. A server from /// another library is that library's own refusal ([`Error::Usage`]). pub server: Option, + /// `schema_options`' `posture` (ABI v2): how the statement is compiled, + /// [`Posture::Create`] (the default) as a server's CREATE TABLE, or + /// [`Posture::Load`] as a server loads the stored definition of a table it + /// holds. `None` leaves the member out. The caller's obligation under + /// `Load` is `spec/abi-v2/docs.md`'s (`input:schema_options`). + pub posture: Option, + /// A `schema_options` member this binding has no name for: its key and its + /// value as JSON text, passed through as given. The library refuses a key + /// it does not know; text that is not valid JSON is [`Error::Usage`]. + pub schema_options_extra: Vec<(String, String)>, +} + +/// Implement `insert_options_document` over a struct's six insert-option +/// fields, so the three structs that carry them build one document one way. +macro_rules! insert_options_document { + ($t:ty) => { + impl $t { + fn insert_options_document(&self) -> Result> { + InsertMembers { + insertable_columns: self.insertable_columns.as_deref(), + column_defaults: self.column_defaults.as_deref(), + verdict_basis: self.verdict_basis, + defer_defaults: self.defer_defaults, + extra: &self.insert_options_extra, + } + .document() + } + } + }; } /// What `row` and `parse_block` take. @@ -129,8 +148,29 @@ pub struct RowOptions { pub session_timezone: Option, /// The INSERT column list, each name as bytes; empty for none. pub columns: Vec>, + /// `insert_options`' `insertable_columns` (ABI v2): the columns an INSERT + /// may supply (the server's column-level INSERT grant), checked on the + /// INSERT's header before any byte of the body is read. `None` leaves the + /// member out (every column); `Some` of an empty list is sent as `[]` (no + /// column). Each name is bytes. + pub insertable_columns: Option>>, + /// `insert_options`' `column_defaults` (ABI v2): per-call DEFAULT + /// overrides. `None` leaves the member out; `Some` of an empty list is sent + /// as `[]`. + pub column_defaults: Option>, + /// `insert_options`' `verdict_basis` (ABI v2). + pub verdict_basis: Option, + /// `insert_options`' `defer_defaults` (ABI v2). + pub defer_defaults: Option, + /// An `insert_options` member this binding has no name for: its key and + /// its value as JSON text, passed through as given (rule r1). The library + /// refuses a key it does not know; text that is not valid JSON is + /// [`Error::Usage`]. A key set twice keeps the last value. + pub insert_options_extra: Vec<(String, String)>, } +insert_options_document!(RowOptions); + /// What `rows` takes. #[derive(Debug, Clone, Default)] pub struct RowsOptions { @@ -140,6 +180,25 @@ pub struct RowsOptions { pub session_timezone: Option, /// The INSERT column list, each name as bytes; empty for none. pub columns: Vec>, + /// `insert_options`' `insertable_columns` (ABI v2): the columns an INSERT + /// may supply (the server's column-level INSERT grant), checked on the + /// INSERT's header before any byte of the body is read. `None` leaves the + /// member out (every column); `Some` of an empty list is sent as `[]` (no + /// column). Each name is bytes. + pub insertable_columns: Option>>, + /// `insert_options`' `column_defaults` (ABI v2): per-call DEFAULT + /// overrides. `None` leaves the member out; `Some` of an empty list is sent + /// as `[]`. + pub column_defaults: Option>, + /// `insert_options`' `verdict_basis` (ABI v2). + pub verdict_basis: Option, + /// `insert_options`' `defer_defaults` (ABI v2). + pub defer_defaults: Option, + /// An `insert_options` member this binding has no name for: its key and + /// its value as JSON text, passed through as given (rule r1). The library + /// refuses a key it does not know; text that is not valid JSON is + /// [`Error::Usage`]. A key set twice keeps the last value. + pub insert_options_extra: Vec<(String, String)>, /// An attached filter. Its WHERE runs in the zone it was compiled under. pub filter: Option, /// An export format; `None` asks for no export. @@ -148,6 +207,8 @@ pub struct RowsOptions { pub doc_flags: Option, } +insert_options_document!(RowsOptions); + /// What `compile_filter` takes. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct FilterOptions { @@ -158,17 +219,48 @@ pub struct FilterOptions { pub settings: Vec<(String, String)>, /// The `session_timezone` key: the filter's own zone, fixed at compile. pub session_timezone: Option, + /// `filter_options`' `param_encoding` (ABI v2): how a query parameter's + /// value is read. `None` leaves the member out. + pub param_encoding: Option, + /// A `filter_options` member this binding has no name for: its key and its + /// value as JSON text, passed through as given (rule r1). + pub filter_options_extra: Vec<(String, String)>, } -/// What `Filter::rows` takes: the body's PARSE settings, never the WHERE's. +/// What `Filter::rows` takes: the body's PARSE settings, never the WHERE's, +/// and the same INSERT the schema's `rows` models (`columns` and the +/// insert options, ABI v2). #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct EvalOptions { /// The body's parse settings. Values are strings. pub settings: Vec<(String, String)>, /// The `session_timezone` key: the zone the body is parsed in. pub session_timezone: Option, + /// The INSERT column list, each name as bytes; empty for none. + pub columns: Vec>, + /// `insert_options`' `insertable_columns` (ABI v2): the columns an INSERT + /// may supply (the server's column-level INSERT grant), checked on the + /// INSERT's header before any byte of the body is read. `None` leaves the + /// member out (every column); `Some` of an empty list is sent as `[]` (no + /// column). Each name is bytes. + pub insertable_columns: Option>>, + /// `insert_options`' `column_defaults` (ABI v2): per-call DEFAULT + /// overrides. `None` leaves the member out; `Some` of an empty list is sent + /// as `[]`. + pub column_defaults: Option>, + /// `insert_options`' `verdict_basis` (ABI v2). + pub verdict_basis: Option, + /// `insert_options`' `defer_defaults` (ABI v2). + pub defer_defaults: Option, + /// An `insert_options` member this binding has no name for: its key and + /// its value as JSON text, passed through as given (rule r1). The library + /// refuses a key it does not know; text that is not valid JSON is + /// [`Error::Usage`]. A key set twice keeps the last value. + pub insert_options_extra: Vec<(String, String)>, } +insert_options_document!(EvalOptions); + /// A compiled table. Immutable; `Clone` shares one handle. #[derive(Clone)] pub struct Schema { @@ -228,9 +320,17 @@ impl Schema { pub fn row(&self, format: Format, body: &[u8], options: &RowOptions) -> Result { let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; let columns = columns_array(&options.columns)?; - let doc = self - .library - .call(|api| api.preview_row(&self.handle, format.code(), body, &settings, &columns))?; + let insert = options.insert_options_document()?; + let doc = self.library.call(|api| { + api.preview_row( + &self.handle, + format.code(), + body, + &settings, + &columns, + &insert, + ) + })?; decode::row(&doc) } @@ -239,6 +339,7 @@ impl Schema { pub fn rows(&self, format: Format, body: &[u8], options: &RowsOptions) -> Result { let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; let columns = columns_array(&options.columns)?; + let insert = options.insert_options_document()?; let filter = options.filter.as_ref().map(|f| &*f.handle); let export = options.export.map_or(EXPORT_NONE, Format::code); let flags = options.doc_flags.unwrap_or(DocFlags::ALL).bits(); @@ -252,6 +353,7 @@ impl Schema { filter, export, flags, + &insert, ) })?; decode::batch(&doc, payload) @@ -266,9 +368,10 @@ impl Schema { ) -> Result { let params = string_object(&options.params, None, "params")?; let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; - let handle = self - .library - .call(|api| api.filter_create(&self.handle, expr.as_ref(), ¶ms, &settings))?; + let document = filter_options(options.param_encoding, &options.filter_options_extra)?; + let handle = self.library.call(|api| { + api.filter_create(&self.handle, expr.as_ref(), ¶ms, &settings, &document) + })?; Ok(Filter { library: Arc::clone(&self.library), handle: Arc::new(handle), @@ -279,9 +382,17 @@ impl Schema { pub fn parse_block(&self, format: Format, body: &[u8], options: &RowOptions) -> Result { let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; let columns = columns_array(&options.columns)?; - let handle = self - .library - .call(|api| api.block_create(&self.handle, format.code(), body, &settings, &columns))?; + let insert = options.insert_options_document()?; + let handle = self.library.call(|api| { + api.block_create( + &self.handle, + format.code(), + body, + &settings, + &columns, + &insert, + ) + })?; Ok(Block { handle: Arc::new(handle), }) @@ -290,13 +401,23 @@ impl Schema { impl Filter { /// Evaluate over a body (`chs_filter_eval_body`). The options are the - /// body's parse settings; the WHERE runs in the zone the filter was - /// compiled under. + /// body's parse settings and the INSERT it models (`columns` and the insert + /// options, as on [`Schema::rows`]); the WHERE runs in the zone the filter + /// was compiled under. pub fn rows(&self, format: Format, body: &[u8], options: &EvalOptions) -> Result { let settings = settings_object(&options.settings, options.session_timezone.as_deref())?; - let doc = self - .library - .call(|api| api.filter_eval_body(&self.handle, format.code(), body, &settings))?; + let columns = columns_array(&options.columns)?; + let insert = options.insert_options_document()?; + let doc = self.library.call(|api| { + api.filter_eval_body( + &self.handle, + format.code(), + body, + &settings, + &columns, + &insert, + ) + })?; decode::filter_result(&doc) } @@ -374,6 +495,72 @@ mod tests { assert!(v[1].get("name").is_none()); } + #[test] + fn the_four_insert_calls_build_one_insert_options_document() { + let members = || { + ( + Some(vec![b"a".to_vec()]), + Some(vec![ColumnDefault { + name: b"b".to_vec(), + value: b"7".to_vec(), + }]), + Some(VerdictBasis::Export), + Some(DeferDefaults::Server), + vec![("x_future".to_string(), "{\"k\":[1]}".to_string())], + ) + }; + let (ic, cd, vb, dd, ex) = members(); + let row = RowOptions { + insertable_columns: ic, + column_defaults: cd, + verdict_basis: vb, + defer_defaults: dd, + insert_options_extra: ex, + ..Default::default() + }; + let (ic, cd, vb, dd, ex) = members(); + let rows = RowsOptions { + insertable_columns: ic, + column_defaults: cd, + verdict_basis: vb, + defer_defaults: dd, + insert_options_extra: ex, + ..Default::default() + }; + let (ic, cd, vb, dd, ex) = members(); + let eval = EvalOptions { + insertable_columns: ic, + column_defaults: cd, + verdict_basis: vb, + defer_defaults: dd, + insert_options_extra: ex, + ..Default::default() + }; + let want = row.insert_options_document().unwrap(); + assert_eq!( + String::from_utf8(want.clone()).unwrap(), + concat!( + r#"{"column_defaults":[{"name":"b","value":"7"}],"defer_defaults":"server","#, + r#""insertable_columns":[{"name":"a"}],"verdict_basis":"export","x_future":{"k":[1]}}"# + ) + ); + assert_eq!(rows.insert_options_document().unwrap(), want); + assert_eq!(eval.insert_options_document().unwrap(), want); + // Nothing set sends nothing: the call before the document existed. + assert!( + RowOptions::default() + .insert_options_document() + .unwrap() + .is_empty() + ); + assert!( + EvalOptions::default() + .insert_options_document() + .unwrap() + .is_empty() + ); + } + #[test] fn the_handles_are_clone_send_sync_and_static() { fn shareable() {} diff --git a/rust/tests/reader_rules.rs b/rust/tests/reader_rules.rs index eb3dc0a5..1309e71f 100644 --- a/rust/tests/reader_rules.rs +++ b/rust/tests/reader_rules.rs @@ -137,10 +137,9 @@ fn r2_unknown_members_are_ignored(dir: &Path, variants: &serde_json::Map) plus a free Known function, false for it (the same shape as reasonLossy: a const object cannot carry a member); statusName spells an unlisted status unknown(). docs/reference/bindings-v1.md is the 1.x surface and names none of them; the v2 surface document lands with the other dev bindings (public issue #511), and this entry goes with it." }, + { + "binding": "rust", + "kind": "extra", + "names": [ + "VerdictBasis.Input", + "VerdictBasis.Export", + "VerdictBasis.as_str", + "DeferDefaults.None", + "DeferDefaults.Server", + "DeferDefaults.as_str", + "ParamEncoding.Escaped", + "ParamEncoding.Raw", + "ParamEncoding.as_str", + "ColumnDefault.name", + "ColumnDefault.value" + ], + "reason": "\u00a72, the call options: the options documents' hand-written value types (spec batch 2, ABI v2). The doc names VerdictBasis, DeferDefaults, ParamEncoding and ColumnDefault in the Rust option fields' types and gives Go's members (VerdictBasisInput, DeferDefaultsNone, ParamEncodingEscaped, ColumnDefault{Name, Value}) in its Go column; Rust spells each as the enum variant or struct field of the same word, and as_str is the wire spelling the options document is built from (the same fact the generated vocabularies' as_str gives)." + }, { "binding": "rust", "kind": "extra", @@ -431,7 +452,16 @@ "DiscoverQueryParam.Unknown", "DiscoverQueryParam.from_wire", "DiscoverQueryParam.is_known", - "DiscoverQueryParam.ch_type" + "DiscoverQueryParam.ch_type", + "RefusalSite.Unknown", + "RefusalSite.from_wire", + "RefusalSite.is_known", + "Posture.Unknown", + "Posture.from_wire", + "Posture.is_known", + "DefaultSupport.Unknown", + "DefaultSupport.from_wire", + "DefaultSupport.is_known" ], "issue": 511, "reason": "The ABI v2 dev binding's rule r3 (spec/abi-v2/docs.md): every vocabulary has an explicit unknown(n) member, an unlisted value kept per field, which Rust spells as an Unknown variant carrying the raw value, with is_known() false and from_wire/from_code to read one; the call status, the transform reasons and the value sources gain types of their own (Status, Reason, Source; the 1.x status, reason and source modules stay beside them). DiscoverQueryParam.ch_type is the description's ch_type fact of the discovery query's parameters, which Rust alone exposes; the v2 surface document settles it. Refusal.dev_message is rule r6's exact fingerprint message. docs/reference/bindings-v1.md is the 1.x surface and names none of them; the v2 surface document lands with the other dev bindings (public issue #511), and this entry goes with it." From 1279296190d195f4779de7ed282ed23ccf289871 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:30:14 -0400 Subject: [PATCH 13/24] ts: spec batch 2 port (options documents, b64 values, refusal members, error reason) Mirrors the Go naming table: insert/filter/schema options, the {"b64": ...} form for a non-UTF-8 settings value, reason/refusedAt/refusedRow/verdictReason, export columns and decline reason, nullable row spans, partRow, schema posture and default support, CallError.reason read by hand in abi2/raw.ts. Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/bindings-v1.md | 2 +- scripts/parity-surface-allow.json | 38 +++- ts/CHANGELOG.md | 10 + ts/src/abi2/errors.ts | 10 +- ts/src/abi2/loader.ts | 4 +- ts/src/abi2/raw.ts | 5 +- ts/src/documents.ts | 119 +++++++++-- ts/src/index.ts | 9 +- ts/src/library.ts | 9 +- ts/src/schema.ts | 78 +++++++- ts/src/settings.ts | 283 ++++++++++++++++++++++----- ts/src/setup.ts | 10 +- ts/test/abi2/public-api.test.ts | 3 +- ts/test/abi2/reader-rules.test.ts | 4 +- ts/test/call-error-reason.test.ts | 51 +++++ ts/test/documents.test.ts | 211 +++++++++++++++++--- ts/test/goldens-v1/runner.test.ts | 13 +- ts/test/goldens-v1/stub-goldens.json | 4 +- ts/test/settings-setup.test.ts | 111 ++++++++++- 19 files changed, 845 insertions(+), 129 deletions(-) create mode 100644 ts/test/call-error-reason.test.ts diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 597e19b9..5b17a4e4 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -199,7 +199,7 @@ The two zones, spelled in each binding: - **A Go `string` is a byte string.** Go has no separate byte-string type, and a `string` holds any bytes, NUL included. The rule is only that it is built from copied bytes (`string(b)`), never through `C.GoString`, which stops at the first NUL. - **A `str` or `string` input is encoded as UTF-8**, because that is the only way a text value becomes bytes. That is not an assumption about the payload: a caller with non-UTF-8 bytes passes them as `bytes`, `Uint8Array` or `&[u8]`. A body never accepts the text type, as in v0. - **Rust keeps v0's `RawText`** (the bytes, `as_bytes()`, `as_str() -> Option<&str>` and a lossy `Display`) for every byte output, so it is the one byte-out type in that binding. -- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object with the language's stock encoder and passes each value verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. **A value whose bytes are not valid UTF-8 is the object form `{"b64": ""}`** (ABI v2, `json_object_byte_values`: a settings value, a query-parameter value, a `chs_set_defaults` default, and a value of a server profile's `settings`), written by the binding in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a JSON string. A key that is not valid UTF-8 cannot be carried (a JSON object has no byte form for a key, and a macro name or value and a zone have none either): the bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the profile) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different key. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. A per-call zone is a settings value, so a zone that is not valid UTF-8 is written in the object form too, and the library validates it. +- **Settings values are strings, and only strings.** The binding serializes the map into a JSON object with the language's stock encoder and passes each value verbatim. It never rewrites a value: no boolean or integer spelling, no float (the v0 Python and TypeScript encoders are deleted, §7). A non-string value is the language's own type error: a compile error in Go and Rust, `TypeError` in Python and TypeScript. **A value whose bytes are not valid UTF-8 is the object form `{"b64": ""}`** (ABI v2, `json_object_byte_values`: a settings value, a query-parameter value, a `chs_set_defaults` default, and a value of a server profile's `settings`), written by the binding in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a JSON string. (In TypeScript a value is a `string` or a `Uint8Array`: the bytes of a `Uint8Array` that are not valid UTF-8 are the object form, and a string holding a lone surrogate has no bytes, so it is refused like a key; the macros of a server profile are text only.) A key that is not valid UTF-8 cannot be carried (a JSON object has no byte form for a key, and a macro name or value and a zone have none either): the bindings refuse one before any call, as a usage error naming the input (settings, query parameters, the profile) and the key in escaped form (a Python `str` or a TypeScript string holding a lone surrogate counts); stock encoders would have replaced it with U+FFFD, so the library would have seen a different key. Rust takes `String` throughout, so a non-UTF-8 value is unreachable there. A per-call zone is a settings value, so a zone that is not valid UTF-8 is written in the object form too, and the library validates it. - **A zone name is text**, because ClickHouse's zone names are, and a name `DateLUT` cannot load is refused by the library whatever its bytes. It crosses the ABI as counted bytes (`chs_initialize`) or as a JSON string (`session_timezone`), UTF-8 encoded. - **A list of column names** (the `columns` option) is serialized as a JSON array of names under the name encoding of §5. diff --git a/scripts/parity-surface-allow.json b/scripts/parity-surface-allow.json index 47e4a021..60744397 100644 --- a/scripts/parity-surface-allow.json +++ b/scripts/parity-surface-allow.json @@ -306,9 +306,40 @@ "CallErrorFields.chCode", "CallErrorFields.chName", "CallErrorFields.messageBytes", - "CallErrorFields.column" + "CallErrorFields.column", + "CallErrorFields.reason" ], - "reason": "The interface of §4's five call-error fields, which CallError implements and its constructors take: a type-level name for the same fields, so it adds no field. The other bindings spell the fields once, on CallError (Go's struct, Python's class, Rust's struct)." + "reason": "The interface of §4's six call-error fields, which CallError implements and its constructors take: a type-level name for the same fields, so it adds no field. The other bindings spell the fields once, on CallError (Go's struct, Python's class, Rust's struct)." + }, + { + "binding": "ts", + "kind": "extra", + "names": [ + "VerdictBasis", + "VerdictBasis.Input", + "VerdictBasis.Export", + "DeferDefaults", + "DeferDefaults.None", + "DeferDefaults.Server", + "ParamEncoding", + "ParamEncoding.Escaped", + "ParamEncoding.Raw", + "ColumnDefault", + "ColumnDefault.name", + "ColumnDefault.value", + "SettingValue" + ], + "reason": "§2's options-documents table names the three option fields that take a closed set (verdictBasis, deferDefaults, paramEncoding) and the column_defaults entry, and Go gives each set a named type (VerdictBasis, DeferDefaults, ParamEncoding, ColumnDefault), which the doc's TypeScript column does not spell. TypeScript spells the three sets as const objects with a type of the same name (the language's idiom for a vocabulary, as Format is), ColumnDefault as the interface of an entry, and SettingValue as the type of one settings or query-parameter value (a string, or the bytes of a value that is not UTF-8, which is sent in the {\"b64\": ...} object form). The ABI v2 dev surface document lands with the other dev bindings (public issue #511), and this entry goes with it.", + "issue": 511 + }, + { + "binding": "ts", + "kind": "missing", + "names": [ + "CODE_DECLINED" + ], + "reason": "§3's 'the decline code' row names CHS_CODE_DECLINED (-2) as CODE_DECLINED in TypeScript. The description's constants are emitted into the vocabulary file by the generator, which does so for Go only (CodeDeclined), so TypeScript has no constant yet and a caller compares a result's errCode with -2. The fix is a generator change (scripts/abi-v1/emit), not a hand-written declaration, and this entry goes with it.", + "issue": 511 }, { "binding": "ts", @@ -387,6 +418,9 @@ "declinedTierKnown", "declinedLayerKnown", "discoverQueryParamKnown", + "refusalSiteKnown", + "postureKnown", + "defaultSupportKnown", "Unknown" ], "issue": 511, diff --git a/ts/CHANGELOG.md b/ts/CHANGELOG.md index 8ae025a2..e856672b 100644 --- a/ts/CHANGELOG.md +++ b/ts/CHANGELOG.md @@ -25,6 +25,11 @@ The four bindings in this repository are released together and give one answer, - A `410 Gone` is a retired repository, and permanent (#571). Any request the fetch layer makes (a tag, an alias, a digest, a blob, `tags/list`, the referrers API or its fallback tag, after a redirect too) that answers `410` is now the new code `CHTYPES_SOURCE_RETIRED` (exit status 10), never retried and never sent to the next base, nor from the dev channel's alias to its tag. It was `CHTYPES_SOURCE_UNREACHABLE` (exit 3) with the registry's message dropped, and Go, Python and Rust moved on to the next base, so a mirror's 404 could hide the retirement. The error names the URL that answered and carries the registry's own message, with control and bidirectional-formatting characters removed and cut to 256 code points: Go `CodeSourceRetired` (`errors.Is(err, ErrSourceRetired)`), Python and TypeScript `SourceRetiredError`, Rust `Error::SourceRetired`. The rule is `docs/guides/fetch-v1.md` section 2, "A retired repository", and the `retired-410-*` and `list-tags-retired-410` conformance cases and one shared message table (`tests/fixtures/retired-message/cases.json`) hold all four bindings to it. - The public server handle (ABI v2): `lib.newServer(profile)` describes one ClickHouse server (`ServerProfile`: `timezone`, `settings`, `macros`), `compileTable(createTable, { server })` compiles a table on it, and `Server` has `close()` and `[Symbol.dispose]()`. A schema description carries `server` (`SchemaServer`) and `replicated` (`SchemaReplicated`: `zookeeperPath`, `replicaName`), each `undefined` when absent. Absent `macros` is not `{}`: absent means the server's macros are unknown, present (even `{}`) is its complete set. The binding validates nothing in the profile. Follows the Go binding. - A 2.0.0-dev SDK prefers the newest build of its own fingerprint (#511). A version request resolves `--fp-` first, the alias the staging dev registry serves beside every dev tag, with the binding's own generated ABI fingerprint, and the tag itself only when every base answers the alias 404; any other failure of the alias request is that failure, with no fallback, and an alias whose index lacks the platform is `CHTYPES_ARTIFACT_UNPUBLISHED`. The dev channel's cache lookups (the installed-build lookup a registry open makes first, the offline answer, and the monotonic rule of a fetch) now see only records whose signed `abi_fingerprint` is the SDK's own, so a newer build of another fingerprint, installed by another dev SDK in the shared cache, is never returned or kept over this one's own, and is never removed or rewritten. So a dev build of a newer fingerprint no longer strands an older dev SDK until its consumers upgrade. Trust is unchanged (the signed predicate and the load checks decide), a listing never shows an alias, nothing records one, `list` and `where` are unchanged, and there is no option: both steps are the dev channel's alone, and the v1 contract and every production channel never take them. The rule is `docs/guides/fetch-v1.md` section 3, and the `dev-alias-*`, `resolve-installed-dev-alias-*` and `list-tags-dev-alias` conformance cases hold all four bindings to it. +- `CallError.reason` (ABI v2): the error's catalog id, read through `chs_error_reason` beside the other `chs_error_*` fields and passed through verbatim (ASCII, empty if none), on all four call classes and on `CallErrorFields`. A misuse the binding detects itself carries the empty string. +- Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): `insertableColumns`, `columnDefaults` (a list of `ColumnDefault`), `verdictBasis` (`VerdictBasis.Input`, `VerdictBasis.Export`) and `deferDefaults` (`DeferDefaults.None`, `DeferDefaults.Server`) build `insert_options` for `schema.row`, `schema.rows`, `schema.parseBlock` and `filter.rows`; `paramEncoding` (`ParamEncoding.Escaped`, `ParamEncoding.Raw`) builds `filter_options` for `compileFilter`; `posture` (`Posture.Create`, `Posture.Load`) builds `schema_options` for `compileTable`. `insertOptionsExtra`, `filterOptionsExtra` and `schemaOptionsExtra` pass a member the binding has no name for through as JSON text, so the library, which refuses an unknown key (rule r1), is the only judge. A document with no member set is not sent. `columns` now applies to `filter.rows` too (`chs_filter_eval_body` takes the column list). +- Spec batch 2, a value that is not UTF-8 (ABI v2): a settings value, a query-parameter value, a `setup` default and a server profile's settings value may be a `Uint8Array` (`SettingValue` is `string | Uint8Array`), and bytes that are not valid UTF-8 are sent as `{"b64": ...}` (the content `json_object_byte_values`), where the binding used to refuse the value; so may `sessionTimezone`. `SchemaServer.settings` decodes the same form back to a `Buffer`. +- Spec batch 2, the decline and refusal members (ABI v2): `RowResult`, `BatchResult` and `FilterResult` carry `reason` (the catalog id, present where the code is -2, `CHS_CODE_DECLINED`), `refusedAt` (a `RefusalSite`: `Call`, `BodyRead`, `Row`, `BodyStep`) and `refusedRow`; `RowResult.verdictReason`; `FilterRowError.reason`; `BatchResult.exportColumns` and `BatchResult.exportDeclinedReason`. All are `undefined` when the document carries none. +- Spec batch 2, the schema description (ABI v2): `SchemaDescription.posture`, and `Column.defaultSupport` (a `DefaultSupport`), `Column.defaultSupportReason` and `Column.reads`, reserved members a build reports when it implements them. `Source.DefaultDeferred`, `Source.DefaultOverride` (stored) and `DeclinedTier.ServerRefuses` join the vocabularies. ### Changed @@ -73,10 +78,15 @@ The four bindings in this repository are released together and give one answer, - **Enforcement gate lifted** for `26.8` on `darwin-arm64`. - **Enforcement gate lifted** for `26.9` on `darwin-arm64`. - `docs/limitations.md` is re-checked against artifact build `20261012.142240` (v2-dev.13), the library half measured on that build: functions that need a server, in a column expression, and a body the server may read serially or in parallel under a non-zero error budget are listed under "Declined by design"; the known divergence for a column expression that calls a function needing a server is retired, since that build declines those schemas (on 26.8 and 26.9 the `AI*` and `ai*` functions compile, as the page says); and the `lossy` raw-NUL TSV gap is marked fixed on that build and still present in the 1.0 libraries. `docs/guides/filters.md` gains a section on subcolumn filters and a JSON value typed by its batch. +- **Breaking (ABI v2 dev): `BatchResult.spans` is `(Span | null)[]`**, one entry per `rows` entry and index-aligned with it: `row_spans` carries `null` for a row the export did not carry (a skipped row, or a row a filter cut). It was the spans of the exported rows only. +- **Breaking (ABI v2 dev): `AtMergeEntry.row` is `AtMergeEntry.partRow`**, the document's `part_row`. +- **Breaking (ABI v2 dev): `Settings` values and `sessionTimezone` take `string | Uint8Array`**, and a string holding a lone surrogate (no bytes) is still refused as a `UsageError`; the refusal of a value that is not UTF-8 is gone for a `Uint8Array`, while a key that is not valid UTF-8 is still refused. +- A document's `code` can be -2 for a row, batch, filter result or verdict this build declined, and every decline names its `reason`; a `Source` the description does not list reads the description's fallback (`skipped`), not stored. ### Removed - `SchemaServer.filterDeclinedSettings` (a breaking change inside v2 dev, #588): the server member no longer carries the list. Read `SchemaDescription.filterDeclinedSettings`, whose `server`-layer entries are the profile's. +- **`RowResult.unsupportedSettings`** (ABI v2 dev): the row document no longer carries `unsupported_settings`; the batch's and the filter result's lists stay. ### Fixed diff --git a/ts/src/abi2/errors.ts b/ts/src/abi2/errors.ts index fb0b015b..b3291201 100644 --- a/ts/src/abi2/errors.ts +++ b/ts/src/abi2/errors.ts @@ -4,7 +4,7 @@ * * Two shapes (spec/abi-v2/sdk.json's two tables): * - * - a CALL error: the five fields of the library's error object, read + * - a CALL error: the six fields of the library's error object, read * verbatim. `errorForStatus` in `./errmap.gen.ts` picks the class from the * raw status. The four classes are PEERS, never one a subtype of another: * a decline that satisfied `instanceof SchemaError` would let a handler @@ -36,7 +36,7 @@ export { FetchV1Error as ArtifactError }; /** Re-exported unchanged: the fetch layer's corruption error, which the loader's step-4 and step-5 refusals are too. */ export { ArtifactCorruptError }; -/** The five fields of a library error object, read verbatim. */ +/** The six fields of a library error object, read verbatim. */ export interface CallErrorFields { /** The raw status value; compare with `Status`. */ readonly status: number; @@ -48,6 +48,8 @@ export interface CallErrorFields { readonly messageBytes: Buffer; /** The column concerned, empty if none; bytes. */ readonly column: Buffer; + /** The catalog id of a decline or a library-side refusal (`chs_error_reason`); ASCII; empty if none. */ + readonly reason: string; } function renderCallMessage(label: string, f: CallErrorFields): string { @@ -63,6 +65,7 @@ export abstract class CallError extends ChtypesError implements CallErrorFields readonly chName: string; readonly messageBytes: Buffer; readonly column: Buffer; + readonly reason: string; protected constructor(label: string, f: CallErrorFields) { super(renderCallMessage(label, f)); @@ -71,6 +74,7 @@ export abstract class CallError extends ChtypesError implements CallErrorFields this.chName = f.chName; this.messageBytes = f.messageBytes; this.column = f.column; + this.reason = f.reason; } } @@ -103,7 +107,7 @@ export class InternalError extends CallError { } function binding(status: number, message: string): CallErrorFields { - return { status, chCode: 0, chName: '', messageBytes: Buffer.from(message, 'utf8'), column: Buffer.alloc(0) }; + return { status, chCode: 0, chName: '', messageBytes: Buffer.from(message, 'utf8'), column: Buffer.alloc(0), reason: '' }; } /** A misuse the binding detects itself: it carries the invalid-argument status, ch_code 0 and an empty name and column, so a handler sees one shape whichever side caught it. */ diff --git a/ts/src/abi2/loader.ts b/ts/src/abi2/loader.ts index 62d69a1e..f76e5289 100644 --- a/ts/src/abi2/loader.ts +++ b/ts/src/abi2/loader.ts @@ -67,8 +67,8 @@ export interface Predicate { export interface ImageSetup { /** The image zone `chs_initialize` is given (an IANA name); absent or empty means UTC. */ readonly timezone?: string; - /** Seeded once with `chs_set_defaults` when there are any. */ - readonly defaults?: Readonly>; + /** Seeded once with `chs_set_defaults` when there are any: each value a JSON string, or `{"b64": ...}` for one that is not valid UTF-8. */ + readonly defaults?: Readonly>; } /** The loader's own input, decoupled from the fetch layer's `Resolved` type: the adapter in `../registry.ts` joins them and passes the predicate verbatim. */ diff --git a/ts/src/abi2/raw.ts b/ts/src/abi2/raw.ts index fe433ba8..882b987e 100644 --- a/ts/src/abi2/raw.ts +++ b/ts/src/abi2/raw.ts @@ -225,8 +225,11 @@ function decodeError(raw: RawApi, status: number, errSlot: Slot): CallErrorField const chName = readOwnedBuf(raw, rawFn(raw, SYMBOL.ERROR_CH_NAME)([errPtr]) as JsExternal).toString('utf8'); const messageBytes = readOwnedBuf(raw, rawFn(raw, SYMBOL.ERROR_MESSAGE)([errPtr]) as JsExternal); const column = readOwnedBuf(raw, rawFn(raw, SYMBOL.ERROR_COLUMN)([errPtr]) as JsExternal); + // The catalog id: ASCII by the ABI's own promise, empty for an error that has none. The generator + // emits no error construction for this binding, so the field is read here, by hand, beside the others. + const reason = readOwnedBuf(raw, rawFn(raw, SYMBOL.ERROR_REASON)([errPtr]) as JsExternal).toString('utf8'); rawFn(raw, SYMBOL.ERROR_FREE)([errPtr]); - return { status, chCode, chName, messageBytes, column }; + return { status, chCode, chName, messageBytes, column, reason }; } /** diff --git a/ts/src/documents.ts b/ts/src/documents.ts index ba8d12e3..94f3e4bc 100644 --- a/ts/src/documents.ts +++ b/ts/src/documents.ts @@ -18,7 +18,7 @@ * `_b64` (standard base64 of the raw bytes, otherwise), and the * decoder surfaces a `Buffer` either way. Both present is an * `InternalError`; for a name, neither present is too. A list of names - * (`unknown_fields`, `unsupported_settings`, header names) holds `{name}` + * (`unknown_fields`, a batch's `unsupported_settings`, `export_columns`, header names) holds `{name}` * or `{name_b64}` objects, never a bare string. `value_b64` is a scalar * String or FixedString value's raw bytes (rule 6), surfaced as `value`. * 4. A vocabulary fact (`Transform.lossy`, `Value.isStored`, a verdict's @@ -40,15 +40,21 @@ import { declinedTierOf, type DefaultKind, defaultKindOf, + type DefaultSupport, + defaultSupportOf, filterOutcomeOf, type FilterOutcome, type MergeReason, mergeReasonOf, type Outcome, outcomeOf, + type Posture, + postureOf, type Reason, reasonLossy, reasonOf, + type RefusalSite, + refusalSiteOf, type Source, sourceIsStored, sourceOf, @@ -98,8 +104,8 @@ export interface EngineCell { * INSERT writer produces, so the two never contradict each other. */ export interface AtMergeEntry { - /** Indexes `engineRows`: the part's row, not the input body's. */ - readonly row: number; + /** Indexes `engineRows`: the part's row, not the input body's (the document's `part_row`). */ + readonly partRow: number; /** A `MergeReason` value; one the description does not list is its unknown(n), kept verbatim (rule r3). */ readonly reason: MergeReason; /** The column a `ttl_column_reset` resets, as bytes (`column` or `column_b64`); `undefined` when the entry names none. */ @@ -137,19 +143,27 @@ export interface Computed { export interface RowResult { readonly outcome: Outcome; + /** The document's `code`: ClickHouse's own error code, 0 only on a success, and -2 (`CHS_CODE_DECLINED`) where this build declined the row. */ readonly errCode: number; readonly errMsg: Buffer; + /** The decline's catalog id (`spec/abi-v2/declines/declines.json`), from `reason`; `undefined` when the document carries none, which it does exactly where `errCode` is -2. Key on it, never on `errMsg`. */ + readonly reason: string | undefined; + /** Where a single-row result's refusal sits, from `refused_at`; `undefined` for an accepted or skipped row. The rows of a batch carry neither this nor `refusedRow`: the batch's own say where. */ + readonly refusedAt: RefusalSite | undefined; + /** The row index of a `row` refusal, from `refused_row`; `undefined` when the document carries none. */ + readonly refusedRow: number | undefined; /** Every entry of `cols`, in document order. */ readonly columns: readonly Value[]; /** The entries of `columns` whose source is stored, in order. */ readonly values: readonly Value[]; readonly transformed: readonly Transform[]; readonly unknownFields: readonly Buffer[]; - readonly unsupportedSettings: readonly Buffer[]; readonly computed: readonly Computed[]; /** Present only with an attached filter. */ readonly verdict: Verdict | undefined; readonly verdictCode: number; + /** The catalog id of a declined verdict (`verdictCode` is -2), from `verdict_reason`; `undefined` when the document carries none. */ + readonly verdictReason: string | undefined; readonly verdictErr: Buffer; readonly partitionId: Buffer | undefined; /** The bytes the reader consumed for this record. */ @@ -172,8 +186,14 @@ export interface Framing { export interface BatchResult { readonly outcome: Outcome; + /** The document's `code`, as `RowResult.errCode`. */ readonly errCode: number; readonly errMsg: Buffer; + /** The catalog id of a declined batch (`errCode` is -2), from `reason`; `undefined` when the document carries none. */ + readonly reason: string | undefined; + /** Where a refusal or decline sits, from `refused_at`; `undefined` for an accepted batch. A `row` one names its row in `refusedRow` (below `rowsRead`). */ + readonly refusedAt: RefusalSite | undefined; + readonly refusedRow: number | undefined; readonly rows: readonly RowResult[]; readonly rowsRead: number; readonly rowsSkipped: number; @@ -191,10 +211,14 @@ export interface BatchResult { readonly atMerge: readonly AtMergeEntry[]; /** The export buffer: present only for an export that ran (possibly empty); absent when none was asked for or it was declined (`exportDeclined`). */ readonly payload: Buffer | undefined; - /** Each exported row's place in `payload`. */ - readonly spans: readonly Span[] | undefined; + /** Each row's place in `payload`, from `row_spans`: one entry per `rows` entry, index-aligned, `null` for a row not exported (a skipped row, or a row a filter cut). `undefined` when absent, and always `undefined` when the export was declined. */ + readonly spans: readonly (Span | null)[] | undefined; + /** The export's one column list, in the order every line carries its columns, from `export_columns` (name objects); `undefined` when the document carries none (no export ran). */ + readonly exportColumns: readonly Buffer[] | undefined; /** Why an asked-for export was declined, as bytes (`export_declined` or `export_declined_b64`); empty when none was. */ readonly exportDeclined: Buffer; + /** The catalog id of a declined export, from `export_declined_reason`; `undefined` when the export ran or none was asked for. */ + readonly exportDeclinedReason: string | undefined; readonly rowsPassed: number; readonly rowsCut: number; readonly partitionCount: number | undefined; @@ -206,13 +230,20 @@ export interface BatchResult { export interface FilterRowError { readonly row: number; readonly code: number; + /** The catalog id where `code` is -2, from `reason`; `undefined` when the entry carries none. */ + readonly reason: string | undefined; readonly msg: Buffer; } export interface FilterResult { readonly outcome: FilterOutcome; + /** The document's `code`, as `RowResult.errCode`. */ readonly errCode: number; readonly errMsg: Buffer; + /** `reason`, `refusedAt` and `refusedRow` are the batch's (see `BatchResult`): where a refused or declined evaluation sits, and its catalog id. */ + readonly reason: string | undefined; + readonly refusedAt: RefusalSite | undefined; + readonly refusedRow: number | undefined; readonly rowsRead: number; readonly unsupportedSettings: readonly Buffer[]; /** One verdict per row, each read through the generated table. A verdict counts only when `outcome` is `ok`; otherwise treat every verdict as `decline`, whatever the verdict string holds. Under `ok`, `error` and `decline` are never answers. */ @@ -226,6 +257,12 @@ export interface Column { readonly type: Buffer; readonly defaultKind: DefaultKind; readonly defaultExpr: Buffer; + /** What this build does with the column's DEFAULT, from `default_support` (reserved: `undefined` when the build reports none). */ + readonly defaultSupport: DefaultSupport | undefined; + /** The catalog id behind a `declined` `defaultSupport`, from `default_support_reason`; `undefined` when absent. */ + readonly defaultSupportReason: string | undefined; + /** The columns the column's expression reads, from `reads` (name objects; reserved: `undefined` when the build reports none). */ + readonly reads: readonly Buffer[] | undefined; } /** One setting the schema compiles under that this build's filters do not honor in a WHERE: an entry of the schema description's `filter_declined_settings` (ABI v2). */ @@ -245,8 +282,8 @@ export interface DeclinedSetting { export interface SchemaServer { /** The zone the schema's types bind: the profile's, else the image zone. */ readonly timezone: string; - /** The server's settings as the profile gave them; `{}` when it gave none. */ - readonly settings: Readonly> | undefined; + /** The server's settings as the profile gave them; `{}` when it gave none. A value the document carries as `{"b64": ...}` (bytes that are not UTF-8) is a `Buffer` of those bytes; any other is its text. */ + readonly settings: Readonly> | undefined; /** The server's macro set: `undefined` exactly when the profile carried no macros (unknown); present, even `{}`, is the complete set. */ readonly macros: Readonly> | undefined; } @@ -259,6 +296,8 @@ export interface SchemaReplicated { export interface SchemaDescription { readonly columns: readonly Column[]; + /** How the schema's statement was compiled, from `posture`, which the document carries only for `load`: `undefined` is `create`. */ + readonly posture: Posture | undefined; /** The settings the schema compiles under that this build's filters do not honor in a WHERE, over every layer known at schema compile (the defaults `setup` set, the server profile and the schema's own settings); empty when it lists none. One entry per name, at the highest layer that sets it, ordered by layer and then by name bytes, as the library wrote it. A schema that lists any is one `compileFilter` declines (`UnsupportedError`), and a consumer that streams rows refuses such a tenant (`spec/abi-v2/where-settings/README.md`). */ readonly filterDeclinedSettings: readonly DeclinedSetting[]; /** The server the schema was compiled on: `undefined` exactly when it was compiled without one. */ @@ -446,13 +485,19 @@ function engineCellOf(doc: string, v: unknown): EngineCell { return { column: nameOf(doc, o, 'name'), text: bytesOr(doc, o, 'stored'), null: nullFlag, value: bytesField(doc, o, 'value') }; } +/** The refusal members a row, a batch and a filter result share: `reason`, `refused_at` and `refused_row`. An unlisted site is its unknown(n): kept, never a failure (r3). */ +function refusalOf(doc: string, o: Obj): { reason: string | undefined; refusedAt: RefusalSite | undefined; refusedRow: number | undefined } { + const site = optStr(doc, o, 'refused_at'); + return { reason: optStr(doc, o, 'reason'), refusedAt: site === undefined ? undefined : refusalSiteOf(site), refusedRow: optInt(doc, o, 'refused_row') }; +} + /** One entry of `at_merge`. An unlisted reason is its unknown(n): kept, never a failure (r3). */ function atMergeOf(doc: string, v: unknown): AtMergeEntry { const o = asObject(doc, 'at_merge[]', v); const reason: MergeReason = mergeReasonOf(strOr(doc, o, 'reason', '')); const rowsRaw = o.input_rows; return { - row: intOr(doc, o, 'row', 0), + partRow: intOr(doc, o, 'part_row', 0), reason, column: bytesField(doc, o, 'column'), stored: bytesField(doc, o, 'stored'), @@ -525,14 +570,15 @@ function rowOf(doc: string, o: Obj): RowResult { outcome, errCode: intOr(doc, o, 'code', 0), errMsg: bytesOr(doc, o, 'err'), + ...refusalOf(doc, o), columns, values: columns.filter((c) => c.isStored), transformed: listOf(doc, o, 'transformed').map((t) => transformOf(doc, t)), unknownFields: bytesListOf(doc, o, 'unknown_fields'), - unsupportedSettings: bytesListOf(doc, o, 'unsupported_settings'), computed: listOf(doc, o, 'computed').map((c) => computedOf(doc, c)), verdict, verdictCode: intOr(doc, o, 'verdict_code', 0), + verdictReason: optStr(doc, o, 'verdict_reason'), verdictErr: bytesOr(doc, o, 'verdict_err'), partitionId, inputSpan: spanRaw === undefined ? undefined : spanOf(doc, 'input_span', spanRaw), @@ -575,6 +621,7 @@ export function decodeBatch(bytes: Uint8Array, payload: Buffer | undefined): Bat const spansRaw = o.row_spans; const engineRaw = o.engine_rows; const exportDeclined = bytesOr(doc, o, 'export_declined'); + const exportColumnsRaw = o.export_columns; // A declined export has no payload and no spans, whatever buffer the call returned: the library // returns a NULL out_export for it (builds up to 20261011.105019 returned a zero-length buffer), and // the document's export_declined decides, never the buffer. @@ -583,6 +630,7 @@ export function decodeBatch(bytes: Uint8Array, payload: Buffer | undefined): Bat outcome, errCode: intOr(doc, o, 'code', 0), errMsg: bytesOr(doc, o, 'err'), + ...refusalOf(doc, o), rows: listOf(doc, o, 'rows').map((r) => rowOf(doc, asObject(doc, 'rows[]', r))), rowsRead: intOr(doc, o, 'rows_read', 0), rowsSkipped: intOr(doc, o, 'rows_skipped', 0), @@ -593,8 +641,13 @@ export function decodeBatch(bytes: Uint8Array, payload: Buffer | undefined): Bat ? undefined : asArray(doc, 'engine_rows', engineRaw).map((row) => asArray(doc, 'engine_rows[]', row).map((c) => engineCellOf(doc, c))), payload: declined ? undefined : payload, - spans: spansRaw === undefined || declined ? undefined : asArray(doc, 'row_spans', spansRaw).map((s) => spanOf(doc, 'row_spans[]', s)), + spans: + spansRaw === undefined || declined + ? undefined + : asArray(doc, 'row_spans', spansRaw).map((s) => (s === null ? null : spanOf(doc, 'row_spans[]', s))), + exportColumns: exportColumnsRaw === undefined || declined ? undefined : bytesListOf(doc, o, 'export_columns'), exportDeclined, + exportDeclinedReason: optStr(doc, o, 'export_declined_reason'), rowsPassed: intOr(doc, o, 'rows_passed', 0), rowsCut: intOr(doc, o, 'rows_cut', 0), partitionCount: optInt(doc, o, 'partition_count'), @@ -618,12 +671,13 @@ export function decodeFilterResult(bytes: Uint8Array): FilterResult { outcome, errCode: intOr(doc, o, 'code', 0), errMsg: bytesOr(doc, o, 'err'), + ...refusalOf(doc, o), rowsRead: intOr(doc, o, 'rows_read', 0), unsupportedSettings: bytesListOf(doc, o, 'unsupported_settings'), verdicts, errors: listOf(doc, o, 'errors').map((e) => { const eo = asObject(doc, 'errors[]', e); - return { row: intOr(doc, eo, 'row', 0), code: intOr(doc, eo, 'code', 0), msg: bytesOr(doc, eo, 'err') }; + return { row: intOr(doc, eo, 'row', 0), code: intOr(doc, eo, 'code', 0), reason: optStr(doc, eo, 'reason'), msg: bytesOr(doc, eo, 'err') }; }), }; } @@ -641,7 +695,7 @@ export function decodeSchemaDescription(bytes: Uint8Array): SchemaDescription { const so = asObject(doc, '$.server', serverValue); server = { timezone: strOr(doc, so, 'timezone', ''), - settings: stringMapOf(doc, so, 'settings', '$.server'), + settings: byteValueMapOf(doc, so, 'settings', '$.server'), macros: stringMapOf(doc, so, 'macros', '$.server'), }; } @@ -650,7 +704,9 @@ export function decodeSchemaDescription(bytes: Uint8Array): SchemaDescription { const ro = asObject(doc, '$.replicated', replicatedValue); replicated = { zookeeperPath: bytesOr(doc, ro, 'zookeeper_path'), replicaName: bytesOr(doc, ro, 'replica_name') }; } + const posture = optStr(doc, o, 'posture'); return { + posture: posture === undefined ? undefined : postureOf(posture), server, replicated, // Over every layer known at schema compile, with or without a server. @@ -659,7 +715,17 @@ export function decodeSchemaDescription(bytes: Uint8Array): SchemaDescription { const co = asObject(doc, 'columns[]', c); // An unlisted default_kind is its unknown(n): kept, never a failure (r3). const defaultKind = defaultKindOf(strOr(doc, co, 'default_kind', '')); - return { name: nameOf(doc, co, 'name'), type: bytesOr(doc, co, 'type'), defaultKind, defaultExpr: bytesOr(doc, co, 'default_expression') }; + // default_support is reserved: absent is not reported, and an unlisted value is its unknown(n) (r3). + const support = optStr(doc, co, 'default_support'); + return { + name: nameOf(doc, co, 'name'), + type: bytesOr(doc, co, 'type'), + defaultKind, + defaultExpr: bytesOr(doc, co, 'default_expression'), + defaultSupport: support === undefined ? undefined : defaultSupportOf(support), + defaultSupportReason: optStr(doc, co, 'default_support_reason'), + reads: co.reads === undefined || co.reads === null ? undefined : bytesListOf(doc, co, 'reads'), + }; }), }; } @@ -691,6 +757,31 @@ function stringMapOf(doc: string, o: Obj, key: string, path: string): Readonly"}` + * (`json_object_byte_values`) is a `Buffer` of the decoded bytes. `undefined` when absent, an object (even an empty one) + * when present. The object form carries the `b64` member and nothing else. + */ +function byteValueMapOf(doc: string, o: Obj, key: string, path: string): Readonly> | undefined { + const v = o[key]; + if (v === undefined || v === null) return undefined; + const m = asObject(doc, `${path}.${key}`, v); + const out: Record = {}; + for (const [k, e] of Object.entries(m)) { + let value: string | Buffer; + if (typeof e === 'string') { + value = e; + } else { + const eo = asObject(doc, `${path}.${key}.${k}`, e); + const b64 = eo.b64; + if (typeof b64 !== 'string' || Object.keys(eo).length !== 1) bad(doc, `${path}.${key}.${k}`, 'a JSON string or an object with one member, b64', e); + value = decodeB64(doc, `${path}.${key}.${k}.b64`, b64); + } + Object.defineProperty(out, k, { value, enumerable: true, writable: true, configurable: true }); + } + return out; +} + /** Decode a `discovery` document. */ export function decodeDiscovery(bytes: Uint8Array): Discovery { const doc = 'discovery'; diff --git a/ts/src/index.ts b/ts/src/index.ts index 2864de36..2aff3e3c 100644 --- a/ts/src/index.ts +++ b/ts/src/index.ts @@ -27,6 +27,8 @@ export { declinedTierKnown, DefaultKind, defaultKindKnown, + DefaultSupport, + defaultSupportKnown, DiscoverQueryParam, discoverQueryParamKnown, DocFlags, @@ -40,9 +42,13 @@ export { mergeReasonKnown, Outcome, outcomeKnown, + Posture, + postureKnown, Reason, reasonKnown, reasonLossy, + RefusalSite, + refusalSiteKnown, Source, sourceIsStored, sourceKnown, @@ -129,5 +135,6 @@ export { Schema, } from './schema.js'; export { Server } from './server.js'; -export type { BytesIn, ServerProfile, Settings } from './settings.js'; +export { DeferDefaults, ParamEncoding, VerdictBasis } from './settings.js'; +export type { BytesIn, ColumnDefault, ServerProfile, SettingValue, Settings } from './settings.js'; export { setup, type SetupOptions } from './setup.js'; diff --git a/ts/src/library.ts b/ts/src/library.ts index f3d872cb..7fb9e833 100644 --- a/ts/src/library.ts +++ b/ts/src/library.ts @@ -18,7 +18,7 @@ import type { Resolved } from './ocifetch/index.js'; import { Schema, type CompileOptions } from './schema.js'; import { commitSetup, latchSetup, settleFailedOpen, setupGeneration } from './setup.js'; import { Server, serverHandleOf } from './server.js'; -import { type BytesIn, bytesIn, encodeServerProfile, encodeSettings, type ServerProfile } from './settings.js'; +import { type BytesIn, bytesIn, encodeSchemaOptions, encodeServerProfile, encodeSettings, type ServerProfile } from './settings.js'; export class Library { readonly #image: LoadedImage; @@ -115,12 +115,13 @@ export class Library { // exactly as before the server profile existed. A closed server is refused // before the call by its handle's guard (its `ptr` raises a UsageError), // because a freed handle would cross as NULL and the library would compile - // on the image's server instead of refusing. No schema option is defined - // yet: length 0, which the library reads as `{}`. + // on the image's server instead of refusing. The schema options document + // is built from the members set (`posture`, and any other as JSON text); + // with none set it is length 0, which the library reads as `{}`. const server = options.server === undefined || options.server === null ? null : serverHandleOf(options.server); return new Schema( this.#calls, - this.#calls.schemaCreate(server, bytesIn(createTable), encodeSettings(options.settings, options.sessionTimezone), Buffer.alloc(0)), + this.#calls.schemaCreate(server, bytesIn(createTable), encodeSettings(options.settings, options.sessionTimezone), encodeSchemaOptions(options)), ); } } diff --git a/ts/src/schema.ts b/ts/src/schema.ts index 15bbea2d..9144ac04 100644 --- a/ts/src/schema.ts +++ b/ts/src/schema.ts @@ -17,7 +17,7 @@ * call is ever in flight while `close` runs. */ -import type { BlockHandle, Calls, FilterHandle, SchemaHandle } from './abi2/index.js'; +import type { BlockHandle, Calls, FilterHandle, Posture, SchemaHandle } from './abi2/index.js'; import { DocFlags, EXPORT_NONE, type Format } from './abi2/index.js'; import { type BatchResult, @@ -30,13 +30,27 @@ import { type SchemaDescription, } from './documents.js'; import type { Server } from './server.js'; -import { type BytesIn, bytesIn, encodeColumns, encodeParams, encodeSettings, type Settings } from './settings.js'; +import { + type BytesIn, + bytesIn, + encodeColumns, + encodeFilterOptions, + encodeInsertOptions, + encodeParams, + encodeSettings, + type ColumnDefault, + type DeferDefaults, + type ParamEncoding, + type SettingValue, + type Settings, + type VerdictBasis, +} from './settings.js'; /** Options of `Library.compileTable`: the profile settings a schema compiles under, and the zone that profile defaults to. */ export interface CompileOptions { readonly settings?: Settings | undefined; - /** In a compile profile it is a default for later calls on the schema; a compiled type always takes the image zone, never the profile's. */ - readonly sessionTimezone?: string | undefined; + /** In a compile profile it is a default for later calls on the schema; a compiled type always takes the image zone, never the profile's. Bytes that are not valid UTF-8 are sent in the `{"b64": ...}` object form. */ + readonly sessionTimezone?: SettingValue | undefined; /** * Compile on this server (`Library.newServer`): the schema's home zone is the * server's timezone, the server's settings layer under the schema's own, and a @@ -46,15 +60,29 @@ export interface CompileOptions { * Absent (or `undefined`) is no server: the schema is on the image's own server. */ readonly server?: Server | undefined; + /** `schema_options.posture` (ABI v2): how the statement is compiled, `Posture.Create` (the default) as a server's CREATE TABLE, or `Posture.Load` as a server loads the stored definition of a table it holds. */ + readonly posture?: Posture | undefined; + /** Any other member of `schema_options`: key to its value as JSON text, passed through as given (rule r1; a key also named above keeps this value). Text that is not valid JSON is a `UsageError` before any call. */ + readonly schemaOptionsExtra?: Readonly> | undefined; } -/** Options of `Schema.row` and `Schema.parseBlock`. */ +/** Options of `Schema.row` and `Schema.parseBlock`: the settings, the zone, the INSERT column list and the `insert_options` members (ABI v2). */ export interface RowOptions { readonly settings?: Settings | undefined; /** The per-call zone: written into the settings as `session_timezone`, verbatim. */ - readonly sessionTimezone?: string | undefined; + readonly sessionTimezone?: SettingValue | undefined; /** The INSERT column list. */ readonly columns?: readonly BytesIn[] | undefined; + /** `insert_options.insertable_columns` (ABI v2): the columns an INSERT may supply. Absent leaves the member out (every column); `[]` is sent as `[]` (no column). */ + readonly insertableColumns?: readonly BytesIn[] | undefined; + /** `insert_options.column_defaults`: per-call DEFAULT overrides. Absent leaves the member out; `[]` is sent as `[]`. */ + readonly columnDefaults?: readonly ColumnDefault[] | undefined; + /** `insert_options.verdict_basis`. */ + readonly verdictBasis?: VerdictBasis | undefined; + /** `insert_options.defer_defaults`. */ + readonly deferDefaults?: DeferDefaults | undefined; + /** Any other member of `insert_options`: key to its value as JSON text, passed through as given (rule r1; a key also named above keeps this value). Text that is not valid JSON is a `UsageError` before any call. */ + readonly insertOptionsExtra?: Readonly> | undefined; } /** Options of `Schema.rows`. */ @@ -73,13 +101,32 @@ export interface FilterOptions { readonly params?: Settings | undefined; /** The filter's own zone and profile: fixed when it is compiled, and every evaluation of it runs its WHERE under them. */ readonly settings?: Settings | undefined; - readonly sessionTimezone?: string | undefined; + readonly sessionTimezone?: SettingValue | undefined; + /** `filter_options.param_encoding` (ABI v2): how a query parameter's value is read. */ + readonly paramEncoding?: ParamEncoding | undefined; + /** Any other member of `filter_options`, as `insertOptionsExtra` is for `insert_options`. */ + readonly filterOptionsExtra?: Readonly> | undefined; } -/** Options of `Filter.rows`. The settings are PARSE settings only: they decide how the body is parsed, never the filter's WHERE. */ +/** + * Options of `Filter.rows`. The settings are PARSE settings only: they decide how the body is parsed, never the filter's WHERE. + * `columns` and the `insert_options` members model the same INSERT as on `Schema.rows` (ABI v2). + */ export interface EvalOptions { readonly settings?: Settings | undefined; - readonly sessionTimezone?: string | undefined; + readonly sessionTimezone?: SettingValue | undefined; + /** The INSERT column list. */ + readonly columns?: readonly BytesIn[] | undefined; + /** `insert_options.insertable_columns` (ABI v2): the columns an INSERT may supply. Absent leaves the member out (every column); `[]` is sent as `[]` (no column). */ + readonly insertableColumns?: readonly BytesIn[] | undefined; + /** `insert_options.column_defaults`: per-call DEFAULT overrides. Absent leaves the member out; `[]` is sent as `[]`. */ + readonly columnDefaults?: readonly ColumnDefault[] | undefined; + /** `insert_options.verdict_basis`. */ + readonly verdictBasis?: VerdictBasis | undefined; + /** `insert_options.defer_defaults`. */ + readonly deferDefaults?: DeferDefaults | undefined; + /** Any other member of `insert_options`: key to its value as JSON text, passed through as given (rule r1; a key also named above keeps this value). Text that is not valid JSON is a `UsageError` before any call. */ + readonly insertOptionsExtra?: Readonly> | undefined; } function bodyIn(body: Uint8Array): Uint8Array { @@ -104,7 +151,14 @@ export class Filter { /** Evaluate over a body. */ rows(format: Format, body: Uint8Array, options: EvalOptions = {}): FilterResult { return decodeFilterResult( - this.#calls.filterEvalBody(this.#h, format, bodyIn(body), encodeSettings(options.settings, options.sessionTimezone)), + this.#calls.filterEvalBody( + this.#h, + format, + bodyIn(body), + encodeSettings(options.settings, options.sessionTimezone), + encodeColumns(options.columns), + encodeInsertOptions(options), + ), ); } @@ -170,6 +224,7 @@ export class Schema { bodyIn(body), encodeSettings(options.settings, options.sessionTimezone), encodeColumns(options.columns), + encodeInsertOptions(options), ), ); } @@ -187,6 +242,7 @@ export class Schema { filter, exporting ? (options.exportFormat as number) : EXPORT_NONE, options.docFlags ?? DocFlags.All, + encodeInsertOptions(options), ); return decodeBatch(out.out, exporting ? out.outExport : undefined); } @@ -200,6 +256,7 @@ export class Schema { bytesIn(expr), encodeParams(options.params), encodeSettings(options.settings, options.sessionTimezone), + encodeFilterOptions(options), ), ); } @@ -213,6 +270,7 @@ export class Schema { bodyIn(body), encodeSettings(options.settings, options.sessionTimezone), encodeColumns(options.columns), + encodeInsertOptions(options), ), ); } diff --git a/ts/src/settings.ts b/ts/src/settings.ts index e2aaabeb..888027d0 100644 --- a/ts/src/settings.ts +++ b/ts/src/settings.ts @@ -2,10 +2,14 @@ * Settings, the per-call zone and the other call inputs, serialized for the * library exactly as `docs/reference/bindings-v1.md` §2 and §3 say. * - * Settings values are strings, and only strings. The map becomes a JSON object - * of string values through the stock encoder and is passed verbatim: no - * boolean or integer spelling, no float. A value that is not a string is the - * language's own type error (`TypeError`). + * Settings values are strings or bytes, and nothing else. The map becomes a JSON + * object through the stock encoder and each value is passed verbatim: no boolean + * or integer spelling, no float. A value that is neither a string nor a + * `Uint8Array` is the language's own type error (`TypeError`). A value whose + * bytes are valid UTF-8 is a JSON string; any other bytes (a `Uint8Array`) are + * the object form `{"b64": ""}` (ABI v2, `json_object_byte_values`), + * so the library sees the exact bytes. A key that is not valid UTF-8 cannot be + * carried at all and is a `UsageError`. * * The per-call zone is a settings key and nothing more: `sessionTimezone` is * written into the object as `session_timezone`, verbatim, with no default, @@ -14,10 +18,16 @@ * `UsageError`, raised before any call and whether or not the two agree. */ -import { asBuffer, usageError } from './abi2/index.js'; +import { asBuffer, type Posture, usageError } from './abi2/index.js'; -/** Settings and query parameters: names to string values. */ -export type Settings = Readonly>; +/** A settings or query-parameter value: text, or the bytes of a value that need not be text (a non-UTF-8 value is sent in the `{"b64": ...}` object form). */ +export type SettingValue = string | Uint8Array; + +/** Settings and query parameters: names to values. */ +export type Settings = Readonly>; + +/** A value as the document carries it: a JSON string, or the object form of bytes that are not UTF-8. */ +export type WireValue = string | { readonly b64: string }; /** Bytes in: a statement, expression, type, name or literal. A string is encoded as UTF-8; pass a `Uint8Array` for bytes that are not text. */ export type BytesIn = Uint8Array | string; @@ -29,25 +39,63 @@ export function bytesIn(value: BytesIn): Buffer { return typeof value === 'string' ? Buffer.from(value, 'utf8') : asBuffer(value); } -const NON_UTF8_HINT = 'this ABI generation cannot carry a non-UTF-8 value in a JSON input; write the bytes as unhex(\'\') in the expression instead'; +const NON_UTF8_HINT = + 'a JSON input cannot carry these bytes here (only a settings or query-parameter value has the {"b64": ...} object form, and a string holding a lone surrogate has no bytes; pass a Uint8Array)'; /** True for a string with a lone surrogate, which no UTF-8 encoding carries: `JSON.stringify` writes it as a `\udXXX` escape and `TextEncoder` as U+FFFD. */ function hasLoneSurrogate(text: string): boolean { return /\p{Surrogate}/u.test(text); } -function checkStrings(what: string, map: Settings): void { - const shown = what === 'params' ? 'query parameters' : what; - for (const [k, v] of Object.entries(map)) { - if (typeof v !== 'string') throw new TypeError(`chtypes: ${what}[${JSON.stringify(k)}] must be a string, got ${typeof v}`); - // The key is shown through JSON.stringify, which escapes a lone surrogate: never raw. - if (hasLoneSurrogate(k)) throw usageError(`${shown}: the key ${JSON.stringify(k)} is not valid UTF-8, and ${NON_UTF8_HINT}`); - if (hasLoneSurrogate(v)) throw usageError(`${shown}: the value for key ${JSON.stringify(k)} is not valid UTF-8, and ${NON_UTF8_HINT}`); +// ignoreBOM: a leading U+FEFF is data, and the decoder must not drop it. +const utf8 = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }); + +/** The text of `bytes` when they are valid UTF-8, else `undefined`. */ +function utf8Text(bytes: Uint8Array): string | undefined { + try { + return utf8.decode(bytes); + } catch { + return undefined; } } -function checkZone(what: string, zone: string): void { - if (hasLoneSurrogate(zone)) throw usageError(`${what}: the value is not valid UTF-8, and ${NON_UTF8_HINT}`); +function isValidUtf8(bytes: Uint8Array): boolean { + return utf8Text(bytes) !== undefined; +} + +/** + * One value of `json_object_byte_values`: a string stays a JSON string, and + * bytes are a JSON string when they are valid UTF-8 and `{"b64": ...}` (standard + * base64, padded) otherwise. `where` names the value for a refusal. + */ +function wireValue(what: string, where: string, v: unknown): WireValue { + if (typeof v === 'string') { + if (hasLoneSurrogate(v)) throw usageError(`${what}: ${where} is not valid UTF-8, and ${NON_UTF8_HINT}`); + return v; + } + if (v instanceof Uint8Array) { + return utf8Text(v) ?? { b64: asBuffer(v).toString('base64') }; + } + throw new TypeError(`chtypes: ${what} ${where} must be a string or a Uint8Array, got ${typeof v}`); +} + +function keyName(k: string): string { + return `the key ${JSON.stringify(k)}`; +} + +/** The wire form of a map of byte values; refuses a key that is not valid UTF-8 (the key is shown through JSON.stringify, which escapes a lone surrogate: never raw). */ +function wireMap(what: string, map: Settings): Record { + const shown = what === 'params' ? 'query parameters' : what; + const out: Record = {}; + for (const [k, v] of Object.entries(map)) { + if (hasLoneSurrogate(k)) throw usageError(`${shown}: ${keyName(k)} is not valid UTF-8, and ${NON_UTF8_HINT}`); + if (typeof v !== 'string' && !(v instanceof Uint8Array)) { + throw new TypeError(`chtypes: ${what}[${JSON.stringify(k)}] must be a string or a Uint8Array, got ${typeof v}`); + } + // defineProperty, so a key such as "__proto__" is an own member and never the prototype. + Object.defineProperty(out, k, { value: wireValue(shown, `the value for key ${JSON.stringify(k)}`, v), enumerable: true, writable: true, configurable: true }); + } + return out; } /** @@ -55,20 +103,20 @@ function checkZone(what: string, zone: string): void { * map nor a zone, else a JSON object of string values with the zone, when * given, written in as `session_timezone`. */ -export function encodeSettings(settings: Settings | undefined, sessionTimezone: string | undefined): Buffer { +export function encodeSettings(settings: Settings | undefined, sessionTimezone: SettingValue | undefined): Buffer { if (settings === undefined && sessionTimezone === undefined) return Buffer.alloc(0); - const out: Record = {}; + let out: Record = {}; if (settings !== undefined) { - checkStrings('settings', settings); if (sessionTimezone !== undefined && Object.hasOwn(settings, ZONE_KEY)) { throw usageError(`the zone was given twice: sessionTimezone and a ${ZONE_KEY} key in settings; pass only one`); } - Object.assign(out, settings); + out = wireMap('settings', settings); } if (sessionTimezone !== undefined) { - if (typeof sessionTimezone !== 'string') throw new TypeError(`chtypes: sessionTimezone must be a string, got ${typeof sessionTimezone}`); - checkZone('the zone', sessionTimezone); - out[ZONE_KEY] = sessionTimezone; + if (typeof sessionTimezone !== 'string' && !(sessionTimezone instanceof Uint8Array)) { + throw new TypeError(`chtypes: sessionTimezone must be a string or a Uint8Array, got ${typeof sessionTimezone}`); + } + out[ZONE_KEY] = wireValue('the zone', 'the value', sessionTimezone); } return Buffer.from(JSON.stringify(out), 'utf8'); } @@ -76,19 +124,7 @@ export function encodeSettings(settings: Settings | undefined, sessionTimezone: /** Query parameters: a JSON object of string values, or nothing when absent. */ export function encodeParams(params: Settings | undefined): Buffer { if (params === undefined) return Buffer.alloc(0); - checkStrings('params', params); - return Buffer.from(JSON.stringify({ ...params }), 'utf8'); -} - -const utf8 = new TextDecoder('utf-8', { fatal: true }); - -function isValidUtf8(bytes: Uint8Array): boolean { - try { - utf8.decode(bytes); - return true; - } catch { - return false; - } + return Buffer.from(JSON.stringify(wireMap('params', params)), 'utf8'); } /** The INSERT column list: a JSON array of names, each an object carrying `name` when its bytes are valid UTF-8 and `name_b64` otherwise, or nothing (length 0) when absent. */ @@ -101,9 +137,9 @@ export function encodeColumns(columns: readonly BytesIn[] | undefined): Buffer { return Buffer.from(JSON.stringify(names), 'utf8'); } -/** The process default settings, as `chs_set_defaults` takes them. */ -export function validateDefaults(defaults: Settings): void { - checkStrings('defaults', defaults); +/** The process default settings, as `chs_set_defaults` takes them: the wire form of each value (a value that is not valid UTF-8 is `{"b64": ...}`). */ +export function validateDefaults(defaults: Settings): Readonly> { + return wireMap('defaults', defaults); } /** @@ -115,15 +151,16 @@ export function validateDefaults(defaults: Settings): void { export interface ServerProfile { /** The server's zone, the one its `timezone()` returns. Absent or `''` omits it: the server does not describe it, and the image zone applies. */ readonly timezone?: string | undefined; - /** What the server's profile applies to every query. Absent omits it (no server settings layer); `{}` is sent as `{}`. */ + /** What the server's profile applies to every query. Absent omits it (no server settings layer); `{}` is sent as `{}`. A value that is not valid UTF-8 is a `Uint8Array`, sent in the `{"b64": ...}` form. */ readonly settings?: Settings | undefined; /** * The server's ``, which a Replicated engine's ZooKeeper path and * replica name expand. ABSENT is not `{}`: absent means the server's macros * are UNKNOWN, and a schema whose engine reads one is declined; present, even - * `{}`, is the server's COMPLETE set. + * `{}`, is the server's COMPLETE set. A macro's name and value are text: they + * have no byte form. */ - readonly macros?: Settings | undefined; + readonly macros?: Readonly> | undefined; } /** The profile document: an empty or absent zone, absent settings and absent macros are omitted; a present map is sent as given, `{}` when empty. */ @@ -131,16 +168,168 @@ export function encodeServerProfile(profile: ServerProfile): Buffer { const out: Record = {}; if (profile.timezone !== undefined && profile.timezone !== '') { if (typeof profile.timezone !== 'string') throw new TypeError(`chtypes: timezone must be a string, got ${typeof profile.timezone}`); - checkZone('server profile: the timezone', profile.timezone); + if (hasLoneSurrogate(profile.timezone)) throw usageError(`server profile: the timezone is not valid UTF-8, and ${NON_UTF8_HINT}`); out.timezone = profile.timezone; } if (profile.settings !== undefined) { - checkStrings('settings', profile.settings); - out.settings = { ...profile.settings }; + out.settings = wireMap('settings', profile.settings); } if (profile.macros !== undefined) { - checkStrings('macros', profile.macros); + for (const [k, v] of Object.entries(profile.macros)) { + if (typeof v !== 'string') throw new TypeError(`chtypes: macros[${JSON.stringify(k)}] must be a string, got ${typeof v}`); + if (hasLoneSurrogate(k)) throw usageError(`macros: ${keyName(k)} is not valid UTF-8, and ${NON_UTF8_HINT}`); + if (hasLoneSurrogate(v)) throw usageError(`macros: the value for key ${JSON.stringify(k)} is not valid UTF-8, and ${NON_UTF8_HINT}`); + } out.macros = { ...profile.macros }; } return Buffer.from(JSON.stringify(out), 'utf8'); } + +// ---- the options documents (rule r1). Each named member is its own option, and a +// key the binding has no name for goes through an `...Extra` record as JSON text. +// The binding builds the document from the members the caller set, sends nothing +// when none was set, and validates no member: the library refuses a key it does not +// know, or a value that is not one of the member's, with CHS_INVALID_ARGUMENT, +// naming it. + +/** `insert_options.verdict_basis`: what row a filter verdict answers for. */ +export const VerdictBasis = { + /** The default: the verdict answers for the input row. */ + Input: 'input', + /** The row the export would store. A build that does not implement it declines it. */ + Export: 'export', +} as const; +export type VerdictBasis = (typeof VerdictBasis)[keyof typeof VerdictBasis]; + +/** `insert_options.defer_defaults`. */ +export const DeferDefaults = { + /** The default: no DEFAULT is deferred. */ + None: 'none', + /** Defer to the server each DEFAULT the build would otherwise decline. A build that does not implement it declines it. */ + Server: 'server', +} as const; +export type DeferDefaults = (typeof DeferDefaults)[keyof typeof DeferDefaults]; + +/** `filter_options.param_encoding`: how a query parameter's value is read. */ +export const ParamEncoding = { + /** The default: ClickHouse's escaped-text reader. */ + Escaped: 'escaped', + /** The bytes as the value. A build that does not implement it declines it. */ + Raw: 'raw', +} as const; +export type ParamEncoding = (typeof ParamEncoding)[keyof typeof ParamEncoding]; + +/** One entry of `insert_options.column_defaults`: the value the call uses in a cell the server would fill from the column's DEFAULT. Both are byte strings: bytes that are not valid UTF-8 are sent in the `name_b64` and `value_b64` forms. */ +export interface ColumnDefault { + readonly name: BytesIn; + readonly value: BytesIn; +} + +/** The `insert_options` members of the four calls that model an INSERT of a body (`row`, `rows`, `parseBlock`, and a filter's `rows`): what `encodeInsertOptions` reads from those calls' options objects. */ +export interface InsertOptions { + /** The columns an INSERT may supply (the server's column-level INSERT grant). Absent leaves the member out (every column); `[]` is sent as `[]` (no column). */ + readonly insertableColumns?: readonly BytesIn[] | undefined; + /** Per-call DEFAULT overrides. Absent leaves the member out; `[]` is sent as `[]`. */ + readonly columnDefaults?: readonly ColumnDefault[] | undefined; + readonly verdictBasis?: VerdictBasis | undefined; + readonly deferDefaults?: DeferDefaults | undefined; + /** Any other member of `insert_options`: key to its value as JSON text, passed through as given (rule r1). A key also named above keeps this value. Text that is not valid JSON is a `UsageError` before any call. */ + readonly insertOptionsExtra?: Readonly> | undefined; +} + +/** The `filter_options` members of `Schema.compileFilter`. */ +export interface FilterOptionMembers { + readonly paramEncoding?: ParamEncoding | undefined; + /** Any other member of `filter_options`, as `insertOptionsExtra` is for `insert_options`. */ + readonly filterOptionsExtra?: Readonly> | undefined; +} + +/** The `schema_options` members of `Library.compileTable`. */ +export interface SchemaOptionMembers { + /** How the statement is compiled: `Posture.Create` (the default), as a server's CREATE TABLE, or `Posture.Load`, as a server loads the stored definition of a table it holds. */ + readonly posture?: Posture | undefined; + /** Any other member of `schema_options`, as `insertOptionsExtra` is for `insert_options`. */ + readonly schemaOptionsExtra?: Readonly> | undefined; +} + +/** A name or value as a byte-string member: `` when it is text, `_b64` otherwise. */ +function byteMember(what: string, key: string, v: BytesIn): Record { + if (typeof v === 'string') { + if (hasLoneSurrogate(v)) throw usageError(`${what}: the ${key} is not valid UTF-8, and ${NON_UTF8_HINT}`); + return { [key]: v }; + } + if (!(v instanceof Uint8Array)) throw new TypeError(`chtypes: ${what}: the ${key} must be a string or a Uint8Array, got ${typeof v}`); + return isValidUtf8(v) ? { [key]: Buffer.from(v.buffer, v.byteOffset, v.byteLength).toString('utf8') } : { [`${key}_b64`]: asBuffer(v).toString('base64') }; +} + +function nameObject(what: string, name: BytesIn): Record { + return byteMember(what, 'name', name); +} + +/** The members of an options document under construction, each as its JSON text, by key; a key set twice keeps the last. */ +class OptionDoc { + readonly #members = new Map(); + readonly what: string; + + constructor(what: string) { + this.what = what; + } + + set(key: string, value: unknown): void { + this.#members.set(key, JSON.stringify(value)); + } + + /** Members given as JSON text, passed through verbatim after a validity check. */ + setRaw(extra: Readonly> | undefined): void { + if (extra === undefined) return; + for (const [key, text] of Object.entries(extra)) { + if (hasLoneSurrogate(key)) throw usageError(`${this.what}: ${keyName(key)} is not valid UTF-8, and ${NON_UTF8_HINT}`); + if (typeof text !== 'string') throw new TypeError(`chtypes: ${this.what}[${JSON.stringify(key)}] must be JSON text (a string), got ${typeof text}`); + try { + JSON.parse(text); + } catch { + throw usageError(`${this.what}: the value for key ${JSON.stringify(key)} is not valid JSON`); + } + this.#members.set(key, text); + } + } + + /** The document, or nothing (length 0, which the library reads as `{}`) when no member was set. */ + encode(): Buffer { + if (this.#members.size === 0) return Buffer.alloc(0); + const members = [...this.#members].map(([k, text]) => `${JSON.stringify(k)}:${text}`); + return Buffer.from(`{${members.join(',')}}`, 'utf8'); + } +} + +/** The `insert_options` document of a call that models an INSERT. */ +export function encodeInsertOptions(o: InsertOptions): Buffer { + const doc = new OptionDoc('insert options'); + if (o.insertableColumns !== undefined) doc.set('insertable_columns', o.insertableColumns.map((n) => nameObject('insert options', n))); + if (o.columnDefaults !== undefined) { + doc.set( + 'column_defaults', + o.columnDefaults.map((d) => ({ ...nameObject('insert options', d.name), ...byteMember('insert options', 'value', d.value) })), + ); + } + if (o.verdictBasis !== undefined) doc.set('verdict_basis', o.verdictBasis); + if (o.deferDefaults !== undefined) doc.set('defer_defaults', o.deferDefaults); + doc.setRaw(o.insertOptionsExtra); + return doc.encode(); +} + +/** The `filter_options` document of `chs_filter_create`. */ +export function encodeFilterOptions(o: FilterOptionMembers): Buffer { + const doc = new OptionDoc('filter options'); + if (o.paramEncoding !== undefined) doc.set('param_encoding', o.paramEncoding); + doc.setRaw(o.filterOptionsExtra); + return doc.encode(); +} + +/** The `schema_options` document of `chs_schema_create`. */ +export function encodeSchemaOptions(o: SchemaOptionMembers): Buffer { + const doc = new OptionDoc('schema options'); + if (o.posture !== undefined) doc.set('posture', o.posture); + doc.setRaw(o.schemaOptionsExtra); + return doc.encode(); +} diff --git a/ts/src/setup.ts b/ts/src/setup.ts index f12d3ad7..1b6ca8a4 100644 --- a/ts/src/setup.ts +++ b/ts/src/setup.ts @@ -27,7 +27,7 @@ */ import { usageError } from './abi2/index.js'; -import { type Settings, validateDefaults } from './settings.js'; +import { type Settings, validateDefaults, type WireValue } from './settings.js'; /** What `setup` takes: the image zone (an IANA name) and the default settings every call starts from. */ export interface SetupOptions { @@ -38,7 +38,8 @@ export interface SetupOptions { /** The setup in effect, as step 7 applies it. */ export interface ProcessSetup { readonly timezone: string; - readonly defaults: Readonly>; + /** Each default as the library takes it: a JSON string, or `{"b64": ...}` for a value that is not valid UTF-8. */ + readonly defaults: Readonly>; } let recorded: ProcessSetup | undefined; @@ -59,15 +60,14 @@ function same(a: ProcessSetup, b: ProcessSetup): boolean { if (a.timezone !== b.timezone) return false; const ak = Object.keys(a.defaults); const bk = Object.keys(b.defaults); - return ak.length === bk.length && ak.every((k) => Object.hasOwn(b.defaults, k) && a.defaults[k] === b.defaults[k]); + return ak.length === bk.length && ak.every((k) => Object.hasOwn(b.defaults, k) && JSON.stringify(a.defaults[k]) === JSON.stringify(b.defaults[k])); } /** Record the process setup, once. Call it before the first open; see this module's header for the rule. */ export function setup(options: SetupOptions = {}): void { const timezone = options.timezone ?? ''; if (typeof timezone !== 'string') throw new TypeError(`chtypes: setup timezone must be a string, got ${typeof timezone}`); - const defaults = options.defaults ?? {}; - validateDefaults(defaults); + const defaults = validateDefaults(options.defaults ?? {}); const next: ProcessSetup = { timezone, defaults: { ...defaults } }; if (recorded !== undefined && same(recorded, next)) return; if (recorded !== undefined && (latched || !replaceable)) { diff --git a/ts/test/abi2/public-api.test.ts b/ts/test/abi2/public-api.test.ts index a29509e9..6cecbd9d 100644 --- a/ts/test/abi2/public-api.test.ts +++ b/ts/test/abi2/public-api.test.ts @@ -103,7 +103,7 @@ describe.skipIf(!stubsAvailable)('the public API over the stub library', () => { expect(() => library.errorCodes()).toThrow(InternalError); }); - it('maps each call status to its class, with the five fields verbatim; the classes are peers, never subtypes', () => { + it('maps each call status to its class, with the six fields verbatim; the classes are peers, never subtypes', () => { const { library } = openStub('ok'); const cases: [string, new (...a: never[]) => CallError, number][] = [ ['CHS_REJECTED', SchemaError, Status.Rejected], @@ -127,6 +127,7 @@ describe.skipIf(!stubsAvailable)('the public API over the stub library', () => { expect(e.chName).toBe('TEST_CODE'); expect(e.messageBytes.toString()).toBe(`forced ${status}`); expect(e.column).toHaveLength(0); + expect(e.reason).toBe(''); // the stub's error object carries no catalog id for (const [, other] of cases) { if (other !== cls) expect(caught).not.toBeInstanceOf(other); } diff --git a/ts/test/abi2/reader-rules.test.ts b/ts/test/abi2/reader-rules.test.ts index 74b2783e..525f89a3 100644 --- a/ts/test/abi2/reader-rules.test.ts +++ b/ts/test/abi2/reader-rules.test.ts @@ -114,7 +114,7 @@ describe.skipIf(!stubsAvailable)('r2: unknown members are ignored at every level expect(r.transformed[0]?.lossy).toBe(true); expect(r.transformed[0]?.row).toBe(0); expect(r.unknownFields.map((b) => b.toString())).toEqual(['u']); - expect(r.unsupportedSettings.map((b) => b.toString())).toEqual(['st']); + // ABI v2 spec batch 2: a row document no longer carries unsupported_settings; only the batch's own does (below). const b = schema.rows(Format.JSONEachRow, body('{"x":1}'), { exportFormat: Format.JSONEachRow }); expect(b.outcome).toBe(Outcome.Accepted); @@ -176,7 +176,7 @@ describe.skipIf(!stubsAvailable)('r3: an unlisted value is kept as its unknown(n const src = schema.row(Format.JSONEachRow, planted('row.cols.src')); expect(src.columns[0]?.source).toBe('x_future_src'); expect(sourceKnown(src.columns[0]?.source ?? '')).toBe(false); - expect(src.columns[0]?.isStored).toBe(false); // value_src names no fallback: unknown(n) reads not stored + expect(src.columns[0]?.isStored).toBe(false); // value_src's fallback is `skipped`: unknown(n) reports its is_stored fact, false expect(src.columns[0]?.text.toString()).toBe('abc'); expect(src.outcome).toBe(Outcome.Accepted); diff --git a/ts/test/call-error-reason.test.ts b/ts/test/call-error-reason.test.ts new file mode 100644 index 00000000..6bb589ad --- /dev/null +++ b/ts/test/call-error-reason.test.ts @@ -0,0 +1,51 @@ +/** + * The sixth field of a call error (`docs/reference/bindings-v1.md` §4): `reason`, the catalog id + * `chs_error_reason` carries for a decline or a library-side refusal. Pure: the classes are built + * from the fields the generic reader (`src/abi2/raw.ts`) hands them, so no library is needed. + */ + +import { describe, expect, it } from 'vitest'; +import { type CallErrorFields, InternalError, internalError, SchemaError, UnsupportedError, usageError, UsageError } from '../src/abi2/errors.js'; +import { errorForStatus } from '../src/abi2/errmap.gen.js'; +import { Status } from '../src/abi2/vocab.gen.js'; + +const fields = (status: number): CallErrorFields => ({ + status, + chCode: 53, + chName: 'TYPE_MISMATCH', + messageBytes: Buffer.from([0x6d, 0xff]), + column: Buffer.from([0x63, 0x00]), + reason: 'option-not-implemented', +}); + +describe('a call error carries the six fields of chs_error, verbatim', () => { + it('every class reads reason, beside the other five', () => { + for (const [status, cls] of [ + [Status.Rejected, SchemaError], + [Status.Declined, UnsupportedError], + [Status.InvalidArgument, UsageError], + [Status.Internal, InternalError], + ] as const) { + const e = errorForStatus(status, fields(status)); + expect(e).toBeInstanceOf(cls); + const c = e as unknown as CallErrorFields; + expect([c.status, c.chCode, c.chName, c.reason]).toEqual([status, 53, 'TYPE_MISMATCH', 'option-not-implemented']); + expect([...c.messageBytes]).toEqual([0x6d, 0xff]); + expect([...c.column]).toEqual([0x63, 0x00]); + } + }); + + it('a status outside the closed set is an InternalError that keeps the reason', () => { + const e = errorForStatus(99, fields(99)); + expect(e).toBeInstanceOf(InternalError); + expect((e as unknown as CallErrorFields).reason).toBe('option-not-implemented'); + }); + + it('a misuse the binding detects itself carries an empty reason, as it carries an empty name and column', () => { + for (const e of [usageError('misuse'), internalError('bug')]) { + expect(e.reason).toBe(''); + expect(e.chName).toBe(''); + expect(e.column).toHaveLength(0); + } + }); +}); diff --git a/ts/test/documents.test.ts b/ts/test/documents.test.ts index afdcfb73..02e1a056 100644 --- a/ts/test/documents.test.ts +++ b/ts/test/documents.test.ts @@ -12,13 +12,18 @@ import { batchOutcomeKnown, DefaultKind, defaultKindKnown, + DefaultSupport, + defaultSupportKnown, filterOutcomeKnown, MergeReason, mergeReasonKnown, Outcome, outcomeKnown, + Posture, Reason, reasonKnown, + RefusalSite, + refusalSiteKnown, Source, sourceKnown, verdictAnswered, @@ -52,7 +57,6 @@ describe('the row document', () => { ], transformed: [{ column: 'a', input: '300', stored: '44', reason: 'overflow_wrap', row: 0 }], unknown_fields: [{ name: 'zz' }], - unsupported_settings: [], computed: [{ name: 'm', kind: 'materialized', stored: '9', value_b64: b64([0x39]) }], verdict: 't', verdict_code: 0, @@ -280,13 +284,13 @@ const AT_MERGE_BATCH = { outcome: 'accepted', unsupported_settings: [{ name: 'session_timezone' }, { name_b64: b64([0xff]) }], at_merge: [ - { row: 0, reason: 'ttl_delete' }, - { row: 1, reason: 'ttl_column_reset', column: 'c', stored: '0' }, - { row: 2, reason: 'ttl_column_reset', column: 't' }, - { row: 3, reason: 'ttl_column_reset', column_b64: b64([0xff, 0x00]), stored_b64: b64([0xff]) }, - { row: 4, reason: 'ttl_delete', input_rows: [4, 7] }, - { row: 5, reason: 'a_reason_nobody_listed', x_future: { a: [1] }, x_future_b64: '/w==' }, - { row: 6, reason: 'ttl_column_reset', column: 'e', stored: '', input_rows: [] }, + { part_row: 0, reason: 'ttl_delete' }, + { part_row: 1, reason: 'ttl_column_reset', column: 'c', stored: '0' }, + { part_row: 2, reason: 'ttl_column_reset', column: 't' }, + { part_row: 3, reason: 'ttl_column_reset', column_b64: b64([0xff, 0x00]), stored_b64: b64([0xff]) }, + { part_row: 4, reason: 'ttl_delete', input_rows: [4, 7] }, + { part_row: 5, reason: 'a_reason_nobody_listed', x_future: { a: [1] }, x_future_b64: '/w==' }, + { part_row: 6, reason: 'ttl_column_reset', column: 'e', stored: '', input_rows: [] }, ], }; @@ -295,7 +299,7 @@ describe('the batch document: at_merge and the batch\'s own unsupported_settings it('decodes each at_merge entry one to one', () => { const got = decodeBatch(enc(AT_MERGE_BATCH), undefined).atMerge.map((e) => ({ - row: e.row, + row: e.partRow, reason: e.reason, known: mergeReasonKnown(e.reason), column: bytes(e.column), @@ -303,17 +307,17 @@ describe('the batch document: at_merge and the batch\'s own unsupported_settings inputRows: e.inputRows, })); expect(got).toEqual([ - { row: 0, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: undefined }, - { row: 1, reason: MergeReason.TtlColumnReset, known: true, column: [0x63], stored: [0x30], inputRows: undefined }, + { part_row: 0, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: undefined }, + { part_row: 1, reason: MergeReason.TtlColumnReset, known: true, column: [0x63], stored: [0x30], inputRows: undefined }, // stored absent: the column's DEFAULT is decided at the merge - { row: 2, reason: MergeReason.TtlColumnReset, known: true, column: [0x74], stored: undefined, inputRows: undefined }, + { part_row: 2, reason: MergeReason.TtlColumnReset, known: true, column: [0x74], stored: undefined, inputRows: undefined }, // both by their _b64 form - { row: 3, reason: MergeReason.TtlColumnReset, known: true, column: [0xff, 0x00], stored: [0xff], inputRows: undefined }, - { row: 4, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: [4, 7] }, + { part_row: 3, reason: MergeReason.TtlColumnReset, known: true, column: [0xff, 0x00], stored: [0xff], inputRows: undefined }, + { part_row: 4, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: [4, 7] }, // an unlisted reason is its unknown(n), kept verbatim, and the unknown members are ignored (r2, r3) - { row: 5, reason: 'a_reason_nobody_listed', known: false, column: undefined, stored: undefined, inputRows: undefined }, + { part_row: 5, reason: 'a_reason_nobody_listed', known: false, column: undefined, stored: undefined, inputRows: undefined }, // an empty stored is not an absent one - { row: 6, reason: MergeReason.TtlColumnReset, known: true, column: [0x65], stored: [], inputRows: [] }, + { part_row: 6, reason: MergeReason.TtlColumnReset, known: true, column: [0x65], stored: [], inputRows: [] }, ]); }); @@ -332,11 +336,11 @@ describe('the batch document: at_merge and the batch\'s own unsupported_settings it('refuses an at_merge that breaks its schema as an InternalError', () => { for (const bad of [ - { at_merge: [{ row: 0, reason: 'ttl_column_reset', column: 'c', column_b64: 'Yw==' }] }, - { at_merge: [{ row: 0, reason: 'ttl_column_reset', column: 'c', stored_b64: '!!' }] }, - { at_merge: [{ row: 0, reason: 'ttl_delete', input_rows: ['x'] }] }, - { at_merge: [{ row: 0, reason: 'ttl_delete', input_rows: [-1] }] }, - { at_merge: [{ row: 0, reason: 7 }] }, + { at_merge: [{ part_row: 0, reason: 'ttl_column_reset', column: 'c', column_b64: 'Yw==' }] }, + { at_merge: [{ part_row: 0, reason: 'ttl_column_reset', column: 'c', stored_b64: '!!' }] }, + { at_merge: [{ part_row: 0, reason: 'ttl_delete', input_rows: ['x'] }] }, + { at_merge: [{ part_row: 0, reason: 'ttl_delete', input_rows: [-1] }] }, + { at_merge: [{ part_row: 0, reason: 7 }] }, { at_merge: { row: 0 } }, { unsupported_settings: ['session_timezone'] }, ]) { @@ -425,16 +429,16 @@ describe('the schema description, discovery, error-code and live-handle document describe('name lists and engine rows (rule 5)', () => { it('reads unknown_fields and unsupported_settings as name objects in either form', () => { - const r = decodeRow( - enc({ outcome: 'accepted', unknown_fields: [{ name: 'a' }, { name_b64: b64([0xff, 0x00]) }], unsupported_settings: [{ name: 'x' }] }), - ); + const r = decodeRow(enc({ outcome: 'accepted', unknown_fields: [{ name: 'a' }, { name_b64: b64([0xff, 0x00]) }] })); expect(r.unknownFields.map((b) => [...b])).toEqual([[0x61], [0xff, 0x00]]); - expect(r.unsupportedSettings.map((b) => b.toString())).toEqual(['x']); + const b = decodeBatch(enc({ outcome: 'accepted', unsupported_settings: [{ name: 'x' }], export_columns: [{ name: 'a' }, { name_b64: b64([0xff]) }] }), undefined); + expect(b.unsupportedSettings.map((n) => n.toString())).toEqual(['x']); + expect(b.exportColumns?.map((n) => [...n])).toEqual([[0x61], [0xff]]); }); it('refuses a bare string in a name list, an element carrying both forms, and one carrying neither', () => { expect(() => decodeRow(enc({ outcome: 'accepted', unknown_fields: ['zz'] }))).toThrow(InternalError); - expect(() => decodeRow(enc({ outcome: 'accepted', unsupported_settings: ['zz'] }))).toThrow(InternalError); + expect(() => decodeBatch(enc({ outcome: 'accepted', unsupported_settings: ['zz'] }), undefined)).toThrow(InternalError); expect(() => decodeRow(enc({ outcome: 'accepted', unknown_fields: [{ name: 'a', name_b64: 'YQ==' }] }))).toThrow(InternalError); expect(() => decodeRow(enc({ outcome: 'accepted', unknown_fields: [{}] }))).toThrow(InternalError); }); @@ -484,3 +488,158 @@ describe('name lists and engine rows (rule 5)', () => { expect([...(f.unsupportedSettings[0] ?? [])]).toEqual([0xff]); }); }); + +// ABI v2 spec batch 2 (the sixth interface batch): a refusal or decline names its code (-2 is +// CHS_CODE_DECLINED), its catalog id and where it sits. +describe('spec batch 2: the refusal members (reason, refused_at, refused_row)', () => { + it('reads a declined batch: code -2, its catalog id, the site and the row', () => { + const b = decodeBatch( + enc({ outcome: 'unsupported', code: -2, reason: 'preview-batch-zone-mismatch', refused_at: 'body_read', refused_row: 3, rows_read: 0, rows: [] }), + undefined, + ); + expect(b.errCode).toBe(-2); + expect(b.reason).toBe('preview-batch-zone-mismatch'); + expect(b.refusedAt).toBe(RefusalSite.BodyRead); + expect(refusalSiteKnown(b.refusedAt ?? '')).toBe(true); + expect(b.refusedRow).toBe(3); + }); + + it('an accepted batch carries no refusal members: they are absent, never empty', () => { + const b = decodeBatch(enc({ outcome: 'accepted', code: 0, rows_read: 1 }), undefined); + expect(b.reason).toBeUndefined(); + expect(b.refusedAt).toBeUndefined(); + expect(b.refusedRow).toBeUndefined(); + }); + + it('keeps an unlisted site as its unknown(n), never a failure (r3)', () => { + const b = decodeBatch(enc({ outcome: 'rejected', refused_at: 'a_site_nobody_listed' }), undefined); + expect(b.refusedAt).toBe('a_site_nobody_listed'); + expect(refusalSiteKnown(b.refusedAt ?? '')).toBe(false); + }); + + it('reads a row: code, reason, site, row, and the declined verdict with its own catalog id', () => { + const r = decodeRow( + enc({ outcome: 'unsupported', code: -2, reason: 'drawn-value', refused_at: 'row', refused_row: 0, verdict: 'd', verdict_code: -2, verdict_reason: 'defer-defaults-reader' }), + ); + expect(r.errCode).toBe(-2); + expect(r.reason).toBe('drawn-value'); + expect(r.refusedAt).toBe(RefusalSite.Row); + expect(r.refusedRow).toBe(0); + expect(r.verdict).toBe(Verdict.Decline); + expect(r.verdictCode).toBe(-2); + expect(r.verdictReason).toBe('defer-defaults-reader'); + const ok = decodeRow(enc({ outcome: 'accepted', code: 0 })); + expect([ok.reason, ok.refusedAt, ok.refusedRow, ok.verdictReason]).toEqual([undefined, undefined, undefined, undefined]); + }); + + it('reads a filter result: code, reason, site, and an errors entry with its own reason', () => { + const f = decodeFilterResult( + enc({ outcome: 'unsupported', code: -2, reason: 'filter-zone', refused_at: 'call', rows_read: 0, verdicts: '', errors: [{ row: 0, code: -2, reason: 'drawn-value', err: 'x' }], unsupported_settings: [] }), + ); + expect(f.errCode).toBe(-2); + expect(f.reason).toBe('filter-zone'); + expect(f.refusedAt).toBe(RefusalSite.Call); + expect(f.refusedRow).toBeUndefined(); + expect(f.errors[0]?.code).toBe(-2); + expect(f.errors[0]?.reason).toBe('drawn-value'); + expect(decodeFilterResult(enc({ outcome: 'ok', errors: [{ row: 1, code: 70 }] })).errors[0]?.reason).toBeUndefined(); + }); + + it('a member of the wrong JSON type is an InternalError', () => { + expect(() => decodeBatch(enc({ outcome: 'rejected', refused_row: 'x' }), undefined)).toThrow(InternalError); + expect(() => decodeRow(enc({ outcome: 'rejected', reason: 7 }))).toThrow(InternalError); + }); +}); + +describe('spec batch 2: the export members (row_spans, export_columns, export_declined_reason)', () => { + it('row_spans is one entry per row, a null for a row not exported, aligned with rows; export_columns are bytes', () => { + const b = decodeBatch( + enc({ outcome: 'accepted', rows_read: 3, row_spans: [{ off: 0, len: 5 }, null, { off: 5, len: 6 }], export_columns: [{ name: 'a' }, { name_b64: b64([0xff]) }] }), + Buffer.from('0123456789a'), + ); + expect(b.spans).toEqual([{ off: 0, len: 5 }, null, { off: 5, len: 6 }]); + expect(b.exportColumns?.map((n) => [...n])).toEqual([[0x61], [0xff]]); + expect(b.exportDeclinedReason).toBeUndefined(); + }); + + it('a declined export has no payload, no spans and no export columns, and names its reason', () => { + const d = decodeBatch( + enc({ outcome: 'unsupported', export_declined: 'nope', export_declined_reason: 'export-batch-not-accepted', row_spans: [null], export_columns: [{ name: 'a' }] }), + Buffer.from('x'), + ); + expect(d.exportDeclined.toString()).toBe('nope'); + expect(d.exportDeclinedReason).toBe('export-batch-not-accepted'); + expect(d.payload).toBeUndefined(); + expect(d.spans).toBeUndefined(); + expect(d.exportColumns).toBeUndefined(); + }); + + it('no export asked for: none of the export members', () => { + const b = decodeBatch(enc({ outcome: 'accepted' }), undefined); + expect([b.spans, b.exportColumns, b.exportDeclinedReason]).toEqual([undefined, undefined, undefined]); + }); +}); + +describe('spec batch 2: the schema description members', () => { + const described = decodeSchemaDescription( + enc({ + columns: [ + { name: 'a', type: 'Int32', default_kind: '', default_support: 'deferred', default_support_reason: 'r-x', reads: [{ name: 'b' }, { name_b64: b64([0xff]) }] }, + { name: 'b', type: 'Int32', default_kind: 'DEFAULT', default_support: 'a_support_nobody_listed' }, + ], + filter_declined_settings: [], + posture: 'load', + server: { timezone: 'UTC', settings: { s: 'v', t: { b64: b64([0xff]) } } }, + }), + ); + + it('reads the posture, and each column\'s default support, its reason and the columns it reads', () => { + expect(described.posture).toBe(Posture.Load); + expect(described.columns[0]?.defaultSupport).toBe(DefaultSupport.Deferred); + expect(described.columns[0]?.defaultSupportReason).toBe('r-x'); + expect(described.columns[0]?.reads?.map((n) => [...n])).toEqual([[0x62], [0xff]]); + }); + + it('keeps an unlisted default support as its unknown(n); an absent reads is undefined', () => { + const c = described.columns[1]; + expect(c?.defaultSupport).toBe('a_support_nobody_listed'); + expect(defaultSupportKnown(c?.defaultSupport ?? '')).toBe(false); + expect(c?.reads).toBeUndefined(); + expect(c?.defaultSupportReason).toBeUndefined(); + }); + + it('reads a server settings value in the {"b64": ...} form as its bytes', () => { + expect(described.server?.settings?.s).toBe('v'); + expect([...((described.server?.settings?.t as Buffer | undefined) ?? [])]).toEqual([0xff]); + }); + + it('a create-posture schema and a build that reports no default support carry none of the members', () => { + const d = decodeSchemaDescription(enc({ columns: [{ name: 'a', type: 'Int32', default_kind: '' }], filter_declined_settings: [] })); + expect(d.posture).toBeUndefined(); + expect(d.columns[0]?.defaultSupport).toBeUndefined(); + expect(d.columns[0]?.reads).toBeUndefined(); + }); + + it('a b64 value that is not the one-member object form is an InternalError', () => { + expect(() => decodeSchemaDescription(enc({ columns: [], server: { settings: { t: { b64: b64([0xff]), x: 1 } } } }))).toThrow(InternalError); + expect(() => decodeSchemaDescription(enc({ columns: [], server: { settings: { t: { b64: 7 } } } }))).toThrow(InternalError); + expect(() => decodeSchemaDescription(enc({ columns: [], server: { settings: { t: 7 } } }))).toThrow(InternalError); + }); +}); + +describe('spec batch 2: value_src gains default_deferred and default_override', () => { + it('reads their stored facts from the description: deferred is not stored, an override is', () => { + const r = decodeRow( + enc({ + outcome: 'accepted', + code: 0, + cols: [ + { name: 'a', null: false, stored: '', src: 'default_deferred' }, + { name: 'b', null: false, stored: '5', src: 'default_override' }, + ], + }), + ); + expect(r.columns.map((c) => c.isStored)).toEqual([false, true]); + expect(r.values.map((c) => c.column.toString())).toEqual(['b']); + }); +}); diff --git a/ts/test/goldens-v1/runner.test.ts b/ts/test/goldens-v1/runner.test.ts index 4319774c..8a1f531f 100644 --- a/ts/test/goldens-v1/runner.test.ts +++ b/ts/test/goldens-v1/runner.test.ts @@ -142,6 +142,9 @@ const configured = overrideMode || fetchMode; const TS_ROOT = path.resolve(import.meta.dirname, '../..'); const SELF = 'test/goldens-v1/runner.test.ts'; +/** The options document the runner sends: none (length 0, which the library reads as `{}`). */ +const NO_OPTIONS = Buffer.alloc(0); + function b64(s: string): Buffer { return Buffer.from(s, 'base64'); } @@ -207,7 +210,7 @@ function runCase(image: LoadedImage, setup: string, platform: string, c: GoldenC let schema: SchemaHandle | undefined; let filter: FilterHandle | undefined; try { - const created = attempt(() => calls.schemaCreate(null, b64(c.schema.create_table_b64), b64(c.schema.settings_b64), Buffer.alloc(0))); + const created = attempt(() => calls.schemaCreate(null, b64(c.schema.create_table_b64), b64(c.schema.settings_b64), NO_OPTIONS)); if (!created.ok) return { id: c.id, setup, result: 'ran', at: 'schema_create', status: created.status, error: created.error, decoded_ok: null }; schema = created.value; @@ -225,7 +228,7 @@ function runCase(image: LoadedImage, setup: string, platform: string, c: GoldenC const columns = b64(body.columns_b64 ?? ''); if (c.call === 'preview_row') { - const row = attempt(() => calls.previewRow(schema as SchemaHandle, body.format, bodyBytes, settings, columns)); + const row = attempt(() => calls.previewRow(schema as SchemaHandle, body.format, bodyBytes, settings, columns, NO_OPTIONS)); if (!row.ok) return { id: c.id, setup, result: 'ran', at: 'call', status: row.status, error: row.error, decoded_ok: null }; const doc = row.value; return { id: c.id, setup, result: 'ran', at: 'call', status: 'CHS_OK', document_b64: enc(doc), decoded_ok: decodes(() => decodeRow(doc)) }; @@ -234,7 +237,7 @@ function runCase(image: LoadedImage, setup: string, platform: string, c: GoldenC if (c.call === 'preview_batch') { const exportFormat = body.export_format ?? -1; const batch = attempt(() => - calls.previewBatch(schema as SchemaHandle, body.format, bodyBytes, settings, columns, null, exportFormat, body.doc_flags ?? 7), + calls.previewBatch(schema as SchemaHandle, body.format, bodyBytes, settings, columns, null, exportFormat, body.doc_flags ?? 7, NO_OPTIONS), ); if (!batch.ok) return { id: c.id, setup, result: 'ran', at: 'call', status: batch.status, error: batch.error, decoded_ok: null }; const { out, outExport } = batch.value; @@ -254,10 +257,10 @@ function runCase(image: LoadedImage, setup: string, platform: string, c: GoldenC // filter_eval_body: chs_filter_create, then chs_filter_eval_body. const f = c.filter; if (f === undefined) throw new Error(`goldens runner: case ${c.id} (filter_eval_body) carries no filter`); - const made = attempt(() => calls.filterCreate(schema as SchemaHandle, b64(f.expr_b64), b64(f.query_params_b64), b64(f.settings_b64))); + const made = attempt(() => calls.filterCreate(schema as SchemaHandle, b64(f.expr_b64), b64(f.query_params_b64), b64(f.settings_b64), NO_OPTIONS)); if (!made.ok) return { id: c.id, setup, result: 'ran', at: 'filter_create', status: made.status, error: made.error, decoded_ok: null }; filter = made.value; - const evald = attempt(() => calls.filterEvalBody(filter as FilterHandle, body.format, bodyBytes, settings)); + const evald = attempt(() => calls.filterEvalBody(filter as FilterHandle, body.format, bodyBytes, settings, columns, NO_OPTIONS)); if (!evald.ok) return { id: c.id, setup, result: 'ran', at: 'call', status: evald.status, error: evald.error, decoded_ok: null }; const doc = evald.value; return { id: c.id, setup, result: 'ran', at: 'call', status: 'CHS_OK', document_b64: enc(doc), decoded_ok: decodes(() => decodeFilterResult(doc)) }; diff --git a/ts/test/goldens-v1/stub-goldens.json b/ts/test/goldens-v1/stub-goldens.json index e595e0f3..7d5a0ce5 100644 --- a/ts/test/goldens-v1/stub-goldens.json +++ b/ts/test/goldens-v1/stub-goldens.json @@ -43,12 +43,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 5 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -150,12 +150,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 5 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", diff --git a/ts/test/settings-setup.test.ts b/ts/test/settings-setup.test.ts index 2e800cdd..791ccd69 100644 --- a/ts/test/settings-setup.test.ts +++ b/ts/test/settings-setup.test.ts @@ -10,10 +10,19 @@ import { beforeEach, describe, expect, it } from 'vitest'; import { UsageError } from '../src/abi2/errors.js'; import { ArtifactMissingError } from '../src/ocifetch/errors.js'; import { Registry } from '../src/registry.js'; -import { bytesIn, encodeColumns, encodeParams, encodeSettings } from '../src/settings.js'; +import { + bytesIn, + encodeColumns, + encodeFilterOptions, + encodeInsertOptions, + encodeParams, + encodeSchemaOptions, + encodeServerProfile, + encodeSettings, +} from '../src/settings.js'; import { commitSetup, latchSetup, resetSetupForTests, settleFailedOpen, setup, setupGeneration } from '../src/setup.js'; -describe('settings values are strings, and only strings', () => { +describe('settings values are strings or bytes, and nothing else', () => { it('serializes a map of strings with the stock encoder, verbatim', () => { expect(encodeSettings({ input_format_defaults_for_omitted_fields: '1', a: 'x' }, undefined).toString()).toBe( '{"input_format_defaults_for_omitted_fields":"1","a":"x"}', @@ -30,6 +39,102 @@ describe('settings values are strings, and only strings', () => { expect(() => encodeSettings({ a: true as unknown as string }, undefined)).toThrow(TypeError); expect(() => encodeSettings({ a: 1 as unknown as string }, undefined)).toThrow(TypeError); expect(() => encodeParams({ p: 1 as unknown as string })).toThrow(TypeError); + expect(() => encodeServerProfile({ settings: { a: 1 as unknown as string } })).toThrow(TypeError); + }); +}); + +describe('a value that is not valid UTF-8 is the {"b64": ...} object form (ABI v2, json_object_byte_values)', () => { + const bad = new Uint8Array([0xff, 0xfe, 0x00]); + const b64 = Buffer.from(bad).toString('base64'); + + it('writes a Uint8Array that is not UTF-8 as {"b64": standard padded base64}, and one that is UTF-8 as a plain string', () => { + expect(JSON.parse(encodeSettings({ a: bad, b: new Uint8Array([0x78, 0x79]), c: 'z' }, undefined).toString())).toEqual({ a: { b64 }, b: 'xy', c: 'z' }); + expect(JSON.parse(encodeParams({ p: bad }).toString())).toEqual({ p: { b64 } }); + expect(JSON.parse(encodeSettings({ a: new Uint8Array([0x61, 0x62, 0x63]) }, undefined).toString())).toEqual({ a: 'abc' }); + }); + + it('keeps a leading U+FEFF (EF BB BF) in a valid-UTF-8 value: the decoder never drops a byte order mark', () => { + expect(JSON.parse(encodeSettings({ a: new Uint8Array([0xef, 0xbb, 0xbf, 0x78]) }, undefined).toString())).toEqual({ a: '\ufeffx' }); + }); + + it('carries a per-call zone that is not UTF-8 in the object form too, and the library validates it', () => { + expect(JSON.parse(encodeSettings(undefined, bad).toString())).toEqual({ session_timezone: { b64 } }); + expect(() => encodeSettings({ session_timezone: 'UTC' }, bad)).toThrow(UsageError); + }); + + it('writes a server profile setting value in the object form, and still refuses a non-UTF-8 key, macro or zone', () => { + expect(JSON.parse(encodeServerProfile({ timezone: 'UTC', settings: { s: bad, t: 'v' }, macros: { shard: '01' } }).toString())).toEqual({ + timezone: 'UTC', + settings: { s: { b64 }, t: 'v' }, + macros: { shard: '01' }, + }); + expect(() => encodeServerProfile({ settings: { ['\udcff']: 'v' } })).toThrow(/settings: the key "\\udcff" is not valid UTF-8/); + expect(() => encodeServerProfile({ macros: { m: '\udcff' } })).toThrow(UsageError); + expect(() => encodeServerProfile({ timezone: '\udcff' })).toThrow(UsageError); + }); + + it('the process defaults take the same form', () => { + resetSetupForTests(); + setup({ defaults: { a: bad, b: '1' } }); + expect(commitSetup().defaults).toEqual({ a: { b64 }, b: '1' }); + expect(() => setup({ defaults: { b: '1', a: bad } })).not.toThrow(); // the same bytes, in any order + expect(() => setup({ defaults: { a: new Uint8Array([0xff]), b: '1' } })).toThrow(UsageError); + resetSetupForTests(); + }); +}); + +describe('the options documents (rule r1)', () => { + const doc = (b: Buffer) => JSON.parse(b.toString()); + + it('sends nothing at all (length 0) when no member is set', () => { + expect(encodeInsertOptions({})).toHaveLength(0); + expect(encodeFilterOptions({})).toHaveLength(0); + expect(encodeSchemaOptions({})).toHaveLength(0); + }); + + it('builds insert_options from the members set: name objects, column defaults in both forms, the two enums', () => { + expect( + doc( + encodeInsertOptions({ + insertableColumns: ['a', new Uint8Array([0xff])], + columnDefaults: [ + { name: 'a', value: '7' }, + { name: new Uint8Array([0xfe]), value: new Uint8Array([0xff, 0x00]) }, + ], + verdictBasis: 'export', + deferDefaults: 'server', + }), + ), + ).toEqual({ + insertable_columns: [{ name: 'a' }, { name_b64: '/w==' }], + column_defaults: [ + { name: 'a', value: '7' }, + { name_b64: '/g==', value_b64: '/wA=' }, + ], + verdict_basis: 'export', + defer_defaults: 'server', + }); + }); + + it('an absent list leaves the member out, and a present empty list is sent as []', () => { + expect(doc(encodeInsertOptions({ insertableColumns: [], columnDefaults: [] }))).toEqual({ insertable_columns: [], column_defaults: [] }); + expect(encodeInsertOptions({ insertableColumns: undefined })).toHaveLength(0); + }); + + it('passes a member it has no name for through verbatim as JSON text, and a repeated key keeps the last value', () => { + expect(encodeInsertOptions({ insertOptionsExtra: { x_future: '{"a":[1,2]}' } }).toString()).toBe('{"x_future":{"a":[1,2]}}'); + expect(doc(encodeInsertOptions({ verdictBasis: 'input', insertOptionsExtra: { verdict_basis: '"export"' } }))).toEqual({ verdict_basis: 'export' }); + expect(() => encodeInsertOptions({ insertOptionsExtra: { x: '{not json' } })).toThrow(UsageError); + expect(() => encodeFilterOptions({ filterOptionsExtra: { x: '' } })).toThrow(UsageError); + }); + + it('builds filter_options and schema_options from their members', () => { + expect(doc(encodeFilterOptions({ paramEncoding: 'raw', filterOptionsExtra: { y: '1' } }))).toEqual({ param_encoding: 'raw', y: 1 }); + expect(doc(encodeSchemaOptions({ posture: 'load', schemaOptionsExtra: { z: 'null' } }))).toEqual({ posture: 'load', z: null }); + }); + + it('validates no member: an unlisted value goes to the library, which refuses it', () => { + expect(doc(encodeInsertOptions({ verdictBasis: 'sideways' as never }))).toEqual({ verdict_basis: 'sideways' }); }); }); @@ -50,7 +155,7 @@ describe('a key or value that is not valid UTF-8 is refused, never replaced', () expect(message).toContain('settings'); expect(message).toContain('the value for key "k"'); expect(message).toContain('not valid UTF-8'); - expect(message).toContain("unhex('')"); + expect(message).toContain('Uint8Array'); }); it('refuses a settings key, a query-parameter key or value, and a zone, each naming its input', () => { From 751b2a108a0ca353543732396f357ef7386c5a58 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:31:05 -0400 Subject: [PATCH 14/24] python: spec batch 2 docs, CHANGELOG, parity allowlist entries, reason test Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/python.md | 24 ++++----- python/CHANGELOG.md | 10 ++++ python/README.md | 2 +- python/tests/abi2/test_public_library.py | 36 ++++++++++++++ scripts/parity-surface-allow.json | 63 +++++++++++++++++++++++- 5 files changed, 121 insertions(+), 14 deletions(-) diff --git a/docs/reference/python.md b/docs/reference/python.md index b5b8dc9d..8b542d97 100644 --- a/docs/reference/python.md +++ b/docs/reference/python.md @@ -42,15 +42,17 @@ A library is never unloaded and never closed: there is no `Registry.close` or `L ## Schema, Filter and Block -| Symbol | C entry point | Returns | -| ---------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------- | -| `Schema.describe()` | `chs_schema_describe` | `SchemaDescription` | -| `Schema.row(format, body, *, settings=None, session_timezone=None, columns=None)` | `chs_preview_row` | `RowResult` | -| `Schema.rows(format, body, *, settings=None, session_timezone=None, columns=None, row_filter=None, export=None, doc_flags=DocFlags.ALL)` | `chs_preview_batch` | `BatchResult` | -| `Schema.compile_filter(expr, *, params=None, settings=None, session_timezone=None)` | `chs_filter_create` | `Filter` | -| `Schema.parse_block(format, body, *, settings=None, session_timezone=None, columns=None)` | `chs_block_create` | `Block` | -| `Filter.rows(format, body, *, settings=None, session_timezone=None)`, `Filter.eval(block)` | `chs_filter_eval_body`, `chs_filter_eval_block` | `FilterResult` | -| `Schema.close()`, `Filter.close()`, `Block.close()` | `chs_*_free` | `None`; idempotent, any order is safe | +| Symbol | C entry point | Returns | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- | ------------------------------------- | +| `Schema.describe()` | `chs_schema_describe` | `SchemaDescription` | +| `Schema.row(format, body, *, settings=None, session_timezone=None, columns=None, insertable_columns=None, column_defaults=None, verdict_basis=None, defer_defaults=None, insert_options_extra=None)` | `chs_preview_row` | `RowResult` | +| `Schema.rows(format, body, *, settings=None, session_timezone=None, columns=None, insertable_columns=None, column_defaults=None, verdict_basis=None, defer_defaults=None, insert_options_extra=None, row_filter=None, export=None, doc_flags=DocFlags.ALL)` | `chs_preview_batch` | `BatchResult` | +| `Schema.compile_filter(expr, *, params=None, settings=None, session_timezone=None, param_encoding=None, filter_options_extra=None)` | `chs_filter_create` | `Filter` | +| `Schema.parse_block(format, body, *, settings=None, session_timezone=None, columns=None, insertable_columns=None, column_defaults=None, verdict_basis=None, defer_defaults=None, insert_options_extra=None)` | `chs_block_create` | `Block` | +| `Filter.rows(format, body, *, settings=None, session_timezone=None, columns=None, insertable_columns=None, column_defaults=None, verdict_basis=None, defer_defaults=None, insert_options_extra=None)`, `Filter.eval(block)` | `chs_filter_eval_body`, `chs_filter_eval_block` | `FilterResult` | +| `Schema.close()`, `Filter.close()`, `Block.close()` | `chs_*_free` | `None`; idempotent, any order is safe | + +**The options documents (ABI v2).** The insert options are keyword-only on the four calls that model an INSERT of a body (`Schema.row`, `Schema.rows`, `Schema.parse_block`, `Filter.rows`): `insertable_columns` (names), `column_defaults` (a sequence of `ColumnDefault(name, value)`), `verdict_basis` (`VerdictBasis`), `defer_defaults` (`DeferDefaults`). `compile_filter` takes `param_encoding` (`ParamEncoding`) and `compile_table` takes `posture` (`Posture`). Each `*_options_extra` mapping passes a member the binding has no name for, as JSON text by key, so the library (which refuses an unknown key) is the only judge. `None` leaves a member out and an empty sequence is sent as `[]`; no document is sent when no member is set. A settings or query-parameter value that is not UTF-8 is a `str` holding the bytes as `surrogateescape` makes them (`b'\xff'.decode('utf-8', 'surrogateescape')`), and travels as `{"b64": ...}`. `Schema`, `Filter` and `Block` are context managers and are freed by a finalizer when abandoned. Using a closed object is a `UsageError`. `BytesIn` is `bytes | str` (a `str` is encoded as UTF-8); a body is `bytes` and never accepts `str`. The per-call zone is the `session_timezone` argument, which is written into `settings` as that one key, and giving both is a `UsageError`. The export channel is the keyword-only `export` of `rows`; there is no separate method. @@ -60,11 +62,11 @@ A library is never unloaded and never closed: there is no `Registry.close` or `L `Value.is_stored` and `Transform.lossy` are the description's own facts, read from the generated tables. `BatchResult.engine_rows` is `None` or a tuple of rows, each a tuple of `EngineCell`; `RowResult.partition_id` and `BatchResult.partition_count` are `None` unless the statement declared a partition key. -**Vocabularies**, generated from the description ([§3](bindings-v1.md)): `Format` (an `IntEnum`, with `.ch_name`), `Status`, `Outcome` and `FilterOutcome` (`StrEnum`), `Verdict` (with `.answered`), `DefaultKind`, `DocFlags`, and the plain-string vocabularies `Reason` and `Source` (a `Source` of `DEFAULT_GENERATED` means insert the library's export, never the original body). `Reason.lossy(reason)` and `Source.is_stored(source)` are the same facts the result fields read. +**Vocabularies**, generated from the description ([§3](bindings-v1.md)): `Format` (an `IntEnum`, with `.ch_name`), `Status`, `Outcome` and `FilterOutcome` (`StrEnum`), `Verdict` (with `.answered`), `DefaultKind`, `RefusalSite`, `Posture`, `DefaultSupport`, `DocFlags`, and the plain-string vocabularies `Reason` and `Source` (a `Source` of `DEFAULT_GENERATED` means insert the library's export, never the original body). `Reason.lossy(reason)` and `Source.is_stored(source)` are the same facts the result fields read. ## Errors -One table ([§4](bindings-v1.md)). The four call errors are peers under `CallError`, which carries `status`, `ch_code`, `ch_name`, `message` (bytes) and `column` (bytes): +One table ([§4](bindings-v1.md)). The four call errors are peers under `CallError`, which carries `status`, `ch_code`, `ch_name`, `message` (bytes), `column` (bytes) and `reason` (the catalog id, ASCII, empty if none): | Class | Raised for | | ----------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index eff4709a..364f4c10 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -8,6 +8,11 @@ The four bindings in this repository are released together and give one answer, ### Added +- `CallError.reason` (ABI v2): the error's catalog id, read through `chs_error_reason` and passed through verbatim (ASCII, empty if none), on all four call classes. +- Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): keyword-only `insertable_columns`, `column_defaults` (a sequence of `ColumnDefault`), `verdict_basis` (`VerdictBasis`: `INPUT`, `EXPORT`) and `defer_defaults` (`DeferDefaults`: `NONE`, `SERVER`) build `insert_options` for `Schema.row`, `Schema.rows`, `Schema.parse_block` and `Filter.rows`; `param_encoding` (`ParamEncoding`: `ESCAPED`, `RAW`) builds `filter_options` for `Schema.compile_filter`; `posture` (`Posture`: `CREATE`, `LOAD`) builds `schema_options` for `Library.compile_table`. `insert_options_extra`, `filter_options_extra` and `schema_options_extra` pass a member the binding has no name for through as JSON text, so the library, which refuses an unknown key (rule r1), is the only judge. A document with no member set is not sent. `Filter.rows` takes `columns` (`chs_filter_eval_body` takes the column list). +- Spec batch 2, a value that is not UTF-8 (ABI v2): a settings value, a query-parameter value, a `setup` default and a server profile's settings value is sent as `{"b64": ...}` (the content `json_object_byte_values`), where the binding used to refuse it, and `SchemaServer.settings` decodes the same form back (as `surrogateescape` text, the spelling the input side takes). +- Spec batch 2, the decline and refusal members (ABI v2): `RowResult`, `BatchResult` and `FilterResult` carry `reason` (the catalog id, present where the code is -2, `CHS_CODE_DECLINED`), `refused_at` (a `RefusalSite`: `CALL`, `BODY_READ`, `ROW`, `BODY_STEP`) and `refused_row`; `RowResult.verdict_reason`; `FilterRowError.reason`; `BatchResult.export_columns` and `BatchResult.export_declined_reason`. +- Spec batch 2, the schema description (ABI v2): `SchemaDescription.posture`, and `Column.default_support` (a `DefaultSupport`), `Column.default_support_reason` and `Column.reads`, reserved members a build reports when it implements them. `Source.DEFAULT_DEFERRED`, `Source.DEFAULT_OVERRIDE` (stored) and `DeclinedTier.SERVER_REFUSES` join the vocabularies. - A fetch-only call (#492): `Registry.fetch(request) -> Resolved` resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing: no library is loaded and the process setup is untouched, so it needs no `setup`. It honors the registry's fetch options (offline, the cache, and under the production channel the bases, the trust and the lock), and a concurrent `fetch` and open of one request share one fetch. The `Resolved` names the installed version, build, digests and library path; errors are the fetch codes' classes. - `chtypes resolve [--json]` (#493): what a line or version resolves to on every platform the registry offers, one `resolved ` line each, or with `--json` one JSON array of `{platform, version, build, manifest}` objects, each signed statement verified exactly as a fetch verifies it and no layer downloaded; `--offline` answers from the cache. The four CLIs print the same bytes, so one parser reads them (a CI cache key, for example). - `chtypes prune [--line ] [--keep ] [--dry-run]` (#494): removes the installed builds of each line and platform that at least n newer installed builds supersede (n defaults to 1), each with its record, its blobs and its `index.json` entry; `--dry-run` prints exactly what would go. It never removes a build a running process holds: a registry now holds every build it opens or fetches, with a shared `flock` on the entry's `verified.json` until the process exits, and prune removes an entry only after taking the exclusive lock without waiting, reporting it `in-use` otherwise. It never touches a system directory, nor, on the dev channel, another fingerprint's builds. The rules are `docs/guides/fetch-v1.md` sections 1 and 3, and the `resolve-build-*` and `prune-*` conformance cases hold all four bindings to them. @@ -26,6 +31,10 @@ The four bindings in this repository are released together and give one answer, ### Changed +- **Breaking (ABI v2 dev): `BatchResult.spans` is a tuple of `Span | None`**, one entry per `rows` entry and index-aligned with it: `row_spans` carries `null` for a row the export did not carry (a skipped row, or a row a filter cut), which reads as `None`. It was the exported rows' spans only. +- **Breaking (ABI v2 dev): `AtMergeEntry.row` is `AtMergeEntry.part_row`**, the document's `part_row`. +- A document's `code` can be -2 (`CHS_CODE_DECLINED`) for a row, batch, filter result or verdict this build declined, and every decline names its `reason`. +- The refusal of a settings, query-parameter, default or server-settings value that is not valid UTF-8 is gone (it is the `{"b64": ...}` form now); a key that is not valid UTF-8, a macro and a server timezone are still refused, as a `UsageError` naming the input. - `chtypes list` and the tag listing name the published lines only (two-part, numeric order), as Go's always did; `chtypes resolve ` prints the exact version a line resolves to (#613). - **Breaking: the public surface the parity gate carried as exceptions is resolved, before the v2 lock (#499, #500).** A convenience exists in every binding or in none (`docs/reference/bindings-v1.md` §1, principle 2). Migration, old → new: - `chtypes.errors.internal` and `chtypes.errors.misuse` → private (#499): they were the binding's own construction helpers, never documented. Catch `InternalError` and `UsageError`, as before. @@ -70,6 +79,7 @@ The four bindings in this repository are released together and give one answer, ### Removed +- **Breaking (ABI v2 dev): `RowResult.unsupported_settings`**: the row document no longer carries `unsupported_settings`; the batch's and the filter result's lists stay. - `SchemaServer.filter_declined_settings` (a breaking change inside v2 dev, #588): the server member no longer carries the list. Read `SchemaDescription.filter_declined_settings`, whose `server`-layer entries are the profile's. ### Fixed diff --git a/python/README.md b/python/README.md index bfe9e872..a28f41b7 100644 --- a/python/README.md +++ b/python/README.md @@ -77,7 +77,7 @@ A bad **row** is a verdict, not an exception: `outcome` becomes `Outcome.REJECTE **`ctypes` releases the GIL for the whole of a foreign call**, so calls on compiled handles run in parallel across threads. The package takes no lock around a call; it keeps a close guard per handle and one setup guard. -**The INSERT column list is a keyword-only `columns`** on `schema.row`, `schema.rows` and `schema.parse_block`: `schema.row(Format.JSON_EACH_ROW, body, columns=["id", "e"])`. `None` or an empty sequence is the no-list shape; a list makes the data supply exactly those columns, with a listed `EPHEMERAL` value read and in scope for the DEFAULTs that reference it but never stored. +**The INSERT column list is a keyword-only `columns`** on `schema.row`, `schema.rows`, `schema.parse_block` and `filter.rows`: `schema.row(Format.JSON_EACH_ROW, body, columns=["id", "e"])`. `None` or an empty sequence is the no-list shape; a list makes the data supply exactly those columns, with a listed `EPHEMERAL` value read and in scope for the DEFAULTs that reference it but never stored. **Known gaps in 1.0** are listed, with workarounds, in [`docs/limitations.md`](https://github.com/wave-rf/chtypes/blob/main/docs/limitations.md#known-gaps-in-10). diff --git a/python/tests/abi2/test_public_library.py b/python/tests/abi2/test_public_library.py index b458f8e3..914ba70c 100644 --- a/python/tests/abi2/test_public_library.py +++ b/python/tests/abi2/test_public_library.py @@ -398,3 +398,39 @@ def test_an_unnamed_member_reaches_the_library_which_refuses_it_by_name(lib) -> schema.compile_filter("a > 1", filter_options_extra={"future": "1"}) with pytest.raises(UsageError, match='unknown key "future"'): lib.compile_table(CREATE, schema_options_extra={"future": "1"}) + + +def test_a_call_error_carries_the_librarys_reason(monkeypatch) -> None: + """`CallError.reason` is `chs_error_reason`, read with the other five fields. + The stub writes an empty reason, so a fake raw table (real memory behind each + buffer) supplies one: the verdict is what the generated `_check` raised.""" + import ctypes + + keep: list[ctypes.Array] = [] + length: dict[int, int] = {} + + def buf(text: bytes) -> int: + b = ctypes.create_string_buffer(text, max(len(text), 1)) + keep.append(b) + length[ctypes.addressof(b)] = len(text) + return ctypes.addressof(b) + + ch_name, message, column, reason = buf(b"CH"), buf(b"msg"), buf(b""), buf(b"zone-declined") + raw = { + "chs_error_status": lambda e: 2, + "chs_error_ch_code": lambda e: 0, + "chs_error_ch_name": lambda e: ch_name, + "chs_error_message": lambda e: message, + "chs_error_column": lambda e: column, + "chs_error_reason": lambda e: reason, + "chs_error_free": lambda e: None, + "chs_buf_data": lambda p: p, + "chs_buf_len": lambda p: length[p], + "chs_buf_free": lambda p: None, + } + api = object.__new__(_decls.Api) + api._raw = raw + with pytest.raises(Exception) as info: + api._check(2, ctypes.c_void_p(1)) + assert info.value.reason == "zone-declined" and info.value.ch_name == "CH" + assert info.value.message == b"msg" diff --git a/scripts/parity-surface-allow.json b/scripts/parity-surface-allow.json index 47e4a021..37341e1f 100644 --- a/scripts/parity-surface-allow.json +++ b/scripts/parity-surface-allow.json @@ -68,7 +68,10 @@ "DefaultKind.of", "MergeReason.of", "DeclinedTier.of", - "DeclinedLayer.of" + "DeclinedLayer.of", + "RefusalSite.of", + "Posture.of", + "DefaultSupport.of" ], "reason": "Generated from the description by scripts/abi-v1/gen.py: each string vocabulary's decoder from a document's text, with the description's fallback. The decoders use it; §3 names the vocabularies and their facts, not their decoders." }, @@ -365,7 +368,10 @@ "DeclinedTier.known", "DeclinedLayer.known", "DiscoverQueryParam.known", - "DiscoverQueryParam.of" + "DiscoverQueryParam.of", + "RefusalSite.known", + "Posture.known", + "DefaultSupport.known" ], "issue": 511, "reason": "The ABI v2 dev binding's rule r3 (spec/abi-v2/docs.md), as Python spells it: an unlisted value of an enum class is that class's unknown(n) member (`Outcome(\"x\")`, named `unknown(x)`), and every vocabulary gains `known`, False for exactly that member (a property on the enum classes, a function on the plain-string Reason and Source); DiscoverQueryParam has `of`, as the other document vocabularies have (their own entry). docs/reference/bindings-v1.md is the 1.x surface and names neither; the v2 surface document lands with the other dev bindings (public issue #511), and this entry goes with it." @@ -435,6 +441,59 @@ ], "issue": 511, "reason": "The ABI v2 dev binding's rule r3 (spec/abi-v2/docs.md): every vocabulary has an explicit unknown(n) member, an unlisted value kept per field, which Rust spells as an Unknown variant carrying the raw value, with is_known() false and from_wire/from_code to read one; the call status, the transform reasons and the value sources gain types of their own (Status, Reason, Source; the 1.x status, reason and source modules stay beside them). DiscoverQueryParam.ch_type is the description's ch_type fact of the discovery query's parameters, which Rust alone exposes; the v2 surface document settles it. Refusal.dev_message is rule r6's exact fingerprint message. docs/reference/bindings-v1.md is the 1.x surface and names none of them; the v2 surface document lands with the other dev bindings (public issue #511), and this entry goes with it." + }, + { + "binding": "python", + "kind": "extra", + "names": [ + "Schema.row(insertable_columns=)", + "Schema.row(column_defaults=)", + "Schema.row(verdict_basis=)", + "Schema.row(defer_defaults=)", + "Schema.row(insert_options_extra=)", + "Schema.rows(insertable_columns=)", + "Schema.rows(column_defaults=)", + "Schema.rows(verdict_basis=)", + "Schema.rows(defer_defaults=)", + "Schema.rows(insert_options_extra=)", + "Schema.parse_block(insertable_columns=)", + "Schema.parse_block(column_defaults=)", + "Schema.parse_block(verdict_basis=)", + "Schema.parse_block(defer_defaults=)", + "Schema.parse_block(insert_options_extra=)", + "Filter.rows(insertable_columns=)", + "Filter.rows(column_defaults=)", + "Filter.rows(verdict_basis=)", + "Filter.rows(defer_defaults=)", + "Filter.rows(insert_options_extra=)", + "Filter.rows(columns=)", + "Schema.compile_filter(param_encoding=)", + "Schema.compile_filter(filter_options_extra=)", + "Library.compile_table(posture=)", + "Library.compile_table(schema_options_extra=)" + ], + "reason": "ABI v2's options documents and the filter column list (bindings-v1.md §2, the call options table): the table names each option once, by its Python keyword, and the gate reads that as 'some call takes it'. These are the per-call spellings: the five insert_options members on the four calls that model an INSERT of a body (row, rows, parse_block, and Filter.rows, which also takes the columns list), param_encoding and filter_options_extra on compile_filter, posture and schema_options_extra on compile_table. Keyword-only, as every Python call option is.", + "issue": 511 + }, + { + "binding": "python", + "kind": "extra", + "names": [ + "ColumnDefault", + "ColumnDefault.name", + "ColumnDefault.value", + "VerdictBasis", + "VerdictBasis.INPUT", + "VerdictBasis.EXPORT", + "DeferDefaults", + "DeferDefaults.NONE", + "DeferDefaults.SERVER", + "ParamEncoding", + "ParamEncoding.ESCAPED", + "ParamEncoding.RAW" + ], + "reason": "The option value types of ABI v2's options documents (bindings-v1.md §2): the hand-written enums VerdictBasis, DeferDefaults and ParamEncoding (Go's VerdictBasis, DeferDefaults and ParamEncoding, spelled as StrEnum members) and the ColumnDefault record of a column_defaults entry. The call options table names the values (`WithVerdictBasis(VerdictBasis)`) and not the Python types; the v2 surface document lands with the other dev bindings (public issue #511), and this entry goes with it.", + "issue": 511 } ] } From 972c97e02bd121a5166eb1c5e1efe3392da84e94 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:31:50 -0400 Subject: [PATCH 15/24] rust: goldens runner passes the new options and columns arguments Co-Authored-By: Claude Sonnet 5.5 --- rust/tests/goldens_v1_runner.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rust/tests/goldens_v1_runner.rs b/rust/tests/goldens_v1_runner.rs index 7e850ada..008e9dab 100644 --- a/rust/tests/goldens_v1_runner.rs +++ b/rust/tests/goldens_v1_runner.rs @@ -548,7 +548,7 @@ fn execute(api: &Arc, case: &Value) -> Result, St match call { "preview_row" => Ok( - match api.preview_row(&schema, format, &body, &settings, &columns) { + match api.preview_row(&schema, format, &body, &settings, &columns, &[]) { Ok(doc) => { let ok = decode::row(&doc).is_ok(); Ok((doc, None, ok)) @@ -575,6 +575,7 @@ fn execute(api: &Arc, case: &Value) -> Result, St None, export_format, doc_flags, + &[], ) { Ok((doc, export)) => { // The public call hands the export to the decoder as the payload. @@ -590,7 +591,7 @@ fn execute(api: &Arc, case: &Value) -> Result, St let expr = dec(&filter_in["expr_b64"], "expr_b64")?; let params = dec(&filter_in["query_params_b64"], "query_params_b64")?; let filter_settings = dec(&filter_in["settings_b64"], "filter settings_b64")?; - let filter = match api.filter_create(&schema, &expr, ¶ms, &filter_settings) { + let filter = match api.filter_create(&schema, &expr, ¶ms, &filter_settings, &[]) { Ok(f) => f, Err(error) => { return Ok(Err(Stopped { @@ -600,7 +601,7 @@ fn execute(api: &Arc, case: &Value) -> Result, St } }; Ok( - match api.filter_eval_body(&filter, format, &body, &settings) { + match api.filter_eval_body(&filter, format, &body, &settings, &columns, &[]) { Ok(doc) => { let ok = decode::filter_result(&doc).is_ok(); Ok((doc, None, ok)) From 0b585394aad44634ca366bb8c01e4c04d7b87d2f Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:36:47 -0400 Subject: [PATCH 16/24] examples(ts): follow the spec batch 2 surface (nullable spans, no row-level unsupportedSettings, reason) Co-Authored-By: Claude Sonnet 5.5 --- examples/ts/demo.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/ts/demo.ts b/examples/ts/demo.ts index a69f164f..606d4194 100644 --- a/examples/ts/demo.ts +++ b/examples/ts/demo.ts @@ -462,7 +462,6 @@ function section6(lib: Library): void { blank(); kv('unknownFields[]', JSON.stringify(r.unknownFields.map((b) => b.toString()))); - kv('unsupportedSettings[]', JSON.stringify(r.unsupportedSettings.map((b) => b.toString()))); note('unknown fields are reported, not judged: whether to 400 on them is gateway policy'); blank(); @@ -914,10 +913,10 @@ function section15(lib: Library): void { note('wire order = declared minus MATERIALIZED/ALIAS/EPHEMERAL, so these bytes are directly'); note("INSERT-able with no column list; DEFAULTs (seq=7, label='unknown') are already applied"); (batch.spans ?? []).forEach((span, i) => { - const line = span.len ? JSON.stringify(payload.subarray(span.off, span.off + span.len).toString()) : '(no bytes: row not accepted)'; - kv(` span[${i}] {off:${span.off} len:${span.len}}`, `${batch.rows[i]?.outcome} -> ${line}`); + const line = span?.len ? JSON.stringify(payload.subarray(span.off, span.off + span.len).toString()) : '(no bytes: row not exported)'; + kv(` span[${i}] ${span === null ? 'null' : `{off:${span.off} len:${span.len}}`}`, `${batch.rows[i]?.outcome} -> ${line}`); }); - note('spans are INDEX-ALIGNED with rows; concatenating the non-zero spans reproduces the payload'); + note('spans are INDEX-ALIGNED with rows, null for a row not exported; concatenating the non-null spans reproduces the payload'); blank(); const lean = schema.rows(Format.JSONEachRow, body, { settings, exportFormat, docFlags: 0 }); @@ -1306,7 +1305,7 @@ async function openOrNote(registry: Registry, line: string): Promise[0] { - return { status: 0, chCode: 0, chName: '', messageBytes: Buffer.alloc(0), column: Buffer.alloc(0) }; + return { status: 0, chCode: 0, chName: '', messageBytes: Buffer.alloc(0), column: Buffer.alloc(0), reason: '' }; } function errName(err: unknown): string { From eb1ffdffd2052af146784472e0879800915786b0 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:40:08 -0400 Subject: [PATCH 17/24] rust: stub goldens fixture follows the row document (code, no row unsupported_settings) Co-Authored-By: Claude Sonnet 5.5 --- rust/tests/goldens_v1_stub/goldens.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/tests/goldens_v1_stub/goldens.json b/rust/tests/goldens_v1_stub/goldens.json index 6614b8cb..a1c46b5a 100644 --- a/rust/tests/goldens_v1_stub/goldens.json +++ b/rust/tests/goldens_v1_stub/goldens.json @@ -68,12 +68,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 8 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -113,12 +113,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 6 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -188,12 +188,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 6 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -232,12 +232,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 9 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", @@ -284,12 +284,12 @@ "status": "CHS_OK", "document": { "outcome": "accepted", + "code": 0, "input_span": { "off": 0, "len": 8 }, "unknown_fields": [], - "unsupported_settings": [], "cols": [ { "name": "s", From 18c2a3bd131c093d52b95b37b08bb111683b0fa0 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:41:47 -0400 Subject: [PATCH 18/24] ts(test): at_merge expectations keep the partRow mapping under the row key Co-Authored-By: Claude Sonnet 5.5 --- ts/test/documents.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ts/test/documents.test.ts b/ts/test/documents.test.ts index 02e1a056..a79bc003 100644 --- a/ts/test/documents.test.ts +++ b/ts/test/documents.test.ts @@ -307,17 +307,17 @@ describe('the batch document: at_merge and the batch\'s own unsupported_settings inputRows: e.inputRows, })); expect(got).toEqual([ - { part_row: 0, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: undefined }, - { part_row: 1, reason: MergeReason.TtlColumnReset, known: true, column: [0x63], stored: [0x30], inputRows: undefined }, + { row: 0, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: undefined }, + { row: 1, reason: MergeReason.TtlColumnReset, known: true, column: [0x63], stored: [0x30], inputRows: undefined }, // stored absent: the column's DEFAULT is decided at the merge - { part_row: 2, reason: MergeReason.TtlColumnReset, known: true, column: [0x74], stored: undefined, inputRows: undefined }, + { row: 2, reason: MergeReason.TtlColumnReset, known: true, column: [0x74], stored: undefined, inputRows: undefined }, // both by their _b64 form - { part_row: 3, reason: MergeReason.TtlColumnReset, known: true, column: [0xff, 0x00], stored: [0xff], inputRows: undefined }, - { part_row: 4, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: [4, 7] }, + { row: 3, reason: MergeReason.TtlColumnReset, known: true, column: [0xff, 0x00], stored: [0xff], inputRows: undefined }, + { row: 4, reason: MergeReason.TtlDelete, known: true, column: undefined, stored: undefined, inputRows: [4, 7] }, // an unlisted reason is its unknown(n), kept verbatim, and the unknown members are ignored (r2, r3) - { part_row: 5, reason: 'a_reason_nobody_listed', known: false, column: undefined, stored: undefined, inputRows: undefined }, + { row: 5, reason: 'a_reason_nobody_listed', known: false, column: undefined, stored: undefined, inputRows: undefined }, // an empty stored is not an absent one - { part_row: 6, reason: MergeReason.TtlColumnReset, known: true, column: [0x65], stored: [], inputRows: [] }, + { row: 6, reason: MergeReason.TtlColumnReset, known: true, column: [0x65], stored: [], inputRows: [] }, ]); }); From 72c16e7d0e7966858cc8ddd464b7177a71136342 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:51:55 -0400 Subject: [PATCH 19/24] abi: the generator emits CODE_DECLINED for Python, TypeScript and Rust bindings-v1.md promised a declined-code constant in every binding; only Go's emitter generated one. Python: CODE_DECLINED in the generated vocab, exported from chtypes. TypeScript: CODE_DECLINED in vocab.gen.ts, exported. Rust: chtypes::CODE_DECLINED (crate root; the code module is the fetch-code table), and the doc row says so. The TS parity allow entry that carried the gap is gone. ABI v2 fingerprint unchanged. Co-Authored-By: Claude Sonnet 5.5 --- docs/reference/bindings-v1.md | 2 +- python/CHANGELOG.md | 1 + python/src/chtypes/__init__.py | 2 ++ python/src/chtypes/_abi2/_vocab.py | 4 ++++ rust/CHANGELOG.md | 1 + rust/src/abi2/vocab_gen.rs | 4 ++++ rust/src/lib.rs | 6 +++--- scripts/abi-v1/emit/python.py | 6 ++++++ scripts/abi-v1/emit/rust.py | 8 ++++++++ scripts/abi-v1/emit/ts.py | 7 +++++++ scripts/parity-surface-allow.json | 9 --------- ts/CHANGELOG.md | 1 + ts/src/abi2/vocab.gen.ts | 3 +++ ts/src/index.ts | 1 + 14 files changed, 42 insertions(+), 13 deletions(-) diff --git a/docs/reference/bindings-v1.md b/docs/reference/bindings-v1.md index 32e876ab..b56ab311 100644 --- a/docs/reference/bindings-v1.md +++ b/docs/reference/bindings-v1.md @@ -271,7 +271,7 @@ Every vocabulary is generated from the description into all four bindings, with | refusal sites | `refusal_site` | `RefusalSite`: `RefusedAtCall`, `RefusedAtBodyRead`, `RefusedAtRow`, `RefusedAtBodyStep` | `RefusalSite` | `RefusalSite` | `RefusalSite`, `#[non_exhaustive]` | ABI v2 dev. Where a batch, a row or a filter result's refusal sits (`refused_at`, §5). The spellings are the description's; each binding reads an underscore as a word boundary in its identifiers | | postures | `posture` | `Posture`: `PostureCreate`, `PostureLoad` | `Posture` | `Posture` | `Posture`, `#[non_exhaustive]` | ABI v2 dev. How a schema's statement was compiled (`schema_options`' `posture`, and a schema description's `posture`). The spellings are the description's | | default support | `default_support` | `DefaultSupport`: `DefaultSupportComputed`, `DefaultSupportGenerated`, `DefaultSupportDeferred`, `DefaultSupportDeclined` | `DefaultSupport` | `DefaultSupport` | `DefaultSupport`, `#[non_exhaustive]` | ABI v2 dev, reserved. What a build does with a column's DEFAULT (`default_support`, §5). The spellings are the description's | -| the decline code | `CHS_CODE_DECLINED` | `CodeDeclined` (`int32`, -2) | `CODE_DECLINED` | `CODE_DECLINED` | `code::DECLINED` (`i32`) | ABI v2 dev. The `code` of a declined row, outcome or verdict, outside ClickHouse's code table; 0 is written only on a success | +| the decline code | `CHS_CODE_DECLINED` | `CodeDeclined` (`int32`, -2) | `CODE_DECLINED` | `CODE_DECLINED` | `CODE_DECLINED` (`i32`, crate root) | ABI v2 dev. The `code` of a declined row, outcome or verdict, outside ClickHouse's code table; 0 is written only on a success | | discover query parameters | `discover_query_param` | `DiscoverQueryParam`: `QueryParamDatabase`, `QueryParamTable` | `DiscoverQueryParam` | `DiscoverQueryParam` | `DiscoverQueryParam`, `#[non_exhaustive]` | ABI v2 dev. The discovery query's two parameters (§2, the discovery query), which a caller binds when it runs the query against its server. The spellings are the description's | | default kinds | `default_kind` | `DefaultKind`: `KindNone`, `KindDefault`, `KindMaterialized`, `KindAlias`, `KindEphemeral` | `DefaultKind` | `DefaultKind` | `DefaultKind` | FIRM; joins the description in the regeneration that freezes the fingerprint (§8 question 3). The spellings are v0's | | document groups | `CHS_DOC_*` | `DocFlags`: `DocValues`, `DocTransforms`, `DocDefaults`, `DocAll` | `DocFlags` | `DocFlags` | `DocFlags` | FIRM | diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index cac8513b..974a8463 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `CODE_DECLINED` (ABI v2, -2): the code a declined row, outcome or verdict carries, generated from the description and exported from `chtypes`, as `bindings-v1.md` section 3 promises. - `CallError.reason` (ABI v2): the error's catalog id, read through `chs_error_reason` and passed through verbatim (ASCII, empty if none), on all four call classes. - Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): keyword-only `insertable_columns`, `column_defaults` (a sequence of `ColumnDefault`), `verdict_basis` (`VerdictBasis`: `INPUT`, `EXPORT`) and `defer_defaults` (`DeferDefaults`: `NONE`, `SERVER`) build `insert_options` for `Schema.row`, `Schema.rows`, `Schema.parse_block` and `Filter.rows`; `param_encoding` (`ParamEncoding`: `ESCAPED`, `RAW`) builds `filter_options` for `Schema.compile_filter`; `posture` (`Posture`: `CREATE`, `LOAD`) builds `schema_options` for `Library.compile_table`. `insert_options_extra`, `filter_options_extra` and `schema_options_extra` pass a member the binding has no name for through as JSON text, so the library, which refuses an unknown key (rule r1), is the only judge. A document with no member set is not sent. `Filter.rows` takes `columns` (`chs_filter_eval_body` takes the column list). - Spec batch 2, a value that is not UTF-8 (ABI v2): a settings value, a query-parameter value, a `setup` default and a server profile's settings value is sent as `{"b64": ...}` (the content `json_object_byte_values`), where the binding used to refuse it, and `SchemaServer.settings` decodes the same form back (as `surrogateescape` text, the spelling the input side takes). diff --git a/python/src/chtypes/__init__.py b/python/src/chtypes/__init__.py index 197fc2e4..48a02c21 100644 --- a/python/src/chtypes/__init__.py +++ b/python/src/chtypes/__init__.py @@ -47,6 +47,7 @@ from importlib.metadata import version as _package_version from ._abi2._vocab import ( + CODE_DECLINED, EXPORT_NONE, DeclinedLayer, DeclinedTier, @@ -149,6 +150,7 @@ "CODE_SOURCE_RETIRED", "CODE_SOURCE_UNAUTHORIZED", "CODE_SOURCE_UNREACHABLE", + "CODE_DECLINED", "EXPORT_NONE", "ArtifactCorruptError", "ArtifactError", diff --git a/python/src/chtypes/_abi2/_vocab.py b/python/src/chtypes/_abi2/_vocab.py index 1c3dee2a..cab9a22f 100644 --- a/python/src/chtypes/_abi2/_vocab.py +++ b/python/src/chtypes/_abi2/_vocab.py @@ -531,6 +531,10 @@ class DocFlags(IntFlag): # `chs_preview_batch`'s `export_format` for no export (`CHS_EXPORT_NONE`). EXPORT_NONE: Final = -1 +# The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside +# ClickHouse's code table. +CODE_DECLINED: Final = -2 + # Every enum the description defines, by its own name, and the Python type that # spells it: rule r3's tests reach every one from here, so a new enum is covered # without a hand-kept list. diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index fc543507..29b60a32 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `chtypes::CODE_DECLINED` (ABI v2, `i32`, -2): the code a declined row, outcome or verdict carries, generated from the description. It is at the crate root, not in `code`, which is the fetch-code table. - A fetch-only call (#492): `Registry::fetch(&self, request: &str) -> Result` resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing: no library is loaded and the process setup is untouched, so it needs no `setup`. It honors the registry's fetch options (offline, the cache, and under the production channel the bases, the trust and the lock), whatever `autofetch` says, and a concurrent `fetch` and open of one request share one fetch. The `Resolved` names the installed version, build, digests and library path; errors are the fetch codes, and a refused spelling or pin is `Error::Usage`, as for an open. - `chtypes resolve [--json]` (#493): what a line or version resolves to on every platform the registry offers, one `resolved ` line each, or with `--json` one JSON array of `{platform, version, build, manifest}` objects, each signed statement verified exactly as a fetch verifies it and no layer downloaded; `--offline` answers from the cache. The four CLIs print the same bytes, so one parser reads them (a CI cache key, for example). - `chtypes prune [--line ] [--keep ] [--dry-run]` (#494): removes the installed builds of each line and platform that at least n newer installed builds supersede (n defaults to 1), each with its record, its blobs and its `index.json` entry; `--dry-run` prints exactly what would go. It never removes a build a running process holds: a `Registry` now holds every build it opens or fetches, with a shared `flock` on the entry's `verified.json` until the process exits, and prune removes an entry only after taking the exclusive lock without waiting, reporting it `in-use` otherwise. It never touches a system directory, nor, on the dev channel, another fingerprint's builds. The rules are `docs/guides/fetch-v1.md` sections 1 and 3, and the `resolve-build-*` and `prune-*` conformance cases hold all four bindings to them. The crate now depends on `libc` directly, for `flock`. diff --git a/rust/src/abi2/vocab_gen.rs b/rust/src/abi2/vocab_gen.rs index 75812e44..3603d5fa 100644 --- a/rust/src/abi2/vocab_gen.rs +++ b/rust/src/abi2/vocab_gen.rs @@ -1279,6 +1279,10 @@ pub(crate) const BYTES_SUFFIX: &str = "_b64"; /// value, beside its rendering. pub(crate) const VALUE_BYTES: &str = "value_b64"; +/// The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside +/// ClickHouse's code table. +pub const CODE_DECLINED: i32 = -2; + /// Every enum the description defines, in its own order: rule r3's tests reach /// each one through [`described_vocabulary`], so a new enum is covered without /// a hand-kept list. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 3a85184c..bb9866d8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -118,9 +118,9 @@ mod server; mod setup; pub use abi2::vocab_gen::{ - DeclinedLayer, DeclinedTier, DefaultKind, DefaultSupport, DiscoverQueryParam, DocFlags, - FilterOutcome, Format, MergeReason, Outcome, Posture, Reason, RefusalSite, Source, Status, - Verdict, reason, source, status, + CODE_DECLINED, DeclinedLayer, DeclinedTier, DefaultKind, DefaultSupport, DiscoverQueryParam, + DocFlags, FilterOutcome, Format, MergeReason, Outcome, Posture, Reason, RefusalSite, Source, + Status, Verdict, reason, source, status, }; pub use error::{CacheFault, CallError, Error, Refusal, Result}; pub use library::Library; diff --git a/scripts/abi-v1/emit/python.py b/scripts/abi-v1/emit/python.py index e5b8d390..a8268601 100644 --- a/scripts/abi-v1/emit/python.py +++ b/scripts/abi-v1/emit/python.py @@ -1475,6 +1475,12 @@ def render_vocab_v2(model: Model) -> str: f"# `chs_preview_batch`'s `export_format` for no export (`CHS_EXPORT_NONE`).\nEXPORT_NONE: Final = {c['CHS_EXPORT_NONE'].value}" ) parts.append("") + if "CHS_CODE_DECLINED" in c: + parts.append( + "# The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside\n" + f"# ClickHouse's code table.\nCODE_DECLINED: Final = {c['CHS_CODE_DECLINED'].value}" + ) + parts.append("") parts.append("# Every enum the description defines, by its own name, and the Python type that") parts.append("# spells it: rule r3's tests reach every one from here, so a new enum is covered") parts.append("# without a hand-kept list.") diff --git a/scripts/abi-v1/emit/rust.py b/scripts/abi-v1/emit/rust.py index a1c88572..383fde95 100644 --- a/scripts/abi-v1/emit/rust.py +++ b/scripts/abi-v1/emit/rust.py @@ -1999,6 +1999,14 @@ def render_vocab_v2(model) -> str: f"pub(crate) const VALUE_BYTES: &str = {_rust_str_any(model.byte_strings['value'])};", "", ] + declined = model.constants.get("CHS_CODE_DECLINED") + if declined is not None: + out += [ + "/// The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside", + "/// ClickHouse's code table.", + f"pub const CODE_DECLINED: i32 = {declined.value};", + "", + ] out += _vocab_v2_described(model) return "\n".join(out) diff --git a/scripts/abi-v1/emit/ts.py b/scripts/abi-v1/emit/ts.py index fcb59ccb..65e983ff 100644 --- a/scripts/abi-v1/emit/ts.py +++ b/scripts/abi-v1/emit/ts.py @@ -985,6 +985,13 @@ def render_vocab_v2(model) -> str: f"export const EXPORT_NONE = {export_none.value};", "", ] + declined = model.constants.get("CHS_CODE_DECLINED") + if declined is not None: + lines += [ + "/** The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside ClickHouse's code table. */", + f"export const CODE_DECLINED = {declined.value};", + "", + ] lines += _render_described(model) return "\n".join(lines) diff --git a/scripts/parity-surface-allow.json b/scripts/parity-surface-allow.json index e85d9e5d..db3263f5 100644 --- a/scripts/parity-surface-allow.json +++ b/scripts/parity-surface-allow.json @@ -338,15 +338,6 @@ "reason": "§2's options-documents table names the three option fields that take a closed set (verdictBasis, deferDefaults, paramEncoding) and the column_defaults entry, and Go gives each set a named type (VerdictBasis, DeferDefaults, ParamEncoding, ColumnDefault), which the doc's TypeScript column does not spell. TypeScript spells the three sets as const objects with a type of the same name (the language's idiom for a vocabulary, as Format is), ColumnDefault as the interface of an entry, and SettingValue as the type of one settings or query-parameter value (a string, or the bytes of a value that is not UTF-8, which is sent in the {\"b64\": ...} object form). The ABI v2 dev surface document lands with the other dev bindings (public issue #511), and this entry goes with it.", "issue": 511 }, - { - "binding": "ts", - "kind": "missing", - "names": [ - "CODE_DECLINED" - ], - "reason": "§3's 'the decline code' row names CHS_CODE_DECLINED (-2) as CODE_DECLINED in TypeScript. The description's constants are emitted into the vocabulary file by the generator, which does so for Go only (CodeDeclined), so TypeScript has no constant yet and a caller compares a result's errCode with -2. The fix is a generator change (scripts/abi-v1/emit), not a hand-written declaration, and this entry goes with it.", - "issue": 511 - }, { "binding": "ts", "kind": "extra", diff --git a/ts/CHANGELOG.md b/ts/CHANGELOG.md index 8f69a709..ea8c667b 100644 --- a/ts/CHANGELOG.md +++ b/ts/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `CODE_DECLINED` (ABI v2, -2): the code a declined row, outcome or verdict carries, generated from the description and exported from the package, as `bindings-v1.md` section 3 promises. - `registry.fetch(request): Promise`, the fetch-only call (#492): it resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing (no library is loaded, the process setup is untouched, nothing appears in `libraries()`). It honors the registry's fetch options, its errors are the fetch codes (a refused spelling is a `UsageError`, as for `for`), and it shares one fetch with a concurrent open or `fetch` of the same request: an open that misses the cache now waits on the same fetch. - `chtypes resolve [--json] [--offline]` (#493): the version, build and platform manifest digest the spelling resolves to on every platform the registry offers, one `resolved ` line each in platform order (`--json`: one compact JSON array of `{platform, version, build, manifest}`), each verified exactly as a fetch verifies it, with no layer downloaded and nothing written. `--offline` answers from the cache. A refused spelling (a `v` prefix, a channel suffix) exits with the usage status. - `chtypes prune [--line ] [--keep ] [--dry-run]`, and the in-use hold (#494): prune removes from the cache the installed builds of each line that newer installed builds of the same line and platform supersede, keeping the newest `n` (default 1), with their `index.json` entries and every blob no remaining build names. It never touches a system directory or another fingerprint's build, and never removes a build a live process holds: every build a registry opens or fetches is held by that process until it exits, by a shared `flock` on the entry's `verified.json` (prune takes the exclusive lock without waiting, and reports a held build `in-use`). diff --git a/ts/src/abi2/vocab.gen.ts b/ts/src/abi2/vocab.gen.ts index 7e75b5dc..c070296b 100644 --- a/ts/src/abi2/vocab.gen.ts +++ b/ts/src/abi2/vocab.gen.ts @@ -594,6 +594,9 @@ export type DocFlags = number; /** The `export_format` of a batch preview that exports nothing. */ export const EXPORT_NONE = -1; +/** The code a declined row, outcome or verdict carries (`CHS_CODE_DECLINED`), outside ClickHouse's code table. */ +export const CODE_DECLINED = -2; + /** One described enum, as the rule r3 tests reach it: its listed values, `known` and `of`. */ export type DescribedVocabulary = | { readonly repr: 'int32'; readonly listed: readonly number[]; readonly known: (value: number) => boolean; readonly of: (value: number) => number } diff --git a/ts/src/index.ts b/ts/src/index.ts index 2aff3e3c..eac0f551 100644 --- a/ts/src/index.ts +++ b/ts/src/index.ts @@ -31,6 +31,7 @@ export { defaultSupportKnown, DiscoverQueryParam, discoverQueryParamKnown, + CODE_DECLINED, DocFlags, EXPORT_NONE, FilterOutcome, From 14dd039208436cd945010a86624cb770d06f4445 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:51:55 -0400 Subject: [PATCH 20/24] parity-surface: read the description of the major the bindings speak DESCRIPTION was hard-coded to spec/abi-v1/abi.json, so every v2 vocabulary growth read as drift (Source: 13 members vs 11). It now follows spec/binding-majors.json (v1 where absent) and refuses mixed majors; selftest covers all three. Co-Authored-By: Claude Sonnet 5.5 --- scripts/parity-surface.py | 46 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/scripts/parity-surface.py b/scripts/parity-surface.py index 4fa5000d..8d51ab53 100755 --- a/scripts/parity-surface.py +++ b/scripts/parity-surface.py @@ -30,7 +30,9 @@ where its call options live. Three things the doc delegates are completed from where it says they live, never from a list kept here: - - a vocabulary's members are the description's (spec/abi-v1/abi.json): each + - a vocabulary's members are the description's (spec/abi-v/abi.json, N the + major spec/binding-majors.json gives the bindings checked; v1 where that + file is absent): each binding's members must number exactly the description's values and agree with the other three bindings, compared without case or underscores; - "one class per code" (§4) is expanded over the codes the row names, with @@ -111,7 +113,25 @@ ROOT = Path(__file__).resolve().parent.parent DOC = ROOT / "docs" / "reference" / "bindings-v1.md" -DESCRIPTION = ROOT / "spec" / "abi-v1" / "abi.json" + + +def description_path(only: Iterable[str], root: Path = ROOT) -> Path: + """The description of the ABI major the bindings checked speak: the + major spec/binding-majors.json gives them (every binding speaks 1 where + the file is absent, as on main). Bindings on different majors cannot share + one vocabulary count, so that is refused rather than checked against the + wrong description.""" + f = root / "spec" / "binding-majors.json" + majors = json.loads(f.read_text(encoding="utf-8")) if f.exists() else {} + wanted = {majors.get(b, 1) for b in only} + if len(wanted) != 1: + raise SystemExit( + f"parity-surface: the bindings checked speak different ABI majors ({sorted(wanted)}); " + "check them one major at a time" + ) + return root / "spec" / f"abi-v{wanted.pop()}" / "abi.json" + + FETCH_DOC = ROOT / "docs" / "guides" / "fetch-v1.md" ALLOWLIST = ROOT / "scripts" / "parity-surface-allow.json" FIXTURES = ROOT / "tests" / "fixtures" / "parity-surface" @@ -1989,6 +2009,26 @@ def merge_surfaces(go_reset: str) -> dict[str, dict[str, str]]: if not any(f.binding == "ts" for f in compare(exp, surfaces, without, description).findings): failures.append("ts: removing its allowlisted spelling difference did not fail") + # The description follows the major the bindings speak: absent file -> v1, + # a file -> that major, mixed majors refused. Driven through description_path itself. + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + if description_path(("go",), root) != root / "spec" / "abi-v1" / "abi.json": + failures.append("with no binding-majors.json the description was not v1's") + (root / "spec").mkdir() + (root / "spec" / "binding-majors.json").write_text( + '{"go": 2, "python": 2, "rust": 1, "ts": 2}', encoding="utf-8" + ) + if description_path(("go", "python"), root) != root / "spec" / "abi-v2" / "abi.json": + failures.append("bindings on v2 did not read v2's description") + if description_path(("rust",), root) != root / "spec" / "abi-v1" / "abi.json": + failures.append("a binding on v1 did not read v1's description") + try: + description_path(("go", "rust"), root) + failures.append("bindings on different majors were not refused") + except SystemExit: + pass + # The allowlist's `issue` field and --check-issues, offline: the lookup is injected. row = '{"binding": "go", "kind": "extra", "names": ["X"], "reason": "%s", "issue": %s}' why = "a reason long enough to be accepted here" @@ -2202,7 +2242,7 @@ def cmd_run(tools: Path, only: tuple[str, ...], save: Path | None, proofs_only: return 1 if any(proof.values()) else 0 exp = expected_surface() allow, problems = load_allowlist(ALLOWLIST) - description = json.loads(DESCRIPTION.read_text(encoding="utf-8")) + description = json.loads(description_path(only).read_text(encoding="utf-8")) for p in problems: print(f"parity-surface: allowlist: {p}", file=sys.stderr) surfaces: dict[str, dict[str, str]] = {} From d7630b8f0df8176850e354a3dbc7b53b9ce6f4d5 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 16:51:56 -0400 Subject: [PATCH 21/24] docs: settings.md describes the b64 form for a non-UTF-8 value Co-Authored-By: Claude Sonnet 5.5 --- docs/guides/settings.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/guides/settings.md b/docs/guides/settings.md index 0cbd44f2..18774227 100644 --- a/docs/guides/settings.md +++ b/docs/guides/settings.md @@ -98,7 +98,16 @@ This has cost real bugs, so v1 makes it one rule in all four bindings: **a setti The reason is one setting in particular. `chtypes_now_epoch_nanos` is a 19-digit nanosecond epoch, and 19 digits do not survive an IEEE double: `1700000000123456789` becomes `1.7e+18`. Sent as a JSON number, the setting would be **silently ignored** — the batch keeps stamping the real wall clock, and nothing anywhere says so. A loud `TypeError` is the whole point. -A value whose bytes are not valid UTF-8 cannot be passed as a setting or a query parameter in 1.0. Inline it in the SQL as `unhex('')`, built from hex digits only ([`limitations.md`](../limitations.md#binary-input-parameters-must-be-utf-8)). +A value whose bytes are not valid UTF-8 is sent in the object form `{"b64": ""}` (ABI v2): a setting value, a query-parameter value, a `Setup` default and a value of a server profile's `settings`. The binding writes it in place of the string, so the library sees the exact bytes; a value that is valid UTF-8 stays a plain string. Three of the bindings can send one: + +| Binding | How a non-UTF-8 value is passed | +| ---------- | -------------------------------------------------------------------------------------------------- | +| Go | a `string` holding the bytes (a Go `string` is a byte string) | +| Python | a `str` carrying the bytes with `surrogateescape` | +| TypeScript | a `Uint8Array` (a `string` holding a lone surrogate has no bytes and is refused, as a key is) | +| Rust | **cannot**: settings and query parameters take `String`, so a non-UTF-8 value is unreachable there | + +A key that is not valid UTF-8 cannot be carried in any binding (a JSON object has no byte form for a key), and neither can a macro's name or value. Each is refused before any call. Where a binding cannot send the bytes, inline them in the SQL as `unhex('')`, built from hex digits only ([`limitations.md`](../limitations.md#binary-input-parameters-must-be-utf-8)). ## An unknown name refuses the whole call From 6892d4d1a21fe74cc52e05763ca783292b86edab Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Mon, 19 Oct 2026 17:10:04 -0400 Subject: [PATCH 22/24] ts test: the fetch-code census skips the numeric CODE_DECLINED Co-Authored-By: Claude Sonnet 5.5 --- ts/test/ocifetch/errors.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ts/test/ocifetch/errors.test.ts b/ts/test/ocifetch/errors.test.ts index 94856254..aac18c01 100644 --- a/ts/test/ocifetch/errors.test.ts +++ b/ts/test/ocifetch/errors.test.ts @@ -80,7 +80,8 @@ describe('the artifact error carries a loader refusal\'s fields, as in Go and Py it('exports one generated constant per code, and no exit status and no isChtypesError', () => { const codes = Object.entries(publicApi) - .filter(([name]) => name.startsWith('CODE_')) + // CODE_DECLINED is the ABI's numeric decline code, not a CHTYPES_* fetch code. + .filter(([name, value]) => name.startsWith('CODE_') && typeof value === 'string') .map(([, value]) => value) .sort(); expect(codes).toEqual(Object.keys(ERROR_EXIT_CODES).sort()); From 1b2a53270016610f7c9518e072894794181a2ba7 Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Tue, 20 Oct 2026 14:03:18 -0400 Subject: [PATCH 23/24] test+docs: column_defaults wire form and default_override decode, in all four bindings Unit tests assert the exact insert_options JSON (UTF-8, value_b64/name_b64, empty value, order) and the default_override decode; the batches guide gains a usage section; CHANGELOG bullets. No change under spec/abi-v2 or include/v2. Co-Authored-By: Claude Sonnet 5.5 --- docs/guides/batches.md | 51 +++++++++++++++++++++++++++++++++ go/CHANGELOG.md | 1 + go/chtypes/decode_test.go | 18 ++++++++++++ go/chtypes/settings_test.go | 21 ++++++++++++++ python/CHANGELOG.md | 1 + python/tests/api/test_decode.py | 19 ++++++++++++ python/tests/api/test_input.py | 23 +++++++++++++++ rust/CHANGELOG.md | 1 + rust/src/decode.rs | 13 +++++++++ rust/src/options.rs | 40 ++++++++++++++++++++++++++ ts/CHANGELOG.md | 1 + ts/test/settings-setup.test.ts | 13 +++++++++ 12 files changed, 202 insertions(+) diff --git a/docs/guides/batches.md b/docs/guides/batches.md index 8e99130f..9421c4a2 100644 --- a/docs/guides/batches.md +++ b/docs/guides/batches.md @@ -291,6 +291,57 @@ Three payload states, and they are distinct: absent means no export was requeste Attaching a compiled filter to this same call narrows the export further, to the rows the filter admits, from the same one parse: [`filters.md` → Exporting only the rows a filter admits](filters.md#exporting-only-the-rows-a-filter-admits). +## Overriding a column's DEFAULT for one call (ABI v2) + +`column_defaults` is a per-call list of `{name, value}`. In a cell the server would have filled from the column's DEFAULT, the call uses your `value` instead. The library decides which cells those are, by ClickHouse's own reader and the call's format settings: a field the record omits, a `null` for a non-Nullable column under `input_format_null_as_default`, an empty CSV field under `input_format_csv_empty_as_default`. A field the record supplied is never overridden, and `""` is a value like any other. A cell an override fills reads `default_override` (`Source` in each binding), which is stored, and the export carries it as a present value, so inserting the export needs no override. + +The binding only passes the list: `name` and `value` are byte strings (UTF-8, else the `name_b64` and `value_b64` forms), the order is kept, and an empty list is sent as `[]`. It validates nothing. An entry that names an unknown, non-insertable, MATERIALIZED or ALIAS column, or a value that does not read as the column's type, is `CHS_INVALID_ARGUMENT` at the call, naming `column_defaults[i]`. A build that does not implement the member declines it. The option exists on the four calls that model an INSERT of a body: row, rows, parse a block and filter rows. + +
Go + +```go +batch, err := schema.Rows(chtypes.JSONEachRow, body, + chtypes.WithColumnDefaults([]chtypes.ColumnDefault{{Name: "region", Value: "eu"}})) +// a cell filled from the override: batch.Rows[i].Columns[j].Source == chtypes.SourceDefaultOverride +``` + +
+ +
Python + +```python +batch = schema.rows( + Format.JSON_EACH_ROW, body, column_defaults=[ColumnDefault("region", "eu")] +) +# a cell filled from the override has source == Source.DEFAULT_OVERRIDE +``` + +
+ +
TypeScript + +```ts +const batch = schema.rows(Format.JSONEachRow, body, { + columnDefaults: [{ name: 'region', value: 'eu' }], +}); +// a cell filled from the override has source Source.DefaultOverride +``` + +
+ +
Rust + +```rust +let options = RowsOptions { + column_defaults: Some(vec![ColumnDefault { name: b"region".to_vec(), value: b"eu".to_vec() }]), + ..Default::default() +}; +let batch = schema.rows(Format::JsonEachRow, body, &options)?; +// a cell filled from the override has source Source::DefaultOverride +``` + +
+ ## Next - [`transformations.md`](transformations.md) — what the accepted rows were silently changed into. diff --git a/go/CHANGELOG.md b/go/CHANGELOG.md index 704c41c9..47de7371 100644 --- a/go/CHANGELOG.md +++ b/go/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `column_defaults` is covered by unit tests (ABI v2): the exact `insert_options` JSON for a UTF-8 value, a non-UTF-8 value (`value_b64`), an empty value (sent as `""`) and two entries in order, and the decode of a `default_override` cell (`SourceDefaultOverride`, stored). The batches guide gains a usage section. No behavior changed. - `CallError.Reason` (ABI v2): the error's catalog id, read through `chs_error_reason` by the generated layer and passed through verbatim (ASCII, empty if none), on all four call classes. - A fetch-only call (#492): `(*Registry).Fetch(ctx, request) (Resolved, error)` resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing: no library is loaded and the process setup is untouched, so it needs no `Setup`. It honors the registry's fetch options (offline, the cache, and under the production channel the bases, the trust and the lock), and a concurrent `Fetch` and open of one request share one fetch. The `Resolved` names the installed version, build, digests and library path; errors are the fetch codes, and a done `ctx` is `ctx.Err()`. - `chtypes resolve [--json]` (#493): what a line or version resolves to on every platform the registry offers, one `resolved ` line each, or with `--json` one JSON array of `{platform, version, build, manifest}` objects, each signed statement verified exactly as a fetch verifies it and no layer downloaded; `--offline` answers from the cache. The four CLIs print the same bytes, so one parser reads them (a CI cache key, for example). diff --git a/go/chtypes/decode_test.go b/go/chtypes/decode_test.go index 718d803f..bd46d398 100644 --- a/go/chtypes/decode_test.go +++ b/go/chtypes/decode_test.go @@ -37,6 +37,24 @@ const rowDoc = `{ "partition_id": "all" }` +func TestDecodeRowDefaultOverride(t *testing.T) { + r, err := decodeRow([]byte(`{"outcome":"accepted","code":0,"cols":[ + {"name":"a","null":false,"src":"default_override","stored":"5"}, + {"name":"b","null":false,"src":"default_deferred","stored":""}]}`)) + if err != nil { + t.Fatal(err) + } + if v := r.Columns[0]; v.Source != SourceDefaultOverride || !v.IsStored || v.Text != "5" { + t.Errorf("default_override entry = %+v, want stored with text 5", v) + } + if r.Columns[1].IsStored { + t.Errorf("default_deferred read as stored") + } + if len(r.Values) != 1 || r.Values[0].Column != "a" { + t.Errorf("Values = %+v, want only the override cell", r.Values) + } +} + func TestDecodeRow(t *testing.T) { r, err := decodeRow([]byte(rowDoc)) if err != nil { diff --git a/go/chtypes/settings_test.go b/go/chtypes/settings_test.go index 780dbbc6..48f53641 100644 --- a/go/chtypes/settings_test.go +++ b/go/chtypes/settings_test.go @@ -296,6 +296,27 @@ func TestAFailedOpenUnlocksOnlyItsOwnRecordAndNeverALatchedOne(t *testing.T) { } } +func TestColumnDefaultsDocument(t *testing.T) { + cases := []struct { + name string + in []ColumnDefault + want string + }{ + {"utf8", []ColumnDefault{{Name: "a", Value: "héllo"}}, `{"column_defaults":[{"name":"a","value":"héllo"}]}`}, + {"non-utf8 value is value_b64", []ColumnDefault{{Name: "a", Value: "\xff\x00"}}, `{"column_defaults":[{"name":"a","value_b64":"/wA="}]}`}, + {"non-utf8 name is name_b64", []ColumnDefault{{Name: "\xfe", Value: "v"}}, `{"column_defaults":[{"name_b64":"/g==","value":"v"}]}`}, + {"empty value is sent", []ColumnDefault{{Name: "a", Value: ""}}, `{"column_defaults":[{"name":"a","value":""}]}`}, + {"order is kept", []ColumnDefault{{Name: "z", Value: "1"}, {Name: "a", Value: "2"}}, `{"column_defaults":[{"name":"z","value":"1"},{"name":"a","value":"2"}]}`}, + {"empty list is sent", []ColumnDefault{}, `{"column_defaults":[]}`}, + } + for _, tc := range cases { + b, err := rowConfig([]RowOption{WithColumnDefaults(tc.in)}).insertOpts.json("insert options") + if err != nil || string(b) != tc.want { + t.Errorf("%s: got %s, %v, want %s", tc.name, b, err, tc.want) + } + } +} + func TestInsertOptionsDocument(t *testing.T) { // No member set: no document (length 0). if b, err := rowConfig(nil).insertOpts.json("insert options"); err != nil || b != nil { diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 7bc06b56..2128fdda 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `column_defaults` is covered by unit tests (ABI v2): the exact `insert_options` JSON for a UTF-8 value, a non-UTF-8 value (`value_b64`), an empty value (sent as `""`) and two entries in order, and the decode of a `default_override` cell (`Source.DEFAULT_OVERRIDE`, stored). The batches guide gains a usage section. No behavior changed. - `CODE_DECLINED` (ABI v2, -2): the code a declined row, outcome or verdict carries, generated from the description and exported from `chtypes`, as `bindings-v1.md` section 3 promises. - `CallError.reason` (ABI v2): the error's catalog id, read through `chs_error_reason` and passed through verbatim (ASCII, empty if none), on all four call classes. - Spec batch 2, the options documents (ABI v2; moves the dev fingerprint): keyword-only `insertable_columns`, `column_defaults` (a sequence of `ColumnDefault`), `verdict_basis` (`VerdictBasis`: `INPUT`, `EXPORT`) and `defer_defaults` (`DeferDefaults`: `NONE`, `SERVER`) build `insert_options` for `Schema.row`, `Schema.rows`, `Schema.parse_block` and `Filter.rows`; `param_encoding` (`ParamEncoding`: `ESCAPED`, `RAW`) builds `filter_options` for `Schema.compile_filter`; `posture` (`Posture`: `CREATE`, `LOAD`) builds `schema_options` for `Library.compile_table`. `insert_options_extra`, `filter_options_extra` and `schema_options_extra` pass a member the binding has no name for through as JSON text, so the library, which refuses an unknown key (rule r1), is the only judge. A document with no member set is not sent. `Filter.rows` takes `columns` (`chs_filter_eval_body` takes the column list). diff --git a/python/tests/api/test_decode.py b/python/tests/api/test_decode.py index 8c739c29..17d3a618 100644 --- a/python/tests/api/test_decode.py +++ b/python/tests/api/test_decode.py @@ -603,3 +603,22 @@ def test_a_schema_description_decodes_posture_default_support_reads_and_b64_sett def test_a_bad_server_settings_value_is_an_internal_error(settings) -> None: with pytest.raises(InternalError): decode_schema_description(dumps({"server": {"settings": settings}})) + + +def test_default_override_decodes_as_a_stored_value_source() -> None: + row = decode_row_document( + dumps( + { + "outcome": "accepted", + "code": 0, + "cols": [ + {"name": "a", "null": False, "src": "default_override", "stored": "5"}, + {"name": "b", "null": False, "src": "default_deferred", "stored": ""}, + ], + } + ) + ) + assert row.columns[0].source == "default_override" and row.columns[0].is_stored is True + assert row.columns[0].text == b"5" + assert row.columns[1].is_stored is False + assert [c.column for c in row.values] == [b"a"] diff --git a/python/tests/api/test_input.py b/python/tests/api/test_input.py index ca85e04f..021ad537 100644 --- a/python/tests/api/test_input.py +++ b/python/tests/api/test_input.py @@ -213,3 +213,26 @@ def test_valid_non_ascii_serializes_byte_for_byte_as_before() -> None: assert settings_json({"é": "\ufffd"}, "Asia/Tokyo") == ( b'{"\\u00e9":"\\ufffd","session_timezone":"Asia/Tokyo"}' ) + + +def test_column_defaults_wire_form_utf8_b64_empty_and_order() -> None: + def doc(*entries: ColumnDefault) -> dict[str, object]: + return json.loads(insert_options_json(column_defaults=list(entries)) or b"") + + assert doc(ColumnDefault("a", "h\u00e9llo")) == { + "column_defaults": [{"name": "a", "value": "h\u00e9llo"}] + } + assert doc(ColumnDefault("a", b"\xff\x00")) == { + "column_defaults": [{"name": "a", "value_b64": "/wA="}] + } + assert doc(ColumnDefault(b"\xfe", "v")) == { + "column_defaults": [{"name_b64": "/g==", "value": "v"}] + } + # An empty value is a supplied value: sent as "", never omitted. + assert doc(ColumnDefault("a", "")) == {"column_defaults": [{"name": "a", "value": ""}]} + assert doc(ColumnDefault("a", b"")) == {"column_defaults": [{"name": "a", "value": ""}]} + # The list keeps its order (a JSON list compares in order). + raw = insert_options_json(column_defaults=[ColumnDefault("z", "1"), ColumnDefault("a", "2")]) + assert json.loads(raw or b"") == { + "column_defaults": [{"name": "z", "value": "1"}, {"name": "a", "value": "2"}] + } diff --git a/rust/CHANGELOG.md b/rust/CHANGELOG.md index 79e8f6ae..63c1f3f0 100644 --- a/rust/CHANGELOG.md +++ b/rust/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `column_defaults` is covered by unit tests (ABI v2): the exact `insert_options` JSON for a UTF-8 value, a non-UTF-8 value (`value_b64`), an empty value (sent as `""`) and two entries in order, and the decode of a `default_override` cell (`Source::DefaultOverride`, stored). The batches guide gains a usage section. No behavior changed. - `chtypes::CODE_DECLINED` (ABI v2, `i32`, -2): the code a declined row, outcome or verdict carries, generated from the description. It is at the crate root, not in `code`, which is the fetch-code table. - A fetch-only call (#492): `Registry::fetch(&self, request: &str) -> Result` resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing: no library is loaded and the process setup is untouched, so it needs no `setup`. It honors the registry's fetch options (offline, the cache, and under the production channel the bases, the trust and the lock), whatever `autofetch` says, and a concurrent `fetch` and open of one request share one fetch. The `Resolved` names the installed version, build, digests and library path; errors are the fetch codes, and a refused spelling or pin is `Error::Usage`, as for an open. - `chtypes resolve [--json]` (#493): what a line or version resolves to on every platform the registry offers, one `resolved ` line each, or with `--json` one JSON array of `{platform, version, build, manifest}` objects, each signed statement verified exactly as a fetch verifies it and no layer downloaded; `--offline` answers from the cache. The four CLIs print the same bytes, so one parser reads them (a CI cache key, for example). diff --git a/rust/src/decode.rs b/rust/src/decode.rs index 5e27cc63..20825203 100644 --- a/rust/src/decode.rs +++ b/rust/src/decode.rs @@ -972,6 +972,19 @@ mod tests { assert!(r.columns[2].is_stored && !r.columns[1].is_stored); } + #[test] + fn default_override_is_a_stored_source_and_default_deferred_is_not() { + let r = row(br#"{"outcome":"accepted","cols":[ + {"name":"a","null":false,"src":"default_override","stored":"5"}, + {"name":"b","null":false,"src":"default_deferred","stored":""}]}"#) + .unwrap(); + assert_eq!(r.columns[0].source, Source::DefaultOverride); + assert!(r.columns[0].is_stored && !r.columns[1].is_stored); + assert_eq!(r.columns[0].text.as_bytes(), b"5"); + assert_eq!(r.values.len(), 1); + assert_eq!(r.values[0].column.as_bytes(), b"a"); + } + /// Rule r3: an unlisted source is its `Unknown`, for that column alone, not /// stored, and the row decodes on; an absent one is still malformed. #[test] diff --git a/rust/src/options.rs b/rust/src/options.rs index 90feef61..cf4c1572 100644 --- a/rust/src/options.rs +++ b/rust/src/options.rs @@ -287,6 +287,46 @@ mod tests { ); } + #[test] + fn column_defaults_wire_form_utf8_b64_empty_and_order() { + let doc = |defaults: Vec| { + text( + InsertMembers { + column_defaults: Some(&defaults), + ..members() + } + .document() + .unwrap(), + ) + }; + let cd = |n: &[u8], v: &[u8]| ColumnDefault { + name: n.to_vec(), + value: v.to_vec(), + }; + assert_eq!( + doc(vec![cd(b"a", "h\u{e9}llo".as_bytes())]), + "{\"column_defaults\":[{\"name\":\"a\",\"value\":\"h\u{e9}llo\"}]}" + ); + assert_eq!( + doc(vec![cd(b"a", &[0xff, 0x00])]), + r#"{"column_defaults":[{"name":"a","value_b64":"/wA="}]}"# + ); + assert_eq!( + doc(vec![cd(&[0xfe], b"v")]), + r#"{"column_defaults":[{"name_b64":"/g==","value":"v"}]}"# + ); + // An empty value is a supplied value: sent as "", never omitted. + assert_eq!( + doc(vec![cd(b"a", b"")]), + r#"{"column_defaults":[{"name":"a","value":""}]}"# + ); + // The list keeps its order. + assert_eq!( + doc(vec![cd(b"z", b"1"), cd(b"a", b"2")]), + r#"{"column_defaults":[{"name":"z","value":"1"},{"name":"a","value":"2"}]}"# + ); + } + #[test] fn an_empty_list_is_sent_as_an_empty_list_and_none_is_left_out() { let none: Vec> = Vec::new(); diff --git a/ts/CHANGELOG.md b/ts/CHANGELOG.md index 91191ba6..e3e6c4fe 100644 --- a/ts/CHANGELOG.md +++ b/ts/CHANGELOG.md @@ -8,6 +8,7 @@ The four bindings in this repository are released together and give one answer, ### Added +- `columnDefaults` is covered by unit tests (ABI v2): the exact `insert_options` JSON for a UTF-8 value, a non-UTF-8 value (`value_b64`), an empty value (sent as `""`) and two entries in order. The decode of `default_override` was already tested. The batches guide gains a usage section. No behavior changed. - `CODE_DECLINED` (ABI v2, -2): the code a declined row, outcome or verdict carries, generated from the description and exported from the package, as `bindings-v1.md` section 3 promises. - `registry.fetch(request): Promise`, the fetch-only call (#492): it resolves, fetches, verifies and installs the build a request names into the cache, as `chtypes fetch` does, and opens nothing (no library is loaded, the process setup is untouched, nothing appears in `libraries()`). It honors the registry's fetch options, its errors are the fetch codes (a refused spelling is a `UsageError`, as for `for`), and it shares one fetch with a concurrent open or `fetch` of the same request: an open that misses the cache now waits on the same fetch. - `chtypes resolve [--json] [--offline]` (#493): the version, build and platform manifest digest the spelling resolves to on every platform the registry offers, one `resolved ` line each in platform order (`--json`: one compact JSON array of `{platform, version, build, manifest}`), each verified exactly as a fetch verifies it, with no layer downloaded and nothing written. `--offline` answers from the cache. A refused spelling (a `v` prefix, a channel suffix) exits with the usage status. diff --git a/ts/test/settings-setup.test.ts b/ts/test/settings-setup.test.ts index 791ccd69..e43b0658 100644 --- a/ts/test/settings-setup.test.ts +++ b/ts/test/settings-setup.test.ts @@ -116,6 +116,19 @@ describe('the options documents (rule r1)', () => { }); }); + it('column_defaults wire form: UTF-8, value_b64, name_b64, an empty value, and order', () => { + const cd = (columnDefaults: { name: string | Uint8Array; value: string | Uint8Array }[]) => doc(encodeInsertOptions({ columnDefaults })); + expect(cd([{ name: 'a', value: 'h\u00e9llo' }])).toEqual({ column_defaults: [{ name: 'a', value: 'h\u00e9llo' }] }); + expect(cd([{ name: 'a', value: new Uint8Array([0xff, 0x00]) }])).toEqual({ column_defaults: [{ name: 'a', value_b64: '/wA=' }] }); + expect(cd([{ name: new Uint8Array([0xfe]), value: 'v' }])).toEqual({ column_defaults: [{ name_b64: '/g==', value: 'v' }] }); + // An empty value is a supplied value: sent as "", never omitted. + expect(cd([{ name: 'a', value: '' }])).toEqual({ column_defaults: [{ name: 'a', value: '' }] }); + expect(cd([{ name: 'a', value: new Uint8Array(0) }])).toEqual({ column_defaults: [{ name: 'a', value: '' }] }); + expect(encodeInsertOptions({ columnDefaults: [{ name: 'z', value: '1' }, { name: 'a', value: '2' }] }).toString()).toBe( + '{"column_defaults":[{"name":"z","value":"1"},{"name":"a","value":"2"}]}', + ); + }); + it('an absent list leaves the member out, and a present empty list is sent as []', () => { expect(doc(encodeInsertOptions({ insertableColumns: [], columnDefaults: [] }))).toEqual({ insertable_columns: [], column_defaults: [] }); expect(encodeInsertOptions({ insertableColumns: undefined })).toHaveLength(0); From 2ce065c5486e6a185c76042b4931696eb3fbe47a Mon Sep 17 00:00:00 2001 From: Eric Andrechek Date: Wed, 21 Oct 2026 00:10:09 -0400 Subject: [PATCH 24/24] examples(go,python,rust): follow the spec batch 2 surface (no row-level unsupported settings, nullable spans) Co-Authored-By: Claude Sonnet 5.5 --- examples/go/main.go | 20 ++++++++++++-------- examples/python/demo.py | 12 ++++++------ examples/rust/src/main.rs | 16 +++++++++------- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/examples/go/main.go b/examples/go/main.go index 9ca643e7..35233ebd 100644 --- a/examples/go/main.go +++ b/examples/go/main.go @@ -609,7 +609,6 @@ func section6(lib *chtypes.Library) { blank() kv("UnknownFields[]", fmt.Sprintf("%q", r.UnknownFields)) - kv("UnsupportedSettings[]", fmt.Sprintf("%q", r.UnsupportedSettings)) note("unknown fields are reported, not judged: whether to 400 on them is") note("gateway policy") blank() @@ -1190,15 +1189,20 @@ func section15(lib *chtypes.Library) { note("bytes are directly INSERT-able with no column list; DEFAULTs (seq=7,") note("label='unknown') are already applied: preview == stored") for i, sp := range res.Spans { - l := "(no bytes: row not accepted)" - if sp.Len > 0 { - l = fmt.Sprintf("%q", res.Payload[sp.Off:sp.Off+sp.Len]) + l := "(no bytes: row not exported)" + sd := "nil" + if sp != nil { + sd = fmt.Sprintf("{off:%d len:%d}", sp.Off, sp.Len) + if sp.Len > 0 { + l = fmt.Sprintf("%q", res.Payload[sp.Off:sp.Off+sp.Len]) + } } - kv(fmt.Sprintf(" span[%d] {off:%d len:%d}", i, sp.Off, sp.Len), fmt.Sprintf("%s -> %s", res.Rows[i].Outcome, l)) + kv(fmt.Sprintf(" span[%d] %s", i, sd), fmt.Sprintf("%s -> %s", res.Rows[i].Outcome, l)) } - note("spans are INDEX-ALIGNED with rows; slicing spans out of the payload") - note("IS the per-row payload, and concatenating non-zero spans reproduces") - note("it exactly: batches merge by byte concatenation") + note("spans are INDEX-ALIGNED with rows, nil for a row not exported;") + note("slicing spans out of the payload IS the per-row payload, and") + note("concatenating non-nil spans reproduces it exactly: batches merge") + note("by byte concatenation") blank() // The lean document: fewer doc flags keep the whole verdict channel and diff --git a/examples/python/demo.py b/examples/python/demo.py index 546c5d7d..9041c25d 100644 --- a/examples/python/demo.py +++ b/examples/python/demo.py @@ -472,7 +472,6 @@ def section6(lib: chtypes.Library) -> None: blank() kv("unknown_fields[]", str([show(n) for n in r.unknown_fields])) - kv("unsupported_settings[]", str([show(n) for n in r.unsupported_settings])) blank() with lib.compile_table(table("a UInt8, d UInt8 DEFAULT a + 1")) as dep: @@ -1019,15 +1018,16 @@ def section15(lib: chtypes.Library) -> None: note("wire order = declared minus MATERIALIZED/ALIAS/EPHEMERAL, so these bytes are") note("INSERT-able with no column list; DEFAULTs are already applied") for i, span in enumerate(batch.spans or ()): - line = "(no bytes - row not accepted)" - if span.len: + line = "(no bytes - row not exported)" + if span is not None and span.len: line = repr(payload[span.off : span.off + span.len]) kv( - f" span[{i}] {{off:{span.off} len:{span.len}}}", + f" span[{i}] " + + ("None" if span is None else f"{{off:{span.off} len:{span.len}}}"), f"{batch.rows[i].outcome.value} -> {line}", ) - note("spans are INDEX-ALIGNED with rows; concatenating the non-zero spans") - note("reproduces the payload exactly") + note("spans are INDEX-ALIGNED with rows, None for a row not exported;") + note("concatenating the non-None spans reproduces the payload exactly") blank() lean = schema.rows( diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs index dce640c0..445f9524 100644 --- a/examples/rust/src/main.rs +++ b/examples/rust/src/main.rs @@ -636,10 +636,6 @@ fn section6(lib: &Arc) { } blank(); kv("unknown_fields[]", &format!("{:?}", r.unknown_fields)); - kv( - "unsupported_settings[]", - &format!("{:?}", r.unsupported_settings), - ); blank(); let dep = compile(lib, "a UInt8, d UInt8 DEFAULT a + 1"); @@ -1221,21 +1217,27 @@ fn section15(lib: &Arc) { ), ); for (i, span) in batch.spans.iter().flatten().enumerate() { - let line = if span.len > 0 { + let line = if let Some(span) = span.as_ref().filter(|s| s.len > 0) { let (off, len) = (span.off as usize, span.len as usize); format!( "{:?}", String::from_utf8_lossy(payload.get(off..off + len).unwrap_or_default()) ) } else { - "(no bytes: row not accepted)".to_string() + "(no bytes: row not exported)".to_string() }; let outcome = batch .rows .get(i) .map_or("?".to_string(), |r| r.outcome.to_string()); kv( - &format!(" span[{i}] {{off:{} len:{}}}", span.off, span.len), + &format!( + " span[{i}] {}", + span.as_ref().map_or("None".to_string(), |s| format!( + "{{off:{} len:{}}}", + s.off, s.len + )) + ), &format!("{outcome} -> {line}"), ); }