From b23b97d71d56040ee7d165783b2c403ddab2e76d Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 25 Jul 2026 03:45:17 +0000 Subject: [PATCH] pg-clickhouse-c --- .github/workflows/bake.yml | 2 +- .github/workflows/clickhouse.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/postgres.yml | 2 +- .gitmodules | 6 +- CHANGELOG.md | 8 + Makefile | 27 +- src/binary/binary.c | 70 +- src/binary/binary_internal.h | 176 +--- src/binary/connection.c | 4 +- src/binary/convert.c | 875 -------------------- src/binary/decode.c | 840 ------------------- src/binary/encode.c | 411 ++-------- src/binary/insert.c | 1228 +--------------------------- src/binary/select.c | 93 +-- src/custom_types.c | 1 - src/include/binary.h | 68 +- src/include/fdw.h | 5 +- src/pglink.c | 188 ++--- test/expected/binary.out | 18 +- test/expected/binary_1.out | 18 +- test/expected/binary_inserts.out | 9 +- test/expected/binary_inserts_1.out | 9 +- test/expected/functions.out | 7 +- test/expected/functions_0.out | 7 +- test/expected/functions_1.out | 7 +- test/expected/functions_2.out | 7 +- test/expected/functions_3.out | 7 +- test/expected/functions_4.out | 7 +- test/expected/functions_5.out | 7 +- test/expected/functions_6.out | 7 +- test/expected/functions_7.out | 7 +- test/expected/functions_8.out | 7 +- test/expected/functions_9.out | 7 +- test/expected/import_schema.out | 2 +- test/expected/import_schema_1.out | 2 +- test/expected/import_schema_2.out | 2 +- test/expected/json_4.out | 4 +- test/expected/json_5.out | 4 +- test/expected/json_6.out | 4 +- test/sql/binary_inserts.sql | 5 +- vendor/clickhouse-c | 1 - vendor/pg-clickhouse-c | 1 + 43 files changed, 392 insertions(+), 3772 deletions(-) delete mode 100644 src/binary/convert.c delete mode 100644 src/binary/decode.c delete mode 160000 vendor/clickhouse-c create mode 160000 vendor/pg-clickhouse-c diff --git a/.github/workflows/bake.yml b/.github/workflows/bake.yml index 8d216a24..16123a5f 100644 --- a/.github/workflows/bake.yml +++ b/.github/workflows/bake.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout the Repository uses: actions/checkout@v6 - with: { submodules: true } + with: { submodules: recursive } - name: Set up Depot CLI uses: depot/setup-action@v1 - name: Log in to GHCR diff --git a/.github/workflows/clickhouse.yml b/.github/workflows/clickhouse.yml index 23166573..5f56aeec 100644 --- a/.github/workflows/clickhouse.yml +++ b/.github/workflows/clickhouse.yml @@ -36,7 +36,7 @@ jobs: run: pgxn install re2 - name: Checkout the Repository uses: actions/checkout@v6 - with: { submodules: true } + with: { submodules: recursive } - name: Start ClickHouse env: { CH_RELEASE: "${{ matrix.ch }}" } run: .github/ubuntu/clickhouse.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 63abcd99..ffb0fe9f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,7 +17,7 @@ jobs: run: NO_CLUSTER=1 pg-start ${{ env.pg }} libcurl4-openssl-dev uuid-dev liblz4-dev libzstd-dev clang-tidy bear shellcheck - name: Checkout the Repository uses: actions/checkout@v6 - with: { submodules: true } + with: { submodules: recursive } - name: Install pre-commit run: make debian-install-lint - name: Add Postgres to Path diff --git a/.github/workflows/postgres.yml b/.github/workflows/postgres.yml index ee24a16f..6c15294a 100644 --- a/.github/workflows/postgres.yml +++ b/.github/workflows/postgres.yml @@ -27,7 +27,7 @@ jobs: run: pgxn install re2 - name: Checkout the Repository uses: actions/checkout@v6 - with: { submodules: true } + with: { submodules: recursive } - name: Start ClickHouse run: .github/ubuntu/clickhouse.sh - name: Test diff --git a/.gitmodules b/.gitmodules index bd173a51..7474743c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "vendor/clickhouse-c"] - path = vendor/clickhouse-c - url = https://github.com/ClickHouse/clickhouse-c +[submodule "vendor/pg-clickhouse-c"] + path = vendor/pg-clickhouse-c + url = https://github.com/ClickHouse/pg-clickhouse-c diff --git a/CHANGELOG.md b/CHANGELOG.md index 453c6112..c2cbadfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,12 @@ All notable changes to this project will be documented in this file. It uses the rather than understating foreign scan cost ([#310]). * Added mapping to push down the re2 v0.4 `@~` operator to ClickHouse as the `match()` function ([#318]). +* Binary driver errors now name the column and type involved, for example + `cannot encode integer into ClickHouse Array(Array(Int32)) (column "c2")` + in place of `unexpected PG/CH type pair for column 0` ([#326]). +* Coerce array elements in binary driver. `Array(Int32)` to `bigint[]`, or + `quantilesExactLow()` results into `double precision[]`, no longer fails + with `could not cast value from integer[] to bigint[]` ([#326]). ### 🐞 Bug Fixes @@ -160,6 +166,8 @@ All notable changes to this project will be documented in this file. It uses the "ClickHouse/pg_clickhouse#317 Push down the array IN family unconditionally" [#319]: https://github.com/ClickHouse/pg_clickhouse/pull/319 "ClickHouse/pg_clickhouse#319 Fix foreign scan RTE selection" + [#326]: https://github.com/ClickHouse/pg_clickhouse/pull/326 + "ClickHouse/pg_clickhouse#326 pg-clickhouse-c" ## [v0.3.2] — 2026-06-16 diff --git a/Makefile b/Makefile index 8df87978..55328fed 100644 --- a/Makefile +++ b/Makefile @@ -19,13 +19,18 @@ ARCH = $(shell uname -m) # Collect all the C files to compile into MODULE_big. OBJS = $(subst .c,.o, $(wildcard src/*.c src/*/*.c)) -# clickhouse-c is a header-only single-header library. Override -# CH_C_DIR to point elsewhere when developing against a local checkout. -CH_C_DIR ?= vendor/clickhouse-c +# clickhouse-c and pg-clickhouse-c are header-only. pg-clickhouse-c vendors the +# clickhouse-c commit its own signatures compile against, so take that pin +# rather than a second checkout. Override CH_C_DIR / PGCH_DIR to point +# elsewhere when developing against a local checkout. +PGCH_DIR = vendor/pg-clickhouse-c +CH_C_DIR = $(PGCH_DIR)/clickhouse-c # Own headers via -I; vendored and PG server headers via -isystem so their # own warnings (eg -Wsign-compare in PG array.h macros) don't trip -Werror. -PG_CPPFLAGS = -I./src/include -isystem $(CH_C_DIR) -isystem $(shell $(PG_CONFIG) --includedir-server) +# PGCH_MSG_PREFIX prefixes every message pg-clickhouse-c raises; it expands +# inside the ereport macros, so every TU must see the same definition. +PG_CPPFLAGS = -I./src/include -isystem $(CH_C_DIR) -isystem $(PGCH_DIR) -isystem $(shell $(PG_CONFIG) --includedir-server) -DPGCH_MSG_PREFIX='"pg_clickhouse: "' # Link OpenSSL (for TLS in the binary driver), curl (for the HTTP driver), # libuuid (for http_streaming.c's query-id generator), and lz4 / zstd @@ -50,13 +55,15 @@ EXTRA_CLEAN = sql/$(EXTENSION)--$(EXTVERSION).sql src/include/version.h compile_ PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# Clone clickhouse-c submodule. -$(CH_C_DIR)/clickhouse.h: .gitmodules - git submodule update --init +# Clone vendored submodules, clickhouse-c nested inside pg-clickhouse-c. Spelled +# out rather than via CH_C_DIR / PGCH_DIR: an override points at a checkout git +# knows nothing about. +vendor/pg-clickhouse-c/clickhouse-c/clickhouse.h vendor/pg-clickhouse-c/pg-clickhouse.h: .gitmodules + git submodule update --init --recursive -# Require clickhouse-c and the version header. The bitcode twins compile the -# same sources, so they need the same generated header. -$(OBJS) $(OBJS:.o=.bc): $(CH_C_DIR)/clickhouse.h src/include/version.h +# Require both vendored libraries and the version header. The bitcode twins +# compile the same sources, so they need the same generated header. +$(OBJS) $(OBJS:.o=.bc): $(CH_C_DIR)/clickhouse.h $(PGCH_DIR)/pg-clickhouse.h src/include/version.h # Require the versioned SQL script. all: sql/$(EXTENSION)--$(EXTVERSION).sql diff --git a/src/binary/binary.c b/src/binary/binary.c index 87566024..819868c6 100644 --- a/src/binary/binary.c +++ b/src/binary/binary.c @@ -1,15 +1,11 @@ /* * binary.c * - * Core glue for the binary driver. Owns the chc_alloc thunks (palloc on - * CurrentMemoryContext, MCXT_ALLOC_HUGE so block buffers can escape the 1GB - * cap), error mapping into ereport, and a couple of small type helpers shared - * across the subdir. - * - * This also holds the CHC_IMPLEMENTATION define for the clickhouse-c - * header-only library; the other .c files in src/binary include the - * same headers without that define and link against the bodies emitted - * here. + * Core glue for the binary driver. Holds the CHC_IMPLEMENTATION and + * PGCH_IMPLEMENTATION defines for the header-only clickhouse-c and + * pg-clickhouse-c libraries; the other .c files in src/binary include the + * same headers without those defines and link against the bodies emitted + * here. pgch_in_alloc needs both in the same TU, so they stay together. * * API lives in src/include/binary.h. Driver internals shared between * connection.c / select.c / insert.c live in binary_internal.h @@ -28,50 +24,11 @@ #include "clickhouse-posix-io.h" #include "clickhouse.h" -#include "binary_internal.h" - -static void* -pg_chc_palloc(void* ud pg_attribute_unused(), size_t n) { - return MemoryContextAllocExtended(CurrentMemoryContext, n, MCXT_ALLOC_HUGE); -} - -static void* -pg_chc_repalloc( - void* ud pg_attribute_unused(), - void* p, - size_t old_bytes pg_attribute_unused(), - size_t new_bytes -) { - if (!p) { - return MemoryContextAllocExtended( - CurrentMemoryContext, new_bytes, MCXT_ALLOC_HUGE - ); - } - return repalloc_huge(p, new_bytes); -} +#define PGCH_IMPLEMENTATION +#include "pg-clickhouse-decode.h" +#include "pg-clickhouse-encode.h" -static void -pg_chc_pfree( - void* ud pg_attribute_unused(), - void* p, - size_t bytes pg_attribute_unused() -) { - if (p) { - pfree(p); - } -} - -const chc_alloc pg_chc_alloc = { - .ud = NULL, - .alloc = pg_chc_palloc, - .realloc = pg_chc_repalloc, - .free = pg_chc_pfree, -}; - -/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ -const int64_t pow10i[10] = { - 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, -}; +#include "binary_internal.h" /* * output_format_native_write_json_as_string exists on the server from @@ -93,12 +50,3 @@ server_supports_json_as_string(const chc_client* c) { } return false; } - -void -raise_chc(const chc_err* err, int sqlstate, const char* prefix) { - const char* m = (err && err->msg[0]) ? err->msg : "unknown error"; - - ereport( - ERROR, errcode(sqlstate), errmsg("pg_clickhouse: %s%s", prefix ? prefix : "", m) - ); -} diff --git a/src/binary/binary_internal.h b/src/binary/binary_internal.h index d1fcf832..af95f5a9 100644 --- a/src/binary/binary_internal.h +++ b/src/binary/binary_internal.h @@ -24,53 +24,17 @@ #define pg_noreturn pg_attribute_noreturn() #endif -typedef struct { - Datum* datums; - bool* nulls; - size_t len; - Oid* types; - const char* ch_type_name; -} ch_binary_tuple_t; - -/* - * Holds an array decoded from ClickHouse or built for INSERT. For nested - * arrays (Array(Array(...))) ndim > 1 and datums[i] points to a child - * ch_binary_array_t with ndim-1. item_type is leaf scalar PG type, - * array_type is postgres array type (same across nesting depths since - * postgres uses one array type per element type regardless of ndim). - */ -typedef struct { - Datum* datums; - bool* nulls; - size_t len; - int ndim; /* nesting depth, >=1 */ - Oid item_type; /* leaf scalar PG type */ - Oid array_type; /* PG array type (same at every level) */ -} ch_binary_array_t; - -/* Column metadata returned from ch_binary_begin_insert. */ -typedef struct ch_binary_column_info { - const char* name; - const chc_type* type; /* type unwrapped of Nullable + LowCardinality */ - bool is_nullable; -} ch_binary_column_info; - /* - * Pump next non-empty Data block off wire. Returned pointer is borrowed, - * valid until next fetch_next_block or ch_binary_response_free. NULL when - * stream ends (eos, error, canceled), ch_binary_response_error reports + * Pump next Data block off wire (header block may carry zero rows). Caller + * takes ownership, destroys via chc_block_destroy with pgch_alloc. NULL + * when stream ends (eos, error, canceled), ch_binary_response_error reports * cause if any. */ extern const chc_block* ch_binary_response_fetch_next_block(ch_binary_response_t* resp); extern ch_binary_insert_handle* -ch_binary_begin_insert( - ch_binary_connection_t* conn, - const ch_query* query, - ch_binary_column_info** out_cols, - size_t* out_n -); +ch_binary_begin_insert(ch_binary_connection_t* conn, const ch_query* query); /* * Tear down handle. Never raises and never talks to server, safe to call @@ -85,7 +49,7 @@ ch_binary_release_insert(ch_binary_insert_handle* h); * * Lives in own context `cxt` (child of CacheMemoryContext) so connection * survives transaction boundaries. Result block buffers don't live here: - * pg_chc_alloc routes through CurrentMemoryContext, so blocks land in + * pgch_alloc routes through CurrentMemoryContext, so blocks land in * whichever per-query context the caller has switched to. */ struct ch_binary_state { @@ -131,134 +95,20 @@ conn_state(ch_binary_connection_t* conn) { return (struct ch_binary_state*)conn->client; } -/* chc allocator wired through palloc; defined in binary.c. */ -extern const chc_alloc pg_chc_alloc; - -/* Scalar CH kind -> PG type OID; InvalidOid for wrapper/unsupported kinds. */ -extern const Oid ch_scalar_oids[CHC_KIND_COUNT]; - -/* power-of-10 lookup; CH bounds DateTime64 / Decimal scale to [0, 9] */ -extern const int64_t pow10i[10]; - -/* ereport ERROR carrying chc_err->msg with sqlstate / prefix. */ -pg_noreturn extern void -raise_chc(const chc_err* err, int sqlstate, const char* prefix); - /* True if server advertises output_format_native_write_json_as_string. */ extern bool server_supports_json_as_string(const chc_client* c); -/* - * Per-row, per-column append. Set isnull for NULL values (column must be - * Nullable). ereports on type mismatch / NULL-into-NOT-NULL. - */ -extern void -ch_binary_append_int(ch_binary_insert_handle* h, size_t col, int64_t val, bool isnull); -extern void -ch_binary_append_uint( - ch_binary_insert_handle* h, - size_t col, - uint64_t val, - bool isnull -); -extern void -ch_binary_append_bool(ch_binary_insert_handle* h, size_t col, bool val, bool isnull); -extern void -ch_binary_append_double( - ch_binary_insert_handle* h, - size_t col, - double val, - bool isnull -); -extern void -ch_binary_append_float(ch_binary_insert_handle* h, size_t col, float val, bool isnull); -extern void -ch_binary_append_bytes( - ch_binary_insert_handle* h, - size_t col, - const void* p, - size_t n, - bool isnull -); -extern void -ch_binary_append_decimal( - ch_binary_insert_handle* h, - size_t col, - const char* digits, - bool isnull -); -extern void -ch_binary_append_uuid( - ch_binary_insert_handle* h, - size_t col, - const uint8_t bytes[16], - bool isnull -); +/* Column buffers behind the handle; encode.c appends through this. */ +extern pgch_writer* +ch_binary_insert_writer(ch_binary_insert_handle* h); -/* - * IPv4: addr_be is 4 BE bytes (matches PG inet ip_addr layout). - * IPv6: addr_be is 16 BE bytes. Pass NULL with isnull=true. - */ -extern void -ch_binary_append_inet( - ch_binary_insert_handle* h, - size_t col, - const uint8_t* addr_be, - size_t addrlen, - bool isnull -); - -/* - * Per-row Date/DateTime/DateTime64 sent as seconds-since-epoch (int64). - * For DateTime64 value is wire-level integer at column's scale; - * encode.c does scaling. - */ -extern void -ch_binary_append_date_seconds( - ch_binary_insert_handle* h, - size_t col, - int64_t seconds, - bool isnull -); -extern void -ch_binary_append_datetime_seconds( - ch_binary_insert_handle* h, - size_t col, - int64_t seconds, - bool isnull -); -extern void -ch_binary_append_datetime64_raw( - ch_binary_insert_handle* h, - size_t col, - int64_t raw, - bool isnull -); +extern size_t +ch_binary_insert_ncols(const ch_binary_insert_handle* h); -/* - * Array element append. Open with array_begin. All subsequent - * ch_binary_append_* calls target inner items column regardless of - * `col` until ch_binary_array_end. - */ -extern void -ch_binary_array_begin(ch_binary_insert_handle* h, size_t col); -extern void -ch_binary_array_end(ch_binary_insert_handle* h); - -/* True when handle has an active array context (for assertions). */ -extern bool -ch_binary_array_active(const ch_binary_insert_handle* h); - -/* - * Inspect underlying CH column kind. Used by encode.c to - * dispatch on (Oid pg, chc_kind ch). When an array context is open - * (between ch_binary_array_begin/_end) returned kind is element kind, - * not CHC_ARRAY. - */ -extern chc_kind -ch_binary_column_kind(const ch_binary_insert_handle* h, size_t col); -extern uint32_t -ch_binary_column_datetime64_precision(const ch_binary_insert_handle* h, size_t col); +/* Name the server gave column i of the INSERT, "" when unnamed. */ +extern const char* +ch_binary_insert_column_name(const ch_binary_insert_handle* h, size_t i); /* Send buffered rows and clear; ready for next batch. */ extern void diff --git a/src/binary/connection.c b/src/binary/connection.c index 95daca59..dec0b256 100644 --- a/src/binary/connection.c +++ b/src/binary/connection.c @@ -329,10 +329,10 @@ ch_binary_connect(ch_connection_details* details) { .compression = comp, }; chc_err err = {}; - int rc = chc_client_init(&s->client, &opts, &pg_chc_alloc, &s->io, &err); + int rc = chc_client_init(&s->client, &opts, &pgch_alloc, &s->io, &err); if (rc != CHC_OK) { - raise_chc( + pgch_raise( &err, ERRCODE_FDW_UNABLE_TO_ESTABLISH_CONNECTION, "connection error: " ); } diff --git a/src/binary/convert.c b/src/binary/convert.c deleted file mode 100644 index c5a1dd39..00000000 --- a/src/binary/convert.c +++ /dev/null @@ -1,875 +0,0 @@ -/* Conversion functions for the binary driver. */ - -#include "postgres.h" - -#include "access/tupconvert.h" -#include "access/tupdesc.h" -#include "catalog/pg_type.h" -#include "catalog/pg_type_d.h" -#include "executor/tuptable.h" -#include "funcapi.h" -#include "parser/parse_coerce.h" -#include "parser/parse_type.h" -#include "utils/arrayaccess.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/syscache.h" -#include "utils/typcache.h" - -#include "binary_internal.h" -#include "fdw.h" -#include - -/* - * Scalar CH column kind -> PG type OID, indexed by chc_kind. InvalidOid for - * wrapper kinds (Nullable, LowCardinality, Array, Tuple) and unset entries. - */ -const Oid ch_scalar_oids[CHC_KIND_COUNT] = { - [CHC_INT8] = INT2OID, - [CHC_INT16] = INT2OID, - [CHC_UINT8] = INT2OID, - [CHC_BOOL] = BOOLOID, - [CHC_INT32] = INT4OID, - [CHC_UINT16] = INT4OID, - [CHC_INT64] = INT8OID, - [CHC_UINT32] = INT8OID, - [CHC_UINT64] = INT8OID, - [CHC_FLOAT32] = FLOAT4OID, - [CHC_FLOAT64] = FLOAT8OID, - [CHC_DECIMAL32] = NUMERICOID, - [CHC_DECIMAL64] = NUMERICOID, - [CHC_DECIMAL128] = NUMERICOID, - [CHC_DECIMAL256] = NUMERICOID, - [CHC_STRING] = TEXTOID, - [CHC_FIXED_STRING] = TEXTOID, - [CHC_ENUM8] = TEXTOID, - [CHC_ENUM16] = TEXTOID, - [CHC_JSON] = JSONBOID, - [CHC_OBJECT] = JSONBOID, - [CHC_DATE] = DATEOID, - [CHC_DATE32] = DATEOID, - [CHC_DATETIME] = TIMESTAMPTZOID, - [CHC_DATETIME64] = TIMESTAMPTZOID, - [CHC_UUID] = UUIDOID, - [CHC_IPV4] = INETOID, - [CHC_IPV6] = INETOID, -}; - -typedef struct ch_convert_state ch_convert_state; -typedef struct ch_convert_output_state ch_convert_output_state; -typedef Datum (*convert_func)(ch_convert_state*, Datum); -typedef Datum (*out_convert_func)(ch_convert_output_state*, Datum); - -typedef struct ch_convert_state { - Oid intype; - Oid outtype; - convert_func func; - - /* record */ - TupleConversionMap* tupmap; - CustomObjectDef* cdef; - TupleDesc indesc; /* for RECORD */ - TupleDesc outdesc; /* for RECORD */ - ch_convert_state** conversion_states; - - /* array */ - int16 typlen; - bool typbyval; - char typalign; - - /* text */ - int32 typmod; - Oid typinput; - Oid typioparam; - - /* generic */ - CoercionPathType ctype; - Oid castfunc; -} ch_convert_state; - -typedef struct ch_convert_output_state { - Oid intype; - Oid outtype; - AttrNumber attnum; - out_convert_func func; - - /* array */ - Oid innertype; /* if intype is array */ - int16 typlen; - bool typbyval; - char typalign; - - /* generic */ - CoercionPathType ctype; - Oid castfunc; -} ch_convert_output_state; - -static Datum -convert_record(ch_convert_state* state, Datum val) { - HeapTuple temptup; - HeapTuple htup; - ch_binary_tuple_t* slot = (ch_binary_tuple_t*)DatumGetPointer(val); - - for (size_t i = 0; i < slot->len; i++) { - ch_convert_state* s = state->conversion_states[i]; - - /* Null fields carry no value to convert or to build state from. */ - if (slot->nulls[i]) { - continue; - } - - if (s == NULL && slot->types[i] == RECORDOID) { - MemoryContext oldcxt = MemoryContextSwitchTo(GetMemoryChunkContext(state)); - - s = ch_binary_init_convert_state(slot->datums[i], RECORDOID, RECORDOID); - MemoryContextSwitchTo(oldcxt); - state->conversion_states[i] = s; - } - - if (s) { - slot->datums[i] = s->func(s, slot->datums[i]); - } - } - - htup = heap_form_tuple(state->indesc, slot->datums, slot->nulls); - if (!state->outdesc) { - val = heap_copy_tuple_as_datum(htup, state->indesc); - - if (state->cdef && state->cdef->rowfunc != InvalidOid) { - /* there is converter from row to outtype */ - val = OidFunctionCall1(state->cdef->rowfunc, val); - } else if (state->outtype == TEXTOID) { - /* a lot of allocations, not so efficient */ - val = CStringGetTextDatum( - DatumGetCString(OidFunctionCall1(F_RECORD_OUT, val)) - ); - } - } else { - if (state->tupmap) { - temptup = execute_attr_map_tuple(htup, state->tupmap); - } else { - temptup = htup; - } - - val = heap_copy_tuple_as_datum(temptup, state->outdesc); - } - - return val; -} - -inline static Datum -convert_generic(ch_convert_state* state, Datum val) { - if (state->ctype == COERCION_PATH_FUNC) { - Assert(state->castfunc != InvalidOid); - val = OidFunctionCall1(state->castfunc, val); - } - - return val; -} - -inline static Datum -convert_out_generic(ch_convert_output_state* state, Datum val) { - if (state->ctype == COERCION_PATH_FUNC) { - Assert(state->castfunc != InvalidOid); - val = OidFunctionCall1(state->castfunc, val); - } - - return val; -} - -/* - * Walk nested ch_binary_array_t into a flat datum buffer, verifying each - * level matches the dims taken from the first child. Returns false if the - * shape is jagged so the caller can fall back to a slower path. - */ -static bool -flatten_nested_array( - ch_binary_array_t* slot, - int* dims, - int level, - Datum* values, - bool* nulls, - size_t* idx -) { - if ((int)slot->len != dims[level]) { - return false; - } - - if (slot->ndim == 1) { - for (size_t i = 0; i < slot->len; i++) { - values[*idx] = slot->datums[i]; - nulls[*idx] = slot->nulls[i]; - (*idx)++; - } - } else { - for (size_t i = 0; i < slot->len; i++) { - ch_binary_array_t* child = - (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); - - if (!flatten_nested_array(child, dims, level + 1, values, nulls, idx)) { - return false; - } - } - } - return true; -} - -/* - * Emit a nested ch_binary_array_t as a postgres array text literal, quoting - * each leaf and escaping `\` and `"`. Used as the jagged fallback so binary - * surfaces the same array_in malformed-literal error as the http path. - */ -static void -emit_nested_array_text(ch_binary_array_t* slot, FmgrInfo* outfn, StringInfo buf) { - appendStringInfoChar(buf, '{'); - for (size_t i = 0; i < slot->len; i++) { - if (i > 0) { - appendStringInfoChar(buf, ','); - } - - if (slot->ndim > 1) { - ch_binary_array_t* child = - (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); - - emit_nested_array_text(child, outfn, buf); - } else if (slot->nulls[i]) { - appendStringInfoString(buf, "NULL"); - } else { - char* s = OutputFunctionCall(outfn, slot->datums[i]); - - appendStringInfoChar(buf, '"'); - for (char* p = s; *p; p++) { - if (*p == '"' || *p == '\\') { - appendStringInfoChar(buf, '\\'); - } - appendStringInfoChar(buf, *p); - } - appendStringInfoChar(buf, '"'); - pfree(s); - } - } - appendStringInfoChar(buf, '}'); -} - -static Datum -convert_array(ch_convert_state* state, Datum val) { - ch_binary_array_t* slot = (ch_binary_array_t*)DatumGetPointer(val); - - if (slot->len == 0) { - val = PointerGetDatum(construct_empty_array(slot->item_type)); - } else if (slot->ndim <= 1) { - int dims[1] = { (int)slot->len }; - int lbs[1] = { 1 }; - - val = PointerGetDatum(construct_md_array( - slot->datums, - slot->nulls, - 1, - dims, - lbs, - slot->item_type, - state->typlen, - state->typbyval, - state->typalign - )); - } else { - int dims[MAXDIM] = {}; - int lbs[MAXDIM] = {}; - size_t idx = 0; - Datum* flat; - bool* flatnulls; - ch_binary_array_t* probe = slot; - - if (slot->ndim > MAXDIM) { - ereport( - ERROR, - errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg( - "pg_clickhouse: nested array depth %d exceeds maximum %d", - slot->ndim, - MAXDIM - ) - ); - } - - for (int d = 0; d < slot->ndim; d++) { - dims[d] = (int)probe->len; - lbs[d] = 1; - if (probe->ndim > 1 && probe->len > 0) { - probe = (ch_binary_array_t*)DatumGetPointer(probe->datums[0]); - } - } - - size_t total = ArrayGetNItems(slot->ndim, dims); - - if (total == 0) { - val = PointerGetDatum(construct_empty_array(slot->item_type)); - } else { - flat = palloc(sizeof(Datum) * total); - flatnulls = palloc0(sizeof(bool) * total); - - if (flatten_nested_array(slot, dims, 0, flat, flatnulls, &idx)) { - val = PointerGetDatum(construct_md_array( - flat, - flatnulls, - slot->ndim, - dims, - lbs, - slot->item_type, - state->typlen, - state->typbyval, - state->typalign - )); - } else { - /* - * Jagged shape: format as text and route through array_in so - * binary surfaces the same malformed-literal error as http. - */ - StringInfoData buf; - FmgrInfo outfn; - Oid out_func; - Oid in_func; - Oid ioparam; - bool varlena; - - pfree(flat); - pfree(flatnulls); - - getTypeOutputInfo(slot->item_type, &out_func, &varlena); - fmgr_info(out_func, &outfn); - - initStringInfo(&buf); - emit_nested_array_text(slot, &outfn, &buf); - - getTypeInputInfo(state->intype, &in_func, &ioparam); - val = OidInputFunctionCall(in_func, buf.data, ioparam, -1); - - pfree(buf.data); - } - } - } - - return convert_generic(state, val); -} - -static Datum -convert_remote_text(ch_convert_state* state, Datum val) { - return OidInputFunctionCall( - state->typinput, TextDatumGetCString(val), state->typioparam, state->typmod - ); -} - -/* - * We imply that corresponding type for UInt8 (bool in ClickHouse) is - * SMALLINT and this function covers this case - */ -static Datum -convert_bool(ch_convert_state* state, Datum val) { - int16 dat = DatumGetInt16(val); - - return BoolGetDatum(dat); -} - -Datum -ch_binary_convert_datum(void* state, Datum val) { - return state ? ((ch_convert_state*)state)->func(state, val) : val; -} - -/* - * Render a value decoded by ch_binary_read_row (typed by state->coltypes) to a - * palloc'd cstring. Arrays and tuples arrive as the binary engine's wrapper - * structs rather than native datums, so route them through the conversion path - * to a native datum before applying the output function. Used by - * clickhouse_raw_query, which has no foreign table to pin output types. - */ -char* -ch_binary_value_to_cstring(Oid coltype, Datum value) { - Oid out_func; - bool varlena; - - if (coltype == ANYARRAYOID) { - ch_binary_array_t* slot = (ch_binary_array_t*)DatumGetPointer(value); - void* state = - ch_binary_init_convert_state(value, ANYARRAYOID, slot->array_type); - Datum arr = ch_binary_convert_datum(state, value); - - getTypeOutputInfo(slot->array_type, &out_func, &varlena); - if (state) { - ch_binary_free_convert_state(state); - } - return OidOutputFunctionCall(out_func, arr); - } - - if (coltype == RECORDOID) { - void* state = ch_binary_init_convert_state(value, RECORDOID, TEXTOID); - Datum txt = ch_binary_convert_datum(state, value); - - if (state) { - ch_binary_free_convert_state(state); - } - return TextDatumGetCString(txt); - } - - getTypeOutputInfo(coltype, &out_func, &varlena); - return OidOutputFunctionCall(out_func, value); -} - -/* input */ - -/* - * Convert val from intype to outtype. No conversion for binary-compatible - * types or when intype and outtype are the same. All others converted via the - * appropriate CAST function. - */ -void* -ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype) { - /* make_datum() copies all bytes, no cast needed. */ - if (intype == TEXTOID && outtype == BYTEAOID) { - return NULL; - } - - ch_convert_state* state = palloc0(sizeof(ch_convert_state)); - - state->intype = intype; - state->outtype = outtype; - state->cdef = chfdw_check_for_custom_type(outtype); - state->typmod = -1; - state->ctype = COERCION_PATH_NONE; - - if (intype == ANYARRAYOID) { - ch_binary_array_t* slot = (ch_binary_array_t*)DatumGetPointer(val); - - get_typlenbyvalalign( - slot->item_type, &state->typlen, &state->typbyval, &state->typalign - ); - - /* restore intype */ - state->intype = slot->array_type; - intype = slot->array_type; - state->func = convert_array; - } - - if (intype == RECORDOID) { - ch_binary_tuple_t* slot = (ch_binary_tuple_t*)DatumGetPointer(val); - - state->func = convert_record; - -#if PG_VERSION_NUM < 120000 - state->indesc = CreateTemplateTupleDesc(slot->len, false); -#else - state->indesc = CreateTemplateTupleDesc(slot->len); -#endif - state->conversion_states = palloc(sizeof(void*) * slot->len); - - for (size_t i = 0; i < slot->len; ++i) { - Oid item_type = slot->types[i]; - - if (!slot->nulls[i] && slot->types[i] == ANYARRAYOID) { - ch_binary_array_t* arr = - (ch_binary_array_t*)DatumGetPointer(slot->datums[i]); - - item_type = arr->array_type; - } - - /* Null fields hold Datum 0, convert_record lazily fills these. */ - state->conversion_states[i] = - slot->nulls[i] ? NULL - : ch_binary_init_convert_state( - slot->datums[i], slot->types[i], item_type - ); - - TupleDescInitEntry(state->indesc, (AttrNumber)i + 1, "", item_type, -1, 0); - } - -#if PG_VERSION_NUM >= 190000 - TupleDescFinalize(state->indesc); -#endif - state->indesc = BlessTupleDesc(state->indesc); - - if (!(state->cdef || outtype == RECORDOID || outtype == TEXTOID)) { - TypeCacheEntry* typentry; - TupleDesc tupdesc; - - typentry = lookup_type_cache( - outtype, TYPECACHE_TUPDESC | TYPECACHE_DOMAIN_BASE_INFO - ); - - if (typentry->typtype == TYPTYPE_DOMAIN) { - tupdesc = lookup_rowtype_tupdesc_noerror( - typentry->domainBaseType, typentry->domainBaseTypmod, false - ); - } else { - if (typentry->tupDesc == NULL) { - ereport( - ERROR, - errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg( - "pg_clickhouse: cannot return %s as %s", - slot->ch_type_name ? slot->ch_type_name : "?", - format_type_be(outtype) - ) - ); - } - - tupdesc = typentry->tupDesc; - PinTupleDesc(tupdesc); - } - state->outdesc = CreateTupleDescCopy(tupdesc); - state->tupmap = convert_tuples_by_position( - state->indesc, - state->outdesc, - "pg_clickhouse: could not map tuple to returned type" - ); - ReleaseTupleDesc(tupdesc); - } - } else if (intype != outtype) { - if (!state->func) { - state->func = convert_generic; - } - - if (intype == TEXTOID) { - Type baseType; - Oid baseTypeId; - Form_pg_type typform; - - baseTypeId = getBaseTypeAndTypmod(outtype, &state->typmod); - if (baseTypeId != INTERVALOID) { - state->typmod = -1; - } - - baseType = typeidType(baseTypeId); - typform = (Form_pg_type)GETSTRUCT(baseType); - state->typinput = typform->typinput; - state->typioparam = getTypeIOParam(baseType); - state->func = convert_remote_text; - ReleaseSysCache(baseType); - } else if (outtype == BOOLOID && intype == INT2OID) { - state->func = convert_bool; - } else { - /* try to convert */ - state->ctype = find_coercion_pathway( - outtype, intype, COERCION_EXPLICIT, &state->castfunc - ); - switch (state->ctype) { - case COERCION_PATH_FUNC: - break; - case COERCION_PATH_RELABELTYPE: - - /* - * if the conversion func was not previously set, then no - * conversion needed - */ - if (state->func == NULL) { - goto no_conversion; - } - - /* all good */ - break; - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: could not cast value from %s to %s", - format_type_be(intype), - format_type_be(outtype) - ) - ); - } - } - } else if (!state->func) { - no_conversion: - /* no conversion needed */ - pfree(state); - state = NULL; - } - - return state; -} - -void -ch_binary_free_convert_state(void* s) { - ch_convert_state* state = s; - - pfree(state); -} - -/* output */ - -static void -init_output_convert_state(ch_convert_output_state* state) { - if (state->outtype == state->intype) { - return; - } - - /* column_append() copies all bytes, no cast needed. */ - if (state->intype == BYTEAOID && state->outtype == TEXTOID) { - return; - } - - state->func = convert_out_generic; - - state->ctype = find_coercion_pathway( - state->outtype, state->intype, COERCION_EXPLICIT, &state->castfunc - ); - - switch (state->ctype) { - case COERCION_PATH_FUNC: - break; - case COERCION_PATH_RELABELTYPE: - state->func = NULL; - return; - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: could not find a casting path from %s to %s", - format_type_be(state->intype), - format_type_be(state->outtype) - ) - ); - } -} - -void* -ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid) { - ch_convert_output_state* states; - int n; - int i; - - n = outdesc->natts; - states = (ch_convert_output_state*)palloc0(n * sizeof(ch_convert_output_state)); - - for (i = 0; i < n; i++) { - ch_convert_output_state* curstate = &states[i]; - - Form_pg_attribute attout = TupleDescAttr(outdesc, i); - char* outattname; - int j; - - outattname = NameStr(attout->attname); - curstate->outtype = attout->atttypid; - - if (NameStr(TupleDescAttr(indesc, 0)->attname)[0] == '\0') { - Form_pg_attribute attin = TupleDescAttr(indesc, i); - - curstate->intype = attin->atttypid; - init_output_convert_state(curstate); - curstate->attnum = (AttrNumber)(i + 1); - } else { - for (j = 0; j < indesc->natts; j++) { - Form_pg_attribute attin = TupleDescAttr(indesc, j); - char* inattname; - CustomColumnInfo* cinfo; - - if (attin->attisdropped) { - continue; - } - - /* Honor column_name FDW option, falls through to attname */ - cinfo = OidIsValid(relid) ? chfdw_get_custom_column_info(relid, j + 1) - : NULL; - inattname = (cinfo && cinfo->colname[0]) ? cinfo->colname - : NameStr(attin->attname); - - curstate->intype = attin->atttypid; - - if (strcmp(outattname, inattname) == 0) { - init_output_convert_state(curstate); - curstate->attnum = (AttrNumber)(j + 1); - break; - } - } - } - - curstate->innertype = get_element_type(curstate->outtype); - if (curstate->innertype != InvalidOid) { - curstate->outtype = ANYARRAYOID; - get_typlenbyvalalign( - curstate->innertype, - &curstate->typlen, - &curstate->typbyval, - &curstate->typalign - ); - } - - if (curstate->attnum == 0) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: could not create conversion map"), - errdetail( - "Attribute \"%s\" of type %s does not exist in type %s.", - outattname, - format_type_be(indesc->tdtypeid), - format_type_be(outdesc->tdtypeid) - ) - ); - } - } - - return states; -} - -/* - * Chunk a flat postgres array (already extracted into `flat`/`flatnulls` in - * row-major order) into the nested ch_binary_array_t tree the binary engine - * expects for Array(Array(...)) columns. Each interior node carries - * ndim>1 with datums[i] = PointerGetDatum(child); leaves carry ndim==1 with - * scalar datums copied from the flat buffer. - */ -static ch_binary_array_t* -build_nested_binary_array( - int level, - int ndim, - int* dims, - Oid item_type, - Datum* flat, - bool* flatnulls, - size_t* idx -) { - ch_binary_array_t* arr = palloc(sizeof(ch_binary_array_t)); - - arr->len = dims[level]; - arr->ndim = ndim - level; - arr->item_type = item_type; - arr->array_type = InvalidOid; - arr->datums = palloc(sizeof(Datum) * arr->len); - arr->nulls = palloc0(sizeof(bool) * arr->len); - - if (level + 1 == ndim) { - for (size_t i = 0; i < arr->len; i++) { - arr->datums[i] = flat[*idx]; - arr->nulls[i] = flatnulls[*idx]; - (*idx)++; - } - } else { - for (size_t i = 0; i < arr->len; i++) { - ch_binary_array_t* child = build_nested_binary_array( - level + 1, ndim, dims, item_type, flat, flatnulls, idx - ); - - arr->datums[i] = PointerGetDatum(child); - } - } - return arr; -} - -/* - * For each output value, convert it, if necessary, from the Postgres Datum - * type defined for the foreign table to a Datum that the binary INSERT - * path in encode.c knows how to convert to a ClickHouse type. No conversion - * for binary-compatible types; other types require a CAST. - * ch_binary_make_tuple_map() makes this determination for each type, stored - * in insert_state->conversion_states) - */ -void -ch_binary_do_output_conversion( - ch_binary_insert_state* insert_state, - TupleTableSlot* slot -) { - Datum* out_values = insert_state->values; - bool* out_nulls = insert_state->nulls; - - for (size_t i = 0; i < (size_t)insert_state->outdesc->natts; i++) { - ch_convert_output_state* cstate = - &((ch_convert_output_state*)insert_state->conversion_states)[i]; - AttrNumber attnum = cstate->attnum; - - out_values[i] = slot_getattr(slot, attnum, &out_nulls[i]); - if (!out_nulls[i]) { - if (cstate->func) { - out_values[i] = cstate->func(cstate, out_values[i]); - } else if (cstate->outtype == ANYARRAYOID) { - AnyArrayType* v = DatumGetAnyArrayP(out_values[i]); - ch_binary_array_t* arr; - array_iter iter; - int ndim = AARR_NDIM(v); - int* dims = AARR_DIMS(v); - size_t total = ArrayGetNItems(ndim, dims); - - if (ndim > MAXDIM) { - ereport( - ERROR, - errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), - errmsg( - "pg_clickhouse: inserted array depth %d exceeds maximum " - "%d", - ndim, - MAXDIM - ) - ); - } - - if (ndim <= 1) { - arr = palloc(sizeof(ch_binary_array_t)); - arr->len = total; - arr->ndim = 1; - arr->item_type = cstate->innertype; - arr->array_type = InvalidOid; - arr->datums = total ? palloc(sizeof(Datum) * total) : NULL; - arr->nulls = total ? palloc(sizeof(bool) * total) : NULL; - -#if PG_VERSION_NUM < 190000 - array_iter_setup(&iter, v); - for (size_t j = 0; j < total; j++) { - arr->datums[j] = array_iter_next( - &iter, - &arr->nulls[j], - j, - cstate->typlen, - cstate->typbyval, - cstate->typalign - ); - } -#else - array_iter_setup( - &iter, v, cstate->typlen, cstate->typbyval, cstate->typalign - ); - for (size_t j = 0; j < total; j++) { - arr->datums[j] = array_iter_next(&iter, &arr->nulls[j], j); - } -#endif - } else { - Datum* flat = palloc(sizeof(Datum) * total); - bool* flatnulls = palloc0(sizeof(bool) * total); - size_t idx = 0; - -#if PG_VERSION_NUM < 190000 - array_iter_setup(&iter, v); - for (size_t j = 0; j < total; j++) { - flat[j] = array_iter_next( - &iter, - &flatnulls[j], - j, - cstate->typlen, - cstate->typbyval, - cstate->typalign - ); - } -#else - array_iter_setup( - &iter, v, cstate->typlen, cstate->typbyval, cstate->typalign - ); - for (size_t j = 0; j < total; j++) { - flat[j] = array_iter_next(&iter, &flatnulls[j], j); - } -#endif - arr = build_nested_binary_array( - 0, ndim, dims, cstate->innertype, flat, flatnulls, &idx - ); - - pfree(flat); - pfree(flatnulls); - } - out_values[i] = PointerGetDatum(arr); - - /* hack: mark as unified array */ - TupleDescAttr(insert_state->outdesc, i)->atttypid = ANYARRAYOID; - } - } - } -} diff --git a/src/binary/decode.c b/src/binary/decode.c deleted file mode 100644 index 9f5b1644..00000000 --- a/src/binary/decode.c +++ /dev/null @@ -1,840 +0,0 @@ -/* - * decode.c - * - * Build PG Datums from a chc_column produced by clickhouse-c. Walks the - * wire-shaped column accessors directly; per-row transforms (decimal to - * text, IPv4/IPv6 text, UUID byteswap, enum name lookup, LowCardinality - * key->dict deref, Nullable strip) happen inline at read time. - */ - -#include "postgres.h" - -#include -#include /* AF_INET, expanded by PG inet macros */ - -#include "catalog/pg_type_d.h" -#include "fmgr.h" -#include "port/pg_bswap.h" -#include "utils/builtins.h" -#include "utils/date.h" -#include "utils/inet.h" -#include "utils/lsyscache.h" -#include "utils/memutils.h" -#include "utils/numeric.h" -#include "utils/timestamp.h" -#include "utils/uuid.h" -#if PG_VERSION_NUM >= 190000 -#include "varatt.h" -#endif - -#include "binary_internal.h" - -/* CH Date / Date32 epoch is unix; offset to PG epoch (2000-01-01) */ -#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) - -/* Little-endian fixed-width reads at row offset. */ -static inline int8_t -rd_i8(const uint8_t* p, uint64_t row) { - return (int8_t)p[row]; -} - -static inline uint8_t -rd_u8(const uint8_t* p, uint64_t row) { - return p[row]; -} - -static inline bool -rd_bool(const bool* p, uint64_t row) { - return (bool)p[row]; -} - -#define RD_FIXED(suffix, T) \ - static inline T rd_##suffix(const uint8_t* p, uint64_t row) { \ - T v; \ - memcpy(&v, p + row * sizeof(T), sizeof(T)); \ - return v; \ - } - -RD_FIXED(i16, int16_t) -RD_FIXED(u16, uint16_t) -RD_FIXED(i32, int32_t) -RD_FIXED(u32, uint32_t) -RD_FIXED(i64, int64_t) -RD_FIXED(u64, uint64_t) -RD_FIXED(f32, float) -RD_FIXED(f64, double) - -static inline void -slice_str(const chc_column* col, uint64_t row, const char** out_ptr, size_t* out_len) { - const uint64_t* offs = chc_column_string_offsets(col); - const uint8_t* data = chc_column_string_data(col); - uint64_t start = row == 0 ? 0 : offs[row - 1]; - uint64_t end = offs[row]; - - *out_ptr = (const char*)data + start; - *out_len = (size_t)(end - start); -} - -static Oid -ch_kind_to_pg_oid(const chc_type* type) { - chc_kind kind = chc_type_kind(type); - Oid oid = ch_scalar_oids[kind]; - - if (OidIsValid(oid)) { - return oid; - } - - switch (kind) { - case CHC_VOID: - case CHC_NOTHING: - return InvalidOid; - case CHC_NULLABLE: - case CHC_LOW_CARDINALITY: - return ch_kind_to_pg_oid(chc_type_child(type, 0)); - case CHC_ARRAY: - return ANYARRAYOID; - case CHC_TUPLE: - return RECORDOID; - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported column type") - ); - } - /* unreachable */ - return InvalidOid; -} - -static Datum -read_value( - const chc_column* col, - const chc_type* type, - uint64_t row, - Oid* valtype, - bool* is_null -); - -/* - * Format a ClickHouse Decimal (two's-complement signed integer in LE bytes of - * width 4/8/16/32 for Decimal32/64/128/256, with `scale` fractional digits - * carried on the column type) into `out`. Returns bytes written, -1 on overflow. - */ -static int -format_decimal_text( - const uint8_t* bytes, - size_t width, - uint32_t scale, - char* out, - size_t out_cap -) { - uint32_t mag[8]; - char buf[80]; - size_t nwords = width / 4; - bool neg = false; - - if (width == 0 || width > 32 || width % 4 != 0 || scale >= sizeof(buf)) { - return -1; - } - memcpy(mag, bytes, width); - /* top bit of MSW is sign; negate two's-complement to get magnitude */ - if (mag[nwords - 1] & 0x80000000u) { - neg = true; - for (size_t i = 0; i < nwords; i++) { - mag[i] = ~mag[i]; - } - uint64_t carry = 1; - - for (size_t i = 0; i < nwords && carry; i++) { - uint64_t v = (uint64_t)mag[i] + carry; - - mag[i] = (uint32_t)v; - carry = v >> 32; - } - } - - int n = 0; - bool nonzero; - - /* base-10 division of mag yields digits LSB-first */ - do { - uint64_t rem = 0; - - nonzero = false; - for (ssize_t i = (ssize_t)nwords - 1; i >= 0; i--) { - uint64_t v = (rem << 32) | mag[i]; - - mag[i] = (uint32_t)(v / 10); - rem = v % 10; - if (mag[i]) { - nonzero = true; - } - } - buf[n++] = (char)('0' + (uint32_t)rem); - } while (nonzero && n < (int)sizeof(buf)); - - /* pad leading zeros so digit count covers fractional portion */ - while (n <= (int)scale) { - buf[n++] = '0'; - } - - size_t need = (size_t)neg + (size_t)n + (scale ? 1 : 0) + 1; - - if (need > out_cap) { - return -1; - } - char* p = out; - - if (neg) { - *p++ = '-'; - } - - /* emit MSD-first, inserting '.' before `scale` trailing digits */ - for (int i = n - 1; i >= 0; i--) { - if (i + 1 == (int)scale) { - *p++ = '.'; - } - *p++ = buf[i]; - } - *p = '\0'; - return (int)(p - out); -} - -static Datum -read_decimal(const chc_column* col, const chc_type* type, uint64_t row) { - size_t es; - const uint8_t* p = (const uint8_t*)chc_column_fixed_data(col, &es); - uint32_t scale = (uint32_t)chc_type_decimal_scale(type); - char buf[80]; - int rc; - -#if PG_VERSION_NUM >= 140000 - /* Decimal32/64 fit in int64; skip the byte-array text path. */ - if (es == 4) { - return NumericGetDatum(int64_div_fast_to_numeric(rd_i32(p, row), (int)scale)); - } - if (es == 8) { - return NumericGetDatum(int64_div_fast_to_numeric(rd_i64(p, row), (int)scale)); - } -#endif - - rc = format_decimal_text(p + row * es, es, scale, buf, sizeof(buf)); - if (rc < 0) { - ereport( - ERROR, errcode(ERRCODE_FDW_ERROR), errmsg("pg_clickhouse: decimal too wide") - ); - } - return DirectFunctionCall3( - numeric_in, CStringGetDatum(buf), ObjectIdGetDatum(0), Int32GetDatum(-1) - ); -} - -static Datum -read_string_as_text(const chc_column* col, uint64_t row) { - const char* p; - size_t len; - - slice_str(col, row, &p, &len); - return PointerGetDatum(cstring_to_text_with_len(p, len)); -} - -static Datum -read_fixedstring_as_text(const chc_column* col, uint64_t row) { - size_t width; - const uint8_t* base = chc_column_fixed_data(col, &width); - - return PointerGetDatum( - cstring_to_text_with_len((const char*)base + row * width, width) - ); -} - -static Datum -read_uuid(const chc_column* col, uint64_t row) { - pg_uuid_t* u = (pg_uuid_t*)palloc(sizeof(pg_uuid_t)); - const uint8_t* p = (uint8_t*)chc_column_fixed_data(col, NULL) + row * 16; - uint64_t a, b; - - memcpy(&a, p, 8); - memcpy(&b, p + 8, 8); - a = pg_hton64(a); - b = pg_hton64(b); - memcpy(u->data, &a, 8); - memcpy(u->data + 8, &b, 8); - return UUIDPGetDatum(u); -} - -/* - * IPv4 wire format: native uint32 (LE on supported hosts). PG inet wants - * BE bytes of dotted-quad, so pg_hton32 the value into ip_addr directly. - */ -static Datum -read_ipv4(const chc_column* col, uint64_t row) { - inet* res = (inet*)palloc0(sizeof(inet)); - uint32_t addr; - - memcpy(&addr, (const uint8_t*)chc_column_fixed_data(col, NULL) + row * 4, 4); - addr = pg_hton32(addr); - ip_family(res) = PGSQL_AF_INET; - ip_bits(res) = 32; - memcpy(ip_addr(res), &addr, 4); - SET_INET_VARSIZE(res); - return InetPGetDatum(res); -} - -/* IPv6 wire is already network order; same layout as PG inet ip_addr. */ -static Datum -read_ipv6(const chc_column* col, uint64_t row) { - inet* res = (inet*)palloc0(sizeof(inet)); - - ip_family(res) = PGSQL_AF_INET6; - ip_bits(res) = 128; - memcpy( - ip_addr(res), (const uint8_t*)chc_column_fixed_data(col, NULL) + row * 16, 16 - ); - SET_INET_VARSIZE(res); - return InetPGetDatum(res); -} - -static Datum -read_enum_as_text(const chc_column* col, const chc_type* type, uint64_t row) { - size_t es; - const uint8_t* p = (const uint8_t*)chc_column_fixed_data(col, &es); - int64_t v = 0; - - if (es == 1) { - v = (int8_t)p[row]; - } else { - int16_t t; - - memcpy(&t, p + row * 2, 2); - v = t; - } - - size_t n = chc_type_enum_count(type); - - for (size_t i = 0; i < n; i++) { - const char* en; - size_t el; - int64_t ev; - - chc_type_enum_at(type, i, &en, &el, &ev); - if (ev == v) { - return PointerGetDatum(cstring_to_text_with_len(en ? en : "", el)); - } - } - return PointerGetDatum(cstring_to_text_with_len("", 0)); -} - -/* - * JSON body bytes are STRING-serialized JSON document text. Run them - * through json_in / jsonb_in depending on the caller's target valtype - * (set by ch_kind_to_pg_oid to JSONBOID, overridable when the foreign - * table declares the column as `json`). - */ -static Datum -read_json(const chc_column* col, uint64_t row, Oid valtype) { - const char* p; - size_t len; - char* cstr; - Datum ret; - - slice_str(col, row, &p, &len); - cstr = palloc(len + 1); - memcpy(cstr, p, len); - cstr[len] = '\0'; - ret = DirectFunctionCall1( - valtype == JSONOID ? json_in : jsonb_in, CStringGetDatum(cstr) - ); - pfree(cstr); - return ret; -} - -/* - * LowCardinality(String) or LowCardinality(Nullable(String)). Key for - * row indexes into the dict column; the dict's first slot is the null - * sentinel for the Nullable variant. - */ -static Datum -read_lc_string( - const chc_column* col, - const chc_type* type, - uint64_t row, - Oid* valtype, - bool* is_null -) { - int ks = chc_column_lc_key_size(col); - const uint8_t* kp = (const uint8_t*)chc_column_lc_keys(col) + (size_t)row * ks; - uint64_t k = 0; - - switch (ks) { - case 1: - k = kp[0]; - break; - case 2: { - uint16_t v; - - memcpy(&v, kp, 2); - k = v; - break; - } - case 4: { - uint32_t v; - - memcpy(&v, kp, 4); - k = v; - break; - } - case 8: - memcpy(&k, kp, 8); - break; - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected LowCardinality key size %d", ks) - ); - } - - const chc_column* dict = chc_column_lc_dict(col); - const chc_type* inner_t = chc_type_child(type, 0); - - if (chc_type_kind(inner_t) == CHC_NULLABLE && - chc_column_layout(dict) == CHC_COL_NULLABLE) { - const uint8_t* dnm = chc_column_null_map(dict); - - if (dnm && dnm[k]) { - *valtype = TEXTOID; - *is_null = true; - return (Datum)0; - } - dict = chc_column_nullable_inner(dict); - } - - *valtype = TEXTOID; - *is_null = false; - if (chc_column_layout(dict) != CHC_COL_STRING) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported LowCardinality inner type") - ); - } - return read_string_as_text(dict, k); -} - -static Datum -read_array( - const chc_column* col, - const chc_type* type, - uint64_t row, - Oid* valtype, - bool* is_null -) { - const uint64_t* offs = chc_column_array_offsets(col); - uint64_t start = row == 0 ? 0 : offs[row - 1]; - uint64_t end = offs[row]; - uint64_t len = end - start; - const chc_type* inner_t = chc_type_child(type, 0); - const chc_column* inner = chc_column_array_values(col); - ch_binary_array_t* slot = (ch_binary_array_t*)palloc(sizeof(ch_binary_array_t)); - const chc_type* leaf = type; - int ndim = 0; - - /* - * postgres uses one array type per element type regardless of nesting, so - * walk past nested Array layers to the leaf scalar type. - */ - while (chc_type_kind(leaf) == CHC_ARRAY) { - ndim++; - leaf = chc_type_child(leaf, 0); - } - - slot->len = len; - slot->ndim = ndim; - slot->item_type = ch_kind_to_pg_oid(leaf); - slot->array_type = get_array_type(slot->item_type); - if (slot->array_type == InvalidOid) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: could not find array type for column type \"%s\"", - chc_type_name(leaf, NULL) - ) - ); - } - - if (len > 0) { - Oid scratch = slot->item_type; - - slot->datums = (Datum*)palloc0(sizeof(Datum) * len); - slot->nulls = (bool*)palloc0(sizeof(bool) * len); - - /* - * For ndim==1 read_value returns leaf scalars; for ndim>1 inner_t is - * itself CHC_ARRAY so recursion produces nested ch_binary_array_t*. - * Use a scratch valtype to avoid clobbering slot->item_type. - */ - for (uint64_t i = 0; i < len; ++i) { - slot->datums[i] = - read_value(inner, inner_t, start + i, &scratch, &slot->nulls[i]); - } - } else { - slot->datums = NULL; - slot->nulls = NULL; - } - - *valtype = ANYARRAYOID; - *is_null = false; - return PointerGetDatum(slot); -} - -static Datum -read_tuple( - const chc_column* col, - const chc_type* type, - uint64_t row, - Oid* valtype, - bool* is_null -) { - size_t n = chc_type_n_children(type); - ch_binary_tuple_t* slot; - - if (n == 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: returned tuple is empty") - ); - } - - slot = (ch_binary_tuple_t*)palloc(sizeof(ch_binary_tuple_t)); - slot->datums = (Datum*)palloc(sizeof(Datum) * n); - slot->nulls = (bool*)palloc0(sizeof(bool) * n); - slot->types = (Oid*)palloc0(sizeof(Oid) * n); - slot->len = n; - slot->ch_type_name = chc_type_name(type, NULL); - - for (size_t i = 0; i < n; ++i) { - const chc_type* ft = chc_type_child(type, i); - const chc_column* fc = chc_column_tuple_child(col, i); - - slot->datums[i] = read_value(fc, ft, row, &slot->types[i], &slot->nulls[i]); - } - - *valtype = RECORDOID; - *is_null = false; - return PointerGetDatum(slot); -} - -static Datum -read_value( - const chc_column* col, - const chc_type* type, - uint64_t row, - Oid* valtype, - bool* is_null -) { - /* Unwrap outer Nullable, handling nulls here. */ - if (chc_type_kind(type) == CHC_NULLABLE) { - const chc_type* inner_t = chc_type_child(type, 0); - - if (chc_column_layout(col) == CHC_COL_NULLABLE) { - const uint8_t* nm = chc_column_null_map(col); - - if (nm && nm[row]) { - *valtype = ch_kind_to_pg_oid(inner_t); - *is_null = true; - return (Datum)0; - } - col = chc_column_nullable_inner(col); - } - type = inner_t; - } - *is_null = false; - - switch (chc_type_kind(type)) { - case CHC_VOID: - case CHC_NOTHING: - *valtype = InvalidOid; - *is_null = true; - return (Datum)0; - case CHC_UINT8: - *valtype = INT2OID; - return (Datum)rd_u8((const uint8_t*)chc_column_fixed_data(col, NULL), row); - case CHC_BOOL: - *valtype = BOOLOID; - return (Datum)rd_bool((const bool*)chc_column_fixed_data(col, NULL), row); - case CHC_INT8: - *valtype = INT2OID; - return (Datum)rd_i8((const uint8_t*)chc_column_fixed_data(col, NULL), row); - case CHC_INT16: - *valtype = INT2OID; - return (Datum)rd_i16((const uint8_t*)chc_column_fixed_data(col, NULL), row); - case CHC_UINT16: - *valtype = INT4OID; - return (Datum)rd_u16((const uint8_t*)chc_column_fixed_data(col, NULL), row); - case CHC_INT32: - *valtype = INT4OID; - return (Datum)rd_i32((const uint8_t*)chc_column_fixed_data(col, NULL), row); - case CHC_UINT32: - *valtype = INT8OID; - return Int64GetDatum( - (int64)rd_u32((const uint8_t*)chc_column_fixed_data(col, NULL), row) - ); - case CHC_INT64: - *valtype = INT8OID; - return Int64GetDatum( - rd_i64((const uint8_t*)chc_column_fixed_data(col, NULL), row) - ); - case CHC_UINT64: { - uint64_t v = rd_u64((const uint8_t*)chc_column_fixed_data(col, NULL), row); - - if (v > (uint64_t)PG_INT64_MAX) { - ereport( - ERROR, - errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("value %" PRIu64 " is out of range of bigint", v) - ); - } - *valtype = INT8OID; - return Int64GetDatum((int64)v); - } - case CHC_FLOAT32: - *valtype = FLOAT4OID; - return Float4GetDatum( - rd_f32((const uint8_t*)chc_column_fixed_data(col, NULL), row) - ); - case CHC_FLOAT64: - *valtype = FLOAT8OID; - return Float8GetDatum( - rd_f64((const uint8_t*)chc_column_fixed_data(col, NULL), row) - ); - case CHC_DECIMAL32: - case CHC_DECIMAL64: - case CHC_DECIMAL128: - case CHC_DECIMAL256: - *valtype = NUMERICOID; - return read_decimal(col, type, row); - case CHC_STRING: - *valtype = TEXTOID; - return read_string_as_text(col, row); - case CHC_ENUM8: - case CHC_ENUM16: - *valtype = TEXTOID; - return read_enum_as_text(col, type, row); - case CHC_JSON: - case CHC_OBJECT: { - /* - * *valtype arrives set to JSONBOID by default (from - * ch_kind_to_pg_oid) but the caller's foreign-table - * declaration may have requested JSONOID; honor it. - */ - Oid target = (*valtype == JSONOID) ? JSONOID : JSONBOID; - - *valtype = target; - return read_json(col, row, target); - } - case CHC_FIXED_STRING: - *valtype = TEXTOID; - return read_fixedstring_as_text(col, row); - case CHC_DATE: - *valtype = DATEOID; - return DateADTGetDatum( - (DateADT)rd_u16((const uint8_t*)chc_column_fixed_data(col, NULL), row) - - CH_TO_PG_DATE_OFFSET - ); - case CHC_DATE32: - *valtype = DATEOID; - return DateADTGetDatum( - (DateADT)rd_i32((const uint8_t*)chc_column_fixed_data(col, NULL), row) - - CH_TO_PG_DATE_OFFSET - ); - case CHC_DATETIME: { - uint32_t secs = rd_u32((const uint8_t*)chc_column_fixed_data(col, NULL), row); - - *valtype = TIMESTAMPTZOID; - return TimestampTzGetDatum(time_t_to_timestamptz((pg_time_t)secs)); - } - case CHC_DATETIME64: { - int64 raw = rd_i64((const uint8_t*)chc_column_fixed_data(col, NULL), row); - uint32_t scale = chc_type_datetime64_scale(type); - - if (scale >= lengthof(pow10i)) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: DateTime64 scale %u out of range", scale) - ); - } - int64 power = pow10i[scale]; - - *valtype = TIMESTAMPTZOID; - /* multiply before divide so scale > 6 keeps sub-second us */ - return TimestampTzGetDatum( - time_t_to_timestamptz(raw / power) + (raw % power) * USECS_PER_SEC / power - ); - } - case CHC_UUID: - *valtype = UUIDOID; - return read_uuid(col, row); - case CHC_IPV4: - *valtype = INETOID; - return read_ipv4(col, row); - case CHC_IPV6: - *valtype = INETOID; - return read_ipv6(col, row); - case CHC_LOW_CARDINALITY: - return read_lc_string(col, type, row, valtype, is_null); - case CHC_ARRAY: - return read_array(col, type, row, valtype, is_null); - case CHC_TUPLE: - return read_tuple(col, type, row, valtype, is_null); - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: unsupported type in binary protocol") - ); - } - /* unreachable */ - return (Datum)0; -} - -/* ---- read state ----------------------------------------------------- */ - -static bool -load_block(ch_binary_read_state_t* state) { - state->cur = ch_binary_response_fetch_next_block(state->resp); - if (state->cur == NULL) { - const char* resp_err = ch_binary_response_error(state->resp); - - if (resp_err) { - state->error = pstrdup(resp_err); - } - state->done = true; - return false; - } - return true; -} - -void -ch_binary_read_state_init(ch_binary_read_state_t* state, ch_binary_response_t* resp) { - const char* resp_err; - size_t ncols; - - state->resp = resp; - state->block = 0; - state->row = 0; - state->done = false; - state->error = NULL; - state->coltypes = NULL; - state->values = NULL; - state->nulls = NULL; - state->cur = NULL; - - resp_err = ch_binary_response_error(resp); - if (resp_err) { - state->done = true; - state->error = pstrdup(resp_err); - return; - } - - ncols = ch_binary_response_columns(resp); - if (ncols == 0) { - state->done = true; - return; - } - - state->coltypes = palloc0(sizeof(Oid) * ncols); - state->values = palloc0(sizeof(Datum) * ncols); - state->nulls = palloc0(sizeof(bool) * ncols); - - if (!load_block(state)) { - return; - } - - for (size_t i = 0; i < ncols; i++) { - state->coltypes[i] = ch_kind_to_pg_oid(chc_block_column_type(state->cur, i)); - } -} - -bool -ch_binary_read_row(ch_binary_read_state_t* state) { - size_t ncols; - - if (state->done || state->coltypes == NULL || state->error) { - return false; - } - - ncols = ch_binary_response_columns(state->resp); - -again: - if (state->cur == NULL) { - if (!load_block(state)) { - return false; - } - } - - if (state->row >= chc_block_n_rows(state->cur)) { - state->row = 0; - state->cur = NULL; - goto again; - } - - PG_TRY(); - { - for (size_t i = 0; i < ncols; i++) { - /* - * Currently, read_value overwrites *valtype for most types - * canonical PG type for that CH kind. For CHC_JSON we honor the - * incoming value so callers (binary_fetch_row) can pin the Datum - * type to JSONOID for `data json` foreign columns and avoid the - * jsonb_in -> jsonb_out round-trip that would strip CH's verbatim - * STRING- serialized formatting. - */ - Oid t = state->coltypes[i]; - const chc_column* col = chc_block_column(state->cur, i); - const chc_type* ct = chc_block_column_type(state->cur, i); - - state->values[i] = read_value(col, ct, state->row, &t, &state->nulls[i]); - } - } - PG_CATCH(); - { - MemoryContext mcxt = GetMemoryChunkContext(state); - MemoryContext oldcxt; - ErrorData* edata; - const char* msg; - static const char prefix[] = "pg_clickhouse: "; - - oldcxt = MemoryContextSwitchTo(mcxt); - edata = CopyErrorData(); - msg = edata->message ? edata->message : "unknown error"; - - /* - * binary_fetch_row re-prefixes with "pg_clickhouse: error while - * reading row:"; drop a leading "pg_clickhouse: " here so the final - * message doesn't carry it twice. - */ - if (strncmp(msg, prefix, sizeof(prefix) - 1) == 0) { - msg += sizeof(prefix) - 1; - } - state->error = pstrdup(msg); - FlushErrorState(); - FreeErrorData(edata); - MemoryContextSwitchTo(oldcxt); - state->done = true; - return false; - } - PG_END_TRY(); - - state->row++; - return true; -} - -void -ch_binary_read_state_free(ch_binary_read_state_t* state) { - /* state->error is palloc'd; freed with surrounding memory context. */ - state->error = NULL; -} diff --git a/src/binary/encode.c b/src/binary/encode.c index 7ec3aa4c..30dc6572 100644 --- a/src/binary/encode.c +++ b/src/binary/encode.c @@ -1,94 +1,21 @@ /* * encode.c * - * PG-side INSERT path. Reads PG Datums and dispatches into the typed - * ch_binary_append_* shims exposed by insert.c. + * PG-side INSERT path. Matches slot attributes onto the ClickHouse columns + * the server announced, then hands each Datum to pg-clickhouse-c's writer, + * which dispatches on (PG type, CH kind) and casts when no direct pair fits. */ #include "postgres.h" #include -#include /* AF_INET, expanded by PG inet macros */ #include "access/tupdesc.h" -#include "catalog/pg_type_d.h" -#include "fmgr.h" -#include "nodes/pg_list.h" +#include "executor/tuptable.h" #include "utils/builtins.h" -#include "utils/date.h" -#include "utils/inet.h" -#include "utils/lsyscache.h" -#include "utils/palloc.h" -#include "utils/timestamp.h" -#include "utils/uuid.h" -#if PG_VERSION_NUM >= 190000 -#include "varatt.h" -#endif #include "binary_internal.h" - -/* CH Date epoch is unix; offset to PG epoch (2000-01-01) */ -#define CH_TO_PG_DATE_OFFSET (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) - -/* - * Map a CH column kind to the PG type our import path uses. Used when - * constructing the TupleDesc for INSERT...VALUES. - */ -static Oid -ch_kind_to_pg_oid_for_insert(const chc_type* type, const char* colname) { - chc_kind kind = chc_type_kind(type); - Oid oid = ch_scalar_oids[kind]; - - if (OidIsValid(oid)) { - return oid; - } - - switch (kind) { - case CHC_ARRAY: { - /* - * postgres uses one array type per element type regardless of - * nesting, so unwrap nested Array layers before looking up. - */ - const chc_type* leaf = type; - Oid item_type; - Oid array_type; - - while (chc_type_kind(leaf) == CHC_ARRAY) { - leaf = chc_type_child(leaf, 0); - } - item_type = ch_kind_to_pg_oid_for_insert(leaf, colname); - array_type = get_array_type(item_type); - - if (array_type == InvalidOid) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: could not find array type for column \"%s\"", - colname ? colname : "?" - ) - ); - } - return array_type; - } - case CHC_TUPLE: - return RECORDOID; - case CHC_NULLABLE: - case CHC_LOW_CARDINALITY: - return ch_kind_to_pg_oid_for_insert(chc_type_child(type, 0), colname); - default: - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: unsupported column type \"%s\" for \"%s\"", - chc_type_name(type, NULL), - colname ? colname : "?" - ) - ); - } - return InvalidOid; -} +#include "fdw.h" void ch_binary_prepare_insert( @@ -96,297 +23,93 @@ ch_binary_prepare_insert( const ch_query* query, ch_binary_insert_state* state ) { - ch_binary_column_info* cols = NULL; - size_t n = 0; - ch_binary_insert_handle* h; + ch_binary_insert_handle* h = + ch_binary_begin_insert((ch_binary_connection_t*)conn, query); - h = ch_binary_begin_insert((ch_binary_connection_t*)conn, query, &cols, &n); - - state->len = n; state->insert_block = h; - - if (n == 0) { - return; - } - - state->outdesc = CreateTemplateTupleDesc(n); - for (size_t i = 0; i < n; i++) { - Oid pg_type = ch_kind_to_pg_oid_for_insert(cols[i].type, cols[i].name); - - /* - * CHC_JSON defaults to JSONBOID; honor foreign-table's JSONOID - * declaration when present so outdesc matches the source slot and no - * json↔jsonb conversion is needed. - */ - if (pg_type == JSONBOID && query->tupdesc && query->attr_nums) { - AttrNumber attnum = list_nth_int((List*)query->attr_nums, i); - - if (attnum >= 1 && attnum <= query->tupdesc->natts && - TupleDescAttr(query->tupdesc, attnum - 1)->atttypid == JSONOID) { - pg_type = JSONOID; - } - } - - TupleDescInitEntry( - state->outdesc, - (AttrNumber)(i + 1), - cols[i].name ? cols[i].name : "", - pg_type, - -1, - 0 - ); - } + state->len = ch_binary_insert_ncols(h); } /* - * Append a single value (already extracted from a Datum + isnull) into the - * current column, dispatching on (PG Oid, CH kind). ereports on mismatch. + * Resolve which slot attribute feeds each ClickHouse column. Matches by name, + * honoring the column_name FDW option; an input descriptor whose first + * attribute is unnamed carries no names at all, so map by position. */ static void -append_one( - ch_binary_insert_handle* h, - size_t colidx, - chc_kind kind, - Datum val, - Oid valtype, - bool isnull -) { - switch (valtype) { - case INT2OID: - case INT4OID: - case INT8OID: { - int64_t v = 0; +build_colmap(ch_binary_insert_state* state, TupleDesc indesc) { + ch_binary_insert_handle* h = state->insert_block; + bool positional = + indesc->natts > 0 && NameStr(TupleDescAttr(indesc, 0)->attname)[0] == '\0'; - /* Support mixing integer types. */ - if (!(kind == CHC_BOOL || (kind >= CHC_INT8 && kind <= CHC_INT64) || - (kind >= CHC_UINT8 && kind <= CHC_UINT64))) { - goto type_mismatch; - } - if (!isnull) { - if (valtype == INT2OID) { - v = (int64_t)DatumGetInt16(val); - } else if (valtype == INT4OID) { - v = (int64_t)DatumGetInt32(val); - } else { - v = DatumGetInt64(val); - } - } - ch_binary_append_int(h, colidx, v, isnull); - return; - } - case BOOLOID: - if (kind != CHC_BOOL && kind != CHC_UINT8) { - goto type_mismatch; - } - ch_binary_append_bool(h, colidx, (bool)val, isnull); - return; - case FLOAT4OID: - if (kind != CHC_FLOAT32) { - goto type_mismatch; - } - ch_binary_append_float(h, colidx, DatumGetFloat4(val), isnull); - return; - case FLOAT8OID: - if (kind != CHC_FLOAT64) { - goto type_mismatch; - } - ch_binary_append_double(h, colidx, DatumGetFloat8(val), isnull); - return; - case NUMERICOID: { - char* s = NULL; - - if (kind != CHC_DECIMAL32 && kind != CHC_DECIMAL64 && kind != CHC_DECIMAL128 && - kind != CHC_DECIMAL256) { - goto type_mismatch; - } - if (!isnull) { - s = DatumGetCString(DirectFunctionCall1(numeric_out, val)); - } - ch_binary_append_decimal(h, colidx, s, isnull); - if (s) { - pfree(s); - } - return; - } - case TEXTOID: { - const char* p = NULL; - size_t len = 0; - text* string = NULL; + state->colmap = palloc0(state->len * sizeof(ch_binary_insert_colmap)); - if (!isnull) { - string = PG_DETOAST_DATUM(val); - p = VARDATA(string); - len = VARSIZE_ANY_EXHDR(string); - } - switch (kind) { - case CHC_FIXED_STRING: - case CHC_STRING: - case CHC_ENUM8: - case CHC_ENUM16: - ch_binary_append_bytes(h, colidx, p, len, isnull); - return; - default: - goto type_mismatch; - } - } - case DATEOID: { - int64_t seconds = 0; + for (size_t i = 0; i < state->len; i++) { + ch_binary_insert_colmap* m = &state->colmap[i]; + const char* chname = ch_binary_insert_column_name(h, i); - if (kind != CHC_DATE && kind != CHC_DATE32) { - goto type_mismatch; - } - if (!isnull) { - seconds = - ((int64_t)DatumGetDateADT(val) + CH_TO_PG_DATE_OFFSET) * SECS_PER_DAY; - } - ch_binary_append_date_seconds(h, colidx, seconds, isnull); - return; - } - case TIMESTAMPOID: - case TIMESTAMPTZOID: { - switch (kind) { - case CHC_DATETIME: { - int64_t seconds = - isnull ? 0 : (int64_t)timestamptz_to_time_t(DatumGetTimestamp(val)); + if (positional) { + if (i < (size_t)indesc->natts) { + m->attnum = (AttrNumber)(i + 1); + } + } else { + for (int j = 0; j < indesc->natts; j++) { + Form_pg_attribute attin = TupleDescAttr(indesc, j); + CustomColumnInfo* cinfo; + const char* inname; - ch_binary_append_datetime_seconds(h, colidx, seconds, isnull); - } break; - case CHC_DATETIME64: { - int64_t raw = 0; + if (attin->attisdropped) { + continue; + } - if (!isnull) { - uint32_t prec = ch_binary_column_datetime64_precision(h, colidx); - Timestamp t = DatumGetTimestamp(val); - int64 power = pow10i[prec]; - int64 secs = t / USECS_PER_SEC; - int64 us_rem = t % USECS_PER_SEC; + /* Honor column_name FDW option, falls through to attname */ + cinfo = OidIsValid(state->relid) + ? chfdw_get_custom_column_info(state->relid, j + 1) + : NULL; + inname = (cinfo && cinfo->colname[0]) ? cinfo->colname + : NameStr(attin->attname); - /* - * floor-divide; C trunc-to-zero leaves - * negative remainder - */ - if (us_rem < 0) { - secs -= 1; - us_rem += USECS_PER_SEC; + if (strcmp(chname, inname) == 0) { + m->attnum = (AttrNumber)(j + 1); + break; } - secs += CH_TO_PG_DATE_OFFSET * SECS_PER_DAY; - raw = secs * power + us_rem * power / USECS_PER_SEC; } - ch_binary_append_datetime64_raw(h, colidx, raw, isnull); - } break; - default: - goto type_mismatch; - } - return; - } - case ANYARRAYOID: { - ch_binary_array_t* arr; - chc_kind item_kind; - Oid child_valtype; - - if (kind != CHC_ARRAY) { - goto type_mismatch; - } - - arr = (ch_binary_array_t*)DatumGetPointer(val); - ch_binary_array_begin(h, colidx); - - /* - * While array_begin is active ch_binary_column_kind targets - * the inner element kind. For nested arrays the children are - * themselves ch_binary_array_t* so recurse with ANYARRAYOID; - * at the leaf level use the scalar item_type. - */ - item_kind = ch_binary_column_kind(h, colidx); - child_valtype = (arr->ndim > 1) ? ANYARRAYOID : arr->item_type; - for (size_t i = 0; i < arr->len; i++) { - append_one(h, 0, item_kind, arr->datums[i], child_valtype, arr->nulls[i]); } - ch_binary_array_end(h); - return; - } - case UUIDOID: { - uint8_t bytes[16]; - - if (kind != CHC_UUID) { - goto type_mismatch; - } - if (!isnull) { - memcpy(bytes, DatumGetUUIDP(val)->data, 16); - } else { - memset(bytes, 0, 16); + if (m->attnum == 0) { + ereport( + ERROR, + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal("pg_clickhouse: could not create conversion map"), + errdetail( + "ClickHouse column \"%s\" has no matching attribute in type %s.", + chname, + format_type_be(indesc->tdtypeid) + ) + ); } - ch_binary_append_uuid(h, colidx, bytes, isnull); - return; + m->atttypid = TupleDescAttr(indesc, m->attnum - 1)->atttypid; } - case INETOID: { - const uint8_t* addr = NULL; - size_t addrlen = 0; +} - if (kind != CHC_IPV4 && kind != CHC_IPV6) { - goto type_mismatch; - } - if (!isnull) { - inet* ipa = DatumGetInetPP(val); - int fam = ip_family(ipa); - int expected = (kind == CHC_IPV4) ? PGSQL_AF_INET : PGSQL_AF_INET6; +void +ch_binary_insert_tuple(ch_binary_insert_state* state, TupleTableSlot* slot) { + pgch_writer* w = ch_binary_insert_writer(state->insert_block); - if (fam != expected) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: inet family mismatch for column %zu", colidx) - ); - } - addr = ip_addr(ipa); - addrlen = ip_addrsize(ipa); - } else { - addrlen = (kind == CHC_IPV4) ? 4 : 16; - } - ch_binary_append_inet(h, colidx, addr, addrlen, isnull); - return; - } - case JSONOID: - case JSONBOID: { - char* s = NULL; - size_t len = 0; + if (state->colmap == NULL) { + /* Outlive the per-tuple context the executor calls us in. */ + MemoryContext old = MemoryContextSwitchTo(state->memcxt); - if (kind != CHC_JSON && kind != CHC_OBJECT) { - goto type_mismatch; - } - if (!isnull) { - s = DatumGetCString( - DirectFunctionCall1(valtype == JSONBOID ? jsonb_out : json_out, val) - ); - len = strlen(s); - } - ch_binary_append_bytes(h, colidx, s, len, isnull); - if (s) { - pfree(s); - } - return; - } - default: - goto type_mismatch; + build_colmap(state, slot->tts_tupleDescriptor); + MemoryContextSwitchTo(old); } -type_mismatch: - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: unexpected PG/CH type pair for column %zu", colidx) - ); -} + for (size_t i = 0; i < state->len; i++) { + ch_binary_insert_colmap* m = &state->colmap[i]; + bool isnull; + Datum val = slot_getattr(slot, m->attnum, &isnull); -void -ch_binary_column_append_data(ch_binary_insert_state* state, size_t colidx) { - Datum val = state->values[colidx]; - Oid valtype = TupleDescAttr(state->outdesc, colidx)->atttypid; - bool isnull = state->nulls[colidx]; - chc_kind kind = ch_binary_column_kind(state->insert_block, colidx); - - append_one(state->insert_block, colidx, kind, val, valtype, isnull); + pgch_append_datum(w, i, val, m->atttypid, isnull); + } } void diff --git a/src/binary/insert.c b/src/binary/insert.c index 9eb43510..66e2dd0d 100644 --- a/src/binary/insert.c +++ b/src/binary/insert.c @@ -1,161 +1,21 @@ /* * insert.c * - * Handle binary INSERT lifecycle - * - * -begin_insert sends query and uses server's empty Data block as schema - * - append functions buffer column values in format chosen from schema - * - flush_block assembles buffered columns and sends one Data packet - * - finalize_insert sends closing empty Data packet and reads final server response - * - release_insert can run during memory context reset and never writes to connection. + * Server-facing half of the binary INSERT: enter insert mode, take the + * schema from the server's initial empty Data block, ship blocks, finish. + * Row buffering is pg-clickhouse-c's pgch_writer, built over the same + * chc_type tree the server sent. */ #include "postgres.h" #include -#include "common/hashfn.h" -#include "port/pg_bswap.h" #include "utils/memutils.h" #include "utils/palloc.h" #include "binary_internal.h" -/* dynamic buffer for raw bytes; palloc-backed, freed by context delete */ -typedef struct dynbuf { - uint8_t* data; - size_t len; - size_t cap; -} dynbuf; - -static void -dynbuf_reserve(dynbuf* b, size_t need) { - if (need <= b->cap) { - return; - } - size_t ncap = b->cap ? b->cap : 64; - - while (ncap < need) { - ncap *= 2; - } - b->data = - b->data - ? repalloc_huge(b->data, ncap) - : MemoryContextAllocExtended(CurrentMemoryContext, ncap, MCXT_ALLOC_HUGE); - b->cap = ncap; -} - -static void -dynbuf_append(dynbuf* b, const void* src, size_t n) { - dynbuf_reserve(b, b->len + n); - if (src && n) { - memcpy(b->data + b->len, src, n); - } - b->len += n; -} - -static void -dynbuf_append_zero(dynbuf* b, size_t n) { - dynbuf_reserve(b, b->len + n); - memset(b->data + b->len, 0, n); - b->len += n; -} - -static void -dynbuf_reset(dynbuf* b) { - b->len = 0; -} - -/* dynamic array of u64 */ -typedef struct u64buf { - uint64_t* data; - size_t len; - size_t cap; -} u64buf; - -static void -u64buf_push(u64buf* b, uint64_t v) { - if (b->len + 1 > b->cap) { - size_t ncap = b->cap ? b->cap * 2 : 16; - size_t bytes = ncap * sizeof(uint64_t); - - b->data = b->data ? repalloc_huge(b->data, bytes) - : MemoryContextAllocExtended( - CurrentMemoryContext, bytes, MCXT_ALLOC_HUGE - ); - b->cap = ncap; - } - b->data[b->len++] = v; -} - -static void -u64buf_reset(u64buf* b) { - b->len = 0; -} - -/* - * Buffer-node tree mirroring the column's chc_type, one node per CH - * structural level. Typed append_* functions write leaves, array begin/end - * records offsets on Array nodes, flush assembles chc_build_* bottom-up - * from the same shape. - */ -typedef enum { - ICN_FIXED, - ICN_STRING, - ICN_NULLABLE, - ICN_ARRAY, - ICN_LC, -} icn_kind; - -typedef struct icn icn; - -struct icn { - icn_kind kind; - const chc_type* type; /* this level's type; source of elem size, enum - * table, decimal scale, dt64 precision */ - union { - struct { - dynbuf data; /* row-aligned values */ - size_t elem_size; - uint32_t dt64_precision; - } fixed; - - struct { - dynbuf data; /* byte-flat rows */ - u64buf offs; /* cumulative ends per row */ - bool is_json; - } str; - - struct { - dynbuf null_map; /* one byte per row */ - icn* inner; - } nullable; - - struct { - u64buf offs; /* cumulative ends per committed row */ - icn* values; - } array; - - struct { - dynbuf data; /* raw rows; dict dedup happens at flush */ - u64buf offs; - dynbuf null_map; /* only when inner_nullable */ - bool inner_nullable; - } lc; - }; -}; - -typedef struct ic_col { - const chc_type* t; /* full column type (incl. Nullable wrapper) */ - icn* root; - - /* - * Cached column info exposed to callers. info.type borrows from the - * initial_block's chc_type tree, unwrapping Nullable + outer LC. - */ - ch_binary_column_info info; -} ic_col; - struct ch_binary_insert_handle { MemoryContext cxt; chc_client* client; @@ -163,98 +23,12 @@ struct ch_binary_insert_handle { * state on error */ chc_block* initial_block; /* schema source (server's empty Data) */ size_t ncols; - ic_col* cols; - icn** cursor; /* stack of open Array nodes; top's values child - * receives appends. Empty at top level */ - size_t cursor_len; - size_t cursor_cap; - size_t cursor_col; /* column that opened cursor[0] */ + char** names; /* column names, borrowed by writer's pgch_col */ + pgch_writer* w; /* row buffers, one per column */ bool started; bool finalized; /* finalize_insert has run (success or raised) */ }; -/* Build one buffer node per structural level of `t` */ -static icn* -build_node(const chc_type* t) { - icn* n = palloc0(sizeof(icn)); - - n->type = t; - switch (chc_type_kind(t)) { - case CHC_NULLABLE: - n->kind = ICN_NULLABLE; - n->nullable.inner = build_node(chc_type_child(t, 0)); - return n; - case CHC_ARRAY: - n->kind = ICN_ARRAY; - n->array.values = build_node(chc_type_child(t, 0)); - return n; - case CHC_LOW_CARDINALITY: { - /* - * Nullable lives inside LowCardinality, not as a wrapper above it: - * resolve_leaf tracks the per-row null bits build_lc_dict reads to - * map nulls onto dict slot 0. - */ - const chc_type* inner = chc_type_child(t, 0); - bool inner_nullable = chc_type_kind(inner) == CHC_NULLABLE; - const chc_type* base = inner_nullable ? chc_type_child(inner, 0) : inner; - - if (chc_type_kind(base) != CHC_STRING) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: unsupported LowCardinality variant: %s", - chc_type_name(base, NULL) - ) - ); - } - n->kind = ICN_LC; - n->lc.inner_nullable = inner_nullable; - return n; - } - case CHC_STRING: - n->kind = ICN_STRING; - return n; - case CHC_JSON: - n->kind = ICN_STRING; - n->str.is_json = true; - return n; - case CHC_DATETIME64: { - int scale = chc_type_datetime64_scale(t); - - if (scale < 0 || (size_t)scale >= lengthof(pow10i)) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg("pg_clickhouse: DateTime64 scale %d out of range", scale) - ); - } - n->kind = ICN_FIXED; - n->fixed.elem_size = chc_type_elem_size(t); - n->fixed.dt64_precision = (uint32_t)scale; - return n; - } - default: { - size_t es = chc_type_elem_size(t); - - if (es == 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_DATA_TYPE), - errmsg( - "pg_clickhouse: could not prepare insert - unsupported column " - "type: %s", - chc_type_name(t, NULL) - ) - ); - } - n->kind = ICN_FIXED; - n->fixed.elem_size = es; - return n; - } - } -} - static void recv_initial_block(struct ch_binary_state* s, ch_binary_insert_handle* h) { for (;;) { @@ -264,7 +38,7 @@ recv_initial_block(struct ch_binary_state* s, ch_binary_insert_handle* h) { if (rc != CHC_OK) { s->broken = true; - raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + pgch_raise(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); } if (pkt.kind == CHC_PKT_EXCEPTION) { const char* msg = "server exception"; @@ -296,7 +70,7 @@ recv_initial_block(struct ch_binary_state* s, ch_binary_insert_handle* h) { } /* - * After build_node ereports the server is mid-INSERT awaiting our + * After pgch_writer_new ereports the server is mid-INSERT awaiting our * Data; send empty Data + drain so the connection stays usable. */ static void @@ -321,12 +95,7 @@ drain_aborted_insert(struct ch_binary_state* s) { } ch_binary_insert_handle* -ch_binary_begin_insert( - ch_binary_connection_t* conn, - const ch_query* query, - ch_binary_column_info** out_cols, - size_t* out_n -) { +ch_binary_begin_insert(ch_binary_connection_t* conn, const ch_query* query) { struct ch_binary_state* s = conn_state(conn); /* @@ -384,7 +153,7 @@ ch_binary_begin_insert( if (rc != CHC_OK) { s->broken = true; - raise_chc(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); + pgch_raise(&err, ERRCODE_FDW_ERROR, "could not prepare insert - "); } recv_initial_block(s, h); @@ -396,54 +165,21 @@ ch_binary_begin_insert( need_drain = true; h->started = true; - size_t nc = chc_block_n_columns(h->initial_block); + size_t nc = chc_block_n_columns(h->initial_block); + pgch_col* cols = nc ? palloc0(nc * sizeof(pgch_col)) : NULL; h->ncols = nc; - h->cols = palloc0(nc * sizeof(ic_col)); + h->names = nc ? palloc0(nc * sizeof(char*)) : NULL; for (size_t i = 0; i < nc; i++) { - ic_col* c = &h->cols[i]; - const chc_type* ct = chc_block_column_type(h->initial_block, i); size_t nlen; - const char* nm; - - c->t = ct; - c->root = build_node(ct); - nm = chc_block_column_name(h->initial_block, i, &nlen); - c->info.name = pnstrdup(nm ? nm : "", nlen); - - /* - * Unwrap Nullable, then LowCardinality and perhaps its inner - * Nullable to expose innermost type. Nullable inside LC counts - * as column nullability. - */ - const chc_type* vt = ct; - bool is_nullable = false; - - if (chc_type_kind(vt) == CHC_NULLABLE) { - is_nullable = true; - vt = chc_type_child(vt, 0); - } - if (chc_type_kind(vt) == CHC_LOW_CARDINALITY) { - vt = chc_type_child(vt, 0); - is_nullable = chc_type_kind(vt) == CHC_NULLABLE; - if (is_nullable) { - vt = chc_type_child(vt, 0); - } - } - c->info.type = vt; - c->info.is_nullable = is_nullable; - } + const char* nm = chc_block_column_name(h->initial_block, i, &nlen); - *out_cols = NULL; - if (nc) { - ch_binary_column_info* arr = palloc(nc * sizeof(*arr)); - - for (size_t i = 0; i < nc; i++) { - arr[i] = h->cols[i].info; - } - *out_cols = arr; + h->names[i] = pnstrdup(nm ? nm : "", nlen); + cols[i].name = h->names[i]; + cols[i].name_len = nlen; + cols[i].type = chc_block_column_type(h->initial_block, i); } - *out_n = nc; + h->w = pgch_writer_new(cxt, cols, nc); } PG_CATCH(); { @@ -460,877 +196,19 @@ ch_binary_begin_insert( return h; } -/* Node receiving values: innermost open array's values child, else column root */ -static inline icn* -cursor_node(const ch_binary_insert_handle* h, size_t col) { - return h->cursor_len ? h->cursor[h->cursor_len - 1]->array.values - : h->cols[col].root; -} - -/* - * Descend from the append target to its leaf FIXED/STRING/LC node, - * recording a null bit on each Nullable level crossed (or on - * LowCardinality(Nullable(...))). ereports on NULL-into-NOT-NULL. - */ -static icn* -resolve_leaf(ch_binary_insert_handle* h, size_t col, bool isnull) { - icn* node = cursor_node(h, col); - uint8_t b = isnull ? 1 : 0; - bool nullable = false; - - while (node->kind == ICN_NULLABLE) { - dynbuf_append(&node->nullable.null_map, &b, 1); - nullable = true; - node = node->nullable.inner; - } - if (node->kind == ICN_LC && node->lc.inner_nullable) { - dynbuf_append(&node->lc.null_map, &b, 1); - nullable = true; - } - if (isnull && !nullable) { - const chc_type* t = h->cols[h->cursor_len ? h->cursor_col : col].t; - size_t tnlen; - const char* tname = chc_type_name(t, &tnlen); - - ereport( - ERROR, - errcode(ERRCODE_NOT_NULL_VIOLATION), - errmsg( - "pg_clickhouse: cannot append NULL to NOT NULL %.*s column", - (int)tnlen, - tname ? tname : "?" - ) - ); - } - if (node->kind == ICN_ARRAY) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: scalar value into Array column") - ); - } - return node; +pgch_writer* +ch_binary_insert_writer(ch_binary_insert_handle* h) { + return h->w; } -/* Leaf buffer for fixed-width appends; guards union access on misdispatch */ -static dynbuf* -fixed_data(icn* node) { - if (node->kind != ICN_FIXED) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: fixed-width value into non-fixed-width column") - ); - } - return &node->fixed.data; +size_t +ch_binary_insert_ncols(const ch_binary_insert_handle* h) { + return h->ncols; } -/* STRING & LC rows: append bytes, record cumulative end */ -static void -append_row_offs(dynbuf* data, u64buf* offs, const void* p, size_t n) { - if (n) { - dynbuf_append(data, p, n); - } - u64buf_push(offs, data->len); -} - -/* - * Convert decimal text "[-]digits[.frac]" into a ClickHouse Decimal wire value: - * two's-complement signed integer in `width` LE bytes (4/8/16/32 for - * Decimal32/64/128/256), with `scale` fractional digits folded into the value. - */ -static void -decimal_text_to_bytes(const char* s, uint32_t scale, size_t width, uint8_t* out) { - const char* input = s; - bool neg = false; - - if (!s) { - ereport( - ERROR, - errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg("pg_clickhouse: decimal parse failure") - ); - } - if (*s == '-') { - neg = true; - s++; - } else if (*s == '+') { - s++; - } - - /* find offsets, going to iterate digits, skipping non-digits */ - const char* dot = strchr(s, '.'); - size_t ilen = dot ? (size_t)(dot - s) : strlen(s); - const char* frac = dot ? dot + 1 : ""; - size_t flen = strlen(frac); - size_t ndig = ilen + scale; - - uint32_t mag[8] = {}; - size_t nwords = width / 4; - - /* accumulate digits (padded/truncated to scale) into mag */ - for (size_t i = 0; i < ndig; i++) { - char c = i < ilen ? s[i] : i - ilen < flen ? frac[i - ilen] : '0'; - - /* reject NaN / Infinity from numeric_out */ - if (c < '0' || c > '9') { - ereport( - ERROR, - errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), - errmsg( - "pg_clickhouse: cannot encode \"%s\" as ClickHouse Decimal", input - ) - ); - } - uint64_t carry = (uint64_t)(c - '0'); - - for (size_t b = 0; b < nwords; b++) { - uint64_t v = (uint64_t)mag[b] * 10 + carry; - - mag[b] = (uint32_t)v; - carry = v >> 32; - } - } - /* two's-complement negation */ - if (neg) { - for (size_t b = 0; b < nwords; b++) { - mag[b] = ~mag[b]; - } - uint64_t carry = 1; - - for (size_t b = 0; b < nwords && carry; b++) { - uint64_t v = (uint64_t)mag[b] + carry; - - mag[b] = (uint32_t)v; - carry = v >> 32; - } - } - - memcpy(out, mag, width); -} - -static void -append_int_kind(icn* node, int64_t val) { - switch (chc_type_kind(node->type)) { - case CHC_INT8: - case CHC_UINT8: - case CHC_BOOL: { - int8_t v = (int8_t)val; - - dynbuf_append(fixed_data(node), &v, 1); - return; - } - case CHC_INT16: - case CHC_UINT16: { - int16_t v = (int16_t)val; - - dynbuf_append(fixed_data(node), &v, 2); - return; - } - case CHC_INT32: - case CHC_UINT32: { - int32_t v = (int32_t)val; - - dynbuf_append(fixed_data(node), &v, 4); - return; - } - case CHC_INT64: - case CHC_UINT64: { - dynbuf_append(fixed_data(node), &val, 8); - return; - } - default: - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: int value into non-integer column") - ); - } -} - -void -ch_binary_append_int(ch_binary_insert_handle* h, size_t col, int64_t val, bool isnull) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - - if (isnull) { - dynbuf* d = fixed_data(node); - - dynbuf_append_zero(d, node->fixed.elem_size); - } else { - append_int_kind(node, val); - } - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_uint( - ch_binary_insert_handle* h, - size_t col, - uint64_t val, - bool isnull -) { - ch_binary_append_int(h, col, (int64_t)val, isnull); -} - -void -ch_binary_append_bool(ch_binary_insert_handle* h, size_t col, bool val, bool isnull) { - ch_binary_append_int(h, col, val, isnull); -} - -void -ch_binary_append_double( - ch_binary_insert_handle* h, - size_t col, - double val, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - - if (isnull) { - dynbuf_append_zero(fixed_data(node), 8); - } else { - dynbuf_append(fixed_data(node), &val, 8); - } - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_float(ch_binary_insert_handle* h, size_t col, float val, bool isnull) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - - if (isnull) { - dynbuf_append_zero(fixed_data(node), 4); - } else { - dynbuf_append(fixed_data(node), &val, 4); - } - MemoryContextSwitchTo(old); -} - -/* FixedString width-pads, Enum maps name to value via node's type table */ -static void -append_bytes_fixed(icn* node, const void* p, size_t n, bool isnull) { - chc_kind k = chc_type_kind(node->type); - dynbuf* data = &node->fixed.data; - - if (k == CHC_FIXED_STRING) { - size_t w = node->fixed.elem_size; - - if (isnull) { - dynbuf_append_zero(data, w); - return; - } - size_t take = n < w ? n : w; - - if (take) { - dynbuf_append(data, p, take); - } - if (take < w) { - dynbuf_append_zero(data, w - take); - } - return; - } - if (k == CHC_ENUM8 || k == CHC_ENUM16) { - if (isnull) { - dynbuf_append_zero(data, node->fixed.elem_size); - return; - } - size_t nenum = chc_type_enum_count(node->type); - int64_t val = 0; - bool found = false; - - for (size_t i = 0; i < nenum; i++) { - const char* en; - size_t el; - int64_t ev; - - chc_type_enum_at(node->type, i, &en, &el, &ev); - if (el == n && memcmp(en, p, n) == 0) { - val = ev; - found = true; - break; - } - } - if (!found) { - ereport( - ERROR, - errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg( - "pg_clickhouse: enum value '%.*s' not found", (int)n, (const char*)p - ) - ); - } - if (k == CHC_ENUM8) { - int8_t v = (int8_t)val; - - dynbuf_append(data, &v, 1); - } else { - int16_t v = (int16_t)val; - - dynbuf_append(data, &v, 2); - } - return; - } - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: bytes into non-text column") - ); -} - -void -ch_binary_append_bytes( - ch_binary_insert_handle* h, - size_t col, - const void* p, - size_t n, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - - switch (node->kind) { - case ICN_LC: - /* Buffer values and offsets, build LowCardinality dictionary during flush */ - append_row_offs( - &node->lc.data, &node->lc.offs, isnull ? NULL : p, isnull ? 0 : n - ); - break; - case ICN_STRING: - if (isnull && node->str.is_json) { - /* CH still parses Nullable's values, choking on invalid JSON */ - append_row_offs(&node->str.data, &node->str.offs, "{}", 2); - } else { - append_row_offs( - &node->str.data, &node->str.offs, isnull ? NULL : p, isnull ? 0 : n - ); - } - break; - case ICN_FIXED: - append_bytes_fixed(node, p, n, isnull); - break; - default: - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: bytes into non-text column") - ); - } - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_decimal( - ch_binary_insert_handle* h, - size_t col, - const char* digits, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - size_t w; - - switch (chc_type_kind(node->type)) { - case CHC_DECIMAL32: - w = 4; - break; - case CHC_DECIMAL64: - w = 8; - break; - case CHC_DECIMAL128: - w = 16; - break; - case CHC_DECIMAL256: - w = 32; - break; - default: - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: decimal into non-decimal column") - ); - } - uint8_t raw[32] = {}; - - if (!isnull && digits) { - uint32_t scale = (uint32_t)chc_type_decimal_scale(node->type); - - decimal_text_to_bytes(digits, scale, w, raw); - } - dynbuf_append(&node->fixed.data, raw, w); - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_uuid( - ch_binary_insert_handle* h, - size_t col, - const uint8_t bytes[16], - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - uint8_t wire[16] = {}; - - if (!isnull) { - uint64_t a, b; - - memcpy(&a, bytes, 8); - memcpy(&b, bytes + 8, 8); - a = pg_ntoh64(a); - b = pg_ntoh64(b); - memcpy(wire, &a, 8); - memcpy(wire + 8, &b, 8); - } - dynbuf_append(fixed_data(node), wire, 16); - MemoryContextSwitchTo(old); -} - -/* - * addr_be is BE bytes (PG inet ip_addr layout). For IPv4 CH wire is a - * host-order uint32; pg_ntoh32 turns BE bytes into the right host value. - * For IPv6 CH wire matches PG byte order. - */ -void -ch_binary_append_inet( - ch_binary_insert_handle* h, - size_t col, - const uint8_t* addr_be, - size_t addrlen, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - chc_kind k = chc_type_kind(node->type); - - if (k == CHC_IPV4 && addrlen == 4) { - uint32_t addr = 0; - - if (!isnull && addr_be) { - uint32_t be; - - memcpy(&be, addr_be, 4); - addr = pg_ntoh32(be); - } - dynbuf_append(&node->fixed.data, &addr, 4); - MemoryContextSwitchTo(old); - return; - } - if (k == CHC_IPV6 && addrlen == 16) { - uint8_t raw[16] = {}; - - if (!isnull && addr_be) { - memcpy(raw, addr_be, 16); - } - dynbuf_append(&node->fixed.data, raw, 16); - MemoryContextSwitchTo(old); - return; - } - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: cannot insert inet into non-inet column") - ); -} - -void -ch_binary_append_date_seconds( - ch_binary_insert_handle* h, - size_t col, - int64_t seconds, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - chc_kind k = chc_type_kind(node->type); - - if (k == CHC_DATE) { - uint16_t days = isnull ? 0 : (uint16_t)(seconds / 86400); - - dynbuf_append(&node->fixed.data, &days, 2); - } else if (k == CHC_DATE32) { - int32_t days = isnull ? 0 : (int32_t)(seconds / 86400); - - dynbuf_append(&node->fixed.data, &days, 4); - } else { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg("pg_clickhouse: date into non-date column") - ); - } - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_datetime_seconds( - ch_binary_insert_handle* h, - size_t col, - int64_t seconds, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - uint32_t v = isnull ? 0 : (uint32_t)seconds; - - dynbuf_append(fixed_data(node), &v, 4); - MemoryContextSwitchTo(old); -} - -void -ch_binary_append_datetime64_raw( - ch_binary_insert_handle* h, - size_t col, - int64_t raw, - bool isnull -) { - MemoryContext old = MemoryContextSwitchTo(h->cxt); - icn* node = resolve_leaf(h, col, isnull); - int64_t v = isnull ? 0 : raw; - - dynbuf_append(fixed_data(node), &v, 8); - MemoryContextSwitchTo(old); -} - -/* Rows committed to a node so far; an array level counts closed subarrays */ -static uint64_t -icn_n_rows(const icn* n) { - switch (n->kind) { - case ICN_FIXED: - return n->fixed.elem_size ? n->fixed.data.len / n->fixed.elem_size : 0; - case ICN_STRING: - return n->str.offs.len; - case ICN_NULLABLE: - return icn_n_rows(n->nullable.inner); - case ICN_ARRAY: - return n->array.offs.len; - case ICN_LC: - return n->lc.offs.len; - } - pg_unreachable(); -} - -void -ch_binary_array_begin(ch_binary_insert_handle* h, size_t col) { - icn* node; - - if (h->cursor_len) { - /* - * Nested arrays recurse via append_one with col=0, so once an array - * is open the caller's col is meaningless, descend from cursor. - */ - node = h->cursor[h->cursor_len - 1]->array.values; - } else { - if (col >= h->ncols) { - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: array_begin: col out of range") - ); - } - node = h->cols[col].root; - } - - MemoryContext old = MemoryContextSwitchTo(h->cxt); - - if (node->kind == ICN_NULLABLE && node->nullable.inner->kind == ICN_ARRAY) { - /* Nullable(Array(...)): the array value itself is non-null */ - uint8_t b = 0; - - dynbuf_append(&node->nullable.null_map, &b, 1); - node = node->nullable.inner; - } - if (node->kind != ICN_ARRAY) { - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: array_begin: column is not Array") - ); - } - if (h->cursor_len == h->cursor_cap) { - h->cursor_cap = h->cursor_cap ? h->cursor_cap * 2 : 4; - h->cursor = h->cursor ? repalloc(h->cursor, h->cursor_cap * sizeof(icn*)) - : palloc(h->cursor_cap * sizeof(icn*)); - } - if (h->cursor_len == 0) { - h->cursor_col = col; - } - h->cursor[h->cursor_len++] = node; - MemoryContextSwitchTo(old); -} - -void -ch_binary_array_end(ch_binary_insert_handle* h) { - if (h->cursor_len == 0) { - return; - } - icn* a = h->cursor[--h->cursor_len]; - MemoryContext old = MemoryContextSwitchTo(h->cxt); - - u64buf_push(&a->array.offs, icn_n_rows(a->array.values)); - MemoryContextSwitchTo(old); -} - -bool -ch_binary_array_active(const ch_binary_insert_handle* h) { - return h && h->cursor_len > 0; -} - -chc_kind -ch_binary_column_kind(const ch_binary_insert_handle* h, size_t col) { - if (col >= h->ncols) { - return CHC_VOID; - } - - /* - * While nested, surface CHC_ARRAY until the innermost layer is open; - * at that point return the leaf kind so encode targets scalars. - */ - const icn* node = cursor_node(h, col); - - if (node->kind == ICN_NULLABLE) { - node = node->nullable.inner; - } - switch (node->kind) { - case ICN_ARRAY: - return CHC_ARRAY; - case ICN_LC: - return CHC_STRING; /* PG side targets TEXT */ - default: - return chc_type_kind(node->type); - } -} - -uint32_t -ch_binary_column_datetime64_precision(const ch_binary_insert_handle* h, size_t col) { - if (col >= h->ncols) { - return 0; - } - const icn* node = cursor_node(h, col); - - for (;;) { - if (node->kind == ICN_NULLABLE) { - node = node->nullable.inner; - } else if (node->kind == ICN_ARRAY) { - node = node->array.values; - } else { - break; - } - } - return node->kind == ICN_FIXED ? node->fixed.dt64_precision : 0; -} - -/* Dedup map for LowCardinality dict */ -typedef struct lcd_key { - const uint8_t* bytes; - size_t len; -} lcd_key; - -typedef struct lcd_entry { - uint32 status; - lcd_key key; - uint32 idx; -} lcd_entry; - -#define SH_PREFIX lcd -#define SH_ELEMENT_TYPE lcd_entry -#define SH_KEY_TYPE lcd_key -#define SH_KEY key -#define SH_HASH_KEY(tb, key) hash_bytes((key).bytes, (int)(key).len) -#define SH_EQUAL(tb, a, b) \ - ((a).len == (b).len && memcmp((a).bytes, (b).bytes, (a).len) == 0) -#define SH_SCOPE static inline -#define SH_DECLARE -#define SH_DEFINE -#include "lib/simplehash.h" - -/* Build LC dict (collect unique strings in insertion order) */ -static void -build_lc_dict( - const icn* node, - uint64_t** out_dict_offs, - uint8_t** out_dict_data, - size_t* out_dict_n, - void** out_keys, - int* out_key_size, - size_t* out_n_rows -) { - bool nullable = node->lc.inner_nullable; - size_t n_rows = node->lc.offs.len; - uint64_t* dict_offs = NULL; - uint8_t* dict_data = NULL; - uint32_t* keys = n_rows ? palloc(n_rows * sizeof(uint32_t)) : NULL; - size_t dict_n = 0; - size_t dict_cap = 0; - size_t data_len = 0; - lcd_hash* ht = - n_rows - ? lcd_create( - CurrentMemoryContext, (uint32)Min(n_rows, (size_t)PG_UINT32_MAX), NULL - ) - : NULL; - - if (nullable) { - /* dict[0] = "" sentinel. */ - dict_cap = 8; - dict_offs = palloc(dict_cap * sizeof(uint64_t)); - dict_offs[0] = 0; - dict_n = 1; - } - - for (size_t i = 0; i < n_rows; i++) { - uint64_t start = i == 0 ? 0 : node->lc.offs.data[i - 1]; - uint64_t end = node->lc.offs.data[i]; - size_t len = (size_t)(end - start); - const uint8_t* bytes = node->lc.data.data + start; - lcd_key k = { bytes, len }; - lcd_entry* entry; - bool found; - - /* If nullable and this row was null (signalled by null bit), key 0. */ - if (nullable && node->lc.null_map.data[i]) { - keys[i] = 0; - continue; - } - - entry = lcd_insert(ht, k, &found); - if (found) { - keys[i] = entry->idx; - continue; - } - - if (dict_n == dict_cap) { - dict_cap = dict_cap ? dict_cap * 2 : 64; - dict_offs = dict_offs ? repalloc(dict_offs, dict_cap * sizeof(uint64_t)) - : palloc(dict_cap * sizeof(uint64_t)); - } - data_len += len; - dict_offs[dict_n] = data_len; - entry->idx = (uint32)dict_n; - keys[i] = (uint32)dict_n; - dict_n++; - } - - if (data_len) { - lcd_iterator it; - lcd_entry* e; - - dict_data = - MemoryContextAllocExtended(CurrentMemoryContext, data_len, MCXT_ALLOC_HUGE); - lcd_start_iterate(ht, &it); - while ((e = lcd_iterate(ht, &it)) != NULL) { - uint64_t s = e->idx == 0 ? 0 : dict_offs[e->idx - 1]; - - memcpy(dict_data + s, e->key.bytes, e->key.len); - } - } - if (ht) { - lcd_destroy(ht); - } - *out_dict_offs = dict_offs; - *out_dict_data = dict_data; - *out_dict_n = dict_n; - *out_keys = keys; - *out_key_size = 4; - *out_n_rows = n_rows; -} - -static inline chc_column* -col_node(chc_column v) { - chc_column* n = palloc(sizeof(*n)); - - *n = v; - return n; -} - -/* - * Build chc_column tree in current context, 1:1 with the node tree. Result - * holds references to nested chc_column objects, buffered column data, and - * any LowCardinality dictionary. Keep context and column buffers alive - * until send_data returns. - */ -static chc_column* -finalize_node(icn* n) { - switch (n->kind) { - case ICN_FIXED: { - size_t n_rows = icn_n_rows(n); - - return col_node( - chc_build_fixed(n->fixed.data.data, n->fixed.elem_size, n_rows) - ); - } - case ICN_STRING: - return col_node( - chc_build_string(n->str.offs.data, n->str.data.data, n->str.offs.len) - ); - case ICN_NULLABLE: - return col_node(chc_build_nullable( - n->nullable.null_map.data, finalize_node(n->nullable.inner) - )); - case ICN_ARRAY: - return col_node(chc_build_array( - n->array.offs.data, n->array.offs.len, finalize_node(n->array.values) - )); - case ICN_LC: { - size_t dict_n, n_rows; - int key_size; - uint64_t* lc_offs; - uint8_t* lc_data; - void* lc_keys; - - build_lc_dict(n, &lc_offs, &lc_data, &dict_n, &lc_keys, &key_size, &n_rows); - chc_column* dict = col_node(chc_build_string(lc_offs, lc_data, dict_n)); - - return col_node(chc_build_lc(key_size, lc_keys, n_rows, dict)); - } - } - pg_unreachable(); -} - -static size_t -node_bytes(const icn* n) { - switch (n->kind) { - case ICN_FIXED: - return n->fixed.data.len; - case ICN_STRING: - return n->str.data.len + n->str.offs.len * sizeof(uint64_t); - case ICN_NULLABLE: - return n->nullable.null_map.len + node_bytes(n->nullable.inner); - case ICN_ARRAY: - return n->array.offs.len * sizeof(uint64_t) + node_bytes(n->array.values); - case ICN_LC: - return n->lc.data.len + n->lc.offs.len * sizeof(uint64_t) + n->lc.null_map.len; - } - pg_unreachable(); -} - -static void -reset_node(icn* n) { - switch (n->kind) { - case ICN_FIXED: - dynbuf_reset(&n->fixed.data); - return; - case ICN_STRING: - dynbuf_reset(&n->str.data); - u64buf_reset(&n->str.offs); - return; - case ICN_NULLABLE: - dynbuf_reset(&n->nullable.null_map); - reset_node(n->nullable.inner); - return; - case ICN_ARRAY: - u64buf_reset(&n->array.offs); - reset_node(n->array.values); - return; - case ICN_LC: - dynbuf_reset(&n->lc.data); - u64buf_reset(&n->lc.offs); - dynbuf_reset(&n->lc.null_map); - return; - } - pg_unreachable(); +const char* +ch_binary_insert_column_name(const ch_binary_insert_handle* h, size_t i) { + return i < h->ncols ? h->names[i] : ""; } /* @@ -1342,61 +220,25 @@ void ch_binary_insert_autoflush(ch_binary_insert_state* state) { ch_binary_insert_handle* h = state->insert_block; - if (h) { - size_t total = 0; - for (size_t i = 0; i < h->ncols; i++) { - total += node_bytes(h->cols[i].root); - } - if (total >= 64 * 1024 * 1024) { - ch_binary_flush_block(h); - } + if (h && pgch_writer_bytes(h->w) >= 64 * 1024 * 1024) { + ch_binary_flush_block(h); } } void ch_binary_flush_block(ch_binary_insert_handle* h) { - /* Unbalanced array_begin/array_end leaves offsets short of leaf rows */ - Assert(h->cursor_len == 0); - - /* - * Allocate chc_column objects, block metadata, and LowCardinality dictionaries - * in per-flush context. - */ - MemoryContext old = MemoryContextSwitchTo(h->cxt); - MemoryContext bcxt = AllocSetContextCreate( - h->cxt, "pg_clickhouse binary insert flush", ALLOCSET_DEFAULT_SIZES - ); - - MemoryContextSwitchTo(bcxt); - - chc_err err = {}; - chc_block_col* bcols = h->ncols ? palloc(h->ncols * sizeof(*bcols)) : NULL; - chc_block_builder bb; - - chc_block_builder_init(&bb, bcols); - - for (size_t i = 0; i < h->ncols; i++) { - ic_col* c = &h->cols[i]; - const char* name = c->info.name ? c->info.name : ""; - chc_block_builder_append(&bb, name, strlen(name), c->t, finalize_node(c->root)); - } - - int rc = chc_client_send_data(h->client, &bb, &err); + MemoryContext old = MemoryContextSwitchTo(h->cxt); + chc_err err = {}; + int rc = chc_client_send_data(h->client, pgch_writer_build(h->w), &err); if (rc != CHC_OK) { if (h->state) { h->state->broken = true; } - raise_chc(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); + pgch_raise(&err, ERRCODE_FDW_ERROR, "could not insert columns - "); } - MemoryContextSwitchTo(h->cxt); - MemoryContextDelete(bcxt); - - /* Reset per-column buffers for next batch. */ - for (size_t i = 0; i < h->ncols; i++) { - reset_node(h->cols[i].root); - } + pgch_writer_reset(h->w); MemoryContextSwitchTo(old); } diff --git a/src/binary/select.c b/src/binary/select.c index cc1c3567..390e68dc 100644 --- a/src/binary/select.c +++ b/src/binary/select.c @@ -39,9 +39,7 @@ struct ch_binary_response_t { bool success; bool eos; /* END_OF_STREAM or exception seen */ - size_t columns_count; - chc_block* staged; /* next block to hand out; owned */ - chc_block* prev; /* last block handed out; freed at next fetch */ + chc_block* staged; /* next block to hand out; ownership passes to caller */ }; static void @@ -71,10 +69,10 @@ resp_set_exception(ch_binary_response_t* resp, const chc_exception* ex) { /* * Read one wire packet under resp->cxt and fold it into the response. Sets - * resp->staged (for non-empty Data), resp->columns_count (on first Data with - * columns), resp->error (cancel/exception/transport), or resp->eos - * (END_OF_STREAM / exception / transport failure). Other packet kinds - * (progress, log, profile, ...) are silently consumed. + * resp->staged (for Data with columns; header block may carry zero rows), + * resp->error (cancel/exception/transport), or resp->eos (END_OF_STREAM / + * exception / transport failure). Other packet kinds (progress, log, + * profile, ...) are silently consumed. */ static void pump_one(ch_binary_response_t* resp) { @@ -97,30 +95,25 @@ pump_one(ch_binary_response_t* resp) { switch (pkt.kind) { case CHC_PKT_DATA: - if (pkt.block && chc_block_n_columns(pkt.block) > 0) { - if (resp->columns_count == 0) { - resp->columns_count = chc_block_n_columns(pkt.block); - } - if (chc_block_n_rows(pkt.block) > 0 && resp->staged == NULL) { - size_t ncols = chc_block_n_columns(pkt.block); - chc_err verr = {}; - int vrc = CHC_OK; - - for (size_t i = 0; i < ncols; i++) { - vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); - if (vrc != CHC_OK) { - break; - } - } + if (pkt.block && chc_block_n_columns(pkt.block) > 0 && resp->staged == NULL) { + size_t ncols = chc_block_n_columns(pkt.block); + chc_err verr = {}; + int vrc = CHC_OK; + + for (size_t i = 0; i < ncols; i++) { + vrc = chc_column_validate(chc_block_column(pkt.block, i), &verr); if (vrc != CHC_OK) { - resp_set_error(resp, verr.msg); - resp->state->broken = true; - resp->eos = true; - } else { - resp->staged = pkt.block; - pkt.block = NULL; + break; } } + if (vrc != CHC_OK) { + resp_set_error(resp, verr.msg); + resp->state->broken = true; + resp->eos = true; + } else { + resp->staged = pkt.block; + pkt.block = NULL; + } } chc_packet_clear(resp->client, &pkt); break; @@ -366,11 +359,11 @@ ch_binary_simple_query( } /* - * Pump until schema is known so callers can call - * ch_binary_response_columns immediately. Empty result sets exit on eos - * with columns_count still set from the header Data block. + * Pump until first Data block or eos so query-time exceptions surface + * before cursor construction. Header block carries schema even for + * empty result sets. */ - while (resp->columns_count == 0 && !resp->eos && !resp->error) { + while (resp->staged == NULL && !resp->eos && !resp->error) { pump_one(resp); } @@ -410,31 +403,39 @@ ch_binary_response_success(const ch_binary_response_t* resp) { return resp && resp->success; } -size_t -ch_binary_response_columns(const ch_binary_response_t* resp) { - return resp ? resp->columns_count : 0; +/* Adapt TCP response to shared block source. */ +static const chc_block* +resp_src_next_block(void* ud) { + return ch_binary_response_fetch_next_block((ch_binary_response_t*)ud); +} + +static const char* +resp_src_error(void* ud) { + return ch_binary_response_error((ch_binary_response_t*)ud); +} + +pgch_block_source +ch_binary_response_block_source(ch_binary_response_t* resp) { + return (pgch_block_source){ + .ud = resp, + .next_block = resp_src_next_block, + .error = resp_src_error, + }; } const chc_block* ch_binary_response_fetch_next_block(ch_binary_response_t* resp) { + chc_block* blk; + if (!resp) { return NULL; } - if (resp->prev) { - chc_block_destroy(resp->prev, &pg_chc_alloc); - resp->prev = NULL; - } - while (resp->staged == NULL && !resp->eos && !resp->error) { pump_one(resp); } - if (resp->staged == NULL) { - return NULL; - } - - resp->prev = resp->staged; + blk = resp->staged; resp->staged = NULL; - return resp->prev; + return blk; } diff --git a/src/custom_types.c b/src/custom_types.c index 32c1bed0..19171d71 100644 --- a/src/custom_types.c +++ b/src/custom_types.c @@ -323,7 +323,6 @@ init_custom_entry(CustomObjectDef* entry) { entry->cf_type = CF_USUAL; entry->custom_name[0] = '\0'; entry->cf_context = NULL; - entry->rowfunc = InvalidOid; entry->paren_count = 1; } diff --git a/src/include/binary.h b/src/include/binary.h index 5a9e6ba4..8ae2d3f9 100644 --- a/src/include/binary.h +++ b/src/include/binary.h @@ -22,6 +22,8 @@ #include "clickhouse.h" #include "engine.h" +#include "pg-clickhouse-decode.h" +#include "pg-clickhouse-encode.h" typedef struct ch_binary_connection_t ch_binary_connection_t; typedef struct ch_binary_response_t ch_binary_response_t; @@ -66,8 +68,6 @@ extern const char* ch_binary_response_error(const ch_binary_response_t* resp); extern bool ch_binary_response_success(const ch_binary_response_t* resp); -extern size_t -ch_binary_response_columns(const ch_binary_response_t* resp); /* INSERT. */ @@ -79,59 +79,29 @@ ch_binary_response_columns(const ch_binary_response_t* resp); extern void ch_binary_finalize_insert(ch_binary_insert_handle* h); -/* PG-typed surface follows. */ +/* PG-typed surface follows. Rows decode through pgch_reader from + * pg-clickhouse-c; this layer only supplies the block stream. */ +/* resp must outlive returned source. */ +extern pgch_block_source +ch_binary_response_block_source(ch_binary_response_t* resp); + +/* Slot attribute feeding one ClickHouse column, resolved on first tuple. */ typedef struct { - ch_binary_response_t* resp; - Oid* coltypes; - Datum* values; - bool* nulls; - - size_t block; /* current block */ - size_t row; /* row in current block */ - const chc_block* cur; /* borrowed from resp; NULL when unloaded */ - void* gc; /* allocated objects while reading */ - char* error; - bool done; -} ch_binary_read_state_t; + AttrNumber attnum; + Oid atttypid; +} ch_binary_insert_colmap; typedef struct { MemoryContext memcxt; /* used for cleanup */ MemoryContextCallback callback; - TupleDesc outdesc; ch_binary_insert_handle* insert_block; - size_t len; - void* conversion_states; - char* table_name; - Oid relid; /* foreign table relid, for column_name - * lookups */ - - Datum* values; - bool* nulls; - bool success; - - ch_binary_connection_t* conn; + size_t len; /* ClickHouse column count */ + ch_binary_insert_colmap* colmap; /* len entries, NULL until first tuple */ + Oid relid; /* foreign table relid, for column_name lookups */ } ch_binary_insert_state; -/* SELECT helpers (decode.c). */ -extern void -ch_binary_read_state_init(ch_binary_read_state_t* state, ch_binary_response_t* resp); -extern void -ch_binary_read_state_free(ch_binary_read_state_t* state); -extern bool -ch_binary_read_row(ch_binary_read_state_t* state); - -/* SELECT/INSERT type conversion (convert.c). */ -extern char* -ch_binary_value_to_cstring(Oid coltype, Datum value); -extern Datum -ch_binary_convert_datum(void* state, Datum val); -extern void* -ch_binary_init_convert_state(Datum val, Oid intype, Oid outtype); -extern void -ch_binary_free_convert_state(void* state); - /* INSERT helpers (encode.c). */ extern void ch_binary_prepare_insert( @@ -141,15 +111,13 @@ ch_binary_prepare_insert( ); extern void ch_binary_insert_columns(ch_binary_insert_state* state); + +/* Append one slot's worth of values, one per ClickHouse column. */ extern void -ch_binary_column_append_data(ch_binary_insert_state* state, size_t colidx); +ch_binary_insert_tuple(ch_binary_insert_state* state, TupleTableSlot* slot); extern void ch_binary_insert_autoflush(ch_binary_insert_state* state); -extern void* -ch_binary_make_tuple_map(TupleDesc indesc, TupleDesc outdesc, Oid relid); extern void ch_binary_insert_state_free(void* c); -extern void -ch_binary_do_output_conversion(ch_binary_insert_state* state, TupleTableSlot* slot); #endif /* CLICKHOUSE_BINARY_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 221ccc5f..5509acb1 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -54,7 +54,9 @@ typedef struct ch_cursor { double request_time; double total_time; size_t columns_count; - uintptr_t* conversion_states; /* for binary */ + /* for binary, per returned column: conversion state, target attribute */ + void** conversion_states; + int* fill_dest; } ch_cursor; typedef struct ChFdwScanRowContext { @@ -457,7 +459,6 @@ typedef struct CustomObjectDef { char custom_name[NAMEDATALEN]; /* \0 - no custom name, \1 - many * names */ int paren_count; /* Number closing parens; defaults to 1 */ - Oid rowfunc; void* cf_context; } CustomObjectDef; diff --git a/src/pglink.c b/src/pglink.c index 465b9eef..91290850 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -940,7 +940,7 @@ static ch_cursor* binary_simple_query(void* conn, const ch_query* query) { MemoryContext tempcxt, oldcxt; ch_cursor* cursor; - ch_binary_read_state_t* state; + pgch_reader* state; ch_binary_response_t* resp = ch_binary_simple_query(conn, query, &is_canceled); @@ -967,15 +967,15 @@ binary_simple_query(void* conn, const ch_query* query) { cursor = palloc0(sizeof(ch_cursor)); cursor->conn = conn; cursor->query_response = resp; - state = (ch_binary_read_state_t*)palloc0(sizeof(ch_binary_read_state_t)); - cursor->query = pstrdup(query->sql); - cursor->read_state = state; - cursor->columns_count = ch_binary_response_columns(resp); - ch_binary_read_state_init(cursor->read_state, resp); - cursor->conversion_states = palloc0(sizeof(uintptr_t) * cursor->columns_count); - cursor->memcxt = tempcxt; - cursor->callback.func = binary_cursor_free; - cursor->callback.arg = cursor; + state = (pgch_reader*)palloc0(sizeof(pgch_reader)); + cursor->query = pstrdup(query->sql); + cursor->read_state = state; + pgch_block_source src = ch_binary_response_block_source(resp); + pgch_reader_init(cursor->read_state, &src); + cursor->columns_count = pgch_reader_columns(state); + cursor->memcxt = tempcxt; + cursor->callback.func = binary_cursor_free; + cursor->callback.arg = cursor; MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); /* @@ -1078,8 +1078,8 @@ append_tsv_escaped(StringInfo buf, const char* s) { */ text* chfdw_binary_fetch_raw_data(ch_cursor* cursor) { - ch_binary_read_state_t* state = cursor->read_state; - size_t ncols = ch_binary_response_columns(state->resp); + pgch_reader* state = cursor->read_state; + size_t ncols = pgch_reader_columns(state); StringInfoData buf; if (ncols == 0) { @@ -1088,7 +1088,7 @@ chfdw_binary_fetch_raw_data(ch_cursor* cursor) { initStringInfo(&buf); - while (ch_binary_read_row(state)) { + while (pgch_reader_next(state)) { for (size_t i = 0; i < ncols; i++) { if (i > 0) { appendStringInfoChar(&buf, '\t'); @@ -1097,8 +1097,7 @@ chfdw_binary_fetch_raw_data(ch_cursor* cursor) { if (state->nulls[i]) { appendStringInfoString(&buf, "\\N"); } else { - char* val = - ch_binary_value_to_cstring(state->coltypes[i], state->values[i]); + char* val = pgch_value_to_cstring(state->coltypes[i], state->values[i]); append_tsv_escaped(&buf, val); pfree(val); @@ -1146,15 +1145,38 @@ binary_fetch_row_errcb(void* arg) { errdetail_internal("Remote Query: %.64000s", sql); } +/* Conversion state and target attribute per returned column. */ +static void +build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { + pgch_reader* state = cursor->read_state; + MemoryContext old = MemoryContextSwitchTo(cursor->memcxt); + size_t ncols = pgch_reader_columns(state); + ListCell* lc; + size_t j = 0; + + cursor->conversion_states = palloc0(ncols * sizeof(void*)); + cursor->fill_dest = palloc0(ncols * sizeof(int)); + foreach (lc, ctx->retrieved_attrs) { + int attnum = lfirst_int(lc); + + cursor->fill_dest[j] = attnum - 1; + cursor->conversion_states[j] = pgch_reader_convert_init( + state, j, TupleDescAttr(ctx->tupdesc, attnum - 1)->atttypid + ); + j++; + } + + MemoryContextSwitchTo(old); +} + static Datum* binary_fetch_row(ChFdwScanRowContext* ctx) { - ListCell* lc; - ch_cursor* cursor = ctx->cursor; - List* attrs = ctx->retrieved_attrs; - TupleDesc tupdesc = ctx->tupdesc; - Datum* values = ctx->values; - bool* nulls = ctx->nulls; - ch_binary_read_state_t* state = cursor->read_state; + ch_cursor* cursor = ctx->cursor; + List* attrs = ctx->retrieved_attrs; + TupleDesc tupdesc = ctx->tupdesc; + Datum* values = ctx->values; + bool* nulls = ctx->nulls; + pgch_reader* state = cursor->read_state; ErrorContextCallback errcallback; errcallback.callback = binary_fetch_row_errcb; @@ -1162,16 +1184,19 @@ binary_fetch_row(ChFdwScanRowContext* ctx) { errcallback.previous = error_context_stack; error_context_stack = &errcallback; - bool have_data = ch_binary_read_row(state); + bool have_data = pgch_reader_next(state); size_t attcount = list_length(attrs); if (state->error) { + error_context_stack = errcallback.previous; + /* Prefer consistent interrupt error message when fetch interrupted */ CHECK_FOR_INTERRUPTS(); ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: error while reading row: %s", state->error) + errmsg("pg_clickhouse: %s", state->error), + errdetail_internal("Remote Query: %.64000s", cursor->query) ); } @@ -1181,7 +1206,7 @@ binary_fetch_row(ChFdwScanRowContext* ctx) { } if (attcount == 0) { - if (ch_binary_response_columns(state->resp) == 1 && state->nulls[0]) { + if (pgch_reader_columns(state) == 1 && state->nulls[0]) { nulls[0] = true; goto ok; } else { @@ -1194,83 +1219,27 @@ binary_fetch_row(ChFdwScanRowContext* ctx) { ) ); } - } else if (attcount != ch_binary_response_columns(state->resp)) { + } else if (attcount != pgch_reader_columns(state)) { ereport( ERROR, errcode(ERRCODE_DATATYPE_MISMATCH), errmsg_internal( "pg_clickhouse: returned %lu columns, expected %lu", - ch_binary_response_columns(state->resp), + pgch_reader_columns(state), attcount ) ); } if (tupdesc) { - size_t j = 0; - Assert(values && nulls); - foreach (lc, attrs) { - int i = lfirst_int(lc); - bool isnull = state->nulls[j]; - intptr_t convstate; - - if (isnull) { - values[i - 1] = (Datum)0; - } else { - again: - convstate = cursor->conversion_states[j]; - switch (convstate) { - case 0: { - MemoryContext old_mcxt; - - Oid outtype = TupleDescAttr(tupdesc, i - 1)->atttypid; - void* s; - - /* - * now we're should be in temporary memory - * context, so make sure conversion states outlive - * it. - */ - old_mcxt = MemoryContextSwitchTo(cursor->memcxt); - - /* - * Convert the Postgres Datum, stored by - * ch_binary_read_row() as the type defined by by - * state->coltypes[j], to the type defined by the - * foreign table. No conversion if they're binary - * compatible, but required to allow a static - * ClickHouse type (which maps to a well-known - * Postgres type) to be converted to a compatible - * Postgres value via a CAST. - */ - s = ch_binary_init_convert_state( - state->values[j], state->coltypes[j], outtype - ); - MemoryContextSwitchTo(old_mcxt); - - if (s == NULL) { - /* no conversion but state is initialized */ - cursor->conversion_states[j] = 1; - } else { - cursor->conversion_states[j] = (uintptr_t)s; - } - goto again; - } - case 1: - /* no conversion */ - values[i - 1] = state->values[j]; - break; - default: - values[i - 1] = - ch_binary_convert_datum((void*)convstate, state->values[j]); - } - } - - nulls[i - 1] = isnull; - j++; + if (cursor->conversion_states == NULL) { + build_conversion(cursor, ctx); } + pgch_reader_fill_map( + state, cursor->conversion_states, cursor->fill_dest, values, nulls + ); } ok: @@ -1282,13 +1251,8 @@ static void binary_cursor_free(void* c) { ch_cursor* cursor = c; - for (size_t i = 0; i < cursor->columns_count; i++) { - if (cursor->conversion_states[i] > 1) { - ch_binary_free_convert_state((void*)cursor->conversion_states[i]); - } - } - - ch_binary_read_state_free(cursor->read_state); + /* Conversion states live in the context this callback fires for. */ + pgch_reader_free(cursor->read_state); ch_binary_response_free(cursor->query_response); } @@ -1319,17 +1283,11 @@ binary_prepare_insert( state->memcxt = tempcxt; state->callback.func = ch_binary_insert_state_free; state->callback.arg = state; - state->conn = conn; - state->table_name = pstrdup(table_name); state->relid = RelationGetRelid(rri->ri_RelationDesc); MemoryContextRegisterResetCallback(tempcxt, &state->callback); - /* time for c++ stuff */ + /* enter insert mode, take the column list from the server */ ch_binary_prepare_insert(conn, query, state); - - /* buffers */ - state->values = (Datum*)palloc0(sizeof(Datum) * state->len); - state->nulls = (bool*)palloc0(sizeof(bool) * state->len); MemoryContextSwitchTo(oldcxt); return state; @@ -1340,31 +1298,7 @@ binary_insert_tuple(void* istate, TupleTableSlot* slot) { ch_binary_insert_state* state = istate; if (slot) { - if (state->conversion_states == NULL) { - MemoryContext old_mcxt; - - old_mcxt = MemoryContextSwitchTo(state->memcxt); - - /* - * Set up conversion states so that Postgres Datums of types - * defined by the foreign table can be converted to a well-known - * Postgres type that the binary INSERT path knows how to convert - * to the appropriate ClickHouse type. Binary compatible types - * don't convert; others require a CAST. Fails if there is no - * cast. - */ - state->conversion_states = ch_binary_make_tuple_map( - slot->tts_tupleDescriptor, state->outdesc, state->relid - ); - MemoryContextSwitchTo(old_mcxt); - } - - ch_binary_do_output_conversion(state, slot); - - for (size_t i = 0; i < (size_t)state->outdesc->natts; i++) { - ch_binary_column_append_data(state, i); - } - + ch_binary_insert_tuple(state, slot); ch_binary_insert_autoflush(state); } else { ch_binary_insert_columns(state); diff --git a/test/expected/binary.out b/test/expected/binary.out index f5da7830..7f7b35e1 100644 --- a/test/expected/binary.out +++ b/test/expected/binary.out @@ -308,8 +308,20 @@ SELECT * FROM farrays ORDER BY c1; (10 rows) SELECT * FROM farrays2 ORDER BY c1; -ERROR: pg_clickhouse: could not cast value from integer[] to bigint[] -DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.arrays ORDER BY c1 ASC NULLS LAST + c1 | c2 +--------+-------------- + {0,1} | {num0,num1} + {1,2} | {num1,num2} + {2,3} | {num2,num3} + {3,4} | {num3,num4} + {4,5} | {num4,num5} + {5,6} | {num5,num6} + {6,7} | {num6,num7} + {7,8} | {num7,num8} + {8,9} | {num8,num9} + {9,10} | {num9,num10} +(10 rows) + -- nested arrays SELECT * FROM fnested_arrays ORDER BY c1; c1 | c2 | c3 @@ -319,7 +331,7 @@ SELECT * FROM fnested_arrays ORDER BY c1; (2 rows) SELECT * FROM fragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{"1","2","3"},{"4"}}" +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- tuples SELECT * FROM ftuples ORDER BY c1; diff --git a/test/expected/binary_1.out b/test/expected/binary_1.out index b92e48c3..4f5309f1 100644 --- a/test/expected/binary_1.out +++ b/test/expected/binary_1.out @@ -308,8 +308,20 @@ SELECT * FROM farrays ORDER BY c1; (10 rows) SELECT * FROM farrays2 ORDER BY c1; -ERROR: pg_clickhouse: could not cast value from integer[] to bigint[] -DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.arrays ORDER BY c1 ASC NULLS LAST + c1 | c2 +--------+-------------- + {0,1} | {num0,num1} + {1,2} | {num1,num2} + {2,3} | {num2,num3} + {3,4} | {num3,num4} + {4,5} | {num4,num5} + {5,6} | {num5,num6} + {6,7} | {num6,num7} + {7,8} | {num7,num8} + {8,9} | {num8,num9} + {9,10} | {num9,num10} +(10 rows) + -- nested arrays SELECT * FROM fnested_arrays ORDER BY c1; c1 | c2 | c3 @@ -319,7 +331,7 @@ SELECT * FROM fnested_arrays ORDER BY c1; (2 rows) SELECT * FROM fragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{"1","2","3"},{"4"}}" +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions DETAIL: Remote Query: SELECT c1, c2 FROM binary_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- tuples SELECT * FROM ftuples ORDER BY c1; diff --git a/test/expected/binary_inserts.out b/test/expected/binary_inserts.out index ae692d2e..36584c53 100644 --- a/test/expected/binary_inserts.out +++ b/test/expected/binary_inserts.out @@ -298,7 +298,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 +ERROR: pg_clickhouse: cannot encode integer into ClickHouse Array(Array(Int32)) (column "c2") /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -481,16 +481,15 @@ $$); IMPORT FOREIGN SCHEMA binary_inserts_test LIMIT TO (default_vals) FROM SERVER binary_inserts_loopback INTO binary_inserts_test; --- Fails on c2, the first column we try to set to NULL. Will cease to fail if --- the `if (isnull && !nullable)` block is removed from column_append(). See --- its comment for details. +-- Fails on c2, the first column set to NULL: pg-clickhouse-c's writer rejects +-- NULL into a non-Nullable column rather than writing the type's zero value. INSERT INTO default_vals VALUES( 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column "c2" SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- diff --git a/test/expected/binary_inserts_1.out b/test/expected/binary_inserts_1.out index 26cdbd9d..9a0d8b07 100644 --- a/test/expected/binary_inserts_1.out +++ b/test/expected/binary_inserts_1.out @@ -289,7 +289,7 @@ SELECT * FROM nested_arrays ORDER BY c1; /* shape mismatch with column type must error rather than silently corrupt */ INSERT INTO nested_arrays VALUES (4, ARRAY[1,2,3], ARRAY['x']); -ERROR: pg_clickhouse: unexpected PG/CH type pair for column 0 +ERROR: pg_clickhouse: cannot encode integer into ClickHouse Array(Array(Int32)) (column "c2") /* Check UUIDs and IPs */ \d addr Foreign table "binary_inserts_test.addr" @@ -472,16 +472,15 @@ $$); IMPORT FOREIGN SCHEMA binary_inserts_test LIMIT TO (default_vals) FROM SERVER binary_inserts_loopback INTO binary_inserts_test; --- Fails on c2, the first column we try to set to NULL. Will cease to fail if --- the `if (isnull && !nullable)` block is removed from column_append(). See --- its comment for details. +-- Fails on c2, the first column set to NULL: pg-clickhouse-c's writer rejects +-- NULL into a non-Nullable column rather than writing the type's zero value. INSERT INTO default_vals VALUES( 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); -ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column +ERROR: pg_clickhouse: cannot append NULL to NOT NULL UInt8 column "c2" SELECT * FROM default_vals; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c18 | c19 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 ----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- diff --git a/test/expected/functions.out b/test/expected/functions.out index fca97f33..c56e3eeb 100644 --- a/test/expected/functions.out +++ b/test/expected/functions.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_0.out b/test/expected/functions_0.out index 2ab0b368..5507c375 100644 --- a/test/expected/functions_0.out +++ b/test/expected/functions_0.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_1.out b/test/expected/functions_1.out index 0da61318..f38389d6 100644 --- a/test/expected/functions_1.out +++ b/test/expected/functions_1.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_2.out b/test/expected/functions_2.out index 158b3a20..6df66b7f 100644 --- a/test/expected/functions_2.out +++ b/test/expected/functions_2.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_3.out b/test/expected/functions_3.out index c348070d..9183e71e 100644 --- a/test/expected/functions_3.out +++ b/test/expected/functions_3.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_4.out b/test/expected/functions_4.out index bb6deca9..d13c961c 100644 --- a/test/expected/functions_4.out +++ b/test/expected/functions_4.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_5.out b/test/expected/functions_5.out index 36c3c5b8..5642efff 100644 --- a/test/expected/functions_5.out +++ b/test/expected/functions_5.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_6.out b/test/expected/functions_6.out index 4cd9a80b..80d9fe1f 100644 --- a/test/expected/functions_6.out +++ b/test/expected/functions_6.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_7.out b/test/expected/functions_7.out index c0425e64..99369cb5 100644 --- a/test/expected/functions_7.out +++ b/test/expected/functions_7.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_8.out b/test/expected/functions_8.out index 02bc4421..2e4abf88 100644 --- a/test/expected/functions_8.out +++ b/test/expected/functions_8.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/functions_9.out b/test/expected/functions_9.out index e1fdc696..58b40763 100644 --- a/test/expected/functions_9.out +++ b/test/expected/functions_9.out @@ -753,8 +753,11 @@ EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITH (4 rows) SELECT percentile_disc(ARRAY[0.9, 0.95, 0.99]) WITHIN GROUP (ORDER BY date_part('epoch', timezone('UTC', c))) FROM t1; -ERROR: pg_clickhouse: could not cast value from bigint[] to double precision[] -DETAIL: Remote Query: SELECT quantilesExactLow(0.9,0.95,0.99)(toUnixTimestamp(toTimeZone(c, 'UTC'))) FROM functions_test.t1 + percentile_disc +------------------------------------ + {1546426800,1546426800,1546426800} +(1 row) + EXPLAIN (VERBOSE, COSTS OFF) SELECT percentile_disc(ARRAY[0.25, 0.5, 0.75]) WITHIN GROUP (ORDER BY a) FILTER (WHERE b = 1) FROM t1; QUERY PLAN --------------------------------------------------------------------------------------------------------------------- diff --git a/test/expected/import_schema.out b/test/expected/import_schema.out index 9a25a619..5774f7c6 100644 --- a/test/expected/import_schema.out +++ b/test/expected/import_schema.out @@ -708,7 +708,7 @@ $$); SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) diff --git a/test/expected/import_schema_1.out b/test/expected/import_schema_1.out index 65adc428..79d2a416 100644 --- a/test/expected/import_schema_1.out +++ b/test/expected/import_schema_1.out @@ -608,7 +608,7 @@ $$); SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) diff --git a/test/expected/import_schema_2.out b/test/expected/import_schema_2.out index b9b86a96..143cd76a 100644 --- a/test/expected/import_schema_2.out +++ b/test/expected/import_schema_2.out @@ -708,7 +708,7 @@ $$); SELECT * FROM clickhouse_bin.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: pg_clickhouse: error while reading row: value 18446744073709551615 is out of range of bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) diff --git a/test/expected/json_4.out b/test/expected/json_4.out index e9d22520..3143cfa1 100644 --- a/test/expected/json_4.out +++ b/test/expected/json_4.out @@ -67,7 +67,7 @@ CREATE FOREIGN TABLE json_http.json_things ( ) SERVER http_json_loopback OPTIONS (table_name 'things'); INSERT INTO json_bin.json_things VALUES (5, '{"id": 5, "name": "bauble", "size": "small", "stocked": true}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; @@ -652,7 +652,7 @@ INSERT INTO json_http.special_keys VALUES (1, '{"my field": "hello", "CamelCase": "world", "select": "reserved"}'); INSERT INTO json_bin.special_keys VALUES (2, E'{"The \\"meaning\\" of life": 42, "back\\\\slash": "bs", "dotted.key": "dot", "it''s": "apos", "key/with!special@chars#": "special", "123numeric": "num"}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') CREATE FOREIGN TABLE json_http.json_special_keys (id integer NOT NULL, data json NOT NULL) SERVER http_json_loopback OPTIONS (database 'json_test', table_name 'special_keys'); CREATE FOREIGN TABLE json_bin.json_special_keys (id integer NOT NULL, data json NOT NULL) diff --git a/test/expected/json_5.out b/test/expected/json_5.out index 05b65248..668c6979 100644 --- a/test/expected/json_5.out +++ b/test/expected/json_5.out @@ -67,7 +67,7 @@ CREATE FOREIGN TABLE json_http.json_things ( ) SERVER http_json_loopback OPTIONS (table_name 'things'); INSERT INTO json_bin.json_things VALUES (5, '{"id": 5, "name": "bauble", "size": "small", "stocked": true}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; @@ -652,7 +652,7 @@ INSERT INTO json_http.special_keys VALUES (1, '{"my field": "hello", "CamelCase": "world", "select": "reserved"}'); INSERT INTO json_bin.special_keys VALUES (2, E'{"The \\"meaning\\" of life": 42, "back\\\\slash": "bs", "dotted.key": "dot", "it''s": "apos", "key/with!special@chars#": "special", "123numeric": "num"}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') CREATE FOREIGN TABLE json_http.json_special_keys (id integer NOT NULL, data json NOT NULL) SERVER http_json_loopback OPTIONS (database 'json_test', table_name 'special_keys'); CREATE FOREIGN TABLE json_bin.json_special_keys (id integer NOT NULL, data json NOT NULL) diff --git a/test/expected/json_6.out b/test/expected/json_6.out index 1aa614e4..8c5d03c1 100644 --- a/test/expected/json_6.out +++ b/test/expected/json_6.out @@ -67,7 +67,7 @@ CREATE FOREIGN TABLE json_http.json_things ( ) SERVER http_json_loopback OPTIONS (table_name 'things'); INSERT INTO json_bin.json_things VALUES (5, '{"id": 5, "name": "bauble", "size": "small", "stocked": true}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') INSERT INTO json_http.json_things VALUES (6, '{"id": 6, "name": "curio", "size": "medium", "stocked": false}'); SELECT * FROM json_bin.json_things ORDER BY id; @@ -652,7 +652,7 @@ INSERT INTO json_http.special_keys VALUES (1, '{"my field": "hello", "CamelCase": "world", "select": "reserved"}'); INSERT INTO json_bin.special_keys VALUES (2, E'{"The \\"meaning\\" of life": 42, "back\\\\slash": "bs", "dotted.key": "dot", "it''s": "apos", "key/with!special@chars#": "special", "123numeric": "num"}'); -ERROR: pg_clickhouse: could not prepare insert - unsupported column type: Object('json') +ERROR: pg_clickhouse: unsupported column type: Object('json') CREATE FOREIGN TABLE json_http.json_special_keys (id integer NOT NULL, data json NOT NULL) SERVER http_json_loopback OPTIONS (database 'json_test', table_name 'special_keys'); CREATE FOREIGN TABLE json_bin.json_special_keys (id integer NOT NULL, data json NOT NULL) diff --git a/test/sql/binary_inserts.sql b/test/sql/binary_inserts.sql index 884c9ed4..fb6993d6 100644 --- a/test/sql/binary_inserts.sql +++ b/test/sql/binary_inserts.sql @@ -246,9 +246,8 @@ $$); IMPORT FOREIGN SCHEMA binary_inserts_test LIMIT TO (default_vals) FROM SERVER binary_inserts_loopback INTO binary_inserts_test; --- Fails on c2, the first column we try to set to NULL. Will cease to fail if --- the `if (isnull && !nullable)` block is removed from column_append(). See --- its comment for details. +-- Fails on c2, the first column set to NULL: pg-clickhouse-c's writer rejects +-- NULL into a non-Nullable column rather than writing the type's zero value. INSERT INTO default_vals VALUES( 1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, diff --git a/vendor/clickhouse-c b/vendor/clickhouse-c deleted file mode 160000 index 87d30a2c..00000000 --- a/vendor/clickhouse-c +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 87d30a2ce3580cded1c794c7579fb8be8ea1c00b diff --git a/vendor/pg-clickhouse-c b/vendor/pg-clickhouse-c new file mode 160000 index 00000000..15bd22b0 --- /dev/null +++ b/vendor/pg-clickhouse-c @@ -0,0 +1 @@ +Subproject commit 15bd22b0e6f4d8efbef53612dcd5479edefc4fcd