From 15ac58dd2b1b9c58afdcbcecc0192e87e96ce685 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Fri, 17 Jul 2026 20:27:05 +0000 Subject: [PATCH 01/11] share code between binary & http by using native format over http partially as http optimization, partially as way to make drivers consistent, partially as way to increase code reuse maybe not worth coupling these code paths when we should just get rid of http --- CHANGELOG.md | 2 + src/binary/http_native.c | 158 +++++++++++ src/fdw.c | 3 +- src/http.c | 8 +- src/http_streaming.c | 269 ++++++++---------- src/include/engine.h | 5 +- src/include/fdw.h | 3 +- src/include/http.h | 17 -- src/include/http_native.h | 31 +++ src/include/http_streaming.h | 13 +- src/parser.c | 294 ------------------- src/pglink.c | 527 ++++++++++++----------------------- 12 files changed, 514 insertions(+), 816 deletions(-) create mode 100644 src/binary/http_native.c create mode 100644 src/include/http_native.h delete mode 100644 src/parser.c diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b87f839..68eb01fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,8 @@ All notable changes to this project will be documented in this file. It uses the * Coerce array elements in binary driver. `Array(Int32)` to `bigint[]`, or `quantilesExactLow()` results into `double precision[]`, no longer fails with `could not cast value from integer[] to bigint[]` ([#326]). +* Made HTTP driver decode row results from ClickHouse's Native format. + `clickhouse_raw_query()` and HTTP inserts keep TabSeparated behavior. ### 🐞 Bug Fixes diff --git a/src/binary/http_native.c b/src/binary/http_native.c new file mode 100644 index 00000000..e9281510 --- /dev/null +++ b/src/binary/http_native.c @@ -0,0 +1,158 @@ +/* Adapt HTTP Native blocks to shared ClickHouse-to-PostgreSQL row decoder. */ + +#include "postgres.h" + +#include + +#include "miscadmin.h" +#include "utils/palloc.h" + +#include "binary_internal.h" +#include "http_native.h" + +struct ch_http_native { + MemoryContext memcxt; + MemoryContextCallback free_cb; + HttpStream* stream; + + chc_io io; + chc_in* in; + chc_block_opts opts; + + bool eof; + bool canceled; + char* error; +}; + +static int +native_io_read(void* ud, void* buf, size_t len, size_t* out_n, chc_err* err) { + HttpStream* s = (HttpStream*)ud; + + if (ch_http_stream_read(s, buf, len, out_n) < 0) { + snprintf(err->msg, sizeof(err->msg), "HTTP transport error"); + return CHC_ERR_IO; + } + return CHC_OK; +} + +static int +native_io_check_cancel(void* ud pg_attribute_unused()) { + return (QueryCancelPending || ProcDiePending) ? 1 : 0; +} + +static void +native_set_error(ch_http_native* h, const char* msg) { + if (h->error) { + return; + } + h->error = pstrdup(msg && *msg ? msg : "native block decode failed"); + h->eof = true; +} + +/* Keep decoded block outside transient per-row context. */ +static const chc_block* +native_src_next_block(void* ud) { + ch_http_native* h = (ch_http_native*)ud; + MemoryContext old; + chc_block* blk = NULL; + chc_err err = {}; + int rc; + + if (h->eof || h->error) { + return NULL; + } + + old = MemoryContextSwitchTo(h->memcxt); + rc = chc_block_read(h->in, &pgch_alloc, &h->opts, &blk, &err); + + if (rc != CHC_OK) { + if (rc == CHC_ERR_CANCELLED || QueryCancelPending || ProcDiePending) { + h->canceled = true; + } + native_set_error(h, err.msg); + blk = NULL; + } else if (!blk) { + h->eof = true; /* clean EOF at a block boundary */ + } else { + for (size_t i = 0; i < chc_block_n_columns(blk); i++) { + chc_err verr = {}; + + if (chc_column_validate(chc_block_column(blk, i), &verr) != CHC_OK) { + native_set_error(h, verr.msg); + chc_block_destroy(blk, &pgch_alloc); + blk = NULL; + break; + } + } + } + + MemoryContextSwitchTo(old); + return blk; +} + +static const char* +native_src_error(void* ud) { + return ((ch_http_native*)ud)->error; +} + +static void +native_ctx_free_cb(void* arg) { + ch_http_native_free((ch_http_native*)arg); +} + +ch_http_native* +ch_http_native_begin(HttpStream* stream, MemoryContext memcxt) { + ch_http_native* h = palloc0(sizeof(*h)); + chc_err err = {}; + + h->memcxt = memcxt; + h->stream = stream; + h->io.ud = stream; + h->io.read = native_io_read; + h->io.write = NULL; + h->io.check_cancel = native_io_check_cancel; + h->opts = + (chc_block_opts){ .has_block_info = false, .has_custom_serialization = false }; + + /* Close malloc-backed stream on any cursor-context reset. */ + h->free_cb.func = native_ctx_free_cb; + h->free_cb.arg = h; + MemoryContextRegisterResetCallback(memcxt, &h->free_cb); + + h->in = pgch_in_alloc(); + if (chc_in_init(h->in, &h->io, &pgch_alloc, 0, &err) != CHC_OK) { + native_set_error(h, err.msg[0] ? err.msg : "native reader init failed"); + } + return h; +} + +pgch_block_source +ch_http_native_block_source(ch_http_native* h) { + return (pgch_block_source){ + .ud = h, + .next_block = native_src_next_block, + .error = native_src_error, + }; +} + +bool +ch_http_native_canceled(const ch_http_native* h) { + return h->canceled; +} + +const char* +ch_http_native_query_id(const ch_http_native* h) { + return h->stream ? ch_http_stream_query_id(h->stream) : NULL; +} + +void +ch_http_native_free(ch_http_native* h) { + if (!h) { + return; + } + /* Decode allocations belong to memcxt. */ + if (h->stream) { + ch_http_stream_end(h->stream); + h->stream = NULL; + } +} diff --git a/src/fdw.c b/src/fdw.c index 9de9db09..1d64d7b0 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -366,8 +366,7 @@ ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name); Datum clickhouse_raw_query(PG_FUNCTION_ARGS) { char* connstring = text_to_cstring(PG_GETARG_TEXT_P(1)); - ch_query query = - new_query(text_to_cstring(PG_GETARG_TEXT_P(0)), 0, NULL, NULL, NULL); + ch_query query = new_raw_query(text_to_cstring(PG_GETARG_TEXT_P(0))); ch_connection_details* details = connstring_parse(connstring); ch_connection conn; diff --git a/src/http.c b/src/http.c index b5751dec..18f4c4f3 100644 --- a/src/http.c +++ b/src/http.c @@ -190,16 +190,14 @@ ch_http_connection(ch_connection_details* details) { /* * ch_http_simple_query — buffer the full response in memory. * - * Built on top of the streaming driver with an effectively-unbounded - * fetch_size, so the whole response lands in one batch that we then hand off - * to the caller as a ch_http_response_t. + * fetch_size 0 buffers complete response. */ ch_http_response_t* ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { HttpStream* stream; ch_http_response_t* resp; - stream = ch_http_stream_begin(conn, query, INT32_MAX); + stream = ch_http_stream_begin(conn, query, 0, false); if (stream == NULL) { return NULL; } @@ -239,7 +237,7 @@ ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* /* conn is calloc'd (see ch_http_connect), so version.major == 0 reliably * means the version has not been fetched and cached yet. */ if (conn->version.major == 0) { - ch_query query = { "SELECT version()", 0, NULL, NULL, NULL, NULL }; + ch_query query = { .sql = "SELECT version()" }; ch_http_response_t* resp = ch_http_simple_query(conn, &query); if (resp != NULL) { diff --git a/src/http_streaming.c b/src/http_streaming.c index daa8310d..3d1c0ea1 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -4,8 +4,7 @@ * Streaming HTTP query driver for pg_clickhouse. * * Uses curl_multi + CURL_WRITEFUNC_PAUSE to receive ClickHouse HTTP - * responses in row-aligned batches of approximately fetch_size bytes, - * keeping memory proportional to batch size instead of full result set. + * responses in byte batches, keeping memory proportional to fetch_size. * * Copyright (c) 2025-2026, ClickHouse, Inc. * @@ -53,10 +52,9 @@ struct HttpStream { size_t buf_allocated; size_t write_pos; size_t parse_pos; - size_t batch_end; int32 fetch_size; /* approximate batch size in bytes */ + bool native; bool paused; - bool started; bool transfer_done; char error_buffer[CURL_ERROR_SIZE]; @@ -73,10 +71,8 @@ static void setup_curl(HttpStream* stream, const ch_query* query); static void capture_transfer_info(HttpStream* stream); -static void -compact_buffer(HttpStream* stream); -static size_t -find_batch_end(const HttpStream* stream); +static int +pump(HttpStream* stream); static size_t write_callback(void* contents, size_t size, size_t nmemb, void* userp); @@ -96,11 +92,29 @@ setup_curl(HttpStream* stream, const ch_query* query) { snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); + /* Settings overridden below win over user settings. */ + static const char* const native_overridden[] = { + "default_format", + "output_format_native_encode_types_in_binary_format", + "output_format_native_write_json_as_string", + NULL, + }; + static const char* const tsv_overridden[] = { + "date_time_output_format", + "format_tsv_null_representation", + "output_format_tsv_crlf_end_of_line", + NULL, + }; + const char* const* overridden = stream->native ? native_overridden : tsv_overridden; + kv_iter iter = new_kv_iter(query->settings); while (kv_iter_next(&iter)) { - if (strcmp(iter.name, "date_time_output_format") == 0 || - strcmp(iter.name, "format_tsv_null_representation") == 0 || - strcmp(iter.name, "output_format_tsv_crlf_end_of_line") == 0) { + const char* const* skip = overridden; + + while (*skip && strcmp(iter.name, *skip) != 0) { + skip++; + } + if (*skip) { continue; } snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); @@ -109,24 +123,56 @@ setup_curl(HttpStream* stream, const ch_query* query) { ); } - curl_url_set( - cu, - CURLUPART_QUERY, - "date_time_output_format=iso", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_set( - cu, - CURLUPART_QUERY, - "format_tsv_null_representation=\\N", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_set( - cu, - CURLUPART_QUERY, - "output_format_tsv_crlf_end_of_line=0", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); + if (stream->native) { + int major, minor, patch; + + /* Keep SQL unchanged so query parameters work. */ + curl_url_set( + cu, + CURLUPART_QUERY, + "default_format=Native", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + + ch_http_server_version(stream->conn, &major, &minor, &patch); + + /* Gate settings by server version, unknown HTTP settings fail queries. */ + if (major > 24 || (major == 24 && minor >= 7)) { + curl_url_set( + cu, + CURLUPART_QUERY, + "output_format_native_encode_types_in_binary_format=0", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } + if (major > 24 || (major == 24 && minor >= 10)) { + curl_url_set( + cu, + CURLUPART_QUERY, + "output_format_native_write_json_as_string=1", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } + } else { + curl_url_set( + cu, + CURLUPART_QUERY, + "date_time_output_format=iso", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + curl_url_set( + cu, + CURLUPART_QUERY, + "format_tsv_null_representation=\\N", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + curl_url_set( + cu, + CURLUPART_QUERY, + "output_format_tsv_crlf_end_of_line=0", + CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } curl_url_get(cu, CURLUPART_URL, &stream->url, 0); curl_url_cleanup(cu); @@ -184,8 +230,7 @@ setup_curl(HttpStream* stream, const ch_query* query) { /* ---------------------------------------------------------------- * write_callback — CURL write callback. Appends data to the stream - * buffer and asks CURL to pause receipt once a row-aligned batch of - * approximately fetch_size bytes is buffered. + * buffer and asks CURL to pause receipt near fetch_size bytes. * ---------------------------------------------------------------- */ static size_t @@ -216,15 +261,7 @@ write_callback(void* contents, size_t size, size_t nmemb, void* userp) { self->write_pos += realsize; self->buf[self->write_pos] = '\0'; - /* - * Once we have buffered at least fetch_size bytes AND at least one - * newline (so a row-aligned batch is ready), pause receipt. Pausing on - * byte-count alone can starve the parser of the newline it needs when - * fetch_size is small. We accept this chunk first and pause afterward via - * curl_easy_pause so CURL does not redeliver bytes we already hold. - */ - if (self->write_pos >= (size_t)self->fetch_size && - memchr(self->buf, '\n', self->write_pos) != NULL) { + if (self->fetch_size > 0 && self->write_pos >= (size_t)self->fetch_size) { self->paused = true; curl_easy_pause(self->curl, CURLPAUSE_RECV); } @@ -243,102 +280,20 @@ capture_transfer_info(HttpStream* stream) { stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time ); curl_easy_getinfo(stream->curl, CURLINFO_TOTAL_TIME, &stream->total_time); - stream->started = true; -} - -/* ---------------------------------------------------------------- - * find_batch_end — find a row-aligned split point near fetch_size - * bytes. Looks forward then backward for nearest newline. Never - * returns a partial row: if no newline is buffered, returns 0 so - * the caller keeps ingesting. - * ---------------------------------------------------------------- - */ -static size_t -find_batch_end(const HttpStream* stream) { - const char* base = stream->buf; - const char* found; - - if (stream->fetch_size <= 0) { - return stream->write_pos; - } - - if (stream->write_pos >= (size_t)stream->fetch_size) { - /* Look forward first */ - found = memchr( - base + stream->fetch_size, '\n', stream->write_pos - stream->fetch_size - ); - if (found) { - return (found - base) + 1; - } - - /* Look backward */ - for (size_t i = stream->fetch_size; i > 0; i--) { - if (base[i - 1] == '\n') { - return i; - } - } - } - - if (stream->transfer_done) { - return stream->write_pos; - } - - return 0; } -/* ---------------------------------------------------------------- - * compact_buffer — shift unparsed data to the front of the buffer. - * ---------------------------------------------------------------- - */ -static void -compact_buffer(HttpStream* stream) { - if (stream->parse_pos > 0) { - size_t remaining = stream->write_pos - stream->parse_pos; - - memmove(stream->buf, stream->buf + stream->parse_pos, remaining); - stream->write_pos = remaining; - stream->parse_pos = 0; - stream->batch_end = 0; - stream->buf[stream->write_pos] = '\0'; - } -} - -/* ---------------------------------------------------------------- - * http_stream_pump — drive curl_multi until the next batch is ready or the - * transfer completes. Returns 0 on success, -1 on error. - * ---------------------------------------------------------------- - */ -int -ch_http_stream_pump(HttpStream* stream) { +static int +pump(HttpStream* stream) { int running_handles; CURLMcode mc; CURLMsg* msg; int msgs_left; - /* - * Drop the already-consumed batch and see if there is enough buffered - * data for the next one before touching the network again. - */ - if (stream->parse_pos > 0) { - compact_buffer(stream); - } - - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || - (stream->transfer_done && stream->write_pos <= stream->parse_pos)) { - if (!stream->started) { - capture_transfer_info(stream); - } - return 0; - } - - /* Resume if paused from a previous batch */ if (stream->paused) { stream->paused = false; curl_easy_pause(stream->curl, CURLPAUSE_CONT); } - /* Drive the transfer */ for (;;) { mc = curl_multi_perform(stream->multi, &running_handles); if (mc != CURLM_OK) { @@ -352,8 +307,9 @@ ch_http_stream_pump(HttpStream* stream) { stream->transfer_done = true; } - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || stream->paused || stream->transfer_done) { + /* fetch_size 0 waits for complete response. */ + if (stream->paused || stream->transfer_done || + (stream->fetch_size > 0 && stream->write_pos > 0)) { break; } @@ -361,9 +317,6 @@ ch_http_stream_pump(HttpStream* stream) { } capture_transfer_info(stream); - stream->batch_end = find_batch_end(stream); - - /* Check for transfer errors */ while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { @@ -384,6 +337,35 @@ ch_http_stream_pump(HttpStream* stream) { return 0; } +/* Blocking byte reader for clickhouse-c chc_io. */ +int +ch_http_stream_read(HttpStream* stream, void* dst, size_t len, size_t* out_n) { + size_t avail; + + *out_n = 0; + + while (stream->parse_pos >= stream->write_pos) { + if (stream->transfer_done) { + return 0; /* clean EOF */ + } + /* Reuse buffer after complete drain. */ + stream->parse_pos = 0; + stream->write_pos = 0; + if (pump(stream) < 0) { + return -1; + } + } + + avail = stream->write_pos - stream->parse_pos; + if (len < avail) { + avail = len; + } + memcpy(dst, stream->buf + stream->parse_pos, avail); + stream->parse_pos += avail; + *out_n = avail; + return 0; +} + /* ---------------------------------------------------------------- * Public API — lifecycle * ---------------------------------------------------------------- @@ -397,7 +379,8 @@ HttpStream* ch_http_stream_begin( ch_http_connection_t* conn, const ch_query* query, - int32 fetch_size + int32 fetch_size, + bool native ) { HttpStream* stream; uuid_t id; @@ -409,6 +392,7 @@ ch_http_stream_begin( stream->conn = conn; stream->fetch_size = fetch_size; + stream->native = native; /* Generate query ID */ uuid_generate(id); @@ -441,8 +425,13 @@ ch_http_stream_begin( } curl_multi_add_handle(stream->multi, stream->curl); - /* Pump until first batch is ready or transfer completes */ - ch_http_stream_pump(stream); + pump(stream); + if (native && stream->http_status > 0 && stream->http_status != CH_HTTP_STATUS_OK && + stream->http_status != CH_HTTP_STATUS_CANCELED && + stream->http_status != CH_HTTP_STATUS_TRANSPORT_ERROR) { + stream->fetch_size = 0; + pump(stream); + } return stream; @@ -501,21 +490,7 @@ ch_http_stream_buffer(HttpStream* stream) { size_t ch_http_stream_available(HttpStream* stream) { - return stream->batch_end > stream->parse_pos ? stream->batch_end - stream->parse_pos - : 0; -} - -void -ch_http_stream_advance(HttpStream* stream, size_t n) { - stream->parse_pos += n; - if (stream->parse_pos > stream->batch_end) { - stream->parse_pos = stream->batch_end; - } -} - -bool -ch_http_stream_transfer_done(HttpStream* stream) { - return stream->transfer_done && (stream->write_pos <= stream->parse_pos); + return stream->write_pos - stream->parse_pos; } long diff --git a/src/include/engine.h b/src/include/engine.h index eee488fb..4287da82 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -50,11 +50,14 @@ typedef struct { const TupleDesc tupdesc; /* The numbers of the attributes in tupdesc that the query selects. */ const List* attr_nums; + const bool raw_result; /* List of settings to pass to ClickHouse upon execution. */ const kv_list* settings; } ch_query; #define new_query(sql, num, vals, tupdesc, attrs) \ - { sql, num, vals, tupdesc, attrs, chfdw_get_session_settings() } + { sql, num, vals, tupdesc, attrs, false, chfdw_get_session_settings() } +#define new_raw_query(sql) \ + { sql, 0, NULL, NULL, NULL, true, chfdw_get_session_settings() } #endif /* CLICKHOUSE_ENGINE_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 5509acb1..90921e64 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -54,9 +54,10 @@ typedef struct ch_cursor { double request_time; double total_time; size_t columns_count; - /* for binary, per returned column: conversion state, target attribute */ + /* for Native readers, per returned column: conversion state, target attribute */ void** conversion_states; int* fill_dest; + void (*read_error)(struct ch_cursor*); } ch_cursor; typedef struct ChFdwScanRowContext { diff --git a/src/include/http.h b/src/include/http.h index 6a4089b9..45283577 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -28,18 +28,6 @@ typedef struct ch_http_response_t { double total_time; } ch_http_response_t; -typedef enum { CH_CONT, CH_EOL, CH_EOF } ch_read_status; - -typedef struct { - char* data; - size_t datalen; - size_t curpos; - StringInfoData val; - bool done; - bool is_null; /* set when the parser saw the wire NULL - * marker `\N` for the field just read */ -} ch_http_read_state; - typedef struct { StringInfoData sql; char* sql_begin; /* beginning part of constructed sql */ @@ -67,11 +55,6 @@ ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* char* ch_http_last_error(void); -/* read */ -void -ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen); -int -ch_http_read_next(ch_http_read_state* state, bool is_array); void ch_http_response_free(ch_http_response_t* resp); diff --git a/src/include/http_native.h b/src/include/http_native.h new file mode 100644 index 00000000..0e99ba06 --- /dev/null +++ b/src/include/http_native.h @@ -0,0 +1,31 @@ +/* Adapt HTTP Native response to shared block decoder. */ + +#ifndef CLICKHOUSE_HTTP_NATIVE_H +#define CLICKHOUSE_HTTP_NATIVE_H + +#include "postgres.h" + +#include "binary.h" /* pgch_block_source */ +#include "http_streaming.h" + +typedef struct ch_http_native ch_http_native; + +/* Take stream ownership, allocate decode state in memcxt. */ +extern ch_http_native* +ch_http_native_begin(HttpStream* stream, MemoryContext memcxt); + +extern pgch_block_source +ch_http_native_block_source(ch_http_native* h); + +extern bool +ch_http_native_canceled(const ch_http_native* h); + +/* NULL after stream release. */ +extern const char* +ch_http_native_query_id(const ch_http_native* h); + +/* Stop transfer, release stream, keep memcxt-owned decode allocations. */ +extern void +ch_http_native_free(ch_http_native* h); + +#endif /* CLICKHOUSE_HTTP_NATIVE_H */ diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index 135ddaad..83691a6e 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -16,22 +16,21 @@ HttpStream* ch_http_stream_begin( ch_http_connection_t* conn, const ch_query* query, - int32 fetch_size + int32 fetch_size, + bool native ); -int -ch_http_stream_pump(HttpStream* stream); void ch_http_stream_end(HttpStream* stream); +/* Return 0 with out_n 0 at clean EOF, -1 on transport error or cancellation. */ +int +ch_http_stream_read(HttpStream* stream, void* dst, size_t len, size_t* out_n); + /* accessors — let pglink.c read stream state without seeing the struct */ char* ch_http_stream_buffer(HttpStream* stream); size_t ch_http_stream_available(HttpStream* stream); -void -ch_http_stream_advance(HttpStream* stream, size_t n); -bool -ch_http_stream_transfer_done(HttpStream* stream); long ch_http_stream_status(HttpStream* stream); const char* diff --git a/src/parser.c b/src/parser.c deleted file mode 100644 index 45a0b050..00000000 --- a/src/parser.c +++ /dev/null @@ -1,294 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include - -static void -ch_http_read_array_string_literal(ch_http_read_state* state); -static void -ch_http_read_array(ch_http_read_state* state); -static int -ch_http_read_eof(ch_http_read_state* state); -static void -ch_http_parse_error(const char* msg); - -/* - * The streaming path can pass the parser a bounded slice rather than a - * NUL-terminated buffer, so reaching datalen is the equivalent of the old - * '\0' checks. - */ -inline static int -ch_http_read_eof(ch_http_read_state* state) { - state->done = true; - return CH_EOF; -} - -inline static void -ch_http_parse_error(const char* msg) { - ereport( - ERROR, - errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("pg_clickhouse: %s", msg) - ); -} - -void -ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen) { - state->data = datalen > 0 ? data : NULL; - state->datalen = datalen; - state->curpos = 0; - state->done = false; - state->is_null = false; - if (state->val.data == NULL) { - initStringInfo(&state->val); - } else { - resetStringInfo(&state->val); - } -} - -/* - * Parse the next tab-separated value from state->data. Parses tab-delimited - * ClickHouse literals including unquoted strings and arrays. Returns CH_CONT - * if there are moe fields on the line to read, CH_EOL if it has reached the - * end of the line, and CH_EOF if it has reached the end of the file. - * - * `is_array` tells the parser whether the destination Postgres column is an - * array. TabSeparated is ambiguous: a `String` value beginning with `[` is not - * escaped on the wire and is indistinguishable byte-for-byte from an array - * literal until you know the column type. The caller knows the type, so it - * makes the call. - */ -int -ch_http_read_next(ch_http_read_state* state, bool is_array) { - char* data = state->data; - - if (state->done) { - return CH_EOF; - } - if (data == NULL) { - return ch_http_read_eof(state); - } - - resetStringInfo(&state->val); - state->is_null = false; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - /* - * Detect the wire NULL marker. ClickHouse's TabSeparated format encodes - * NULL as the bare 2-byte sequence `\N`. A literal backslash in data is - * sent as `\\`, so legitimate non-null output never contains `\N` at the - * start of a field followed by a delimiter. Detect it here, before - * unescaping, so a non-null String value whose unescaped content happens - * to be `\N` is not collapsed with the NULL marker. - */ - if (state->curpos + 1 < state->datalen && data[state->curpos] == '\\' && - data[state->curpos + 1] == 'N' && - (state->curpos + 2 == state->datalen || data[state->curpos + 2] == '\t' || - data[state->curpos + 2] == '\n')) { - state->is_null = true; - state->curpos += 2; - } else if (is_array && data[state->curpos] == '[') { - /* Parse array literal. */ - ch_http_read_array(state); - } else { - while (state->curpos < state->datalen && data[state->curpos] != '\t' && - data[state->curpos] != '\n') { - if (data[state->curpos] == '\\') { - state->curpos++; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - /* unescape some sequences */ - switch (data[state->curpos]) { - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'N': - /* NULL (format_tsv_null_representation) */ - appendStringInfoString(&state->val, "\\N"); - break; - default: - appendStringInfoChar(&state->val, data[state->curpos]); - } - state->curpos++; - } else { - appendStringInfoChar(&state->val, data[state->curpos++]); - } - } - } - - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - if (data[state->curpos] == '\t') { - /* There are more fields. */ - state->curpos++; - return CH_CONT; - } - - /* Should be at the end of the line or the file. */ - if (data[state->curpos] != '\n') { - ereport( - ERROR, - errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("unexpected byte (%d) after array", data[state->curpos]) - ); - } - - state->curpos++; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - return CH_EOL; -} - -/* - * Convert a single-quoted string from a ClickHouse array to a double-quoted - * PostgreSQL array string. Based on `readQuotedStringFieldInto()` from - * `src/IO/ReadHelpers.cpp` in the ClickHouse source. The basic conversion is: - * - * - " => \" - * - \\ => \\ - * - \ followed by b, f, r, n, t, or 0 => literal char - * - \ followed by any other character (including ') => append that character - * - Append any other character - * - * https://clickhouse.com/docs/interfaces/formats/TabSeparated - * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO - */ -void -ch_http_read_array_string_literal(ch_http_read_state* state) { - /* Postgres array string is double-quoted. */ - appendStringInfoChar(&state->val, '"'); - state->curpos++; - - while (state->curpos < state->datalen && state->data[state->curpos] != '\'') { - char ch = state->data[state->curpos]; - - if (ch == '"') { - /* Escape double quotation mark. */ - appendStringInfoChar(&state->val, '\\'); - appendStringInfoChar(&state->val, ch); - state->curpos++; - } else if (ch == '\\') { - /* Emit the escaped character. */ - state->curpos++; - if (state->curpos >= state->datalen) { - ch_http_parse_error("invalid array string"); - } - - switch (state->data[state->curpos]) { - case '\\': - appendStringInfoChar(&state->val, state->data[state->curpos]); - appendStringInfoChar(&state->val, state->data[state->curpos]); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - default: - /* Includes ' and probably no other character. */ - appendStringInfoChar(&state->val, state->data[state->curpos]); - } - state->curpos++; - } else { - /* Append any other character. */ - appendStringInfoChar(&state->val, ch); - state->curpos++; - } - } - - if (state->curpos >= state->datalen || state->data[state->curpos] != '\'') { - ch_http_parse_error("invalid array string"); - } - - appendStringInfoChar(&state->val, '"'); - state->curpos++; -} - -/* - * Convert a ClickHouse array literal to a Postgres array literal. Supports - * nested array. The conversions are: - * - * - [ => { - * - ] => } - * - ' => Start of string, convert to double-quoted string - * - Append any other character - * - * Based on `readQuotedFieldInBracketsInto()` from `src/IO/ReadHelpers.cpp` in - * the ClickHouse source. Additional References: - * - * https://clickhouse.com/docs/interfaces/formats/TabSeparated - * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO - */ -void -ch_http_read_array(ch_http_read_state* state) { - size_t balance = 1; - - /* Postgres arrays are wrapped in { and }. */ - appendStringInfoChar(&state->val, '{'); - state->curpos++; - - while (state->curpos < state->datalen && balance) { - switch (state->data[state->curpos]) { - case '\'': - ch_http_read_array_string_literal(state); - break; - case '[': - ++balance; - appendStringInfoChar(&state->val, '{'); - state->curpos++; - break; - case ']': - --balance; - appendStringInfoChar(&state->val, '}'); - state->curpos++; - break; - default: - appendStringInfoChar(&state->val, state->data[state->curpos]); - state->curpos++; - } - } - - if (balance != 0) { - ch_http_parse_error("malformed array literal"); - } -} diff --git a/src/pglink.c b/src/pglink.c index 504143ef..91075afd 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -19,6 +19,7 @@ #include "binary.h" #include "fdw.h" #include "http.h" +#include "http_native.h" #include "http_streaming.h" #include @@ -31,27 +32,31 @@ static void http_disconnect(void* conn); static ch_cursor* http_simple_query(void* conn, const ch_query* query); -static ch_cursor* -http_streaming_query(void* conn, const ch_query* query, int32 fetch_size); static void http_simple_insert(void* conn, const ch_query* query); static void http_cursor_free(void*); +static ch_cursor* +http_native_cursor(void* conn, const ch_query* query, int32 fetch_size); static void -http_streaming_cursor_free(void*); -static Datum* -http_fetch_row(ChFdwScanRowContext* ctx); +http_native_read_error(ch_cursor* cursor); +static void +native_cursor_state_free(void*); +static void +native_cursor_raise_error(ch_cursor* cursor); static Datum* -http_streaming_fetch_row(ChFdwScanRowContext* ctx); +apply_binary_row(ChFdwScanRowContext* ctx); static Datum* -http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state); +native_fetch_row(ChFdwScanRowContext* ctx); +static void +binary_fetch_row_errcb(void* arg); +static void +configure_native_cursor(ch_cursor* cursor, const ch_query* query); static void* http_prepare_insert(void*, ResultRelInfo*, List*, const ch_query*, char*); static void http_insert_tuple(void*, TupleTableSlot*); static void -char_to_datum(ChFdwScanRowContext* ctx, int attnum, char* data, size_t len); -static void report_http_stream_query_failure(void* conn, const ch_query* query, HttpStream* stream); static ch_server_version http_server_version(void* conn); @@ -59,11 +64,11 @@ http_server_version(void* conn); static libclickhouse_methods http_methods = { .disconnect = http_disconnect, .simple_query = http_simple_query, - .fetch_row = http_fetch_row, + .fetch_row = native_fetch_row, .prepare_insert = http_prepare_insert, .insert_tuple = http_insert_tuple, - .streaming_query = http_streaming_query, - .streaming_fetch_row = http_streaming_fetch_row, + .streaming_query = http_native_cursor, + .streaming_fetch_row = native_fetch_row, .server_version = http_server_version, }; @@ -77,8 +82,6 @@ static bool binary_is_broken(const void* conn); /* static void binary_simple_insert(void *conn, const char *query); */ -static Datum* -binary_fetch_row(ChFdwScanRowContext* ctx); static void binary_insert_tuple(void*, TupleTableSlot* slot); static void @@ -105,7 +108,7 @@ binary_server_version(void* conn); static libclickhouse_methods binary_methods = { .disconnect = binary_disconnect, .simple_query = binary_simple_query, - .fetch_row = binary_fetch_row, + .fetch_row = native_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, .finalize_insert = binary_finalize_insert, @@ -307,6 +310,9 @@ report_http_stream_query_failure( static ch_cursor* http_simple_query(void* conn, const ch_query* query) { int attempts = 0; + if (!query->raw_result) { + return http_native_cursor(conn, query, 0); + } /* * volatile: changed after setjmp (PG_TRY) and read after longjmp * (PG_CATCH); longjmp needn't restore register-cached locals, so a @@ -377,11 +383,9 @@ http_simple_query(void* conn, const ch_query* query) { cursor = palloc0(sizeof(ch_cursor)); cursor->conn = conn; cursor->query_response = resp; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); cursor->query = pstrdup(query->sql); cursor->request_time = resp->pretransfer_time * 1000; cursor->total_time = resp->total_time * 1000; - ch_http_read_state_init(cursor->read_state, resp->data, resp->datasize); cursor->memcxt = tempcxt; cursor->callback.func = http_cursor_free; @@ -445,34 +449,22 @@ http_cursor_free(void* c) { ch_http_response_free(((ch_cursor*)c)->query_response); } -inline static void -http_streaming_cursor_free(void* c) { - if (((ch_cursor*)c)->query_response) { - ch_http_stream_end(((ch_cursor*)c)->query_response); - } -} - -/* - * Create a streaming cursor with row-aligned batches of ~fetch_size bytes - * via CURL pause/resume, keeping memory proportional to batch size. - */ +/* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* -http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { +http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { int attempts = 0; - /* - * volatile: changed after setjmp (PG_TRY) and read after longjmp - * (PG_CATCH); longjmp needn't restore register-cached locals, so a - * non-volatile such local has an indeterminate value per C setjmp rules. - */ + /* volatile: modified inside PG_TRY, read after longjmp in PG_CATCH */ volatile MemoryContext tempcxt = NULL; + HttpStream* volatile stream; MemoryContext oldcxt; ch_cursor* cursor; - HttpStream* stream; + ch_http_native* h; + pgch_reader* state; ch_http_set_progress_func(http_progress_callback); again: - stream = ch_http_stream_begin(conn, query, fetch_size); + stream = ch_http_stream_begin(conn, query, fetch_size, true); if (stream == NULL) { ereport( ERROR, @@ -482,11 +474,10 @@ http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { } attempts++; - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR) { - if (attempts < 3) { - ch_http_stream_end(stream); - goto again; - } + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR && + attempts < 3) { + ch_http_stream_end(stream); + goto again; } if (ch_http_stream_status(stream) != CH_HTTP_STATUS_OK) { report_http_stream_query_failure(conn, query, stream); @@ -494,38 +485,38 @@ http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { PG_TRY(); { - /* - * If any palloc below throws, clean up the stream which is not - * tracked by a memory context yet. - */ tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse streaming cursor", ALLOCSET_DEFAULT_SIZES + PortalContext, "pg_clickhouse native cursor", ALLOCSET_DEFAULT_SIZES ); oldcxt = MemoryContextSwitchTo(tempcxt); - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = stream; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); - cursor->query = pstrdup(query->sql); - cursor->request_time = ch_http_stream_request_time(stream); - cursor->total_time = ch_http_stream_total_time(stream); + cursor = palloc0(sizeof(ch_cursor)); + cursor->conn = conn; + cursor->query = pstrdup(query->sql); + cursor->request_time = ch_http_stream_request_time(stream); + cursor->total_time = ch_http_stream_total_time(stream); - ch_http_read_state_init( - cursor->read_state, - ch_http_stream_buffer(stream), - ch_http_stream_available(stream) - ); + /* Transfer ownership before callback registration can fail. */ + { + HttpStream* owned = stream; + + stream = NULL; + h = ch_http_native_begin(owned, tempcxt); + } + cursor->query_response = h; + cursor->read_error = http_native_read_error; + state = palloc0(sizeof(pgch_reader)); + cursor->read_state = state; + pgch_block_source src = ch_http_native_block_source(h); + pgch_reader_init(state, &src); + cursor->columns_count = pgch_reader_columns(state); cursor->memcxt = tempcxt; - cursor->callback.func = http_streaming_cursor_free; + cursor->callback.func = native_cursor_state_free; cursor->callback.arg = cursor; MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); MemoryContextSwitchTo(oldcxt); - - /* Ownership transferred to the cursor callback */ - stream = NULL; } PG_CATCH(); { @@ -539,207 +530,24 @@ http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { } PG_END_TRY(); - return cursor; -} - -/* - * Streaming variant of http_fetch_row. When the parser exhausts the current - * buffer and the transfer isn't done, pump more data from curl and - * reinitialize the parser on the refilled buffer. - */ -static Datum* -http_streaming_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - ch_http_read_state* state = cursor->read_state; - HttpStream* stream = cursor->query_response; - - /* Pump the next batch when the current one has been exhausted. */ - if (state->done || state->data == NULL) { - /* Sync parse position: tell stream how far the parser advanced */ - ch_http_stream_advance(stream, state->curpos); - - if (ch_http_stream_pump(stream) < 0) { - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED) { - char qid[CH_HTTP_QUERY_ID_LEN]; - - memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); - ch_http_stream_end(stream); - cursor->query_response = NULL; - kill_query(cursor->conn, qid); - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: query was aborted") - ); - } - ereport( - ERROR, - errcode(ERRCODE_CONNECTION_FAILURE), - errmsg( - "pg_clickhouse: streaming error - %s", - ch_http_stream_error(stream) ? ch_http_stream_error(stream) - : "unknown" - ) - ); - } - - /* Reinitialize parser on the (possibly compacted) buffer */ - ch_http_read_state_init( - state, ch_http_stream_buffer(stream), ch_http_stream_available(stream) - ); - } - - return http_fetch_row_from_state(ctx, state); -} - -static Datum* -http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state) { - int rc = CH_CONT; - size_t attcount = list_length(ctx->retrieved_attrs); - Datum* values; - - /* All rows or empty table. */ - if (state->done || state->data == NULL) { - return NULL; - } - - /* Special case: SELECT NULL. */ - if (attcount == 0) { - Assert(ctx->values && ctx->nulls); - rc = ch_http_read_next(state, false); - if (rc != CH_CONT && state->is_null) { - ctx->nulls[0] = true; - ctx->values[0] = (Datum)0; - return ctx->values; - } - - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected response for a zero-column result"), - errdetail("Expected a NULL marker (\\N) in the TabSeparated response.") - ); - } - - /* - * Create Datums based on the retrieved_attrs for the TupleDesc. - * ctx->values and ctx->nulls must already be initialized with memory for - * ctx->tupdesc->natts Datums. - */ - if (ctx->tupdesc) { - values = ctx->values; - ListCell* lc; - int i; - - Assert(ctx->values && ctx->nulls && ctx->attinmeta); - foreach (lc, ctx->retrieved_attrs) { - Oid pgtype; - - i = lfirst_int(lc) - 1; - pgtype = TupleDescAttr(ctx->tupdesc, i)->atttypid; - rc = ch_http_read_next(state, type_is_array(pgtype)); - char_to_datum( - ctx, i, state->is_null ? NULL : state->val.data, state->val.len - ); - } - } - /* No TupleDesc, everything is text. */ - else { - values = palloc(attcount * sizeof(Datum)); - for (size_t idx = 0; idx < attcount; idx++) { - rc = ch_http_read_next(state, false); - if (state->is_null) { - values[idx] = (Datum)0; - } else { - values[idx] = PointerGetDatum(cstring_to_text(state->val.data)); - } - } - } - - if (attcount > 0 && rc != CH_EOL && rc != CH_EOF) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: columns mismatch"), - errdetail( - "Number of returned columns does not match " - "expected column count (%lu).", - attcount - ) - ); + if (state->error) { + native_cursor_raise_error(cursor); } - return values; -} - -/* - * Fetch a row from the http response and return its values. - * - * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values - * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts - * values. - * - * Use ctx->tupdesc and ctx->attinmeta to convert the values to the - * appropriate Datums, and store them and the indication of their NULLness in - * ctx->values and ctx->nulls, respectively, then return ctx->values. - * - * If ctx->tupdesc is not set, treat all values as text and return them as - * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, - * which only cares about text. - */ -static Datum* -http_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - ch_http_read_state* state = cursor->read_state; - - return http_fetch_row_from_state(ctx, state); -} + configure_native_cursor(cursor, query); -/* - * Convert the raw data of length len to a Datum identified by attidx. - * Determines the Postgres type and input function from the attidx values in - * ctx->tupdesc and ctx->attinmeta. - */ -static void -char_to_datum(ChFdwScanRowContext* ctx, int attidx, char* data, size_t len) { - static const char time_prefix[] = "1970-01-01T"; - Oid pgtype = TupleDescAttr(ctx->tupdesc, attidx)->atttypid; - - if (data && len > sizeof(time_prefix) - 1 && - (pgtype == TIMEOID || pgtype == TIMETZOID) && data[len - 1] == 'Z') { - /* - * date_time_output_format=iso formats times as ISO timestamps. Remove - * the leading `YYYY-mm-ddT`. - */ - data += sizeof(time_prefix) - 1; - } else if (pgtype == BYTEAOID) { - /* Postgres input function won't work, we have raw data. */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = - data == NULL ? (Datum)0 - : PointerGetDatum((bytea*)cstring_to_text_with_len(data, len)); - return; - } - - /* Apply the input function even to nulls, to support domains */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = InputFunctionCall( - &ctx->attinmeta->attinfuncs[attidx], - data, - ctx->attinmeta->attioparams[attidx], - ctx->attinmeta->atttypmods[attidx] - ); + return cursor; } text* chfdw_http_fetch_raw_data(ch_cursor* cursor) { - ch_http_read_state* state = cursor->read_state; + ch_http_response_t* resp = cursor->query_response; - if (state->data == NULL) { + if (resp->data == NULL) { return NULL; } - return cstring_to_text(state->data); + return cstring_to_text_with_len(resp->data, resp->datasize); } /* @@ -978,59 +786,12 @@ binary_simple_query(void* conn, const ch_query* query) { cursor->callback.arg = cursor; MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - /* - * Validate declared shape before any per-column access. Empty attr_nums - * keeps the zero-attribute NULL sentinel handled at fetch time. Ignore - * columns_count == 0 (DDL) to support callers passing a placeholder - * column list since clickhouse_query() requires one syntactically. - */ - if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && - (size_t)list_length(query->attr_nums) != cursor->columns_count) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal( - "pg_clickhouse: returned %lu columns, expected %lu", - (unsigned long)cursor->columns_count, - (unsigned long)list_length(query->attr_nums) - ), - errdetail_internal("Remote Query: %.64000s", query->sql) - ); - } - - /* - * CH JSON columns default to JSONBOID in state->coltypes. When foreign - * table column is declared `json` (JSONOID), override so - * binary_make_datum returns json Datum from CH's STRING bytes, skipping - * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break - * expected outputs that pin CH's exact formatting. - */ - if (query->tupdesc && state->coltypes) { - ListCell* lc; - size_t j = 0; - - foreach (lc, query->attr_nums) { - int i = lfirst_int(lc); - - if (state->coltypes[j] == JSONBOID && - TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { - state->coltypes[j] = JSONOID; - } - j++; - } - } + configure_native_cursor(cursor, query); MemoryContextSwitchTo(oldcxt); if (state->error) { - /* Prefer consistent interrupt error message when query interrupted */ - CHECK_FOR_INTERRUPTS(); - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", query->sql) - ); + native_cursor_raise_error(cursor); } return cursor; @@ -1169,56 +930,65 @@ build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { MemoryContextSwitchTo(old); } -static Datum* -binary_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - List* attrs = ctx->retrieved_attrs; - TupleDesc tupdesc = ctx->tupdesc; - Datum* values = ctx->values; - bool* nulls = ctx->nulls; +static void +configure_native_cursor(ch_cursor* cursor, const ch_query* query) { pgch_reader* state = cursor->read_state; - ErrorContextCallback errcallback; - - errcallback.callback = binary_fetch_row_errcb; - errcallback.arg = (void*)cursor->query; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - bool have_data = pgch_reader_next(state); - size_t attcount = list_length(attrs); - if (state->error) { - error_context_stack = errcallback.previous; - - /* Prefer consistent interrupt error message when fetch interrupted */ - CHECK_FOR_INTERRUPTS(); + if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && + (size_t)list_length(query->attr_nums) != cursor->columns_count) { ereport( ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", cursor->query) + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + (unsigned long)cursor->columns_count, + (unsigned long)list_length(query->attr_nums) + ), + errdetail_internal("Remote Query: %.64000s", query->sql) ); } - if (!have_data) { - error_context_stack = errcallback.previous; - return NULL; + /* Preserve JSON text when PostgreSQL destination uses json, not jsonb. */ + if (query->tupdesc && state->coltypes) { + ListCell* lc; + size_t j = 0; + + foreach (lc, query->attr_nums) { + int i = lfirst_int(lc); + + if (state->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { + state->coltypes[j] = JSONOID; + } + j++; + } } +} + +/* Apply PostgreSQL conversions to fetched Native row. */ +static Datum* +apply_binary_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + List* attrs = ctx->retrieved_attrs; + TupleDesc tupdesc = ctx->tupdesc; + Datum* values = ctx->values; + bool* nulls = ctx->nulls; + pgch_reader* state = cursor->read_state; + size_t attcount = list_length(attrs); if (attcount == 0) { if (pgch_reader_columns(state) == 1 && state->nulls[0]) { nulls[0] = true; - goto ok; - } else { - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg( - "pg_clickhouse: unexpected state: attributes " - "count == 0 and haven't got NULL in the response" - ) - ); + return state->values; } + ereport( + ERROR, + errcode(ERRCODE_FDW_ERROR), + errmsg( + "pg_clickhouse: unexpected state: attributes " + "count == 0 and haven't got NULL in the response" + ) + ); } else if (attcount != pgch_reader_columns(state)) { ereport( ERROR, @@ -1242,20 +1012,93 @@ binary_fetch_row(ChFdwScanRowContext* ctx) { ); } -ok: - error_context_stack = errcallback.previous; return state->values; } +/* Raise decoder error; read_error hook may convert to cancellation report. */ +static void +native_cursor_raise_error(ch_cursor* cursor) { + pgch_reader* state = cursor->read_state; + + if (cursor->read_error) { + cursor->read_error(cursor); + } + /* Prefer consistent interrupt error message when fetch interrupted */ + CHECK_FOR_INTERRUPTS(); + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", state->error), + errdetail_internal("Remote Query: %.64000s", cursor->query) + ); +} + +static Datum* +native_fetch_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + pgch_reader* state = cursor->read_state; + ErrorContextCallback errcallback; + bool have_data; + Datum* result; + + errcallback.callback = binary_fetch_row_errcb; + errcallback.arg = (void*)cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + have_data = pgch_reader_next(state); + + if (state->error) { + error_context_stack = errcallback.previous; + native_cursor_raise_error(cursor); + } + + result = have_data ? apply_binary_row(ctx) : NULL; + + error_context_stack = errcallback.previous; + return result; +} + +static void +http_native_read_error(ch_cursor* cursor) { + ch_http_native* h = cursor->query_response; + + if (ch_http_native_canceled(h) || QueryCancelPending || ProcDiePending) { + const char* qid_src = ch_http_native_query_id(h); + char qid[CH_HTTP_QUERY_ID_LEN]; + + qid[0] = '\0'; + if (qid_src) { + memcpy(qid, qid_src, sizeof(qid)); + } + ch_http_native_free(h); + if (qid[0]) { + kill_query(cursor->conn, qid); + } + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted") + ); + } +} + static void binary_cursor_free(void* c) { ch_cursor* cursor = c; - /* Conversion states live in the context this callback fires for. */ - pgch_reader_free(cursor->read_state); + native_cursor_state_free(cursor); ch_binary_response_free(cursor->query_response); } +/* Conversion states live in the context this callback fires for. */ +static void +native_cursor_state_free(void* c) { + ch_cursor* cursor = c; + + pgch_reader_free(cursor->read_state); +} + static void* binary_prepare_insert( void* conn, From e846193d5f8eaae72a8af86a7750804bb1c2f28e Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:00:00 +0000 Subject: [PATCH 02/11] use native for inserts too --- CHANGELOG.md | 9 ++- doc/pg_clickhouse.md | 2 +- src/http_streaming.c | 7 +- src/include/engine.h | 5 ++ src/include/http.h | 10 --- src/pglink.c | 184 +++++++++++++++++++++++++++++-------------- 6 files changed, 146 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 68eb01fe..02f0993b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,8 +84,13 @@ All notable changes to this project will be documented in this file. It uses the * Coerce array elements in binary driver. `Array(Int32)` to `bigint[]`, or `quantilesExactLow()` results into `double precision[]`, no longer fails with `could not cast value from integer[] to bigint[]` ([#326]). -* Made HTTP driver decode row results from ClickHouse's Native format. - `clickhouse_raw_query()` and HTTP inserts keep TabSeparated behavior. +* HTTP driver now uses ClickHouse's Native format, sharing encode/decode with + binary driver. `clickhouse_raw_query()` keeps TabSeparated behavior. + Native `INSERT` declares column types from PostgreSQL and leaves the + conversion to the destination types to ClickHouse, which requires + `input_format_native_allow_types_conversion`, on by default since + ClickHouse 23.3. The minimum supported ClickHouse version is therefore + now 23.3, the 23.x LTS release, in place of 23. ### 🐞 Bug Fixes diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index 8d2e560b..b2bd79f3 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -11,7 +11,7 @@ CREATE EXTENSION pg_clickhouse; This library contains PostgreSQL extension that enables remote query execution on ClickHouse databases, including a [foreign data wrapper]. It supports -PostgreSQL 13 and higher and ClickHouse 23 and higher. +PostgreSQL 13 and higher and ClickHouse 23.3 and higher. ## Getting Started diff --git a/src/http_streaming.c b/src/http_streaming.c index 3d1c0ea1..63f80660 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -208,7 +208,12 @@ setup_curl(HttpStream* stream, const ch_query* query) { } /* POST body or MIME form */ - if (query->num_params == 0) { + if (query->body != NULL) { + curl_easy_setopt( + stream->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)query->body_len + ); + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->body); + } else if (query->num_params == 0) { curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); } else { curl_mimepart* part; diff --git a/src/include/engine.h b/src/include/engine.h index 4287da82..66662c06 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -53,11 +53,16 @@ typedef struct { const bool raw_result; /* List of settings to pass to ClickHouse upon execution. */ const kv_list* settings; + /* Posted verbatim, already prefixed with sql; sql stays set for errors. */ + const void* body; + const size_t body_len; } ch_query; #define new_query(sql, num, vals, tupdesc, attrs) \ { sql, num, vals, tupdesc, attrs, false, chfdw_get_session_settings() } #define new_raw_query(sql) \ { sql, 0, NULL, NULL, NULL, true, chfdw_get_session_settings() } +#define new_body_query(sql, body, len) \ + { sql, 0, NULL, NULL, NULL, false, chfdw_get_session_settings(), body, len } #endif /* CLICKHOUSE_ENGINE_H */ diff --git a/src/include/http.h b/src/include/http.h index 45283577..f152ee5c 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -4,8 +4,6 @@ #include "postgres.h" #include "engine.h" -#include "lib/stringinfo.h" -#include "nodes/pg_list.h" #include #define CH_HTTP_QUERY_ID_LEN 37 @@ -28,14 +26,6 @@ typedef struct ch_http_response_t { double total_time; } ch_http_response_t; -typedef struct { - StringInfoData sql; - char* sql_begin; /* beginning part of constructed sql */ - List* target_attrs; /* list of target attribute numbers */ - int p_nums; /* number of parameters to transmit */ - ch_http_connection_t* conn; -} ch_http_insert_state; - void ch_http_init(int verbose, uint32_t query_id_prefix); void diff --git a/src/pglink.c b/src/pglink.c index 91075afd..1a6251b3 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -9,6 +9,7 @@ #include "parser/parse_coerce.h" #include "parser/parse_type.h" #include "utils/builtins.h" +#include "utils/date.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -28,6 +29,17 @@ static bool initialized = false; +/* Rows buffered for one Native INSERT over HTTP. */ +typedef struct { + char* sql; /* INSERT statement, for error reporting */ + char* sql_begin; /* sql plus the FORMAT clause the body follows */ + pgch_writer* writer; + AttrNumber* attnums; /* slot attribute feeding each column */ + Oid* atttypids; + size_t ncols; + ch_http_connection_t* conn; +} ch_http_insert_state; + static void http_disconnect(void* conn); static ch_cursor* @@ -622,58 +634,31 @@ chfdw_datum_to_ch_literal(Datum value, Oid type) { } /* - * extend_insert_query - * Construct values part of INSERT query + * Serialize buffered rows as a Native block and POST them. + * + * Column types come from PostgreSQL, so they rarely match the destination + * exactly. ClickHouse casts them per column name under + * input_format_native_allow_types_conversion, on by default since 23.3. */ static void -extend_insert_query(ch_http_insert_state* state, TupleTableSlot* slot) { -#ifdef USE_ASSERT_CHECKING - int pindex = 0; -#endif - bool first = true; - - if (state->sql.len == 0) { - appendStringInfoString(&state->sql, state->sql_begin); +http_flush_insert(ch_http_insert_state* state) { + /* HTTP Native omits block info and custom serialization, as the reader + * in ch_http_native_begin expects. */ + static const chc_block_opts opts = { .has_block_info = false, + .has_custom_serialization = false }; + pgch_buf body = {}; + + if (pgch_writer_rows(state->writer) == 0) { + return; } - /* get following parameters from slot */ - if (slot != NULL && state->target_attrs != NIL) { - ListCell* lc; - - foreach (lc, state->target_attrs) { - int attnum = lfirst_int(lc); - Datum value; - Oid type; - bool isnull; - char* string; + pgch_buf_append(&body, state->sql_begin, strlen(state->sql_begin)); + pgch_writer_flush(state->writer, &body, &opts); - value = slot_getattr(slot, attnum, &isnull); - type = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1)->atttypid; + ch_query query = new_body_query(state->sql, body.data, body.len); - if (!first) { - appendStringInfoChar(&state->sql, '\t'); - } - first = false; - - if (isnull) { - appendStringInfoString(&state->sql, "\\N"); -#ifdef USE_ASSERT_CHECKING - pindex++; -#endif - continue; - } - - string = chfdw_datum_to_ch_literal(value, type); - appendStringInfoString(&state->sql, string); - pfree(string); -#ifdef USE_ASSERT_CHECKING - pindex++; -#endif - } - appendStringInfoChar(&state->sql, '\n'); - - Assert(pindex == state->p_nums); - } + http_simple_insert(state->conn, &query); + pgch_buf_reset(&body); } static void* @@ -685,12 +670,75 @@ http_prepare_insert( char* table_name ) { ch_http_insert_state* state = palloc0(sizeof(ch_http_insert_state)); + Relation rel = rri->ri_RelationDesc; + TupleDesc tupdesc = RelationGetDescr(rel); + Oid relid = RelationGetRelid(rel); + size_t ncols = list_length(target_attrs); + pgch_col* cols = palloc0(ncols * sizeof(pgch_col)); + ListCell* lc; + size_t i = 0; + + state->ncols = ncols; + state->attnums = palloc0(ncols * sizeof(AttrNumber)); + state->atttypids = palloc0(ncols * sizeof(Oid)); + + foreach (lc, target_attrs) { + AttrNumber attnum = lfirst_int(lc); + Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); + CustomColumnInfo* cinfo = chfdw_get_custom_column_info(relid, attnum); + /* Name must match the INSERT column list chfdw_deparse_insert_sql built */ + const char* colname = + (cinfo && cinfo->colname[0]) ? cinfo->colname : NameStr(attr->attname); + const char* chtype; + chc_err err = {}; + chc_type* coltype; + + /* + * ClickHouse gained Time64 in 25.6 and casts it to none of the types + * a table holds a time of day in, so send a timestamp on the epoch + * date, as the TabSeparated payload did. + */ + if (attr->atttypid == TIMEOID) { + chtype = attr->attnotnull ? "DateTime64(6, 'UTC')" + : "Nullable(DateTime64(6, 'UTC'))"; + } else { + chtype = pgch_ch_type_for( + attr->atttypid, attr->atttypmod, attr->attnotnull, NULL + ); + } - initStringInfo(&state->sql); - state->sql_begin = psprintf("%s FORMAT TSV\n", query->sql); - state->target_attrs = target_attrs; - state->p_nums = list_length(state->target_attrs); - state->conn = conn; + /* A PostgreSQL array type carries no dimension count, only the + * declared attndims does, and ClickHouse nests one Array per + * dimension */ + for (int dim = 1; dim < attr->attndims; dim++) { + chtype = psprintf("Array(%s)", chtype); + } + + if (chc_type_parse(chtype, strlen(chtype), &pgch_alloc, &coltype, &err) != + CHC_OK) { + ereport( + ERROR, + errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not build ClickHouse type for column \"%s\"", + colname + ), + errdetail_internal("%s: %s", chtype, err.msg) + ); + } + + state->attnums[i] = attnum; + state->atttypids[i] = attr->atttypid; + cols[i].name = colname; + cols[i].name_len = strlen(colname); + cols[i].type = coltype; + i++; + } + + state->writer = pgch_writer_new(CurrentMemoryContext, cols, ncols); + state->sql = pstrdup(query->sql); + state->sql_begin = psprintf("%s FORMAT Native\n", query->sql); + state->conn = conn; return state; } @@ -699,15 +747,37 @@ static void http_insert_tuple(void* istate, TupleTableSlot* slot) { ch_http_insert_state* state = istate; - extend_insert_query(state, slot); + if (slot != NULL) { + for (size_t i = 0; i < state->ncols; i++) { + bool isnull; + Datum value = slot_getattr(slot, state->attnums[i], &isnull); + Oid valtype = state->atttypids[i]; + + /* PostgreSQL casts inet to text through network_show, which + * appends a netmask ClickHouse rejects for IPv4 and IPv6. The + * output function omits it for single hosts. */ + if (valtype == INETOID && !isnull) { + value = CStringGetTextDatum(OidOutputFunctionCall(F_INET_OUT, value)); + valtype = TEXTOID; + } else if (valtype == TIMEOID && !isnull) { + /* Pair with the DateTime64 column http_prepare_insert declares */ + value = TimestampTzGetDatum( + DatumGetTimeADT(value) - + (TimestampTz)(POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * + USECS_PER_DAY + ); + valtype = TIMESTAMPTZOID; + } - if ((slot == NULL && state->sql.len > 0) || - (size_t)state->sql.len > (MaxAllocSize / 2 /* 512MB */)) { - ch_query query = new_query(state->sql.data, 0, NULL, NULL, NULL); + pgch_append_datum(state->writer, i, value, valtype, isnull); + } - http_simple_insert(state->conn, &query); - resetStringInfo(&state->sql); + /* Flush at 64MiB so bulk loads stream instead of buffering every row */ + if (pgch_writer_bytes(state->writer) < 64 * 1024 * 1024) { + return; + } } + http_flush_insert(state); } /*** BINARY PROTOCOL ***/ From 527e4f4735a9cc303b57433282d4d83906321aee Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:25:33 +0000 Subject: [PATCH 03/11] simplify HTTP to being chunk source --- src/binary/http_native.c | 158 ----------------------------------- src/http_streaming.c | 61 ++++++-------- src/include/http_native.h | 31 ------- src/include/http_streaming.h | 16 ++-- src/pglink.c | 82 ++++++++++-------- 5 files changed, 82 insertions(+), 266 deletions(-) delete mode 100644 src/binary/http_native.c delete mode 100644 src/include/http_native.h diff --git a/src/binary/http_native.c b/src/binary/http_native.c deleted file mode 100644 index e9281510..00000000 --- a/src/binary/http_native.c +++ /dev/null @@ -1,158 +0,0 @@ -/* Adapt HTTP Native blocks to shared ClickHouse-to-PostgreSQL row decoder. */ - -#include "postgres.h" - -#include - -#include "miscadmin.h" -#include "utils/palloc.h" - -#include "binary_internal.h" -#include "http_native.h" - -struct ch_http_native { - MemoryContext memcxt; - MemoryContextCallback free_cb; - HttpStream* stream; - - chc_io io; - chc_in* in; - chc_block_opts opts; - - bool eof; - bool canceled; - char* error; -}; - -static int -native_io_read(void* ud, void* buf, size_t len, size_t* out_n, chc_err* err) { - HttpStream* s = (HttpStream*)ud; - - if (ch_http_stream_read(s, buf, len, out_n) < 0) { - snprintf(err->msg, sizeof(err->msg), "HTTP transport error"); - return CHC_ERR_IO; - } - return CHC_OK; -} - -static int -native_io_check_cancel(void* ud pg_attribute_unused()) { - return (QueryCancelPending || ProcDiePending) ? 1 : 0; -} - -static void -native_set_error(ch_http_native* h, const char* msg) { - if (h->error) { - return; - } - h->error = pstrdup(msg && *msg ? msg : "native block decode failed"); - h->eof = true; -} - -/* Keep decoded block outside transient per-row context. */ -static const chc_block* -native_src_next_block(void* ud) { - ch_http_native* h = (ch_http_native*)ud; - MemoryContext old; - chc_block* blk = NULL; - chc_err err = {}; - int rc; - - if (h->eof || h->error) { - return NULL; - } - - old = MemoryContextSwitchTo(h->memcxt); - rc = chc_block_read(h->in, &pgch_alloc, &h->opts, &blk, &err); - - if (rc != CHC_OK) { - if (rc == CHC_ERR_CANCELLED || QueryCancelPending || ProcDiePending) { - h->canceled = true; - } - native_set_error(h, err.msg); - blk = NULL; - } else if (!blk) { - h->eof = true; /* clean EOF at a block boundary */ - } else { - for (size_t i = 0; i < chc_block_n_columns(blk); i++) { - chc_err verr = {}; - - if (chc_column_validate(chc_block_column(blk, i), &verr) != CHC_OK) { - native_set_error(h, verr.msg); - chc_block_destroy(blk, &pgch_alloc); - blk = NULL; - break; - } - } - } - - MemoryContextSwitchTo(old); - return blk; -} - -static const char* -native_src_error(void* ud) { - return ((ch_http_native*)ud)->error; -} - -static void -native_ctx_free_cb(void* arg) { - ch_http_native_free((ch_http_native*)arg); -} - -ch_http_native* -ch_http_native_begin(HttpStream* stream, MemoryContext memcxt) { - ch_http_native* h = palloc0(sizeof(*h)); - chc_err err = {}; - - h->memcxt = memcxt; - h->stream = stream; - h->io.ud = stream; - h->io.read = native_io_read; - h->io.write = NULL; - h->io.check_cancel = native_io_check_cancel; - h->opts = - (chc_block_opts){ .has_block_info = false, .has_custom_serialization = false }; - - /* Close malloc-backed stream on any cursor-context reset. */ - h->free_cb.func = native_ctx_free_cb; - h->free_cb.arg = h; - MemoryContextRegisterResetCallback(memcxt, &h->free_cb); - - h->in = pgch_in_alloc(); - if (chc_in_init(h->in, &h->io, &pgch_alloc, 0, &err) != CHC_OK) { - native_set_error(h, err.msg[0] ? err.msg : "native reader init failed"); - } - return h; -} - -pgch_block_source -ch_http_native_block_source(ch_http_native* h) { - return (pgch_block_source){ - .ud = h, - .next_block = native_src_next_block, - .error = native_src_error, - }; -} - -bool -ch_http_native_canceled(const ch_http_native* h) { - return h->canceled; -} - -const char* -ch_http_native_query_id(const ch_http_native* h) { - return h->stream ? ch_http_stream_query_id(h->stream) : NULL; -} - -void -ch_http_native_free(ch_http_native* h) { - if (!h) { - return; - } - /* Decode allocations belong to memcxt. */ - if (h->stream) { - ch_http_stream_end(h->stream); - h->stream = NULL; - } -} diff --git a/src/http_streaming.c b/src/http_streaming.c index 63f80660..af1eb1bb 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -51,7 +51,6 @@ struct HttpStream { char* buf; size_t buf_allocated; size_t write_pos; - size_t parse_pos; int32 fetch_size; /* approximate batch size in bytes */ bool native; bool paused; @@ -342,33 +341,29 @@ pump(HttpStream* stream) { return 0; } -/* Blocking byte reader for clickhouse-c chc_io. */ -int -ch_http_stream_read(HttpStream* stream, void* dst, size_t len, size_t* out_n) { - size_t avail; +/* Blocking chunk reader; the decoder tracks its position within the chunk. */ +bool +ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error) { + HttpStream* stream = (HttpStream*)ud; - *out_n = 0; + *data = NULL; + *len = 0; - while (stream->parse_pos >= stream->write_pos) { + /* Caller is done with the previous chunk, so refill from offset 0. */ + stream->write_pos = 0; + while (stream->write_pos == 0) { if (stream->transfer_done) { - return 0; /* clean EOF */ + return true; /* clean EOF */ } - /* Reuse buffer after complete drain. */ - stream->parse_pos = 0; - stream->write_pos = 0; if (pump(stream) < 0) { - return -1; + *error = stream->error_msg; + return false; } } - avail = stream->write_pos - stream->parse_pos; - if (len < avail) { - avail = len; - } - memcpy(dst, stream->buf + stream->parse_pos, avail); - stream->parse_pos += avail; - *out_n = avail; - return 0; + *data = stream->buf; + *len = stream->write_pos; + return true; } /* ---------------------------------------------------------------- @@ -490,12 +485,12 @@ ch_http_stream_end(HttpStream* stream) { */ char* ch_http_stream_buffer(HttpStream* stream) { - return stream->buf + stream->parse_pos; + return stream->buf; } size_t ch_http_stream_available(HttpStream* stream) { - return stream->write_pos - stream->parse_pos; + return stream->write_pos; } long @@ -528,16 +523,14 @@ ch_http_stream_total_time(HttpStream* stream) { * * On return, *out_data is a malloc()'d buffer the caller must free(). When * status is CH_HTTP_STATUS_TRANSPORT_ERROR the body is the strdup'd libcurl - * error message; otherwise it is the accumulated response bytes (shifted to - * offset 0 and NUL-terminated). *out_size is set to the length in bytes, - * excluding the NUL. Sets *out_data to NULL and *out_size to 0 when there is - * nothing to hand off. Safe to call at most once per stream; the stream - * itself should still be released with ch_http_stream_end(). + * error message; otherwise it is the accumulated response bytes, NUL + * terminated. *out_size is set to the length in bytes, excluding the NUL. + * Sets *out_data to NULL and *out_size to 0 when there is nothing to hand + * off. Safe to call at most once per stream; the stream itself should still + * be released with ch_http_stream_end(). */ void ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size) { - size_t avail; - if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { *out_data = stream->error_msg; *out_size = strlen(stream->error_msg); @@ -545,19 +538,13 @@ ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size) return; } - avail = ch_http_stream_available(stream); - if (avail == 0 || !stream->buf) { + if (stream->write_pos == 0 || !stream->buf) { *out_data = NULL; *out_size = 0; return; } - if (stream->parse_pos > 0) { - memmove(stream->buf, stream->buf + stream->parse_pos, avail); - } - stream->buf[avail] = '\0'; - *out_data = stream->buf; - *out_size = avail; + *out_size = stream->write_pos; stream->buf = NULL; } diff --git a/src/include/http_native.h b/src/include/http_native.h deleted file mode 100644 index 0e99ba06..00000000 --- a/src/include/http_native.h +++ /dev/null @@ -1,31 +0,0 @@ -/* Adapt HTTP Native response to shared block decoder. */ - -#ifndef CLICKHOUSE_HTTP_NATIVE_H -#define CLICKHOUSE_HTTP_NATIVE_H - -#include "postgres.h" - -#include "binary.h" /* pgch_block_source */ -#include "http_streaming.h" - -typedef struct ch_http_native ch_http_native; - -/* Take stream ownership, allocate decode state in memcxt. */ -extern ch_http_native* -ch_http_native_begin(HttpStream* stream, MemoryContext memcxt); - -extern pgch_block_source -ch_http_native_block_source(ch_http_native* h); - -extern bool -ch_http_native_canceled(const ch_http_native* h); - -/* NULL after stream release. */ -extern const char* -ch_http_native_query_id(const ch_http_native* h); - -/* Stop transfer, release stream, keep memcxt-owned decode allocations. */ -extern void -ch_http_native_free(ch_http_native* h); - -#endif /* CLICKHOUSE_HTTP_NATIVE_H */ diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index 83691a6e..20d25fb3 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -22,9 +22,14 @@ ch_http_stream_begin( void ch_http_stream_end(HttpStream* stream); -/* Return 0 with out_n 0 at clean EOF, -1 on transport error or cancellation. */ -int -ch_http_stream_read(HttpStream* stream, void* dst, size_t len, size_t* out_n); +/* + * pgch_chunk_source next_chunk over the response body. Bytes stay valid until + * the following call. Sets *len 0 at clean EOF; returns false with *error on + * transport failure or cancellation. Takes void* so it can be assigned to the + * callback slot without this header knowing pg-clickhouse-c. + */ +bool +ch_http_stream_next_chunk(void* stream, const void** data, size_t* len, char** error); /* accessors — let pglink.c read stream state without seeing the struct */ char* @@ -45,8 +50,9 @@ ch_http_stream_total_time(HttpStream* stream); /* * Transfer ownership of the response body to the caller. On return, *out_data * is a malloc()'d buffer (or the strdup'd transport error message when status - * is CH_HTTP_STATUS_TRANSPORT_ERROR) that the caller must free(). The stream - * itself is unchanged otherwise and should still be released with + * is CH_HTTP_STATUS_TRANSPORT_ERROR) that the caller must free(). Only valid + * before the first ch_http_stream_next_chunk call, which reuses the buffer. + * The stream itself is unchanged otherwise and should still be released with * ch_http_stream_end(). */ void diff --git a/src/pglink.c b/src/pglink.c index 1a6251b3..5e78430c 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -20,7 +20,6 @@ #include "binary.h" #include "fdw.h" #include "http.h" -#include "http_native.h" #include "http_streaming.h" #include @@ -53,6 +52,8 @@ http_native_cursor(void* conn, const ch_query* query, int32 fetch_size); static void http_native_read_error(ch_cursor* cursor); static void +http_native_cursor_free(void*); +static void native_cursor_state_free(void*); static void native_cursor_raise_error(ch_cursor* cursor); @@ -461,6 +462,12 @@ http_cursor_free(void* c) { ch_http_response_free(((ch_cursor*)c)->query_response); } +/* pgch_chunk_source cancellation poll, checked between reads. */ +static bool +native_chunks_cancelled(void* ud pg_attribute_unused()) { + return QueryCancelPending || ProcDiePending; +} + /* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { @@ -470,7 +477,6 @@ http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { HttpStream* volatile stream; MemoryContext oldcxt; ch_cursor* cursor; - ch_http_native* h; pgch_reader* state; ch_http_set_progress_func(http_progress_callback); @@ -507,26 +513,25 @@ http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { cursor->query = pstrdup(query->sql); cursor->request_time = ch_http_stream_request_time(stream); cursor->total_time = ch_http_stream_total_time(stream); + cursor->read_error = http_native_read_error; + state = palloc0(sizeof(pgch_reader)); + cursor->read_state = state; - /* Transfer ownership before callback registration can fail. */ - { - HttpStream* owned = stream; - - stream = NULL; - h = ch_http_native_begin(owned, tempcxt); - } - cursor->query_response = h; - cursor->read_error = http_native_read_error; - state = palloc0(sizeof(pgch_reader)); - cursor->read_state = state; - pgch_block_source src = ch_http_native_block_source(h); - pgch_reader_init(state, &src); - cursor->columns_count = pgch_reader_columns(state); - + /* Register before taking the stream, so unwinding closes it. */ cursor->memcxt = tempcxt; - cursor->callback.func = native_cursor_state_free; + cursor->callback.func = http_native_cursor_free; cursor->callback.arg = cursor; MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); + cursor->query_response = stream; + stream = NULL; + + pgch_chunk_source src = { .ud = cursor->query_response, + .next_chunk = ch_http_stream_next_chunk, + .cancelled = native_chunks_cancelled }; + + /* Blocks decode into tempcxt, outliving the per-row context. */ + pgch_reader_init_chunks(state, &src, NULL); + cursor->columns_count = pgch_reader_columns(state); MemoryContextSwitchTo(oldcxt); } @@ -642,18 +647,15 @@ chfdw_datum_to_ch_literal(Datum value, Oid type) { */ static void http_flush_insert(ch_http_insert_state* state) { - /* HTTP Native omits block info and custom serialization, as the reader - * in ch_http_native_begin expects. */ - static const chc_block_opts opts = { .has_block_info = false, - .has_custom_serialization = false }; - pgch_buf body = {}; + pgch_buf body = {}; if (pgch_writer_rows(state->writer) == 0) { return; } pgch_buf_append(&body, state->sql_begin, strlen(state->sql_begin)); - pgch_writer_flush(state->writer, &body, &opts); + /* NULL opts: no block info or custom serialization, matching the reader. */ + pgch_writer_flush(state->writer, &body, NULL); ch_query query = new_body_query(state->sql, body.data, body.len); @@ -1131,20 +1133,21 @@ native_fetch_row(ChFdwScanRowContext* ctx) { static void http_native_read_error(ch_cursor* cursor) { - ch_http_native* h = cursor->query_response; + HttpStream* stream = cursor->query_response; + + if (stream == NULL) { + return; + } - if (ch_http_native_canceled(h) || QueryCancelPending || ProcDiePending) { - const char* qid_src = ch_http_native_query_id(h); + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED || + QueryCancelPending || ProcDiePending) { char qid[CH_HTTP_QUERY_ID_LEN]; - qid[0] = '\0'; - if (qid_src) { - memcpy(qid, qid_src, sizeof(qid)); - } - ch_http_native_free(h); - if (qid[0]) { - kill_query(cursor->conn, qid); - } + memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); + /* Drop the transfer before asking the server to kill the query. */ + ch_http_stream_end(stream); + cursor->query_response = NULL; + kill_query(cursor->conn, qid); ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), @@ -1153,6 +1156,15 @@ http_native_read_error(ch_cursor* cursor) { } } +static void +http_native_cursor_free(void* c) { + ch_cursor* cursor = c; + + native_cursor_state_free(cursor); + ch_http_stream_end(cursor->query_response); + cursor->query_response = NULL; +} + static void binary_cursor_free(void* c) { ch_cursor* cursor = c; From 1d70cf4b8ac78ac64cad0b9615457d6ff0a7088d Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:55:41 +0000 Subject: [PATCH 04/11] remove fetch_size, native decoder handles streaming one curl chunk at a time --- CHANGELOG.md | 8 +-- doc/pg_clickhouse.md | 7 --- src/fdw.c | 82 +----------------------- src/http.c | 6 +- src/http_streaming.c | 45 ++++++------- src/include/fdw.h | 8 +-- src/include/http_streaming.h | 7 +-- src/option.c | 24 ++----- src/pglink.c | 8 +-- test/expected/http.out | 111 +++------------------------------ test/expected/http_1.out | 111 +++------------------------------ test/expected/http_2.out | 111 +++------------------------------ test/expected/http_3.out | 111 +++------------------------------ test/expected/http_4.out | 111 +++------------------------------ test/expected/http_5.out | 111 +++------------------------------ test/expected/query_cancel.out | 33 +--------- test/expected/stream_out.out | 3 +- test/sql/http.sql | 60 ++---------------- test/sql/query_cancel.sql | 24 +------ test/sql/stream_out.sql | 4 +- 20 files changed, 98 insertions(+), 887 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02f0993b..db8803dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,11 +86,9 @@ All notable changes to this project will be documented in this file. It uses the with `could not cast value from integer[] to bigint[]` ([#326]). * HTTP driver now uses ClickHouse's Native format, sharing encode/decode with binary driver. `clickhouse_raw_query()` keeps TabSeparated behavior. - Native `INSERT` declares column types from PostgreSQL and leaves the - conversion to the destination types to ClickHouse, which requires - `input_format_native_allow_types_conversion`, on by default since - ClickHouse 23.3. The minimum supported ClickHouse version is therefore - now 23.3, the 23.x LTS release, in place of 23. +* Deprecated `fetch_size`. It is ignored and produces a warning. HTTP driver + always streams, handing Native decoder one chunk at a time. Decoded memory + is set by block size ClickHouse writes, not by client-side byte count. ### 🐞 Bug Fixes diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index b2bd79f3..2fd83816 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -142,10 +142,6 @@ The supported options are: "none", "lz4", or "zstd". Defaults to "lz4". Ignored by the "http" driver. * `dbname`: The ClickHouse database to use upon connecting. Defaults to "default". -* `fetch_size`: Approximate batch size in bytes for HTTP streaming. Batches - split on row boundaries. Defaults to `50000000` (50 MB). `0` disables - streaming and buffers the full response. Foreign tables can override this - value. * `host`: The host name of the ClickHouse server. Defaults to "localhost"; * `min_tls_version`: Minimum TLS protocol version to negotiate on connections that use TLS. One of `TLSv1`, `TLSv1.1`, `TLSv1.2`, or `TLSv1.3`. Defaults @@ -315,9 +311,6 @@ The supported table options are: * `database`: The name of the remote database. Defaults to the database defined for the foreign server. -* `fetch_size`: Approximate batch size in bytes for HTTP streaming. - Overrides server-level `fetch_size`. Defaults to `50000000` (50 MB). `0` - disables streaming and buffers the full response. * `table_name`: The name of the remote table. Default to the name specified for the foreign table. * `engine`: The [table engine] used by the ClickHouse table. For diff --git a/src/fdw.c b/src/fdw.c index 1d64d7b0..d3eeb826 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -61,9 +61,6 @@ PG_MODULE_MAGIC; /* If no remote estimates, assume a sort costs 20% extra */ #define DEFAULT_FDW_SORT_MULTIPLIER 1.2 -/* Approximate batch size in bytes for HTTP streaming (50 MB). */ -#define DEFAULT_FETCH_SIZE (50 * 1000 * 1000) - /* * Indexes of FDW-private information stored in fdw_private lists. * @@ -76,8 +73,6 @@ enum FdwScanPrivateIndex { FdwScanPrivateSelectSql, /* Integer list of attribute numbers retrieved by the SELECT */ FdwScanPrivateRetrievedAttrs, - /* Approximate batch size in bytes for HTTP streaming */ - FdwScanPrivateFetchSize, /* * String describing join i.e. names of relations being joined and types @@ -132,7 +127,6 @@ typedef struct ChFdwScanState { MemoryContext batch_cxt; /* context holding current batch of tuples */ MemoryContext temp_cxt; /* context for per-tuple temporary data */ - int32 fetch_size; /* approximate batch size in bytes */ bool is_streaming; /* true when using HTTP streaming */ } ChFdwScanState; @@ -357,10 +351,6 @@ merge_fdw_options( const CHFdwRelationInfo* fpinfo_o, const CHFdwRelationInfo* fpinfo_i ); -static int -get_fetch_size_option(DefElem* def); -static DefElem* -ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name); /* Make one query and close the connection */ Datum @@ -548,53 +538,6 @@ time_diff(struct timeval* prior, struct timeval* latter) { return x; } -static int -get_fetch_size_option(DefElem* def) { - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg( - "invalid value for option \"%s\": %s", def->defname, defGetString(def) - ), - errhint("fetch_size must be greater than or equal to 0") - ); - } - - return fetch_size; -} - -/* - * Utility function to fetch a cascading option definition from `fpinfo`. - * Returns the first option that matches in `fpinfo->table->options` or, if - * none, then the first found in `fpinfo->server->options`. Returns `NULL` - * when none found. - */ -static DefElem* -ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name) { - ListCell* lc; - - foreach (lc, fpinfo->table->options) { - DefElem* def = (DefElem*)lfirst(lc); - - if (strcmp(def->defname, name) == 0) { - return def; - } - } - - foreach (lc, fpinfo->server->options) { - DefElem* def = (DefElem*)lfirst(lc); - - if (strcmp(def->defname, name) == 0) { - return def; - } - } - - return NULL; -} - /* * clickhouseGetForeignRelSize * Estimate # of rows and width of the result of the scan @@ -635,15 +578,6 @@ clickhouseGetForeignRelSize( fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; fpinfo->shippable_extensions = NIL; - /* - * Extract fetch_size: table option overrides server option, default - * DEFAULT_FETCH_SIZE. Value is approximate batch size in bytes; 0 means - * buffer entire response (disable HTTP streaming). - */ - DefElem* def = ch_get_table_or_server_option(fpinfo, "fetch_size"); - - fpinfo->fetch_size = def ? get_fetch_size_option(def) : DEFAULT_FETCH_SIZE; - chfdw_apply_custom_table_options(fpinfo, foreigntableid); fpinfo->user = NULL; @@ -1095,9 +1029,7 @@ clickhouseGetForeignPlan( * Build the fdw_private list that will be available to the executor. * Items in the list must match order in enum FdwScanPrivateIndex. */ - fdw_private = list_make3( - makeString(sql.data), retrieved_attrs, makeInteger(fpinfo->fetch_size) - ); + fdw_private = list_make2(makeString(sql.data), retrieved_attrs); if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) { fdw_private = lappend(fdw_private, makeString(fpinfo->relation_name->data)); } @@ -1197,8 +1129,6 @@ clickhouseBeginForeignScan(ForeignScanState* node, int eflags) { fsstate->query = strVal(list_nth(fsplan->fdw_private, FdwScanPrivateSelectSql)); fsstate->retrieved_attrs = (List*)list_nth(fsplan->fdw_private, FdwScanPrivateRetrievedAttrs); - fsstate->fetch_size = - intVal(list_nth(fsplan->fdw_private, FdwScanPrivateFetchSize)); /* Create contexts for batches of tuples and per-tuple temp workspace. */ fsstate->batch_cxt = AllocSetContextCreate( @@ -1362,12 +1292,11 @@ clickhouseIterateForeignScan(ForeignScanState* node) { } fsstate->is_streaming = - fsstate->fetch_size > 0 && fsstate->scan_conn.gate.methods->streaming_query != NULL; if (fsstate->is_streaming) { fsstate->ch_cursor = fsstate->scan_conn.gate.methods->streaming_query( - fsstate->scan_conn.gate.conn, &query, fsstate->fetch_size + fsstate->scan_conn.gate.conn, &query ); } else { /* Binary still falls back to the simple path for now. */ @@ -2518,7 +2447,6 @@ merge_fdw_options( fpinfo->fdw_tuple_cost = fpinfo_o->fdw_tuple_cost; fpinfo->shippable_extensions = fpinfo_o->shippable_extensions; fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate; - fpinfo->fetch_size = fpinfo_o->fetch_size; /* Merge the table level options from either side of the join. */ if (fpinfo_i) { @@ -2532,12 +2460,6 @@ merge_fdw_options( */ fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate || fpinfo_i->use_remote_estimate; - - /* - * Set fetch size to maximum of the joining sides, since larger joins - * benefit from bigger batches. - */ - fpinfo->fetch_size = Max(fpinfo_o->fetch_size, fpinfo_i->fetch_size); } } diff --git a/src/http.c b/src/http.c index 18f4c4f3..fe4a42a7 100644 --- a/src/http.c +++ b/src/http.c @@ -188,16 +188,14 @@ ch_http_connection(ch_connection_details* details) { } /* - * ch_http_simple_query — buffer the full response in memory. - * - * fetch_size 0 buffers complete response. + * ch_http_simple_query — buffer the full TabSeparated response in memory. */ ch_http_response_t* ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { HttpStream* stream; ch_http_response_t* resp; - stream = ch_http_stream_begin(conn, query, 0, false); + stream = ch_http_stream_begin(conn, query, false); if (stream == NULL) { return NULL; } diff --git a/src/http_streaming.c b/src/http_streaming.c index af1eb1bb..dbea67eb 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -3,8 +3,8 @@ * http_streaming.c * Streaming HTTP query driver for pg_clickhouse. * - * Uses curl_multi + CURL_WRITEFUNC_PAUSE to receive ClickHouse HTTP - * responses in byte batches, keeping memory proportional to fetch_size. + * Uses curl_multi + curl_easy_pause to hand ClickHouse HTTP responses to + * the caller one receive chunk at a time, keeping memory bounded. * * Copyright (c) 2025-2026, ClickHouse, Inc. * @@ -51,8 +51,7 @@ struct HttpStream { char* buf; size_t buf_allocated; size_t write_pos; - int32 fetch_size; /* approximate batch size in bytes */ - bool native; + bool streaming; /* hand out one chunk at a time, else buffer whole body */ bool paused; bool transfer_done; char error_buffer[CURL_ERROR_SIZE]; @@ -67,7 +66,7 @@ struct HttpStream { /* Forward declarations of static helpers */ static void -setup_curl(HttpStream* stream, const ch_query* query); +setup_curl(HttpStream* stream, const ch_query* query, bool native); static void capture_transfer_info(HttpStream* stream); static int @@ -81,7 +80,7 @@ write_callback(void* contents, size_t size, size_t nmemb, void* userp); * ---------------------------------------------------------------- */ static void -setup_curl(HttpStream* stream, const ch_query* query) { +setup_curl(HttpStream* stream, const ch_query* query, bool native) { CURLU* cu = curl_url(); char temp_buf[512]; @@ -104,7 +103,7 @@ setup_curl(HttpStream* stream, const ch_query* query) { "output_format_tsv_crlf_end_of_line", NULL, }; - const char* const* overridden = stream->native ? native_overridden : tsv_overridden; + const char* const* overridden = native ? native_overridden : tsv_overridden; kv_iter iter = new_kv_iter(query->settings); while (kv_iter_next(&iter)) { @@ -122,7 +121,7 @@ setup_curl(HttpStream* stream, const ch_query* query) { ); } - if (stream->native) { + if (native) { int major, minor, patch; /* Keep SQL unchanged so query parameters work. */ @@ -234,7 +233,7 @@ setup_curl(HttpStream* stream, const ch_query* query) { /* ---------------------------------------------------------------- * write_callback — CURL write callback. Appends data to the stream - * buffer and asks CURL to pause receipt near fetch_size bytes. + * buffer and, when streaming, pauses receipt so the caller drains it. * ---------------------------------------------------------------- */ static size_t @@ -265,7 +264,7 @@ write_callback(void* contents, size_t size, size_t nmemb, void* userp) { self->write_pos += realsize; self->buf[self->write_pos] = '\0'; - if (self->fetch_size > 0 && self->write_pos >= (size_t)self->fetch_size) { + if (self->streaming) { self->paused = true; curl_easy_pause(self->curl, CURLPAUSE_RECV); } @@ -311,9 +310,8 @@ pump(HttpStream* stream) { stream->transfer_done = true; } - /* fetch_size 0 waits for complete response. */ - if (stream->paused || stream->transfer_done || - (stream->fetch_size > 0 && stream->write_pos > 0)) { + /* Buffered mode waits for complete response. */ + if (stream->paused || stream->transfer_done) { break; } @@ -376,12 +374,7 @@ ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error * Returns NULL on failure. */ HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - int32 fetch_size, - bool native -) { +ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool native) { HttpStream* stream; uuid_t id; @@ -390,9 +383,9 @@ ch_http_stream_begin( return NULL; } - stream->conn = conn; - stream->fetch_size = fetch_size; - stream->native = native; + stream->conn = conn; + /* Native responses decode incrementally; other formats want one buffer. */ + stream->streaming = native; /* Generate query ID */ uuid_generate(id); @@ -416,7 +409,7 @@ ch_http_stream_begin( stream->buf_allocated = INITIAL_BUF_SIZE; stream->buf[0] = '\0'; - setup_curl(stream, query); + setup_curl(stream, query, native); /* Create multi handle and kick off the transfer */ stream->multi = curl_multi_init(); @@ -426,10 +419,12 @@ ch_http_stream_begin( curl_multi_add_handle(stream->multi, stream->curl); pump(stream); - if (native && stream->http_status > 0 && stream->http_status != CH_HTTP_STATUS_OK && + /* Error bodies are reported whole, so stop streaming and buffer the rest. */ + if (stream->streaming && stream->http_status > 0 && + stream->http_status != CH_HTTP_STATUS_OK && stream->http_status != CH_HTTP_STATUS_CANCELED && stream->http_status != CH_HTTP_STATUS_TRANSPORT_ERROR) { - stream->fetch_size = 0; + stream->streaming = false; pump(stream); } diff --git a/src/include/fdw.h b/src/include/fdw.h index 90921e64..4d3f9b76 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -83,11 +83,7 @@ typedef void* (*prepare_insert_method)( ); typedef void (*insert_tuple_method)(void* state, TupleTableSlot* slot); typedef void (*finalize_insert_method)(void* state); -typedef ch_cursor* (*streaming_query_method)( - void* conn, - const ch_query* query, - int32 fetch_size -); +typedef ch_cursor* (*streaming_query_method)(void* conn, const ch_query* query); typedef bool (*is_broken_method)(const void* conn); typedef ch_server_version (*server_version_method)(void* conn); @@ -203,8 +199,6 @@ typedef struct CHFdwRelationInfo { ForeignServer* server; UserMapping* user; /* only set in use_remote_estimate mode */ - int32 fetch_size; /* fetch size for this remote table */ - /* * Name of the relation while EXPLAINing ForeignScan. It is used for join * relations but is set for all relations. For join relation, the name diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index 20d25fb3..50f6c506 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -13,12 +13,7 @@ typedef struct HttpStream HttpStream; /* lifecycle */ HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - int32 fetch_size, - bool native -); +ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool native); void ch_http_stream_end(HttpStream* stream); diff --git a/src/option.c b/src/option.c index 130eb8cd..0a5d2252 100644 --- a/src/option.c +++ b/src/option.c @@ -83,8 +83,6 @@ static bool is_valid_option(const char* keyword, Oid context); static bool is_ch_option(const char* keyword); -static void -validate_fetch_size_option(DefElem* def); static bool parse_min_tls_version(const char* val, tls_version* out); @@ -138,7 +136,11 @@ clickhouse_fdw_validator(PG_FUNCTION_ARGS) { } if (strcmp(def->defname, "fetch_size") == 0) { - validate_fetch_size_option(def); + ereport( + WARNING, + errcode(ERRCODE_WARNING_DEPRECATED_FEATURE), + errmsg("option \"fetch_size\" is deprecated and ignored") + ); } if (strcmp(def->defname, "secure") == 0) { @@ -248,22 +250,6 @@ InitChFdwOptions(void) { popt++; } -static void -validate_fetch_size_option(DefElem* def) { - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg( - "invalid value for option \"%s\": %s", def->defname, defGetString(def) - ), - errhint("fetch_size must be greater than or equal to 0") - ); - } -} - /* * Map a min_tls_version option value to tls_version. Accepts the same spellings * as Postgres's ssl_min_protocol_version GUC, case-insensitively. Returns false diff --git a/src/pglink.c b/src/pglink.c index 5e78430c..09731c45 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -48,7 +48,7 @@ http_simple_insert(void* conn, const ch_query* query); static void http_cursor_free(void*); static ch_cursor* -http_native_cursor(void* conn, const ch_query* query, int32 fetch_size); +http_native_cursor(void* conn, const ch_query* query); static void http_native_read_error(ch_cursor* cursor); static void @@ -324,7 +324,7 @@ static ch_cursor* http_simple_query(void* conn, const ch_query* query) { int attempts = 0; if (!query->raw_result) { - return http_native_cursor(conn, query, 0); + return http_native_cursor(conn, query); } /* * volatile: changed after setjmp (PG_TRY) and read after longjmp @@ -470,7 +470,7 @@ native_chunks_cancelled(void* ud pg_attribute_unused()) { /* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* -http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { +http_native_cursor(void* conn, const ch_query* query) { int attempts = 0; /* volatile: modified inside PG_TRY, read after longjmp in PG_CATCH */ volatile MemoryContext tempcxt = NULL; @@ -482,7 +482,7 @@ http_native_cursor(void* conn, const ch_query* query, int32 fetch_size) { ch_http_set_progress_func(http_progress_callback); again: - stream = ch_http_stream_begin(conn, query, fetch_size, true); + stream = ch_http_stream_begin(conn, query, true); if (stream == NULL) { ereport( ERROR, diff --git a/test/expected/http.out b/test/expected/http.out index d97798b3..76640131 100644 --- a/test/expected/http.out +++ b/test/expected/http.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -858,60 +827,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -944,20 +870,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1178,11 +1090,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1239,16 +1146,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_1.out b/test/expected/http_1.out index aa1769e6..a7642d3a 100644 --- a/test/expected/http_1.out +++ b/test/expected/http_1.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -856,60 +825,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1176,11 +1088,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1144,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_2.out b/test/expected/http_2.out index 9d7cf836..8893b7a4 100644 --- a/test/expected/http_2.out +++ b/test/expected/http_2.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -856,60 +825,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1176,11 +1088,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1144,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_3.out b/test/expected/http_3.out index 463ad947..02a15616 100644 --- a/test/expected/http_3.out +++ b/test/expected/http_3.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -856,60 +825,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1176,11 +1088,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1144,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_4.out b/test/expected/http_4.out index 85e671f2..b6240caf 100644 --- a/test/expected/http_4.out +++ b/test/expected/http_4.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -856,60 +825,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1176,11 +1088,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1144,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_5.out b/test/expected/http_5.out index ab3dfd5c..21d91175 100644 --- a/test/expected/http_5.out +++ b/test/expected/http_5.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -858,60 +827,17 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser @@ -944,20 +870,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1178,11 +1090,6 @@ ERROR: invalid input syntax for type integer: "abc" -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1239,16 +1146,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/query_cancel.out b/test/expected/query_cancel.out index 4b78aa45..6cbc332b 100644 --- a/test/expected/query_cancel.out +++ b/test/expected/query_cancel.out @@ -1,13 +1,9 @@ -- Test that query cancellation (e.g. Ctrl+C / statement_timeout) works for -- both the HTTP and binary drivers during remote query execution. --- HTTP driver (streaming path) +-- HTTP driver CREATE SERVER cancel_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'cancel_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http; --- HTTP driver (buffered path) -CREATE SERVER cancel_http_buf FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; -- Binary driver CREATE SERVER cancel_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'cancel_test', driver 'binary'); @@ -40,8 +36,6 @@ SELECT clickhouse_raw_query('INSERT INTO cancel_test.t1 CREATE FOREIGN TABLE cancel_http_ft (c1 int) SERVER cancel_http OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE cancel_http_buf_ft (c1 int) - SERVER cancel_http_buf OPTIONS (table_name 't1'); CREATE FOREIGN TABLE cancel_binary_ft (c1 int) SERVER cancel_binary OPTIONS (table_name 't1'); -- Warm up connections so the cancel test only covers query execution. @@ -51,12 +45,6 @@ SELECT count(*) FROM cancel_http_ft; 100 (1 row) -SELECT count(*) FROM cancel_http_buf_ft; - count -------- - 100 -(1 row) - SELECT count(*) FROM cancel_binary_ft; count ------- @@ -73,14 +61,6 @@ SELECT count(*) FROM cancel_http_ft a CROSS JOIN cancel_http_ft b CROSS JOIN cancel_http_ft c CROSS JOIN cancel_http_ft d; ERROR: pg_clickhouse: query was aborted COMMIT; --- HTTP buffered: exercises the curl progress-callback cancel path via --- the legacy non-streaming (simple_query) implementation. -BEGIN; -SET LOCAL statement_timeout = '10ms'; -SELECT count(*) FROM cancel_http_buf_ft a CROSS JOIN cancel_http_buf_ft b - CROSS JOIN cancel_http_buf_ft c CROSS JOIN cancel_http_buf_ft d; -ERROR: pg_clickhouse: query was aborted -COMMIT; -- Binary: same test, exercising the OnProgress cancel path. BEGIN; SET LOCAL statement_timeout = '10ms'; @@ -96,12 +76,6 @@ SELECT count(*) FROM cancel_http_ft; 100 (1 row) -SELECT count(*) FROM cancel_http_buf_ft; - count -------- - 100 -(1 row) - SELECT count(*) FROM cancel_binary_ft; count ------- @@ -110,13 +84,10 @@ SELECT count(*) FROM cancel_binary_ft; -- Cleanup DROP FOREIGN TABLE cancel_http_ft; -DROP FOREIGN TABLE cancel_http_buf_ft; DROP FOREIGN TABLE cancel_binary_ft; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http; -DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_binary; DROP SERVER cancel_http; -DROP SERVER cancel_http_buf; DROP SERVER cancel_binary; SELECT clickhouse_raw_query('DROP DATABASE cancel_test'); clickhouse_raw_query diff --git a/test/expected/stream_out.out b/test/expected/stream_out.out index c9a46397..3a6510aa 100644 --- a/test/expected/stream_out.out +++ b/test/expected/stream_out.out @@ -1,5 +1,5 @@ CREATE SERVER try_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'try_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'try_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER try_http; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS try_test'); clickhouse_raw_query @@ -3534,7 +3534,6 @@ SELECT * FROM try_http_ft; 3499 (3500 rows) -ALTER SERVER try_http OPTIONS (SET fetch_size '3'); SELECT count(*), min(c1), max(c1), sum(c1::bigint) FROM try_http_ft; count | min | max | sum -------+-----+------+--------- diff --git a/test/sql/http.sql b/test/sql/http.sql index 18bfa73e..82243ec0 100644 --- a/test/sql/http.sql +++ b/test/sql/http.sql @@ -37,17 +37,6 @@ CREATE FOREIGN TABLE ft1 ( c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); - ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( @@ -120,9 +109,6 @@ INSERT INTO ft4 (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; COPY ftcopy FROM stdin; @@ -263,43 +249,17 @@ CREATE FOREIGN TABLE bad_name ( ) SERVER http_loopback_bad OPTIONS ( table_name 't3' ); SELECT * FROM bad_name; -/* ===== fetch_size option tests ===== */ - -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; - -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); - -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); - -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); + +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* * TabSeparated does not escape `[` or `]` in String values, so a value like @@ -322,11 +282,6 @@ INSERT INTO ft_brk VALUES SELECT id, term FROM ft_brk ORDER BY id; -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -448,9 +403,6 @@ SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; - /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ diff --git a/test/sql/query_cancel.sql b/test/sql/query_cancel.sql index 22b0af9f..62e4e488 100644 --- a/test/sql/query_cancel.sql +++ b/test/sql/query_cancel.sql @@ -1,16 +1,11 @@ -- Test that query cancellation (e.g. Ctrl+C / statement_timeout) works for -- both the HTTP and binary drivers during remote query execution. --- HTTP driver (streaming path) +-- HTTP driver CREATE SERVER cancel_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'cancel_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http; --- HTTP driver (buffered path) -CREATE SERVER cancel_http_buf FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; - -- Binary driver CREATE SERVER cancel_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'cancel_test', driver 'binary'); @@ -25,14 +20,11 @@ SELECT clickhouse_raw_query('INSERT INTO cancel_test.t1 CREATE FOREIGN TABLE cancel_http_ft (c1 int) SERVER cancel_http OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE cancel_http_buf_ft (c1 int) - SERVER cancel_http_buf OPTIONS (table_name 't1'); CREATE FOREIGN TABLE cancel_binary_ft (c1 int) SERVER cancel_binary OPTIONS (table_name 't1'); -- Warm up connections so the cancel test only covers query execution. SELECT count(*) FROM cancel_http_ft; -SELECT count(*) FROM cancel_http_buf_ft; SELECT count(*) FROM cancel_binary_ft; -- The repeated CROSS JOINs intentionally create a large remote result set, @@ -45,14 +37,6 @@ SELECT count(*) FROM cancel_http_ft a CROSS JOIN cancel_http_ft b CROSS JOIN cancel_http_ft c CROSS JOIN cancel_http_ft d; COMMIT; --- HTTP buffered: exercises the curl progress-callback cancel path via --- the legacy non-streaming (simple_query) implementation. -BEGIN; -SET LOCAL statement_timeout = '10ms'; -SELECT count(*) FROM cancel_http_buf_ft a CROSS JOIN cancel_http_buf_ft b - CROSS JOIN cancel_http_buf_ft c CROSS JOIN cancel_http_buf_ft d; -COMMIT; - -- Binary: same test, exercising the OnProgress cancel path. BEGIN; SET LOCAL statement_timeout = '10ms'; @@ -63,17 +47,13 @@ COMMIT; -- Verify each connection is still usable after the canceled query tears down -- any remote state. SELECT count(*) FROM cancel_http_ft; -SELECT count(*) FROM cancel_http_buf_ft; SELECT count(*) FROM cancel_binary_ft; -- Cleanup DROP FOREIGN TABLE cancel_http_ft; -DROP FOREIGN TABLE cancel_http_buf_ft; DROP FOREIGN TABLE cancel_binary_ft; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http; -DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_binary; DROP SERVER cancel_http; -DROP SERVER cancel_http_buf; DROP SERVER cancel_binary; SELECT clickhouse_raw_query('DROP DATABASE cancel_test'); diff --git a/test/sql/stream_out.sql b/test/sql/stream_out.sql index 9c30982e..7bedbd3c 100644 --- a/test/sql/stream_out.sql +++ b/test/sql/stream_out.sql @@ -1,5 +1,5 @@ CREATE SERVER try_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'try_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'try_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER try_http; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS try_test'); @@ -14,8 +14,6 @@ CREATE FOREIGN TABLE try_http_ft (c1 int) SELECT * FROM try_http_ft; -ALTER SERVER try_http OPTIONS (SET fetch_size '3'); - SELECT count(*), min(c1), max(c1), sum(c1::bigint) FROM try_http_ft; DROP USER MAPPING FOR CURRENT_USER SERVER try_http; From 581466b9ca8f4bfa07840a418de05d2bcde52166 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:44:02 +0000 Subject: [PATCH 05/11] small cleanups: 1. use curl_url 2. add cancel support to streams, replaces progress callback 3. chfdw_version_ge for version checks 4. don't be persistent about ch_http_server_version --- src/http.c | 168 ++++++++++++----------------------- src/http_streaming.c | 46 +++++++--- src/include/engine.h | 6 ++ src/include/fdw.h | 1 - src/include/http.h | 26 +++--- src/include/http_streaming.h | 7 +- src/include/internal.h | 10 +-- src/pglink.c | 64 +++++-------- 8 files changed, 146 insertions(+), 182 deletions(-) diff --git a/src/http.c b/src/http.c index fe4a42a7..4265123d 100644 --- a/src/http.c +++ b/src/http.c @@ -8,17 +8,12 @@ #include #include -static char curl_error_buffer[CURL_ERROR_SIZE]; -static bool curl_error_happened = false; -static long curl_verbose = 0; -static curl_xferinfo_callback curl_progressfunc = NULL; -static bool curl_initialized = false; -static char ch_query_id_prefix[5]; +static long curl_verbose = 0; +static bool curl_initialized = false; void -ch_http_init(int verbose, uint32_t query_id_prefix) { +ch_http_init(int verbose) { curl_verbose = verbose; - snprintf(ch_query_id_prefix, 5, "%x", query_id_prefix); if (!curl_initialized) { curl_initialized = true; @@ -26,16 +21,6 @@ ch_http_init(int verbose, uint32_t query_id_prefix) { } } -void -ch_http_set_progress_func(curl_xferinfo_callback progressfunc) { - curl_progressfunc = progressfunc; -} - -curl_xferinfo_callback -ch_http_get_progress_func(void) { - return curl_progressfunc; -} - long ch_http_get_verbose(void) { return curl_verbose; @@ -67,24 +52,17 @@ curl_min_tls_version(tls_version v) { } ch_http_connection_t* -ch_http_connection(ch_connection_details* details) { - int n; - char* connstring = NULL; - size_t len = 20; /* all symbols from url string + some extra */ - char *host = details->host, *username = details->username, - *password = details->password; - int port = details->port; +ch_http_connection(ch_connection_details* details, const char** error) { + CURLU* cu = NULL; + char* host = details->host; + int port = details->port; + char port_buf[12]; - curl_error_happened = false; - ch_http_connection_t* conn = calloc(sizeof(ch_http_connection_t), 1); + ch_http_connection_t* conn = calloc(1, sizeof(ch_http_connection_t)); + *error = "out of memory"; if (!conn) { - goto cleanup; - } - - conn->curl = curl_easy_init(); - if (!conn->curl) { - goto cleanup; + return NULL; } conn->ssl_version = curl_min_tls_version(details->min_tls_version); @@ -123,66 +101,47 @@ ch_http_connection(ch_connection_details* details) { break; } - len += strlen(host) + snprintf(NULL, 0, "%d", port); - - if (username) { - username = curl_easy_escape(conn->curl, username, 0); - if (username == NULL) { - goto cleanup; - } - len += strlen(username); - } + snprintf(port_buf, sizeof(port_buf), "%d", port); - if (password) { - password = curl_easy_escape(conn->curl, password, 0); - if (password == NULL) { - curl_free(username); - goto cleanup; - } - len += strlen(password); + cu = curl_url(); + if (cu == NULL) { + goto cleanup; } - connstring = calloc(len, 1); - if (!connstring) { + /* Credentials go in as components so curl escapes them for us. */ + *error = "could not build ClickHouse URL"; + if (curl_url_set(cu, CURLUPART_SCHEME, use_tls ? "https" : "http", 0) != + CURLUE_OK || + curl_url_set(cu, CURLUPART_HOST, host, 0) != CURLUE_OK || + curl_url_set(cu, CURLUPART_PORT, port_buf, 0) != CURLUE_OK || + curl_url_set(cu, CURLUPART_PATH, "/", 0) != CURLUE_OK) { goto cleanup; } - char* scheme = use_tls ? "https" : "http"; - - if (username && password) { - n = snprintf( - connstring, len, "%s://%s:%s@%s:%d/", scheme, username, password, host, port - ); - curl_free(username); - curl_free(password); - } else if (username) { - n = snprintf(connstring, len, "%s://%s@%s:%d/", scheme, username, host, port); - curl_free(username); - } else { - n = snprintf(connstring, len, "%s://%s:%d/", scheme, host, port); + if (details->username) { + if (curl_url_set(cu, CURLUPART_USER, details->username, CURLU_URLENCODE) != + CURLUE_OK) { + goto cleanup; + } + + if (details->password && + curl_url_set(cu, CURLUPART_PASSWORD, details->password, CURLU_URLENCODE) != + CURLUE_OK) { + goto cleanup; + } } - if (n < 0) { + if (curl_url_get(cu, CURLUPART_URL, &conn->base_url, 0) != CURLUE_OK) { goto cleanup; } - conn->base_url = connstring; - + curl_url_cleanup(cu); return conn; cleanup: - snprintf(curl_error_buffer, CURL_ERROR_SIZE, "OOM"); - curl_error_happened = true; - if (connstring) { - free(connstring); - } - - if (conn) { - if (conn->dbname) { - free(conn->dbname); - } - free(conn); - } + curl_url_cleanup(cu); + free(conn->dbname); + free(conn); return NULL; } @@ -191,11 +150,15 @@ ch_http_connection(ch_connection_details* details) { * ch_http_simple_query — buffer the full TabSeparated response in memory. */ ch_http_response_t* -ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { +ch_http_simple_query( + ch_http_connection_t* conn, + const ch_query* query, + ch_cancel_check cancel +) { HttpStream* stream; ch_http_response_t* resp; - stream = ch_http_stream_begin(conn, query, false); + stream = ch_http_stream_begin(conn, query, false, cancel); if (stream == NULL) { return NULL; } @@ -222,22 +185,22 @@ ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { /* * Fetches and caches the ClickHouse server version via SELECT version(). - * Writes 0 to all out-params when the version cannot be determined. Caches the - * result on the connection, so only the first call issues a query. + * Returns zeros when the version cannot be determined; a failed lookup counts + * as fetched, so it is not retried and its warning is raised once. */ -void -ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* patch) { - *major = *minor = *patch = 0; +ch_server_version +ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel) { + ch_server_version none = { 0, 0, 0 }; + if (conn == NULL) { - return; + return none; } - /* conn is calloc'd (see ch_http_connect), so version.major == 0 reliably - * means the version has not been fetched and cached yet. */ - if (conn->version.major == 0) { + if (!conn->version_fetched) { ch_query query = { .sql = "SELECT version()" }; - ch_http_response_t* resp = ch_http_simple_query(conn, &query); + ch_http_response_t* resp = ch_http_simple_query(conn, &query, cancel); + conn->version_fetched = true; if (resp != NULL) { if (resp->http_status == CH_HTTP_STATUS_OK && resp->data != NULL) { int parsed, v_tweak; @@ -266,7 +229,7 @@ ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* } if (parsed < 2) { /* Version string probably trash; zero out. */ - conn->version.major = 0; + conn->version = none; } } else if (resp->http_status != CH_HTTP_STATUS_OK) { elog( @@ -281,27 +244,14 @@ ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* } } - *major = conn->version.major; - *minor = conn->version.minor; - *patch = conn->version.patch; + return conn->version; } void ch_http_close(ch_http_connection_t* conn) { - free(conn->base_url); - if (conn->dbname) { - free(conn->dbname); - } - curl_easy_cleanup(conn->curl); -} - -char* -ch_http_last_error(void) { - if (curl_error_happened) { - return curl_error_buffer; - } - - return NULL; + curl_free(conn->base_url); + free(conn->dbname); + free(conn); } void diff --git a/src/http_streaming.c b/src/http_streaming.c index dbea67eb..a3b02176 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -54,6 +54,7 @@ struct HttpStream { bool streaming; /* hand out one chunk at a time, else buffer whole body */ bool paused; bool transfer_done; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ char error_buffer[CURL_ERROR_SIZE]; /* Public state readable via C accessors */ @@ -74,6 +75,20 @@ pump(HttpStream* stream); static size_t write_callback(void* contents, size_t size, size_t nmemb, void* userp); +/* CURLOPT_XFERINFOFUNCTION adapter over the caller's cancellation check. */ +static int +xferinfo_callback( + void* clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow +) { + HttpStream* stream = (HttpStream*)clientp; + + return stream->cancel() ? 1 : 0; +} + /* ---------------------------------------------------------------- * setup_curl — configure the CURL easy handle for this query. * Mirrors the setup portion of ch_http_simple_query() in http.c. @@ -122,7 +137,7 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { } if (native) { - int major, minor, patch; + ch_server_version version; /* Keep SQL unchanged so query parameters work. */ curl_url_set( @@ -132,10 +147,10 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { CURLU_APPENDQUERY | CURLU_URLENCODE ); - ch_http_server_version(stream->conn, &major, &minor, &patch); + version = ch_http_server_version(stream->conn, stream->cancel); /* Gate settings by server version, unknown HTTP settings fail queries. */ - if (major > 24 || (major == 24 && minor >= 7)) { + if (chfdw_version_ge(version, 24, 7)) { curl_url_set( cu, CURLUPART_QUERY, @@ -143,7 +158,7 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { CURLU_APPENDQUERY | CURLU_URLENCODE ); } - if (major > 24 || (major == 24 && minor >= 10)) { + if (chfdw_version_ge(version, 24, 10)) { curl_url_set( cu, CURLUPART_QUERY, @@ -187,12 +202,10 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); } - if (ch_http_get_progress_func()) { + if (stream->cancel) { curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt( - stream->curl, CURLOPT_XFERINFOFUNCTION, ch_http_get_progress_func() - ); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream->conn); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_callback); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream); } else { curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); } @@ -374,7 +387,12 @@ ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error * Returns NULL on failure. */ HttpStream* -ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool native) { +ch_http_stream_begin( + ch_http_connection_t* conn, + const ch_query* query, + bool native, + ch_cancel_check cancel +) { HttpStream* stream; uuid_t id; @@ -383,7 +401,8 @@ ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool nat return NULL; } - stream->conn = conn; + stream->conn = conn; + stream->cancel = cancel; /* Native responses decode incrementally; other formats want one buffer. */ stream->streaming = native; @@ -392,9 +411,8 @@ ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool nat uuid_unparse(id, stream->query_id); /* - * Create our own CURL easy handle so that multiple HttpStream instances - * (e.g. concurrent foreign scans in subqueries or joins) do not fight - * over the single handle in conn->curl. + * Each HttpStream owns its easy handle, so concurrent foreign scans in + * subqueries or joins do not fight over one. */ stream->curl = curl_easy_init(); if (!stream->curl) { diff --git a/src/include/engine.h b/src/include/engine.h index 66662c06..59d1548d 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -36,6 +36,12 @@ typedef struct { * when not specified */ } ch_connection_details; +/* + * Polled by a transport while a request is in flight; return true to abort it. + * NULL means the request cannot be cancelled. + */ +typedef bool (*ch_cancel_check)(void); + /* * ch_query an SQL query to execute on ClickHouse. */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 4d3f9b76..b6504240 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -72,7 +72,6 @@ typedef struct ChFdwScanRowContext { typedef void (*disconnect_method)(void* conn); typedef void (*check_conn_method)(const char* password, UserMapping* user); typedef ch_cursor* (*simple_query_method)(void* conn, const ch_query* query); -typedef void (*simple_insert_method)(void* conn, const ch_query* query); typedef Datum* (*cursor_fetch_row_method)(ChFdwScanRowContext* ctx); typedef void* (*prepare_insert_method)( void* conn, diff --git a/src/include/http.h b/src/include/http.h index f152ee5c..b38f6c43 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -4,6 +4,7 @@ #include "postgres.h" #include "engine.h" +#include "server_version.h" #include #define CH_HTTP_QUERY_ID_LEN 37 @@ -27,23 +28,26 @@ typedef struct ch_http_response_t { } ch_http_response_t; void -ch_http_init(int verbose, uint32_t query_id_prefix); -void -ch_http_set_progress_func(curl_xferinfo_callback progressfunc); -curl_xferinfo_callback -ch_http_get_progress_func(void); +ch_http_init(int verbose); long ch_http_get_verbose(void); +/* Returns NULL and sets *error to a static message on failure. */ ch_http_connection_t* -ch_http_connection(ch_connection_details* details); +ch_http_connection(ch_connection_details* details, const char** error); void ch_http_close(ch_http_connection_t* conn); ch_http_response_t* -ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query); -void -ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* patch); -char* -ch_http_last_error(void); +ch_http_simple_query( + ch_http_connection_t* conn, + const ch_query* query, + ch_cancel_check cancel +); +/* + * Fetch and cache the server version, returning {0, 0, 0} when it cannot be + * determined. Only the first call issues a query. + */ +ch_server_version +ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel); void ch_http_response_free(ch_http_response_t* resp); diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index 50f6c506..e53ad0dc 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -13,7 +13,12 @@ typedef struct HttpStream HttpStream; /* lifecycle */ HttpStream* -ch_http_stream_begin(ch_http_connection_t* conn, const ch_query* query, bool native); +ch_http_stream_begin( + ch_http_connection_t* conn, + const ch_query* query, + bool native, + ch_cancel_check cancel +); void ch_http_stream_end(HttpStream* stream); diff --git a/src/include/internal.h b/src/include/internal.h index c8e54bb1..d2208717 100644 --- a/src/include/internal.h +++ b/src/include/internal.h @@ -1,17 +1,17 @@ #ifndef CLICKHOUSE_INTERNAL_H #define CLICKHOUSE_INTERNAL_H -#include "curl/curl.h" +#include + #include "server_version.h" typedef struct ch_http_connection_t { - CURL* curl; char* dbname; - char* base_url; + char* base_url; /* built by curl_url_get, freed with curl_free */ long ssl_version; /* CURLOPT_SSLVERSION min; DEFAULT means unset */ - /* Server version, fetched lazily via SELECT version() then cached; a major - * of 0 means not fetched yet. */ + /* Server version, fetched lazily via SELECT version() then cached. */ ch_server_version version; + bool version_fetched; /* version lookup ran, even if it failed */ } ch_http_connection_t; typedef struct ch_binary_connection_t { diff --git a/src/pglink.c b/src/pglink.c index 09731c45..ba0b63f1 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -21,6 +21,8 @@ #include "fdw.h" #include "http.h" #include "http_streaming.h" +#include "pg-clickhouse-decode.h" +#include "pg-clickhouse-encode.h" #include #include @@ -131,19 +133,10 @@ static libclickhouse_methods binary_methods = { .server_version = binary_server_version, }; -static int -http_progress_callback( - void* clientp, - curl_off_t dltotal, - curl_off_t dlnow, - curl_off_t ultotal, - curl_off_t ulnow -) { - if (ProcDiePending || QueryCancelPending) { - return 1; - } - - return 0; +/* ch_cancel_check for the HTTP transport, polled while a request is in flight. */ +static bool +http_canceled(void) { + return QueryCancelPending || ProcDiePending; } static bool @@ -160,10 +153,11 @@ ch_connection chfdw_http_connect(ch_connection_details* details) { ch_connection res; ch_http_connection_t* conn; + const char* error; if (!initialized) { initialized = true; - ch_http_init(0, (uint32_t)MyProcPid); + ch_http_init(0); } /* @@ -188,14 +182,8 @@ chfdw_http_connect(ch_connection_details* details) { } } - conn = ch_http_connection(details); + conn = ch_http_connection(details, &error); if (conn == NULL) { - char* error = ch_http_last_error(); - - if (error == NULL) { - error = "undefined"; - } - ereport( ERROR, errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), @@ -221,10 +209,7 @@ http_disconnect(void* conn) { static ch_server_version http_server_version(void* conn) { - ch_server_version v = { 0, 0, 0 }; - - ch_http_server_version((ch_http_connection_t*)conn, &v.major, &v.minor, &v.patch); - return v; + return ch_http_server_version((ch_http_connection_t*)conn, http_canceled); } /* @@ -262,8 +247,8 @@ kill_query(void* conn, const char* query_id) { NULL ); - ch_http_set_progress_func(NULL); - resp = ch_http_simple_query(conn, &query); + /* Not cancellable: it is the cleanup for a query already cancelled. */ + resp = ch_http_simple_query(conn, &query, NULL); if (resp != NULL) { ch_http_response_free(resp); } @@ -336,10 +321,8 @@ http_simple_query(void* conn, const ch_query* query) { ch_cursor* cursor; ch_http_response_t* resp; - ch_http_set_progress_func(http_progress_callback); - again: - resp = ch_http_simple_query(conn, query); + resp = ch_http_simple_query(conn, query, http_canceled); if (resp == NULL) { ereport(ERROR, errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory")); } @@ -423,19 +406,20 @@ http_simple_query(void* conn, const ch_query* query) { static void http_simple_insert(void* conn, const ch_query* query) { - ch_http_response_t* resp = ch_http_simple_query(conn, query); + ch_http_response_t* resp = ch_http_simple_query(conn, query, http_canceled); if (resp == NULL) { - char* error = ch_http_last_error(); + ereport(ERROR, errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory")); + } - if (error == NULL) { - error = "undefined"; - } + if (resp->http_status == CH_HTTP_STATUS_CANCELED) { + kill_query(conn, resp->query_id); + ch_http_response_free(resp); ereport( ERROR, - errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: communication error: %s", error) + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted") ); } @@ -465,7 +449,7 @@ http_cursor_free(void* c) { /* pgch_chunk_source cancellation poll, checked between reads. */ static bool native_chunks_cancelled(void* ud pg_attribute_unused()) { - return QueryCancelPending || ProcDiePending; + return http_canceled(); } /* Create shared-decoder cursor over HTTP Native response. */ @@ -479,10 +463,8 @@ http_native_cursor(void* conn, const ch_query* query) { ch_cursor* cursor; pgch_reader* state; - ch_http_set_progress_func(http_progress_callback); - again: - stream = ch_http_stream_begin(conn, query, true); + stream = ch_http_stream_begin(conn, query, true, http_canceled); if (stream == NULL) { ereport( ERROR, From 0d470664b2b14f7e203646ca2a33f1349ab94fc2 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:08:04 +0000 Subject: [PATCH 06/11] don't mix native format knowledge into http transport --- src/http.c | 14 +++- src/http_streaming.c | 129 ++++++++++++----------------------- src/include/http_streaming.h | 27 ++++++-- src/pglink.c | 36 +++++++++- 4 files changed, 112 insertions(+), 94 deletions(-) diff --git a/src/http.c b/src/http.c index 4265123d..504f8102 100644 --- a/src/http.c +++ b/src/http.c @@ -148,6 +148,9 @@ ch_http_connection(ch_connection_details* details, const char** error) { /* * ch_http_simple_query — buffer the full TabSeparated response in memory. + * + * Server default format is TabSeparated, so only its dialect needs pinning: + * ISO timestamps, \N for NULL and LF line ends, as the text parsers expect. */ ch_http_response_t* ch_http_simple_query( @@ -155,10 +158,19 @@ ch_http_simple_query( const ch_query* query, ch_cancel_check cancel ) { + static const ch_setting tsv_overrides[] = { + { "date_time_output_format", "iso" }, + { "format_tsv_null_representation", "\\N" }, + { "output_format_tsv_crlf_end_of_line", "0" }, + }; + const ch_http_request req = { .query = query, + .overrides = tsv_overrides, + .num_overrides = lengthof(tsv_overrides), + .cancel = cancel }; HttpStream* stream; ch_http_response_t* resp; - stream = ch_http_stream_begin(conn, query, false, cancel); + stream = ch_http_stream_begin(conn, &req); if (stream == NULL) { return NULL; } diff --git a/src/http_streaming.c b/src/http_streaming.c index a3b02176..1de5eea8 100644 --- a/src/http_streaming.c +++ b/src/http_streaming.c @@ -67,7 +67,7 @@ struct HttpStream { /* Forward declarations of static helpers */ static void -setup_curl(HttpStream* stream, const ch_query* query, bool native); +setup_curl(HttpStream* stream, const ch_http_request* req); static void capture_transfer_info(HttpStream* stream); static int @@ -89,14 +89,26 @@ xferinfo_callback( return stream->cancel() ? 1 : 0; } +/* True when an override replaces the user setting of this name. */ +static bool +is_overridden(const ch_http_request* req, const char* name) { + for (int i = 0; i < req->num_overrides; i++) { + if (strcmp(req->overrides[i].name, name) == 0) { + return true; + } + } + + return false; +} + /* ---------------------------------------------------------------- * setup_curl — configure the CURL easy handle for this query. - * Mirrors the setup portion of ch_http_simple_query() in http.c. * ---------------------------------------------------------------- */ static void -setup_curl(HttpStream* stream, const ch_query* query, bool native) { - CURLU* cu = curl_url(); +setup_curl(HttpStream* stream, const ch_http_request* req) { + const ch_query* query = req->query; + CURLU* cu = curl_url(); char temp_buf[512]; /* Build URL with query_id and settings */ @@ -105,29 +117,9 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); - /* Settings overridden below win over user settings. */ - static const char* const native_overridden[] = { - "default_format", - "output_format_native_encode_types_in_binary_format", - "output_format_native_write_json_as_string", - NULL, - }; - static const char* const tsv_overridden[] = { - "date_time_output_format", - "format_tsv_null_representation", - "output_format_tsv_crlf_end_of_line", - NULL, - }; - const char* const* overridden = native ? native_overridden : tsv_overridden; - kv_iter iter = new_kv_iter(query->settings); while (kv_iter_next(&iter)) { - const char* const* skip = overridden; - - while (*skip && strcmp(iter.name, *skip) != 0) { - skip++; - } - if (*skip) { + if (is_overridden(req, iter.name)) { continue; } snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); @@ -136,56 +128,19 @@ setup_curl(HttpStream* stream, const ch_query* query, bool native) { ); } - if (native) { - ch_server_version version; - - /* Keep SQL unchanged so query parameters work. */ - curl_url_set( - cu, - CURLUPART_QUERY, - "default_format=Native", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - - version = ch_http_server_version(stream->conn, stream->cancel); - - /* Gate settings by server version, unknown HTTP settings fail queries. */ - if (chfdw_version_ge(version, 24, 7)) { - curl_url_set( - cu, - CURLUPART_QUERY, - "output_format_native_encode_types_in_binary_format=0", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - } - if (chfdw_version_ge(version, 24, 10)) { - curl_url_set( - cu, - CURLUPART_QUERY, - "output_format_native_write_json_as_string=1", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - } - } else { - curl_url_set( - cu, - CURLUPART_QUERY, - "date_time_output_format=iso", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_set( - cu, - CURLUPART_QUERY, - "format_tsv_null_representation=\\N", - CURLU_APPENDQUERY | CURLU_URLENCODE + for (int i = 0; i < req->num_overrides; i++) { + snprintf( + temp_buf, + sizeof(temp_buf), + "%s=%s", + req->overrides[i].name, + req->overrides[i].value ); curl_url_set( - cu, - CURLUPART_QUERY, - "output_format_tsv_crlf_end_of_line=0", - CURLU_APPENDQUERY | CURLU_URLENCODE + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE ); } + curl_url_get(cu, CURLUPART_URL, &stream->url, 0); curl_url_cleanup(cu); @@ -382,17 +337,23 @@ ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error * ---------------------------------------------------------------- */ +/* True for a status whose body the caller reads as an error message. */ +static bool +error_status(long status) { + /* Synthetic statuses carry no server body. */ + if (status == CH_HTTP_STATUS_CANCELED || status == CH_HTTP_STATUS_TRANSPORT_ERROR) { + return false; + } + + return status > 0 && status != CH_HTTP_STATUS_OK; +} + /* * ch_http_stream_begin — allocate and initialize a streaming HTTP query. * Returns NULL on failure. */ HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - bool native, - ch_cancel_check cancel -) { +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req) { HttpStream* stream; uuid_t id; @@ -401,10 +362,9 @@ ch_http_stream_begin( return NULL; } - stream->conn = conn; - stream->cancel = cancel; - /* Native responses decode incrementally; other formats want one buffer. */ - stream->streaming = native; + stream->conn = conn; + stream->cancel = req->cancel; + stream->streaming = req->stream_chunks; /* Generate query ID */ uuid_generate(id); @@ -427,7 +387,7 @@ ch_http_stream_begin( stream->buf_allocated = INITIAL_BUF_SIZE; stream->buf[0] = '\0'; - setup_curl(stream, query, native); + setup_curl(stream, req); /* Create multi handle and kick off the transfer */ stream->multi = curl_multi_init(); @@ -438,10 +398,7 @@ ch_http_stream_begin( pump(stream); /* Error bodies are reported whole, so stop streaming and buffer the rest. */ - if (stream->streaming && stream->http_status > 0 && - stream->http_status != CH_HTTP_STATUS_OK && - stream->http_status != CH_HTTP_STATUS_CANCELED && - stream->http_status != CH_HTTP_STATUS_TRANSPORT_ERROR) { + if (stream->streaming && error_status(stream->http_status)) { stream->streaming = false; pump(stream); } diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h index e53ad0dc..d90a9a7e 100644 --- a/src/include/http_streaming.h +++ b/src/include/http_streaming.h @@ -11,14 +11,29 @@ typedef struct ch_http_connection_t ch_http_connection_t; */ typedef struct HttpStream HttpStream; +/* A ClickHouse setting the caller sends as a URL parameter. */ +typedef struct ch_setting { + const char* name; + const char* value; +} ch_setting; + +/* + * One HTTP request: the SQL to run plus the response policy its caller needs. + * Overrides win over a query setting of the same name, so a caller pins what + * its decoder requires while user settings fill in the rest. + */ +typedef struct ch_http_request { + const ch_query* query; + const ch_setting* overrides; + int num_overrides; + /* Hand out the body one receive chunk at a time, else buffer it whole */ + bool stream_chunks; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ +} ch_http_request; + /* lifecycle */ HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - bool native, - ch_cancel_check cancel -); +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req); void ch_http_stream_end(HttpStream* stream); diff --git a/src/pglink.c b/src/pglink.c index ba0b63f1..5d51233b 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -452,6 +452,34 @@ native_chunks_cancelled(void* ud pg_attribute_unused()) { return http_canceled(); } +/* Room for every setting native_overrides writes. */ +#define NATIVE_OVERRIDES_MAX 3 + +/* + * Settings the shared decoder needs from a Native response: the pair + * PGCH_NATIVE_SETTINGS joins, plus the format itself. Listed one by one + * because each needs its own server version gate; an unknown HTTP setting + * fails the query. + */ +static int +native_overrides(void* conn, ch_setting out[NATIVE_OVERRIDES_MAX]) { + ch_server_version version = + ch_http_server_version((ch_http_connection_t*)conn, http_canceled); + int n = 0; + + /* Format as a setting keeps SQL unchanged, so query parameters work. */ + out[n++] = (ch_setting){ "default_format", "Native" }; + if (chfdw_version_ge(version, 24, 7)) { + out[n++] = + (ch_setting){ "output_format_native_encode_types_in_binary_format", "0" }; + } + if (chfdw_version_ge(version, 24, 10)) { + out[n++] = (ch_setting){ "output_format_native_write_json_as_string", "1" }; + } + + return n; +} + /* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* http_native_cursor(void* conn, const ch_query* query) { @@ -462,9 +490,15 @@ http_native_cursor(void* conn, const ch_query* query) { MemoryContext oldcxt; ch_cursor* cursor; pgch_reader* state; + ch_setting overrides[NATIVE_OVERRIDES_MAX]; + ch_http_request req = { .query = query, + .overrides = overrides, + .num_overrides = native_overrides(conn, overrides), + .stream_chunks = true, + .cancel = http_canceled }; again: - stream = ch_http_stream_begin(conn, query, true, http_canceled); + stream = ch_http_stream_begin(conn, &req); if (stream == NULL) { ereport( ERROR, From be876bc31a1d647ed0ef7d2fde366d807d20cea5 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:15:50 +0000 Subject: [PATCH 07/11] separate raw query logic from rest --- src/fdw.c | 11 ++--- src/include/engine.h | 7 +-- src/include/fdw.h | 15 +++--- src/pglink.c | 113 +++++++++++++++---------------------------- 4 files changed, 50 insertions(+), 96 deletions(-) diff --git a/src/fdw.c b/src/fdw.c index d3eeb826..60d378b5 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -356,7 +356,8 @@ merge_fdw_options( Datum clickhouse_raw_query(PG_FUNCTION_ARGS) { char* connstring = text_to_cstring(PG_GETARG_TEXT_P(1)); - ch_query query = new_raw_query(text_to_cstring(PG_GETARG_TEXT_P(0))); + ch_query query = + new_query(text_to_cstring(PG_GETARG_TEXT_P(0)), 0, NULL, NULL, NULL); ch_connection_details* details = connstring_parse(connstring); ch_connection conn; @@ -378,13 +379,7 @@ clickhouse_raw_query(PG_FUNCTION_ARGS) { } PG_TRY(); - { - ch_cursor* cursor = conn.methods->simple_query(conn.conn, &query); - - res = conn.is_binary ? chfdw_binary_fetch_raw_data(cursor) - : chfdw_http_fetch_raw_data(cursor); - MemoryContextDelete(cursor->memcxt); - } + { res = conn.methods->raw_query(conn.conn, &query); } PG_CATCH(); { conn.methods->disconnect(conn.conn); diff --git a/src/include/engine.h b/src/include/engine.h index 59d1548d..e99256d8 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -56,7 +56,6 @@ typedef struct { const TupleDesc tupdesc; /* The numbers of the attributes in tupdesc that the query selects. */ const List* attr_nums; - const bool raw_result; /* List of settings to pass to ClickHouse upon execution. */ const kv_list* settings; /* Posted verbatim, already prefixed with sql; sql stays set for errors. */ @@ -65,10 +64,8 @@ typedef struct { } ch_query; #define new_query(sql, num, vals, tupdesc, attrs) \ - { sql, num, vals, tupdesc, attrs, false, chfdw_get_session_settings() } -#define new_raw_query(sql) \ - { sql, 0, NULL, NULL, NULL, true, chfdw_get_session_settings() } + { sql, num, vals, tupdesc, attrs, chfdw_get_session_settings() } #define new_body_query(sql, body, len) \ - { sql, 0, NULL, NULL, NULL, false, chfdw_get_session_settings(), body, len } + { sql, 0, NULL, NULL, NULL, chfdw_get_session_settings(), body, len } #endif /* CLICKHOUSE_ENGINE_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index b6504240..0f49e213 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -41,20 +41,20 @@ */ #define CH_ESCAPED_NAMEDATALEN NAMEDATALEN * 2 -/* pglink.c */ +/* pglink.c: an open Native result, whichever driver produced it */ typedef struct ch_cursor ch_cursor; typedef struct ch_cursor { MemoryContext memcxt; /* used for cleanup */ MemoryContextCallback callback; - void* query_response; - void* read_state; + void* query_response; /* driver response the reader decodes */ + void* read_state; /* pgch_reader over query_response */ void* conn; char* query; double request_time; double total_time; size_t columns_count; - /* for Native readers, per returned column: conversion state, target attribute */ + /* per returned column: conversion state, target attribute */ void** conversion_states; int* fill_dest; void (*read_error)(struct ch_cursor*); @@ -72,6 +72,7 @@ typedef struct ChFdwScanRowContext { typedef void (*disconnect_method)(void* conn); typedef void (*check_conn_method)(const char* password, UserMapping* user); typedef ch_cursor* (*simple_query_method)(void* conn, const ch_query* query); +typedef text* (*raw_query_method)(void* conn, const ch_query* query); typedef Datum* (*cursor_fetch_row_method)(ChFdwScanRowContext* ctx); typedef void* (*prepare_insert_method)( void* conn, @@ -90,6 +91,7 @@ typedef ch_server_version (*server_version_method)(void* conn); typedef struct { disconnect_method disconnect; simple_query_method simple_query; + raw_query_method raw_query; cursor_fetch_row_method fetch_row; prepare_insert_method prepare_insert; insert_tuple_method insert_tuple; @@ -104,7 +106,6 @@ typedef struct { typedef struct { libclickhouse_methods* methods; void* conn; - bool is_binary; } ch_connection; ch_connection_details* @@ -120,10 +121,6 @@ chfdw_binary_connect(ch_connection_details* details); */ ch_server_version chfdw_get_server_version(UserMapping* user); -text* -chfdw_http_fetch_raw_data(ch_cursor* cursor); -text* -chfdw_binary_fetch_raw_data(ch_cursor* cursor); List* chfdw_construct_create_tables(ImportForeignSchemaStmt* stmt, ForeignServer* server); char* diff --git a/src/pglink.c b/src/pglink.c index 5d51233b..8a236e47 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -43,12 +43,10 @@ typedef struct { static void http_disconnect(void* conn); -static ch_cursor* -http_simple_query(void* conn, const ch_query* query); +static text* +http_raw_query(void* conn, const ch_query* query); static void http_simple_insert(void* conn, const ch_query* query); -static void -http_cursor_free(void*); static ch_cursor* http_native_cursor(void* conn, const ch_query* query); static void @@ -78,7 +76,8 @@ http_server_version(void* conn); static libclickhouse_methods http_methods = { .disconnect = http_disconnect, - .simple_query = http_simple_query, + .simple_query = http_native_cursor, + .raw_query = http_raw_query, .fetch_row = native_fetch_row, .prepare_insert = http_prepare_insert, .insert_tuple = http_insert_tuple, @@ -91,6 +90,8 @@ static void binary_disconnect(void* conn); static ch_cursor* binary_simple_query(void* conn, const ch_query* query); +static text* +binary_raw_query(void* conn, const ch_query* query); static void binary_cursor_free(void* cursor); static bool @@ -123,6 +124,7 @@ binary_server_version(void* conn); static libclickhouse_methods binary_methods = { .disconnect = binary_disconnect, .simple_query = binary_simple_query, + .raw_query = binary_raw_query, .fetch_row = native_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, @@ -191,9 +193,8 @@ chfdw_http_connect(ch_connection_details* details) { ); } - res.conn = conn; - res.methods = &http_methods; - res.is_binary = false; + res.conn = conn; + res.methods = &http_methods; return res; } @@ -305,21 +306,13 @@ report_http_stream_query_failure( PG_END_TRY(); } -static ch_cursor* -http_simple_query(void* conn, const ch_query* query) { +/* Whole response body as one text value, for clickhouse_raw_query() */ +static text* +http_raw_query(void* conn, const ch_query* query) { int attempts = 0; - if (!query->raw_result) { - return http_native_cursor(conn, query); - } - /* - * volatile: changed after setjmp (PG_TRY) and read after longjmp - * (PG_CATCH); longjmp needn't restore register-cached locals, so a - * non-volatile such local has an indeterminate value per C setjmp rules. - */ - volatile MemoryContext tempcxt = NULL; - MemoryContext oldcxt; - ch_cursor* cursor; ch_http_response_t* resp; + /* volatile: assigned inside PG_TRY, so longjmp may leave it in a register */ + text* volatile result; again: resp = ch_http_simple_query(conn, query, http_canceled); @@ -368,40 +361,14 @@ http_simple_query(void* conn, const ch_query* query) { PG_TRY(); { - /* - * If any palloc below throws, use PG_CATCH to free the Curl response. - */ - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES - ); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - cursor->query = pstrdup(query->sql); - cursor->request_time = resp->pretransfer_time * 1000; - cursor->total_time = resp->total_time * 1000; - - cursor->memcxt = tempcxt; - cursor->callback.func = http_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - MemoryContextSwitchTo(oldcxt); - } - PG_CATCH(); - { - if (resp) { - ch_http_response_free(resp); - } - if (tempcxt) { - MemoryContextDelete(tempcxt); - } - PG_RE_THROW(); + result = + resp->data ? cstring_to_text_with_len(resp->data, resp->datasize) : NULL; } + PG_FINALLY(); + { ch_http_response_free(resp); } PG_END_TRY(); - return cursor; + return result; } static void @@ -441,11 +408,6 @@ http_simple_insert(void* conn, const ch_query* query) { ch_http_response_free(resp); } -inline static void -http_cursor_free(void* c) { - ch_http_response_free(((ch_cursor*)c)->query_response); -} - /* pgch_chunk_source cancellation poll, checked between reads. */ static bool native_chunks_cancelled(void* ud pg_attribute_unused()) { @@ -572,17 +534,6 @@ http_native_cursor(void* conn, const ch_query* query) { return cursor; } -text* -chfdw_http_fetch_raw_data(ch_cursor* cursor) { - ch_http_response_t* resp = cursor->query_response; - - if (resp->data == NULL) { - return NULL; - } - - return cstring_to_text_with_len(resp->data, resp->datasize); -} - /* * Convert a Datum to a ClickHouse literal string. Returns NULL if the value * cannot be converted to a literal. @@ -804,9 +755,8 @@ ch_connection chfdw_binary_connect(ch_connection_details* details) { ch_connection res; - res.conn = ch_binary_connect(details); - res.methods = &binary_methods; - res.is_binary = true; + res.conn = ch_binary_connect(details); + res.methods = &binary_methods; return res; } @@ -919,14 +869,14 @@ append_tsv_escaped(StringInfo buf, const char* s) { } /* - * Drain a binary cursor into tab-separated rows, mirroring the single-text + * Drain a Native cursor into tab-separated rows, mirroring the single-text * result of the http path. Nulls render as \N, other values escape the * control characters CH's TabSeparated format does so they stay unambiguous. * Output formatting otherwise differs from the http driver since values pass * through PG output functions rather than ClickHouse's wire formatting. */ -text* -chfdw_binary_fetch_raw_data(ch_cursor* cursor) { +static text* +render_native_tsv(ch_cursor* cursor) { pgch_reader* state = cursor->read_state; size_t ncols = pgch_reader_columns(state); StringInfoData buf; @@ -972,6 +922,21 @@ chfdw_binary_fetch_raw_data(ch_cursor* cursor) { return cstring_to_text_with_len(buf.data, buf.len); } +static text* +binary_raw_query(void* conn, const ch_query* query) { + ch_cursor* cursor = binary_simple_query(conn, query); + /* volatile: assigned inside PG_TRY, so longjmp may leave it in a register */ + text* volatile result; + + PG_TRY(); + { result = render_native_tsv(cursor); } + PG_FINALLY(); + { MemoryContextDelete(cursor->memcxt); } + PG_END_TRY(); + + return result; +} + /* * Fetch a row from the binary cursor and return its values. * From 9b2a97a035c013fc2028c734201c14a14a886377 Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:31:58 +0000 Subject: [PATCH 08/11] consolidate cursor code into cursor.c --- src/cursor.c | 338 +++++++++++++++++++++++++++++++++ src/fdw.c | 1 + src/include/cursor.h | 70 +++++++ src/include/fdw.h | 18 +- src/pglink.c | 440 +++++-------------------------------------- 5 files changed, 455 insertions(+), 412 deletions(-) create mode 100644 src/cursor.c create mode 100644 src/include/cursor.h diff --git a/src/cursor.c b/src/cursor.c new file mode 100644 index 00000000..37ff0a64 --- /dev/null +++ b/src/cursor.c @@ -0,0 +1,338 @@ +/* + * cursor.c + * + * Shared lifecycle for a cursor over a ClickHouse Native block stream: open + * over a driver's response, fetch rows through the decoder, release both on + * teardown. API lives in src/include/cursor.h; the drivers in src/pglink.c + * only supply the response and how to read it. + */ + +#include "postgres.h" + +#include "catalog/pg_type_d.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/portal.h" + +#include "cursor.h" + +/* Release the reader before the response its blocks came from. */ +static void +cursor_free(void* c) { + ch_cursor* cursor = c; + + pgch_reader_free(&cursor->reader); + cursor->free_response(cursor->response); + cursor->response = NULL; +} + +/* Report decoder error; a driver hook may report cancellation instead. */ +static void +raise_reader_error(ch_cursor* cursor) { + if (cursor->raise_response_error) { + cursor->raise_response_error(cursor); + } + /* Prefer consistent interrupt error message when fetch interrupted */ + CHECK_FOR_INTERRUPTS(); + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", cursor->reader.error), + errdetail_internal("Remote Query: %.64000s", cursor->query) + ); +} + +/* Match the returned columns against the destination the query asked for. */ +static void +configure_columns(ch_cursor* cursor, const ch_query* query) { + pgch_reader* reader = &cursor->reader; + + if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && + (size_t)list_length(query->attr_nums) != cursor->columns_count) { + ereport( + ERROR, + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + (unsigned long)cursor->columns_count, + (unsigned long)list_length(query->attr_nums) + ), + errdetail_internal("Remote Query: %.64000s", query->sql) + ); + } + + /* Preserve JSON text when PostgreSQL destination uses json, not jsonb. */ + if (query->tupdesc && reader->coltypes) { + ListCell* lc; + size_t j = 0; + + foreach (lc, query->attr_nums) { + int i = lfirst_int(lc); + + if (reader->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { + reader->coltypes[j] = JSONOID; + } + j++; + } + } +} + +ch_cursor* +chfdw_cursor_open(void* conn, const ch_query* query, const ch_cursor_source* src) { + /* volatile: assigned inside PG_TRY, read after longjmp in PG_CATCH */ + ch_cursor* volatile cursor = NULL; + volatile MemoryContext cxt = NULL; + volatile bool owns_response = false; + MemoryContext oldcxt = CurrentMemoryContext; + + PG_TRY(); + { + cxt = AllocSetContextCreate( + PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES + ); + MemoryContextSwitchTo(cxt); + + cursor = palloc0(sizeof(ch_cursor)); + cursor->memcxt = cxt; + cursor->conn = conn; + cursor->query = pstrdup(query->sql); + cursor->free_response = src->free_response; + cursor->raise_response_error = src->raise_response_error; + + /* Register before taking the response, so unwinding releases it. */ + cursor->callback.func = cursor_free; + cursor->callback.arg = (void*)cursor; + MemoryContextRegisterResetCallback(cxt, &cursor->callback); + cursor->response = src->response; + owns_response = true; + + /* Blocks decode into cxt, outliving the per-row context. */ + src->init_reader(&cursor->reader, cursor->response); + cursor->columns_count = pgch_reader_columns(&cursor->reader); + + if (cursor->reader.error) { + raise_reader_error(cursor); + } + configure_columns(cursor, query); + + MemoryContextSwitchTo(oldcxt); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + if (!owns_response) { + src->free_response(src->response); + } + if (cxt != NULL) { + MemoryContextDelete(cxt); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return cursor; +} + +/* Conversion state and target attribute per returned column. */ +static void +build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { + pgch_reader* reader = &cursor->reader; + MemoryContext old = MemoryContextSwitchTo(cursor->memcxt); + size_t ncols = pgch_reader_columns(reader); + ListCell* lc; + size_t j = 0; + + cursor->conversion_states = palloc0(ncols * sizeof(void*)); + cursor->fill_dest = palloc0(ncols * sizeof(int)); + foreach (lc, ctx->retrieved_attrs) { + int attnum = lfirst_int(lc); + Form_pg_attribute att = TupleDescAttr(ctx->tupdesc, attnum - 1); + + cursor->fill_dest[j] = attnum - 1; + cursor->conversion_states[j] = + pgch_reader_convert_init(reader, j, att->atttypid, att->atttypmod); + j++; + } + + MemoryContextSwitchTo(old); +} + +/* Apply PostgreSQL conversions to fetched Native row. */ +static Datum* +apply_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + List* attrs = ctx->retrieved_attrs; + TupleDesc tupdesc = ctx->tupdesc; + Datum* values = ctx->values; + bool* nulls = ctx->nulls; + pgch_reader* reader = &cursor->reader; + size_t attcount = list_length(attrs); + + if (attcount == 0) { + if (pgch_reader_columns(reader) == 1 && reader->nulls[0]) { + nulls[0] = true; + return reader->values; + } + ereport( + ERROR, + errcode(ERRCODE_FDW_ERROR), + errmsg( + "pg_clickhouse: unexpected state: attributes " + "count == 0 and haven't got NULL in the response" + ) + ); + } else if (attcount != pgch_reader_columns(reader)) { + ereport( + ERROR, + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + pgch_reader_columns(reader), + attcount + ) + ); + } + + if (tupdesc) { + Assert(values && nulls); + + if (cursor->conversion_states == NULL) { + build_conversion(cursor, ctx); + } + pgch_reader_fill_map( + reader, cursor->conversion_states, cursor->fill_dest, values, nulls + ); + } + + return reader->values; +} + +static void +fetch_row_errcb(void* arg) { + const char* sql = (const char*)arg; + + errdetail_internal("Remote Query: %.64000s", sql); +} + +/* + * Fetch a row from the cursor and return its values. + * + * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values + * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts + * values. + * + * Use ctx->tupdesc and ctx->attinmeta to convert the values to the + * appropriate Datums, and store them and the indication of their NULLness in + * ctx->values and ctx->nulls, respectively, then return ctx->values. + * + * If ctx->tupdesc is not set, treat all values as text and return them as + * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, + * which only cares about text. + */ +Datum* +chfdw_cursor_fetch_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + ErrorContextCallback errcallback; + bool have_data; + Datum* result; + + errcallback.callback = fetch_row_errcb; + errcallback.arg = (void*)cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + have_data = pgch_reader_next(&cursor->reader); + + if (cursor->reader.error) { + error_context_stack = errcallback.previous; + raise_reader_error(cursor); + } + + result = have_data ? apply_row(ctx) : NULL; + + error_context_stack = errcallback.previous; + return result; +} + +/* + * Escape characters that would otherwise corrupt the tab/newline framing or + * collide with the \N null marker. Matches CH's TabSeparated escaping; \0 is + * unreachable since values arrive as cstrings, so it needs no case. + */ +static void +append_tsv_escaped(StringInfo buf, const char* s) { + for (; *s != '\0'; s++) { + switch (*s) { + case '\\': + appendStringInfoString(buf, "\\\\"); + break; + case '\b': + appendStringInfoString(buf, "\\b"); + break; + case '\f': + appendStringInfoString(buf, "\\f"); + break; + case '\n': + appendStringInfoString(buf, "\\n"); + break; + case '\r': + appendStringInfoString(buf, "\\r"); + break; + case '\t': + appendStringInfoString(buf, "\\t"); + break; + default: + appendStringInfoChar(buf, *s); + } + } +} + +text* +chfdw_cursor_render_tsv(ch_cursor* cursor) { + pgch_reader* reader = &cursor->reader; + size_t ncols = pgch_reader_columns(reader); + StringInfoData buf; + + if (ncols == 0) { + return NULL; + } + + initStringInfo(&buf); + + while (pgch_reader_next(reader)) { + for (size_t i = 0; i < ncols; i++) { + if (i > 0) { + appendStringInfoChar(&buf, '\t'); + } + + if (reader->nulls[i]) { + appendStringInfoString(&buf, "\\N"); + } else { + char* val = + pgch_value_to_cstring(reader->coltypes[i], reader->values[i]); + + append_tsv_escaped(&buf, val); + pfree(val); + } + } + appendStringInfoChar(&buf, '\n'); + CHECK_FOR_INTERRUPTS(); + } + + if (reader->error) { + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", reader->error) + ); + } + + if (buf.len == 0) { + pfree(buf.data); + return NULL; + } + + return cstring_to_text_with_len(buf.data, buf.len); +} diff --git a/src/fdw.c b/src/fdw.c index 60d378b5..58906e1e 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -41,6 +41,7 @@ #endif /* extension includes. */ +#include "cursor.h" #include "fdw.h" #include "utils/builtins.h" #include "version.h" diff --git a/src/include/cursor.h b/src/include/cursor.h new file mode 100644 index 00000000..374ef6c7 --- /dev/null +++ b/src/include/cursor.h @@ -0,0 +1,70 @@ +/* + * cursor.h + * + * Cursor for an open ClickHouse Native result + * + * Driver provides response and initializes reader. Cursor validates columns, + * converts rows, and releases resources + */ + +#ifndef CLICKHOUSE_CURSOR_H +#define CLICKHOUSE_CURSOR_H + +#include "postgres.h" + +#include "fdw.h" +#include "pg-clickhouse-decode.h" + +typedef struct ch_cursor { + MemoryContext memcxt; /* Delete this context to close cursor */ + MemoryContextCallback callback; + + pgch_reader reader; + void* response; /* Driver response read by reader */ + void (*free_response)(void* response); + /* + * Report a driver error, such as cancellation, before reporting a decoder + * error. Leave unset if driver needs no special handling + */ + void (*raise_response_error)(ch_cursor* cursor); + + void* conn; + char* query; + double request_time; + size_t columns_count; + /* One conversion state and destination attribute per returned column */ + void** conversion_states; + int* fill_dest; +} ch_cursor; + +/* Describes driver response and how cursor reads it */ +typedef struct ch_cursor_source { + void* response; + /* Initialize reader from blocks or byte chunks */ + void (*init_reader)(pgch_reader* reader, void* response); + /* Accept NULL because error callback may release response first */ + void (*free_response)(void* response); + /* Report driver-specific errors, or NULL if not needed */ + void (*raise_response_error)(ch_cursor* cursor); +} ch_cursor_source; + +/* + * Open cursor and take ownership of src->response. Cursor context releases + * response even if opening fails. Delete cursor->memcxt to close cursor + */ +extern ch_cursor* +chfdw_cursor_open(void* conn, const ch_query* query, const ch_cursor_source* src); + +/* Fetch next row for either driver */ +extern Datum* +chfdw_cursor_fetch_row(ChFdwScanRowContext* ctx); + +/* + * Read remaining rows and return tab-separated text. Use \N for null values. + * Return NULL if there are no rows. Output can differ from HTTP driver because + * PostgreSQL, rather than ClickHouse, formats each value + */ +extern text* +chfdw_cursor_render_tsv(ch_cursor* cursor); + +#endif /* CLICKHOUSE_CURSOR_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 0f49e213..e6603091 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -41,24 +41,8 @@ */ #define CH_ESCAPED_NAMEDATALEN NAMEDATALEN * 2 -/* pglink.c: an open Native result, whichever driver produced it */ +/* cursor.h: an open Native result, whichever driver produced it */ typedef struct ch_cursor ch_cursor; -typedef struct ch_cursor { - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - void* query_response; /* driver response the reader decodes */ - void* read_state; /* pgch_reader over query_response */ - void* conn; - char* query; - double request_time; - double total_time; - size_t columns_count; - /* per returned column: conversion state, target attribute */ - void** conversion_states; - int* fill_dest; - void (*read_error)(struct ch_cursor*); -} ch_cursor; typedef struct ChFdwScanRowContext { TupleDesc tupdesc; /* tuple descriptor for row */ diff --git a/src/pglink.c b/src/pglink.c index 8a236e47..54817f1c 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -18,6 +18,7 @@ #include "utils/uuid.h" #include "binary.h" +#include "cursor.h" #include "fdw.h" #include "http.h" #include "http_streaming.h" @@ -51,20 +52,6 @@ static ch_cursor* http_native_cursor(void* conn, const ch_query* query); static void http_native_read_error(ch_cursor* cursor); -static void -http_native_cursor_free(void*); -static void -native_cursor_state_free(void*); -static void -native_cursor_raise_error(ch_cursor* cursor); -static Datum* -apply_binary_row(ChFdwScanRowContext* ctx); -static Datum* -native_fetch_row(ChFdwScanRowContext* ctx); -static void -binary_fetch_row_errcb(void* arg); -static void -configure_native_cursor(ch_cursor* cursor, const ch_query* query); static void* http_prepare_insert(void*, ResultRelInfo*, List*, const ch_query*, char*); static void @@ -78,11 +65,11 @@ static libclickhouse_methods http_methods = { .disconnect = http_disconnect, .simple_query = http_native_cursor, .raw_query = http_raw_query, - .fetch_row = native_fetch_row, + .fetch_row = chfdw_cursor_fetch_row, .prepare_insert = http_prepare_insert, .insert_tuple = http_insert_tuple, .streaming_query = http_native_cursor, - .streaming_fetch_row = native_fetch_row, + .streaming_fetch_row = chfdw_cursor_fetch_row, .server_version = http_server_version, }; @@ -92,8 +79,6 @@ static ch_cursor* binary_simple_query(void* conn, const ch_query* query); static text* binary_raw_query(void* conn, const ch_query* query); -static void -binary_cursor_free(void* cursor); static bool binary_is_broken(const void* conn); @@ -125,7 +110,7 @@ static libclickhouse_methods binary_methods = { .disconnect = binary_disconnect, .simple_query = binary_simple_query, .raw_query = binary_raw_query, - .fetch_row = native_fetch_row, + .fetch_row = chfdw_cursor_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, .finalize_insert = binary_finalize_insert, @@ -442,16 +427,26 @@ native_overrides(void* conn, ch_setting out[NATIVE_OVERRIDES_MAX]) { return n; } +static void +http_stream_reader_init(pgch_reader* reader, void* response) { + pgch_chunk_source src = { .ud = response, + .next_chunk = ch_http_stream_next_chunk, + .cancelled = native_chunks_cancelled }; + + pgch_reader_init_chunks(reader, &src, NULL); +} + +static void +http_stream_free(void* response) { + ch_http_stream_end(response); +} + /* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* http_native_cursor(void* conn, const ch_query* query) { int attempts = 0; - /* volatile: modified inside PG_TRY, read after longjmp in PG_CATCH */ - volatile MemoryContext tempcxt = NULL; - HttpStream* volatile stream; - MemoryContext oldcxt; + HttpStream* stream; ch_cursor* cursor; - pgch_reader* state; ch_setting overrides[NATIVE_OVERRIDES_MAX]; ch_http_request req = { .query = query, .overrides = overrides, @@ -479,57 +474,13 @@ http_native_cursor(void* conn, const ch_query* query) { report_http_stream_query_failure(conn, query, stream); } - PG_TRY(); - { - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse native cursor", ALLOCSET_DEFAULT_SIZES - ); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query = pstrdup(query->sql); - cursor->request_time = ch_http_stream_request_time(stream); - cursor->total_time = ch_http_stream_total_time(stream); - cursor->read_error = http_native_read_error; - state = palloc0(sizeof(pgch_reader)); - cursor->read_state = state; - - /* Register before taking the stream, so unwinding closes it. */ - cursor->memcxt = tempcxt; - cursor->callback.func = http_native_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - cursor->query_response = stream; - stream = NULL; - - pgch_chunk_source src = { .ud = cursor->query_response, - .next_chunk = ch_http_stream_next_chunk, - .cancelled = native_chunks_cancelled }; - - /* Blocks decode into tempcxt, outliving the per-row context. */ - pgch_reader_init_chunks(state, &src, NULL); - cursor->columns_count = pgch_reader_columns(state); - - MemoryContextSwitchTo(oldcxt); - } - PG_CATCH(); - { - if (stream) { - ch_http_stream_end(stream); - } - if (tempcxt) { - MemoryContextDelete(tempcxt); - } - PG_RE_THROW(); - } - PG_END_TRY(); - - if (state->error) { - native_cursor_raise_error(cursor); - } + ch_cursor_source src = { .response = stream, + .init_reader = http_stream_reader_init, + .free_response = http_stream_free, + .raise_response_error = http_native_read_error }; - configure_native_cursor(cursor, query); + cursor = chfdw_cursor_open(conn, query, &src); + cursor->request_time = ch_http_stream_request_time(stream); return cursor; } @@ -782,12 +733,20 @@ binary_server_version(void* conn) { return v; } +static void +binary_reader_init(pgch_reader* reader, void* response) { + pgch_block_source src = ch_binary_response_block_source(response); + + pgch_reader_init(reader, &src); +} + +static void +binary_response_free(void* response) { + ch_binary_response_free(response); +} + static ch_cursor* binary_simple_query(void* conn, const ch_query* query) { - MemoryContext tempcxt, oldcxt; - ch_cursor* cursor; - pgch_reader* state; - ch_binary_response_t* resp = ch_binary_simple_query(conn, query, &is_canceled); if (!ch_binary_response_success(resp)) { @@ -805,121 +764,11 @@ binary_simple_query(void* conn, const ch_query* query) { ); } - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES - ); - - oldcxt = MemoryContextSwitchTo(tempcxt); - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - state = (pgch_reader*)palloc0(sizeof(pgch_reader)); - cursor->query = pstrdup(query->sql); - cursor->read_state = state; - pgch_block_source src = ch_binary_response_block_source(resp); - pgch_reader_init(cursor->read_state, &src); - cursor->columns_count = pgch_reader_columns(state); - cursor->memcxt = tempcxt; - cursor->callback.func = binary_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - - configure_native_cursor(cursor, query); - - MemoryContextSwitchTo(oldcxt); - - if (state->error) { - native_cursor_raise_error(cursor); - } - - return cursor; -} - -/* - * Escape characters that would otherwise corrupt the tab/newline framing or - * collide with the \N null marker. Matches CH's TabSeparated escaping; \0 is - * unreachable since values arrive as cstrings, so it needs no case. - */ -static void -append_tsv_escaped(StringInfo buf, const char* s) { - for (; *s != '\0'; s++) { - switch (*s) { - case '\\': - appendStringInfoString(buf, "\\\\"); - break; - case '\b': - appendStringInfoString(buf, "\\b"); - break; - case '\f': - appendStringInfoString(buf, "\\f"); - break; - case '\n': - appendStringInfoString(buf, "\\n"); - break; - case '\r': - appendStringInfoString(buf, "\\r"); - break; - case '\t': - appendStringInfoString(buf, "\\t"); - break; - default: - appendStringInfoChar(buf, *s); - } - } -} - -/* - * Drain a Native cursor into tab-separated rows, mirroring the single-text - * result of the http path. Nulls render as \N, other values escape the - * control characters CH's TabSeparated format does so they stay unambiguous. - * Output formatting otherwise differs from the http driver since values pass - * through PG output functions rather than ClickHouse's wire formatting. - */ -static text* -render_native_tsv(ch_cursor* cursor) { - pgch_reader* state = cursor->read_state; - size_t ncols = pgch_reader_columns(state); - StringInfoData buf; - - if (ncols == 0) { - return NULL; - } - - initStringInfo(&buf); - - while (pgch_reader_next(state)) { - for (size_t i = 0; i < ncols; i++) { - if (i > 0) { - appendStringInfoChar(&buf, '\t'); - } - - if (state->nulls[i]) { - appendStringInfoString(&buf, "\\N"); - } else { - char* val = pgch_value_to_cstring(state->coltypes[i], state->values[i]); - - append_tsv_escaped(&buf, val); - pfree(val); - } - } - appendStringInfoChar(&buf, '\n'); - CHECK_FOR_INTERRUPTS(); - } - - if (state->error) { - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error) - ); - } - - if (buf.len == 0) { - pfree(buf.data); - return NULL; - } + ch_cursor_source src = { .response = resp, + .init_reader = binary_reader_init, + .free_response = binary_response_free }; - return cstring_to_text_with_len(buf.data, buf.len); + return chfdw_cursor_open(conn, query, &src); } static text* @@ -929,7 +778,7 @@ binary_raw_query(void* conn, const ch_query* query) { text* volatile result; PG_TRY(); - { result = render_native_tsv(cursor); } + { result = chfdw_cursor_render_tsv(cursor); } PG_FINALLY(); { MemoryContextDelete(cursor->memcxt); } PG_END_TRY(); @@ -937,184 +786,10 @@ binary_raw_query(void* conn, const ch_query* query) { return result; } -/* - * Fetch a row from the binary cursor and return its values. - * - * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values - * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts - * values. - * - * Use ctx->tupdesc and ctx->attinmeta to convert the values to the - * appropriate Datums, and store them and the indication of their NULLness in - * ctx->values and ctx->nulls, respectively, then return ctx->values. - * - * If ctx->tupdesc is not set, treat all values as text and return them as - * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, - * which only cares about text. - */ -static void -binary_fetch_row_errcb(void* arg) { - const char* sql = (const char*)arg; - - errdetail_internal("Remote Query: %.64000s", sql); -} - -/* Conversion state and target attribute per returned column. */ -static void -build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { - pgch_reader* state = cursor->read_state; - MemoryContext old = MemoryContextSwitchTo(cursor->memcxt); - size_t ncols = pgch_reader_columns(state); - ListCell* lc; - size_t j = 0; - - cursor->conversion_states = palloc0(ncols * sizeof(void*)); - cursor->fill_dest = palloc0(ncols * sizeof(int)); - foreach (lc, ctx->retrieved_attrs) { - int attnum = lfirst_int(lc); - Form_pg_attribute att = TupleDescAttr(ctx->tupdesc, attnum - 1); - - cursor->fill_dest[j] = attnum - 1; - cursor->conversion_states[j] = - pgch_reader_convert_init(state, j, att->atttypid, att->atttypmod); - j++; - } - - MemoryContextSwitchTo(old); -} - -static void -configure_native_cursor(ch_cursor* cursor, const ch_query* query) { - pgch_reader* state = cursor->read_state; - - if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && - (size_t)list_length(query->attr_nums) != cursor->columns_count) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal( - "pg_clickhouse: returned %lu columns, expected %lu", - (unsigned long)cursor->columns_count, - (unsigned long)list_length(query->attr_nums) - ), - errdetail_internal("Remote Query: %.64000s", query->sql) - ); - } - - /* Preserve JSON text when PostgreSQL destination uses json, not jsonb. */ - if (query->tupdesc && state->coltypes) { - ListCell* lc; - size_t j = 0; - - foreach (lc, query->attr_nums) { - int i = lfirst_int(lc); - - if (state->coltypes[j] == JSONBOID && - TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { - state->coltypes[j] = JSONOID; - } - j++; - } - } -} - -/* Apply PostgreSQL conversions to fetched Native row. */ -static Datum* -apply_binary_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - List* attrs = ctx->retrieved_attrs; - TupleDesc tupdesc = ctx->tupdesc; - Datum* values = ctx->values; - bool* nulls = ctx->nulls; - pgch_reader* state = cursor->read_state; - size_t attcount = list_length(attrs); - - if (attcount == 0) { - if (pgch_reader_columns(state) == 1 && state->nulls[0]) { - nulls[0] = true; - return state->values; - } - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg( - "pg_clickhouse: unexpected state: attributes " - "count == 0 and haven't got NULL in the response" - ) - ); - } else if (attcount != pgch_reader_columns(state)) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal( - "pg_clickhouse: returned %lu columns, expected %lu", - pgch_reader_columns(state), - attcount - ) - ); - } - - if (tupdesc) { - Assert(values && nulls); - - if (cursor->conversion_states == NULL) { - build_conversion(cursor, ctx); - } - pgch_reader_fill_map( - state, cursor->conversion_states, cursor->fill_dest, values, nulls - ); - } - - return state->values; -} - -/* Raise decoder error; read_error hook may convert to cancellation report. */ -static void -native_cursor_raise_error(ch_cursor* cursor) { - pgch_reader* state = cursor->read_state; - - if (cursor->read_error) { - cursor->read_error(cursor); - } - /* Prefer consistent interrupt error message when fetch interrupted */ - CHECK_FOR_INTERRUPTS(); - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", cursor->query) - ); -} - -static Datum* -native_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - pgch_reader* state = cursor->read_state; - ErrorContextCallback errcallback; - bool have_data; - Datum* result; - - errcallback.callback = binary_fetch_row_errcb; - errcallback.arg = (void*)cursor->query; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - have_data = pgch_reader_next(state); - - if (state->error) { - error_context_stack = errcallback.previous; - native_cursor_raise_error(cursor); - } - - result = have_data ? apply_binary_row(ctx) : NULL; - - error_context_stack = errcallback.previous; - return result; -} - +/* Report a truncated response as cancellation when that is what caused it. */ static void http_native_read_error(ch_cursor* cursor) { - HttpStream* stream = cursor->query_response; + HttpStream* stream = cursor->response; if (stream == NULL) { return; @@ -1127,7 +802,7 @@ http_native_read_error(ch_cursor* cursor) { memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); /* Drop the transfer before asking the server to kill the query. */ ch_http_stream_end(stream); - cursor->query_response = NULL; + cursor->response = NULL; kill_query(cursor->conn, qid); ereport( ERROR, @@ -1137,31 +812,6 @@ http_native_read_error(ch_cursor* cursor) { } } -static void -http_native_cursor_free(void* c) { - ch_cursor* cursor = c; - - native_cursor_state_free(cursor); - ch_http_stream_end(cursor->query_response); - cursor->query_response = NULL; -} - -static void -binary_cursor_free(void* c) { - ch_cursor* cursor = c; - - native_cursor_state_free(cursor); - ch_binary_response_free(cursor->query_response); -} - -/* Conversion states live in the context this callback fires for. */ -static void -native_cursor_state_free(void* c) { - ch_cursor* cursor = c; - - pgch_reader_free(cursor->read_state); -} - static void* binary_prepare_insert( void* conn, From 54ae51006db397bda0b47453c5fd4c5082f27b8e Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:42:11 +0000 Subject: [PATCH 09/11] consolidate http.c & http_streaming.c --- Makefile | 2 +- src/http.c | 558 +++++++++++++++++++++++++++++++++-- src/http_streaming.c | 520 -------------------------------- src/include/fdw.h | 4 - src/include/http.h | 69 ++++- src/include/http_streaming.h | 76 ----- src/pglink.c | 31 +- test/expected/http.out | 2 + test/expected/http_1.out | 2 + test/expected/http_2.out | 2 + test/expected/http_3.out | 2 + test/expected/http_4.out | 2 + test/expected/http_5.out | 2 + 13 files changed, 616 insertions(+), 656 deletions(-) delete mode 100644 src/http_streaming.c delete mode 100644 src/include/http_streaming.h diff --git a/Makefile b/Makefile index 55328fed..1be86896 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ CH_C_DIR = $(PGCH_DIR)/clickhouse-c PG_CPPFLAGS = -I./src/include -isystem $(CH_C_DIR) -isystem $(PGCH_DIR) -isystem $(shell $(PG_CONFIG) --includedir-server) -DPGCH_MSG_PREFIX='"pg_clickhouse: "' # Link OpenSSL (for TLS in the binary driver), curl (for the HTTP driver), -# libuuid (for http_streaming.c's query-id generator), and lz4 / zstd +# libuuid (for http.c's query-id generator), and lz4 / zstd # (for the binary driver's compressed-frame codecs). PG_LDFLAGS = -lssl -lcrypto -llz4 -lzstd $(shell $(CURL_CONFIG) --libs) diff --git a/src/http.c b/src/http.c index 504f8102..dd64e1a5 100644 --- a/src/http.c +++ b/src/http.c @@ -1,12 +1,36 @@ +/*------------------------------------------------------------------------- + * + * http.c + * HTTP transport for pg_clickhouse. + * + * Owns the connection, request setup, chunk delivery, status, cancellation + * and cleanup. Uses curl_multi + curl_easy_pause to hand ClickHouse + * responses to the caller one receive chunk at a time, keeping memory + * bounded. + * + * Copyright (c) 2025-2026, ClickHouse, Inc. + * + *------------------------------------------------------------------------- + */ #include -#include #include #include #include -#include -#include -#include +#include + +#include "postgres.h" + +#include "http.h" +#include "internal.h" +#include "kv_list.h" + +#ifndef CURL_WRITEFUNC_ERROR +#define CURL_WRITEFUNC_ERROR 0xFFFFFFFF +#endif + +#define DATABASE_HEADER "X-ClickHouse-Database" +#define INITIAL_BUF_SIZE (64 * 1024) static long curl_verbose = 0; static bool curl_initialized = false; @@ -21,10 +45,10 @@ ch_http_init(int verbose) { } } -long -ch_http_get_verbose(void) { - return curl_verbose; -} +/* ---------------------------------------------------------------- + * Connection + * ---------------------------------------------------------------- + */ #define CLICKHOUSE_PORT 8123 #define CLICKHOUSE_TLS_PORT 8443 @@ -146,6 +170,480 @@ ch_http_connection(ch_connection_details* details, const char** error) { return NULL; } +void +ch_http_close(ch_http_connection_t* conn) { + curl_free(conn->base_url); + free(conn->dbname); + free(conn); +} + +/* ---------------------------------------------------------------- + * HttpStream — opaque struct. + * ---------------------------------------------------------------- + */ +struct HttpStream { + /* Connection (borrowed, not owned) */ + ch_http_connection_t* conn; + + /* Owned CURL resources */ + CURL* curl; + CURLM* multi; + struct curl_slist* headers; + curl_mime* form; + char* url; /* allocated by curl_url_get, freed with + * curl_free */ + + /* Stream buffer */ + char* buf; + size_t buf_allocated; + size_t write_pos; + bool streaming; /* hand out one chunk at a time, else buffer whole body */ + bool paused; + bool transfer_done; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ + char error_buffer[CURL_ERROR_SIZE]; + + /* Public state readable via C accessors */ + long http_status; + char query_id[CH_HTTP_QUERY_ID_LEN]; + double pretransfer_time; + char* error_msg; /* strdup'd, freed with free() */ +}; + +/* CURLOPT_XFERINFOFUNCTION adapter over the caller's cancellation check. */ +static int +xferinfo_callback( + void* clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow +) { + HttpStream* stream = (HttpStream*)clientp; + + return stream->cancel() ? 1 : 0; +} + +/* True when an override replaces the user setting of this name. */ +static bool +is_overridden(const ch_http_request* req, const char* name) { + for (int i = 0; i < req->num_overrides; i++) { + if (strcmp(req->overrides[i].name, name) == 0) { + return true; + } + } + + return false; +} + +/* ---------------------------------------------------------------- + * write_callback — CURL write callback. Appends data to the stream + * buffer and, when streaming, pauses receipt so the caller drains it. + * ---------------------------------------------------------------- + */ +static size_t +write_callback(void* contents, size_t size, size_t nmemb, void* userp) { + size_t realsize = size * nmemb; + HttpStream* self = (HttpStream*)userp; + size_t needed = self->write_pos + realsize + 1; + + /* Grow buffer if needed */ + if (needed > self->buf_allocated) { + size_t newsize = self->buf_allocated * 2; + char* newbuf; + + if (newsize < needed) { + newsize = needed; + } + + newbuf = (char*)realloc(self->buf, newsize); + if (!newbuf) { + return CURL_WRITEFUNC_ERROR; + } + + self->buf = newbuf; + self->buf_allocated = newsize; + } + + memcpy(self->buf + self->write_pos, contents, realsize); + self->write_pos += realsize; + self->buf[self->write_pos] = '\0'; + + if (self->streaming) { + self->paused = true; + curl_easy_pause(self->curl, CURLPAUSE_RECV); + } + + return realsize; +} + +/* ---------------------------------------------------------------- + * setup_curl — configure the CURL easy handle for this query. + * ---------------------------------------------------------------- + */ +static void +setup_curl(HttpStream* stream, const ch_http_request* req) { + const ch_query* query = req->query; + CURLU* cu = curl_url(); + char temp_buf[512]; + + /* Build URL with query_id and settings */ + curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); + + snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); + curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); + + kv_iter iter = new_kv_iter(query->settings); + while (kv_iter_next(&iter)) { + if (is_overridden(req, iter.name)) { + continue; + } + snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); + curl_url_set( + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } + + for (int i = 0; i < req->num_overrides; i++) { + snprintf( + temp_buf, + sizeof(temp_buf), + "%s=%s", + req->overrides[i].name, + req->overrides[i].value + ); + curl_url_set( + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } + + curl_url_get(cu, CURLUPART_URL, &stream->url, 0); + curl_url_cleanup(cu); + + /* Configure CURL easy handle */ + curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); + curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); + curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); + curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); + curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, curl_verbose); + + if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) { + curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); + } + + if (stream->cancel) { + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_callback); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream); + } else { + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); + } + + if (stream->conn->dbname) { + snprintf( + temp_buf, sizeof(temp_buf), "%s: %s", DATABASE_HEADER, stream->conn->dbname + ); + stream->headers = curl_slist_append(NULL, temp_buf); + curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); + } + + /* POST body or MIME form */ + if (query->body != NULL) { + curl_easy_setopt( + stream->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)query->body_len + ); + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->body); + } else if (query->num_params == 0) { + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); + } else { + curl_mimepart* part; + + stream->form = curl_mime_init(stream->curl); + part = curl_mime_addpart(stream->form); + curl_mime_name(part, "query"); + curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); + + for (int i = 0; i < query->num_params; i++) { + part = curl_mime_addpart(stream->form); + snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); + curl_mime_name(part, temp_buf); + curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); + } + curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); + } +} + +static int +pump(HttpStream* stream) { + int running_handles; + CURLMcode mc; + CURLMsg* msg; + int msgs_left; + + if (stream->paused) { + stream->paused = false; + curl_easy_pause(stream->curl, CURLPAUSE_CONT); + } + + for (;;) { + mc = curl_multi_perform(stream->multi, &running_handles); + if (mc != CURLM_OK) { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup(curl_multi_strerror(mc)); + return -1; + } + + if (running_handles == 0) { + stream->transfer_done = true; + } + + /* Buffered mode waits for complete response. */ + if (stream->paused || stream->transfer_done) { + break; + } + + curl_multi_wait(stream->multi, NULL, 0, 100, NULL); + } + + curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, &stream->http_status); + curl_easy_getinfo( + stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time + ); + + while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { + if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { + stream->http_status = CH_HTTP_STATUS_CANCELED; + } else { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup( + stream->error_buffer[0] != '\0' + ? stream->error_buffer + : curl_easy_strerror(msg->data.result) + ); + } + return -1; + } + } + + return 0; +} + +/* Blocking chunk reader; the decoder tracks its position within the chunk. */ +bool +ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error) { + HttpStream* stream = (HttpStream*)ud; + + *data = NULL; + *len = 0; + + /* + * ch_http_stream_begin leaves the first chunk buffered, so pump only when + * the buffer is empty + */ + while (stream->write_pos == 0) { + if (stream->transfer_done) { + return true; /* clean EOF */ + } + if (pump(stream) < 0) { + *error = stream->error_msg; + return false; + } + } + + *data = stream->buf; + *len = stream->write_pos; + /* Bytes stay readable until the next pump refills from offset 0 */ + stream->write_pos = 0; + return true; +} + +/* ---------------------------------------------------------------- + * Public API — lifecycle + * ---------------------------------------------------------------- + */ + +/* True for a status whose body the caller reads as an error message. */ +static bool +error_status(long status) { + /* Synthetic statuses carry no server body. */ + if (status == CH_HTTP_STATUS_CANCELED || status == CH_HTTP_STATUS_TRANSPORT_ERROR) { + return false; + } + + return status > 0 && status != CH_HTTP_STATUS_OK; +} + +/* + * ch_http_stream_end — clean up all owned resources. + */ +void +ch_http_stream_end(HttpStream* stream) { + if (!stream) { + return; + } + + if (stream->multi) { + if (stream->curl) { + curl_multi_remove_handle(stream->multi, stream->curl); + } + curl_multi_cleanup(stream->multi); + } + + if (stream->curl) { + curl_easy_cleanup(stream->curl); + } + + if (stream->headers) { + curl_slist_free_all(stream->headers); + } + if (stream->form) { + curl_mime_free(stream->form); + } + if (stream->url) { + curl_free(stream->url); + } + if (stream->buf) { + free(stream->buf); + } + if (stream->error_msg) { + free(stream->error_msg); + } + + free(stream); +} + +/* + * ch_http_stream_begin — allocate and initialize a streaming HTTP query. + * Returns NULL on failure. + */ +HttpStream* +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req) { + HttpStream* stream; + uuid_t id; + + stream = calloc(1, sizeof(HttpStream)); + if (!stream) { + return NULL; + } + + stream->conn = conn; + stream->cancel = req->cancel; + stream->streaming = req->stream_chunks; + + /* Generate query ID */ + uuid_generate(id); + uuid_unparse(id, stream->query_id); + + /* + * Each HttpStream owns its easy handle, so concurrent foreign scans in + * subqueries or joins do not fight over one. + */ + stream->curl = curl_easy_init(); + if (!stream->curl) { + goto fail; + } + + /* Allocate stream buffer */ + stream->buf = (char*)malloc(INITIAL_BUF_SIZE); + if (!stream->buf) { + goto fail; + } + stream->buf_allocated = INITIAL_BUF_SIZE; + stream->buf[0] = '\0'; + + setup_curl(stream, req); + + /* Create multi handle and kick off the transfer */ + stream->multi = curl_multi_init(); + if (!stream->multi) { + goto fail; + } + curl_multi_add_handle(stream->multi, stream->curl); + + pump(stream); + /* Error bodies are reported whole, so stop streaming and buffer the rest. */ + if (stream->streaming && error_status(stream->http_status)) { + stream->streaming = false; + pump(stream); + } + + return stream; + +fail: + ch_http_stream_end(stream); + return NULL; +} + +/* ---------------------------------------------------------------- + * Public API — accessors + * ---------------------------------------------------------------- + */ +char* +ch_http_stream_buffer(HttpStream* stream) { + return stream->buf; +} + +size_t +ch_http_stream_available(HttpStream* stream) { + return stream->write_pos; +} + +long +ch_http_stream_status(HttpStream* stream) { + return stream->http_status; +} + +const char* +ch_http_stream_query_id(HttpStream* stream) { + return stream->query_id; +} + +const char* +ch_http_stream_error(HttpStream* stream) { + return stream->error_msg; +} + +double +ch_http_stream_request_time(HttpStream* stream) { + return stream->pretransfer_time * 1000; +} + +/* + * take_body — transfer ownership of the response body. + * + * On return, *out_data is a malloc()'d buffer the caller must free(). When + * status is CH_HTTP_STATUS_TRANSPORT_ERROR the body is the strdup'd libcurl + * error message; otherwise it is the accumulated response bytes, NUL + * terminated. *out_size is set to the length in bytes, excluding the NUL. + * Sets *out_data to NULL and *out_size to 0 when there is nothing to hand + * off. Safe to call at most once per stream; the stream itself should still + * be released with ch_http_stream_end(). + */ +static void +take_body(HttpStream* stream, char** out_data, size_t* out_size) { + if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { + *out_data = stream->error_msg; + *out_size = strlen(stream->error_msg); + stream->error_msg = NULL; + return; + } + + if (stream->write_pos == 0 || !stream->buf) { + *out_data = NULL; + *out_size = 0; + return; + } + + *out_data = stream->buf; + *out_size = stream->write_pos; + stream->buf = NULL; +} + /* * ch_http_simple_query — buffer the full TabSeparated response in memory. * @@ -181,11 +679,9 @@ ch_http_simple_query( return NULL; } - resp->http_status = ch_http_stream_status(stream); - resp->pretransfer_time = ch_http_stream_request_time(stream) / 1000.0; - resp->total_time = ch_http_stream_total_time(stream) / 1000.0; - memcpy(resp->query_id, ch_http_stream_query_id(stream), CH_HTTP_QUERY_ID_LEN); - ch_http_stream_take_body(stream, &resp->data, &resp->datasize); + resp->http_status = stream->http_status; + memcpy(resp->query_id, stream->query_id, CH_HTTP_QUERY_ID_LEN); + take_body(stream, &resp->data, &resp->datasize); if (curl_verbose && resp->http_status != CH_HTTP_STATUS_OK && resp->data) { fprintf(stderr, "%s", resp->data); @@ -195,6 +691,30 @@ ch_http_simple_query( return resp; } +/* + * Return text before version mentioning + */ +char* +ch_http_format_error(char* errstring) { + size_t n = strlen(errstring); + + for (size_t i = 0; i < n; i++) { + if (strncmp(errstring + i, "version", 7) == 0) { + return pnstrdup(errstring, i - 2); + } + } + + /* + * For some reason ClickHouse 25.12 added a newline to an auth failure + * error. Strip it out. + */ + if (n > 0 && errstring[n - 1] == '\n') { + errstring[--n] = '\0'; + } + + return errstring; +} + /* * Fetches and caches the ClickHouse server version via SELECT version(). * Returns zeros when the version cannot be determined; a failed lookup counts @@ -246,10 +766,9 @@ ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel) { } else if (resp->http_status != CH_HTTP_STATUS_OK) { elog( WARNING, - "pg_clickhouse: SELECT version() failed (HTTP status %d): %.*s", + "pg_clickhouse: SELECT version() failed (HTTP status %d): %s", (int)resp->http_status, - resp->data ? (int)resp->datasize : 0, - resp->data ? resp->data : "" + resp->data ? ch_http_format_error(resp->data) : "" ); } ch_http_response_free(resp); @@ -259,13 +778,6 @@ ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel) { return conn->version; } -void -ch_http_close(ch_http_connection_t* conn) { - curl_free(conn->base_url); - free(conn->dbname); - free(conn); -} - void ch_http_response_free(ch_http_response_t* resp) { if (resp->data) { diff --git a/src/http_streaming.c b/src/http_streaming.c deleted file mode 100644 index 1de5eea8..00000000 --- a/src/http_streaming.c +++ /dev/null @@ -1,520 +0,0 @@ -/*------------------------------------------------------------------------- - * - * http_streaming.c - * Streaming HTTP query driver for pg_clickhouse. - * - * Uses curl_multi + curl_easy_pause to hand ClickHouse HTTP responses to - * the caller one receive chunk at a time, keeping memory bounded. - * - * Copyright (c) 2025-2026, ClickHouse, Inc. - * - *------------------------------------------------------------------------- - */ -#include -#include -#include -#include - -#include - -#include "postgres.h" - -#include "http.h" -#include "http_streaming.h" -#include "internal.h" -#include "kv_list.h" - -#ifndef CURL_WRITEFUNC_ERROR -#define CURL_WRITEFUNC_ERROR 0xFFFFFFFF -#endif - -#define DATABASE_HEADER "X-ClickHouse-Database" -#define INITIAL_BUF_SIZE (64 * 1024) - -/* ---------------------------------------------------------------- - * HttpStream — opaque struct. - * ---------------------------------------------------------------- - */ -struct HttpStream { - /* Connection (borrowed, not owned) */ - ch_http_connection_t* conn; - - /* Owned CURL resources */ - CURL* curl; - CURLM* multi; - struct curl_slist* headers; - curl_mime* form; - char* url; /* allocated by curl_url_get, freed with - * curl_free */ - - /* Stream buffer */ - char* buf; - size_t buf_allocated; - size_t write_pos; - bool streaming; /* hand out one chunk at a time, else buffer whole body */ - bool paused; - bool transfer_done; - ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ - char error_buffer[CURL_ERROR_SIZE]; - - /* Public state readable via C accessors */ - long http_status; - char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; - char* error_msg; /* strdup'd, freed with free() */ -}; - -/* Forward declarations of static helpers */ -static void -setup_curl(HttpStream* stream, const ch_http_request* req); -static void -capture_transfer_info(HttpStream* stream); -static int -pump(HttpStream* stream); -static size_t -write_callback(void* contents, size_t size, size_t nmemb, void* userp); - -/* CURLOPT_XFERINFOFUNCTION adapter over the caller's cancellation check. */ -static int -xferinfo_callback( - void* clientp, - curl_off_t dltotal, - curl_off_t dlnow, - curl_off_t ultotal, - curl_off_t ulnow -) { - HttpStream* stream = (HttpStream*)clientp; - - return stream->cancel() ? 1 : 0; -} - -/* True when an override replaces the user setting of this name. */ -static bool -is_overridden(const ch_http_request* req, const char* name) { - for (int i = 0; i < req->num_overrides; i++) { - if (strcmp(req->overrides[i].name, name) == 0) { - return true; - } - } - - return false; -} - -/* ---------------------------------------------------------------- - * setup_curl — configure the CURL easy handle for this query. - * ---------------------------------------------------------------- - */ -static void -setup_curl(HttpStream* stream, const ch_http_request* req) { - const ch_query* query = req->query; - CURLU* cu = curl_url(); - char temp_buf[512]; - - /* Build URL with query_id and settings */ - curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); - - snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); - curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); - - kv_iter iter = new_kv_iter(query->settings); - while (kv_iter_next(&iter)) { - if (is_overridden(req, iter.name)) { - continue; - } - snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); - curl_url_set( - cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE - ); - } - - for (int i = 0; i < req->num_overrides; i++) { - snprintf( - temp_buf, - sizeof(temp_buf), - "%s=%s", - req->overrides[i].name, - req->overrides[i].value - ); - curl_url_set( - cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE - ); - } - - curl_url_get(cu, CURLUPART_URL, &stream->url, 0); - curl_url_cleanup(cu); - - /* Configure CURL easy handle */ - curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); - curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); - curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); - curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); - curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, ch_http_get_verbose()); - - if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) { - curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); - } - - if (stream->cancel) { - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_callback); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream); - } else { - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); - } - - if (stream->conn->dbname) { - snprintf( - temp_buf, sizeof(temp_buf), "%s: %s", DATABASE_HEADER, stream->conn->dbname - ); - stream->headers = curl_slist_append(NULL, temp_buf); - curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); - } - - /* POST body or MIME form */ - if (query->body != NULL) { - curl_easy_setopt( - stream->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)query->body_len - ); - curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->body); - } else if (query->num_params == 0) { - curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); - } else { - curl_mimepart* part; - - stream->form = curl_mime_init(stream->curl); - part = curl_mime_addpart(stream->form); - curl_mime_name(part, "query"); - curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); - - for (int i = 0; i < query->num_params; i++) { - part = curl_mime_addpart(stream->form); - snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); - curl_mime_name(part, temp_buf); - curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); - } - curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); - } -} - -/* ---------------------------------------------------------------- - * write_callback — CURL write callback. Appends data to the stream - * buffer and, when streaming, pauses receipt so the caller drains it. - * ---------------------------------------------------------------- - */ -static size_t -write_callback(void* contents, size_t size, size_t nmemb, void* userp) { - size_t realsize = size * nmemb; - HttpStream* self = (HttpStream*)userp; - size_t needed = self->write_pos + realsize + 1; - - /* Grow buffer if needed */ - if (needed > self->buf_allocated) { - size_t newsize = self->buf_allocated * 2; - char* newbuf; - - if (newsize < needed) { - newsize = needed; - } - - newbuf = (char*)realloc(self->buf, newsize); - if (!newbuf) { - return CURL_WRITEFUNC_ERROR; - } - - self->buf = newbuf; - self->buf_allocated = newsize; - } - - memcpy(self->buf + self->write_pos, contents, realsize); - self->write_pos += realsize; - self->buf[self->write_pos] = '\0'; - - if (self->streaming) { - self->paused = true; - curl_easy_pause(self->curl, CURLPAUSE_RECV); - } - - return realsize; -} - -/* ---------------------------------------------------------------- - * capture_transfer_info — grab HTTP status and timing from CURL. - * ---------------------------------------------------------------- - */ -static void -capture_transfer_info(HttpStream* stream) { - curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, &stream->http_status); - curl_easy_getinfo( - stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time - ); - curl_easy_getinfo(stream->curl, CURLINFO_TOTAL_TIME, &stream->total_time); -} - -static int -pump(HttpStream* stream) { - int running_handles; - CURLMcode mc; - CURLMsg* msg; - int msgs_left; - - if (stream->paused) { - stream->paused = false; - curl_easy_pause(stream->curl, CURLPAUSE_CONT); - } - - for (;;) { - mc = curl_multi_perform(stream->multi, &running_handles); - if (mc != CURLM_OK) { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = strdup(curl_multi_strerror(mc)); - return -1; - } - - if (running_handles == 0) { - stream->transfer_done = true; - } - - /* Buffered mode waits for complete response. */ - if (stream->paused || stream->transfer_done) { - break; - } - - curl_multi_wait(stream->multi, NULL, 0, 100, NULL); - } - - capture_transfer_info(stream); - while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { - if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { - if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { - stream->http_status = CH_HTTP_STATUS_CANCELED; - } else { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = strdup( - stream->error_buffer[0] != '\0' - ? stream->error_buffer - : curl_easy_strerror(msg->data.result) - ); - } - return -1; - } - } - - return 0; -} - -/* Blocking chunk reader; the decoder tracks its position within the chunk. */ -bool -ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error) { - HttpStream* stream = (HttpStream*)ud; - - *data = NULL; - *len = 0; - - /* Caller is done with the previous chunk, so refill from offset 0. */ - stream->write_pos = 0; - while (stream->write_pos == 0) { - if (stream->transfer_done) { - return true; /* clean EOF */ - } - if (pump(stream) < 0) { - *error = stream->error_msg; - return false; - } - } - - *data = stream->buf; - *len = stream->write_pos; - return true; -} - -/* ---------------------------------------------------------------- - * Public API — lifecycle - * ---------------------------------------------------------------- - */ - -/* True for a status whose body the caller reads as an error message. */ -static bool -error_status(long status) { - /* Synthetic statuses carry no server body. */ - if (status == CH_HTTP_STATUS_CANCELED || status == CH_HTTP_STATUS_TRANSPORT_ERROR) { - return false; - } - - return status > 0 && status != CH_HTTP_STATUS_OK; -} - -/* - * ch_http_stream_begin — allocate and initialize a streaming HTTP query. - * Returns NULL on failure. - */ -HttpStream* -ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req) { - HttpStream* stream; - uuid_t id; - - stream = calloc(1, sizeof(HttpStream)); - if (!stream) { - return NULL; - } - - stream->conn = conn; - stream->cancel = req->cancel; - stream->streaming = req->stream_chunks; - - /* Generate query ID */ - uuid_generate(id); - uuid_unparse(id, stream->query_id); - - /* - * Each HttpStream owns its easy handle, so concurrent foreign scans in - * subqueries or joins do not fight over one. - */ - stream->curl = curl_easy_init(); - if (!stream->curl) { - goto fail; - } - - /* Allocate stream buffer */ - stream->buf = (char*)malloc(INITIAL_BUF_SIZE); - if (!stream->buf) { - goto fail; - } - stream->buf_allocated = INITIAL_BUF_SIZE; - stream->buf[0] = '\0'; - - setup_curl(stream, req); - - /* Create multi handle and kick off the transfer */ - stream->multi = curl_multi_init(); - if (!stream->multi) { - goto fail; - } - curl_multi_add_handle(stream->multi, stream->curl); - - pump(stream); - /* Error bodies are reported whole, so stop streaming and buffer the rest. */ - if (stream->streaming && error_status(stream->http_status)) { - stream->streaming = false; - pump(stream); - } - - return stream; - -fail: - ch_http_stream_end(stream); - return NULL; -} - -/* - * ch_http_stream_end — clean up all owned resources. - */ -void -ch_http_stream_end(HttpStream* stream) { - if (!stream) { - return; - } - - if (stream->multi) { - if (stream->curl) { - curl_multi_remove_handle(stream->multi, stream->curl); - } - curl_multi_cleanup(stream->multi); - } - - if (stream->curl) { - curl_easy_cleanup(stream->curl); - } - - if (stream->headers) { - curl_slist_free_all(stream->headers); - } - if (stream->form) { - curl_mime_free(stream->form); - } - if (stream->url) { - curl_free(stream->url); - } - if (stream->buf) { - free(stream->buf); - } - if (stream->error_msg) { - free(stream->error_msg); - } - - free(stream); -} - -/* ---------------------------------------------------------------- - * Public API — accessors - * ---------------------------------------------------------------- - */ -char* -ch_http_stream_buffer(HttpStream* stream) { - return stream->buf; -} - -size_t -ch_http_stream_available(HttpStream* stream) { - return stream->write_pos; -} - -long -ch_http_stream_status(HttpStream* stream) { - return stream->http_status; -} - -const char* -ch_http_stream_query_id(HttpStream* stream) { - return stream->query_id; -} - -const char* -ch_http_stream_error(HttpStream* stream) { - return stream->error_msg; -} - -double -ch_http_stream_request_time(HttpStream* stream) { - return stream->pretransfer_time * 1000; -} - -double -ch_http_stream_total_time(HttpStream* stream) { - return stream->total_time * 1000; -} - -/* - * ch_http_stream_take_body — transfer ownership of the response body. - * - * On return, *out_data is a malloc()'d buffer the caller must free(). When - * status is CH_HTTP_STATUS_TRANSPORT_ERROR the body is the strdup'd libcurl - * error message; otherwise it is the accumulated response bytes, NUL - * terminated. *out_size is set to the length in bytes, excluding the NUL. - * Sets *out_data to NULL and *out_size to 0 when there is nothing to hand - * off. Safe to call at most once per stream; the stream itself should still - * be released with ch_http_stream_end(). - */ -void -ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size) { - if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { - *out_data = stream->error_msg; - *out_size = strlen(stream->error_msg); - stream->error_msg = NULL; - return; - } - - if (stream->write_pos == 0 || !stream->buf) { - *out_data = NULL; - *out_size = 0; - return; - } - - *out_data = stream->buf; - *out_size = stream->write_pos; - stream->buf = NULL; -} diff --git a/src/include/fdw.h b/src/include/fdw.h index e6603091..115f8b0c 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -261,10 +261,6 @@ extern ch_scan_connection chfdw_get_scan_connection(UserMapping* user); extern void chfdw_release_scan_connection(UserMapping* user, ch_scan_connection sconn); -extern void -chfdw_exec_query(ch_connection conn, const char* query); -extern void -chfdw_report_error(int elevel, ch_connection conn, bool clear, const char* sql); /* in option.c */ extern kv_list* diff --git a/src/include/http.h b/src/include/http.h index b38f6c43..4d69ed98 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -18,24 +18,85 @@ #define CH_HTTP_STATUS_TRANSPORT_ERROR 419L typedef struct ch_http_connection_t ch_http_connection_t; + +/* + * Opaque handle to one in-flight HTTP query. The real type is the HttpStream + * struct, defined in http.c. + */ +typedef struct HttpStream HttpStream; + +/* A ClickHouse setting the caller sends as a URL parameter. */ +typedef struct ch_setting { + const char* name; + const char* value; +} ch_setting; + +/* + * One HTTP request: the SQL to run plus the response policy its caller needs. + * Overrides win over a query setting of the same name, so a caller pins what + * its decoder requires while user settings fill in the rest. + */ +typedef struct ch_http_request { + const ch_query* query; + const ch_setting* overrides; + int num_overrides; + /* Hand out the body one receive chunk at a time, else buffer it whole */ + bool stream_chunks; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ +} ch_http_request; + +/* Response body buffered whole, for callers with no incremental decoder. */ typedef struct ch_http_response_t { char* data; size_t datasize; long http_status; char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; } ch_http_response_t; void ch_http_init(int verbose); -long -ch_http_get_verbose(void); /* Returns NULL and sets *error to a static message on failure. */ ch_http_connection_t* ch_http_connection(ch_connection_details* details, const char** error); void ch_http_close(ch_http_connection_t* conn); + +/* lifecycle */ +HttpStream* +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req); +void +ch_http_stream_end(HttpStream* stream); + +/* + * pgch_chunk_source next_chunk over the response body. Bytes stay valid until + * the following call. Sets *len 0 at clean EOF; returns false with *error on + * transport failure or cancellation. Takes void* so it can be assigned to the + * callback slot without this header knowing pg-clickhouse-c. + */ +bool +ch_http_stream_next_chunk(void* stream, const void** data, size_t* len, char** error); + +/* accessors — let pglink.c read stream state without seeing the struct */ +char* +ch_http_stream_buffer(HttpStream* stream); +size_t +ch_http_stream_available(HttpStream* stream); +long +ch_http_stream_status(HttpStream* stream); +const char* +ch_http_stream_query_id(HttpStream* stream); +const char* +ch_http_stream_error(HttpStream* stream); +double +ch_http_stream_request_time(HttpStream* stream); + +/* + * Trim the ClickHouse build version off an error body so messages stay stable + * across servers. May return errstring itself, modified in place. + */ +char* +ch_http_format_error(char* errstring); + ch_http_response_t* ch_http_simple_query( ch_http_connection_t* conn, diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h deleted file mode 100644 index d90a9a7e..00000000 --- a/src/include/http_streaming.h +++ /dev/null @@ -1,76 +0,0 @@ -#ifndef CLICKHOUSE_HTTP_STREAMING_H -#define CLICKHOUSE_HTTP_STREAMING_H - -#include "engine.h" - -typedef struct ch_http_connection_t ch_http_connection_t; - -/* - * Opaque handle to a streaming HTTP query. The real type is the HttpStream - * struct, defined in http_streaming.c. - */ -typedef struct HttpStream HttpStream; - -/* A ClickHouse setting the caller sends as a URL parameter. */ -typedef struct ch_setting { - const char* name; - const char* value; -} ch_setting; - -/* - * One HTTP request: the SQL to run plus the response policy its caller needs. - * Overrides win over a query setting of the same name, so a caller pins what - * its decoder requires while user settings fill in the rest. - */ -typedef struct ch_http_request { - const ch_query* query; - const ch_setting* overrides; - int num_overrides; - /* Hand out the body one receive chunk at a time, else buffer it whole */ - bool stream_chunks; - ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ -} ch_http_request; - -/* lifecycle */ -HttpStream* -ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req); -void -ch_http_stream_end(HttpStream* stream); - -/* - * pgch_chunk_source next_chunk over the response body. Bytes stay valid until - * the following call. Sets *len 0 at clean EOF; returns false with *error on - * transport failure or cancellation. Takes void* so it can be assigned to the - * callback slot without this header knowing pg-clickhouse-c. - */ -bool -ch_http_stream_next_chunk(void* stream, const void** data, size_t* len, char** error); - -/* accessors — let pglink.c read stream state without seeing the struct */ -char* -ch_http_stream_buffer(HttpStream* stream); -size_t -ch_http_stream_available(HttpStream* stream); -long -ch_http_stream_status(HttpStream* stream); -const char* -ch_http_stream_query_id(HttpStream* stream); -const char* -ch_http_stream_error(HttpStream* stream); -double -ch_http_stream_request_time(HttpStream* stream); -double -ch_http_stream_total_time(HttpStream* stream); - -/* - * Transfer ownership of the response body to the caller. On return, *out_data - * is a malloc()'d buffer (or the strdup'd transport error message when status - * is CH_HTTP_STATUS_TRANSPORT_ERROR) that the caller must free(). Only valid - * before the first ch_http_stream_next_chunk call, which reuses the buffer. - * The stream itself is unchanged otherwise and should still be released with - * ch_http_stream_end(). - */ -void -ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size); - -#endif /* CLICKHOUSE_HTTP_STREAMING_H */ diff --git a/src/pglink.c b/src/pglink.c index 54817f1c..c04eced0 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -21,7 +21,6 @@ #include "cursor.h" #include "fdw.h" #include "http.h" -#include "http_streaming.h" #include "pg-clickhouse-decode.h" #include "pg-clickhouse-encode.h" @@ -198,30 +197,6 @@ http_server_version(void* conn) { return ch_http_server_version((ch_http_connection_t*)conn, http_canceled); } -/* - * Return text before version mentioning - */ -static char* -format_error(char* errstring) { - size_t n = strlen(errstring); - - for (size_t i = 0; i < n; i++) { - if (strncmp(errstring + i, "version", 7) == 0) { - return pnstrdup(errstring, i - 2); - } - } - - /* - * For some reason ClickHouse 25.12 added a newline to an auth failure - * error. Strip it out. - */ - if (n > 0 && errstring[n - 1] == '\n') { - errstring[--n] = '\0'; - } - - return errstring; -} - static void kill_query(void* conn, const char* query_id) { ch_http_response_t* resp; @@ -279,7 +254,7 @@ report_http_stream_query_failure( ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) @@ -338,7 +313,7 @@ http_raw_query(void* conn, const ch_query* query) { ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) ); @@ -384,7 +359,7 @@ http_simple_insert(void* conn, const ch_query* query) { ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) ); diff --git a/test/expected/http.out b/test/expected/http.out index 76640131..786ee8f9 100644 --- a/test/expected/http.out +++ b/test/expected/http.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); diff --git a/test/expected/http_1.out b/test/expected/http_1.out index a7642d3a..afd853bc 100644 --- a/test/expected/http_1.out +++ b/test/expected/http_1.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); diff --git a/test/expected/http_2.out b/test/expected/http_2.out index 8893b7a4..61722e68 100644 --- a/test/expected/http_2.out +++ b/test/expected/http_2.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); diff --git a/test/expected/http_3.out b/test/expected/http_3.out index 02a15616..0cbda64e 100644 --- a/test/expected/http_3.out +++ b/test/expected/http_3.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); diff --git a/test/expected/http_4.out b/test/expected/http_4.out index b6240caf..0dcd66d7 100644 --- a/test/expected/http_4.out +++ b/test/expected/http_4.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); diff --git a/test/expected/http_5.out b/test/expected/http_5.out index 21d91175..ae9d7e4f 100644 --- a/test/expected/http_5.out +++ b/test/expected/http_5.out @@ -187,11 +187,13 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` doesn't exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` doesn't exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); From 7be334896a78ca332e8776e68715c3dd3c42285c Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 05:35:27 +0000 Subject: [PATCH 10/11] update expected --- test/expected/aggregates.out | 78 +++++------ test/expected/aggregates_1.out | 78 +++++------ test/expected/aggregates_2.out | 78 +++++------ test/expected/aggregates_3.out | 78 +++++------ test/expected/aggregates_4.out | 72 +++++----- test/expected/aggregates_5.out | 72 +++++----- test/expected/gucs.out | 8 +- test/expected/http.out | 32 +++-- test/expected/http_1.out | 32 +++-- test/expected/http_2.out | 32 +++-- test/expected/http_3.out | 32 +++-- test/expected/http_4.out | 32 +++-- test/expected/http_5.out | 32 +++-- test/expected/http_inserts.out | 18 +-- test/expected/http_inserts_1.out | 18 +-- test/expected/import_schema.out | 11 +- test/expected/import_schema_1.out | 11 +- test/expected/import_schema_2.out | 11 +- test/expected/json_3.out | 214 ++++++++---------------------- test/expected/json_4.out | 80 +++++------ test/expected/json_5.out | 80 +++++------ test/expected/json_6.out | 80 +++++------ test/sql/http.sql | 14 +- 23 files changed, 514 insertions(+), 679 deletions(-) diff --git a/test/expected/aggregates.out b/test/expected/aggregates.out index 6e53a4c0..085f9508 100644 --- a/test/expected/aggregates.out +++ b/test/expected/aggregates.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1498,9 +1498,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1525,9 +1525,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1552,9 +1552,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1579,9 +1579,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1633,9 +1633,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1660,9 +1660,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1687,9 +1687,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1714,9 +1714,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1741,9 +1741,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_1.out b/test/expected/aggregates_1.out index 09018fd2..f83aa63b 100644 --- a/test/expected/aggregates_1.out +++ b/test/expected/aggregates_1.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1498,9 +1498,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1525,9 +1525,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1552,9 +1552,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1579,9 +1579,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1633,9 +1633,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1660,9 +1660,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1687,9 +1687,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1714,9 +1714,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1741,9 +1741,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_2.out b/test/expected/aggregates_2.out index a12ae333..12b77545 100644 --- a/test/expected/aggregates_2.out +++ b/test/expected/aggregates_2.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1484,9 +1484,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1511,9 +1511,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1538,9 +1538,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1565,9 +1565,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1619,9 +1619,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1646,9 +1646,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1673,9 +1673,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1700,9 +1700,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1727,9 +1727,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_3.out b/test/expected/aggregates_3.out index 795d7480..a4e67d78 100644 --- a/test/expected/aggregates_3.out +++ b/test/expected/aggregates_3.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1488,9 +1488,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1515,9 +1515,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1542,9 +1542,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1569,9 +1569,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1623,9 +1623,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1650,9 +1650,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1677,9 +1677,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1704,9 +1704,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1731,9 +1731,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_4.out b/test/expected/aggregates_4.out index a2b231e2..90b22c49 100644 --- a/test/expected/aggregates_4.out +++ b/test/expected/aggregates_4.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1488,9 +1488,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1515,9 +1515,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1542,9 +1542,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1569,9 +1569,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1645,9 +1645,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1672,9 +1672,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1699,9 +1699,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1726,9 +1726,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_5.out b/test/expected/aggregates_5.out index c2a7522c..fb3aeffc 100644 --- a/test/expected/aggregates_5.out +++ b/test/expected/aggregates_5.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1479,9 +1479,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1506,9 +1506,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1533,9 +1533,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1560,9 +1560,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1636,9 +1636,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1663,9 +1663,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1690,9 +1690,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1717,9 +1717,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/gucs.out b/test/expected/gucs.out index 623cd0e0..b8bbd36f 100644 --- a/test/expected/gucs.out +++ b/test/expected/gucs.out @@ -80,8 +80,8 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i ------------------------------------+---------------------- connect_timeout | 2 count_distinct_implementation | uniq - date_time_output_format | iso - format_tsv_null_representation | \N + date_time_output_format | unix_timestamp + format_tsv_null_representation | NOPE join_algorithm | prefer_partial_merge join_use_nulls | 1 log_queries_min_type | QUERY_FINISH @@ -90,7 +90,7 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i max_result_rows | 1024 metrics_perf_events_list | this,that network_compression_method | ZSTD - output_format_tsv_crlf_end_of_line | 0 + output_format_tsv_crlf_end_of_line | 1 poll_interval | 5 totals_mode | after_having_auto (15 rows) @@ -118,7 +118,7 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i ------------------------------------+---------- connect_timeout | t count_distinct_implementation | t - date_time_output_format | f + date_time_output_format | t format_tsv_null_representation | t join_algorithm | t join_use_nulls | t diff --git a/test/expected/http.out b/test/expected/http.out index 786ee8f9..67e9b01a 100644 --- a/test/expected/http.out +++ b/test/expected/http.out @@ -841,9 +841,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -963,8 +962,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -985,18 +984,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1047,8 +1044,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1084,11 +1081,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_1.out b/test/expected/http_1.out index afd853bc..3d91bdda 100644 --- a/test/expected/http_1.out +++ b/test/expected/http_1.out @@ -839,9 +839,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -961,8 +960,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -983,18 +982,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1045,8 +1042,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1082,11 +1079,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_2.out b/test/expected/http_2.out index 61722e68..6c5acf69 100644 --- a/test/expected/http_2.out +++ b/test/expected/http_2.out @@ -839,9 +839,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -961,8 +960,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -983,18 +982,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1045,8 +1042,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1082,11 +1079,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_3.out b/test/expected/http_3.out index 0cbda64e..ba1433cc 100644 --- a/test/expected/http_3.out +++ b/test/expected/http_3.out @@ -839,9 +839,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -961,8 +960,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -983,18 +982,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1045,8 +1042,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1082,11 +1079,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_4.out b/test/expected/http_4.out index 0dcd66d7..796927ed 100644 --- a/test/expected/http_4.out +++ b/test/expected/http_4.out @@ -839,9 +839,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -961,8 +960,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -983,18 +982,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1045,8 +1042,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1082,11 +1079,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_5.out b/test/expected/http_5.out index ae9d7e4f..dc5cb8a0 100644 --- a/test/expected/http_5.out +++ b/test/expected/http_5.out @@ -841,9 +841,8 @@ WARNING: option "fetch_size" is deprecated and ignored DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -963,8 +962,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -985,18 +984,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1047,8 +1044,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1084,11 +1081,12 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist diff --git a/test/expected/http_inserts.out b/test/expected/http_inserts.out index 21605fd4..9462e64d 100644 --- a/test/expected/http_inserts.out +++ b/test/expected/http_inserts.out @@ -235,9 +235,9 @@ INSERT INTO complex VALUES SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- - 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 - 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 - 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 + 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 17:01:02.123 + 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 18:01:02.234 + 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 19:01:02 4 | 1970-01-01 | 1970-01-01 00:00:00+00 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 5 | 2000-01-01 | 2000-01-01 00:00:00+00 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 (5 rows) @@ -249,8 +249,8 @@ SELECT * FROM complex ORDER BY c1; 1 | 2020-06-01 | 2020-06-02 10:01:02-07 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 2 | 2020-06-02 | 2020-06-03 10:01:02-07 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 3 | 2020-06-03 | 2020-06-04 10:01:02-07 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 - 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 - 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 + 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1969-12-31 16:00:00 + 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 1999-12-31 16:00:00 (5 rows) /* check arrays */ @@ -475,11 +475,11 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); +ERROR: pg_clickhouse: cannot append NULL to NOT NULL Int16 column "c2" SELECT * FROM default_vals; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 -----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------------+------------+------------------------+------------------------+-----+--------------------------------------+---------+----- - 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 0.0 | 0 | | | | 1970-01-01 | 1970-01-01 | 1969-12-31 16:00:00-08 | 1969-12-31 16:00:00-08 | {0} | 00000000-0000-0000-0000-000000000000 | 0.0.0.0 | :: -(1 row) + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 +----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) /* COPY FROM bulk-loads rows into the remote table. */ COPY ints (c1, c2, c3, c4) FROM stdin; diff --git a/test/expected/http_inserts_1.out b/test/expected/http_inserts_1.out index 9db6ce79..e231ec5e 100644 --- a/test/expected/http_inserts_1.out +++ b/test/expected/http_inserts_1.out @@ -226,9 +226,9 @@ INSERT INTO complex VALUES SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- - 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 - 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 - 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 + 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 17:01:02.123 + 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 18:01:02.234 + 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 19:01:02 4 | 1970-01-01 | 1970-01-01 00:00:00+00 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 5 | 2000-01-01 | 2000-01-01 00:00:00+00 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 (5 rows) @@ -240,8 +240,8 @@ SELECT * FROM complex ORDER BY c1; 1 | 2020-06-01 | 2020-06-02 10:01:02-07 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 2 | 2020-06-02 | 2020-06-03 10:01:02-07 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 3 | 2020-06-03 | 2020-06-04 10:01:02-07 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 - 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 - 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 + 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1969-12-31 16:00:00 + 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 1999-12-31 16:00:00 (5 rows) /* check arrays */ @@ -466,11 +466,11 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); +ERROR: pg_clickhouse: cannot append NULL to NOT NULL Int16 column "c2" SELECT * FROM default_vals; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 -----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------------+------------+------------------------+------------------------+-----+--------------------------------------+---------+----- - 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 0.0 | 0 | | | | 1970-01-01 | 1970-01-01 | 1969-12-31 16:00:00-08 | 1969-12-31 16:00:00-08 | {0} | 00000000-0000-0000-0000-000000000000 | 0.0.0.0 | :: -(1 row) + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 +----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) /* COPY FROM bulk-loads rows into the remote table. */ COPY ints (c1, c2, c3, c4) FROM stdin; diff --git a/test/expected/import_schema.out b/test/expected/import_schema.out index 5774f7c6..eda5ce6a 100644 --- a/test/expected/import_schema.out +++ b/test/expected/import_schema.out @@ -364,10 +364,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -713,7 +713,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/import_schema_1.out b/test/expected/import_schema_1.out index 79d2a416..5c626875 100644 --- a/test/expected/import_schema_1.out +++ b/test/expected/import_schema_1.out @@ -328,10 +328,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -613,7 +613,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/import_schema_2.out b/test/expected/import_schema_2.out index 143cd76a..2298d336 100644 --- a/test/expected/import_schema_2.out +++ b/test/expected/import_schema_2.out @@ -364,10 +364,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -713,7 +713,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/json_3.out b/test/expected/json_3.out index ee946eb3..a8224f53 100644 --- a/test/expected/json_3.out +++ b/test/expected/json_3.out @@ -61,12 +61,8 @@ SELECT * FROM json_bin.things ORDER BY id; ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.things ORDER BY id; - id | data -----+------------------------------------------------------------------ - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} - 4 | {"id": "4", "name": "doodad", "size": "large", "stocked": false} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Should also support JSON mapping. CREATE FOREIGN TABLE json_bin.json_things ( id integer NOT NULL, @@ -85,13 +81,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; - id | data -----+----------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} - 4 | {"id":"4","name":"doodad","size":"large","stocked":false} - 6 | {"id":"6","name":"curio","size":"medium","stocked":false} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -106,13 +97,8 @@ SELECT data['name'] FROM json_http.things; (3 rows) SELECT data['name'] FROM json_http.things ORDER BY id; - data ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data['name'] FROM json_bin.things; QUERY PLAN @@ -139,12 +125,8 @@ SELECT DISTINCT data['size'] FROM json_http.things; (6 rows) SELECT DISTINCT data['size'] FROM json_http.things; - data ----------- - "medium" - "large" -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT data FROM json_test.things EXPLAIN (VERBOSE, COSTS OFF) SELECT DISTINCT data['size'] FROM json_bin.things; QUERY PLAN @@ -174,12 +156,8 @@ SELECT data['size'], count(*) FROM json_http.things GROUP BY data['size']; (6 rows) SELECT data['size'], count(*) FROM json_http.things GROUP BY data['size']; - data | count -----------+------- - "medium" | 2 - "large" | 1 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT data FROM json_test.things EXPLAIN (VERBOSE, COSTS OFF) SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; QUERY PLAN @@ -206,13 +184,8 @@ SELECT data ->> 'name' FROM json_http.things; (3 rows) SELECT data ->> 'name' FROM json_http.things ORDER BY id; - ?column? ----------- - gizmo - doodad - curio -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.things; QUERY PLAN @@ -236,13 +209,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; - ?column? ----------- - gizmo - doodad - curio -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -446,11 +414,8 @@ SELECT * FROM json_http.things SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; - id | data -----+----------------------------------------------------------------- - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; @@ -475,11 +440,8 @@ SELECT * FROM json_http.json_things SELECT * FROM json_http.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; - id | data -----+---------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; @@ -501,13 +463,8 @@ SELECT * FROM json_http.things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.things ORDER BY data ->> 'name'; - id | data -----+------------------------------------------------------------------ - 6 | {"id": "6", "name": "curio", "size": "medium", "stocked": false} - 4 | {"id": "4", "name": "doodad", "size": "large", "stocked": false} - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.things ORDER BY data ->> 'name'; QUERY PLAN @@ -540,13 +497,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; - id | data -----+----------------------------------------------------------- - 6 | {"id":"6","name":"curio","size":"medium","stocked":false} - 4 | {"id":"4","name":"doodad","size":"large","stocked":false} - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -579,13 +531,8 @@ SELECT data -> 'name' FROM json_http.things; (3 rows) SELECT data -> 'name' FROM json_http.things ORDER BY id; - ?column? ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.things; QUERY PLAN @@ -608,13 +555,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; - ?column? ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -696,11 +638,8 @@ SELECT * FROM json_http.things WHERE data -> 'stocked' = 'true'::jsonb; (3 rows) SELECT * FROM json_http.things WHERE data -> 'stocked' = 'true'::jsonb ORDER BY id; - id | data -----+----------------------------------------------------------------- - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((toJSONString(data.stocked) = 'true')) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb; QUERY PLAN @@ -724,11 +663,8 @@ SELECT * FROM json_http.json_things WHERE (data -> 'stocked')::text = 'true'; (3 rows) SELECT * FROM json_http.json_things WHERE (data -> 'stocked')::text = 'true' ORDER BY id; - id | data -----+---------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((CAST(toJSONString(data.stocked) AS String) = 'true')) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true'; QUERY PLAN @@ -837,11 +773,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -864,11 +797,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -892,11 +822,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -919,11 +846,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -947,11 +871,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -974,11 +895,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1380,12 +1298,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; - jsonb_extract_path_text -------------------------- - C100 - C200 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1409,12 +1323,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; - jsonb_extract_path_text -------------------------- - Paris - London -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1438,12 +1348,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; - jsonb_extract_path ------------------------------------ - {"zip": "75001", "city": "Paris"} - {"zip": "SW1A", "city": "London"} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1573,12 +1479,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; - json_extract_path_text ------------------------- - C100 - C200 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1602,12 +1504,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; - json_extract_path_text ------------------------- - Paris - London -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1631,12 +1529,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; - json_extract_path --------------------------------- - {"city":"Paris","zip":"75001"} - {"city":"London","zip":"SW1A"} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_4.out b/test/expected/json_4.out index 3143cfa1..be4b616f 100644 --- a/test/expected/json_4.out +++ b/test/expected/json_4.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_5.out b/test/expected/json_5.out index 668c6979..a09da14a 100644 --- a/test/expected/json_5.out +++ b/test/expected/json_5.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_6.out b/test/expected/json_6.out index 8c5d03c1..1a6a7845 100644 --- a/test/expected/json_6.out +++ b/test/expected/json_6.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/sql/http.sql b/test/sql/http.sql index 82243ec0..f5d03e73 100644 --- a/test/sql/http.sql +++ b/test/sql/http.sql @@ -262,9 +262,8 @@ DROP FOREIGN TABLE ft_bad_fetch; DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -336,8 +335,8 @@ CREATE FOREIGN TABLE ft_bytea (id text, v bytea) SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -353,9 +352,8 @@ SELECT v FROM ft_timestr WHERE id = '3'; SELECT v FROM ft_timestr WHERE id = '1'; SELECT v FROM ft_timestr WHERE id = '2'; -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); From 2ca22e611784745a9bdc8f09bbeedf7e1f020d8d Mon Sep 17 00:00:00 2001 From: serprex <159546+serprex@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:44:17 +0000 Subject: [PATCH 11/11] don't warn about version on 404 to reduce noise --- src/http.c | 6 +++++- src/include/http.h | 1 + test/expected/http.out | 1 - test/expected/http_1.out | 1 - test/expected/http_2.out | 1 - test/expected/http_3.out | 1 - test/expected/http_4.out | 1 - test/expected/http_5.out | 1 - 8 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/http.c b/src/http.c index dd64e1a5..570f06be 100644 --- a/src/http.c +++ b/src/http.c @@ -763,7 +763,11 @@ ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel) { /* Version string probably trash; zero out. */ conn->version = none; } - } else if (resp->http_status != CH_HTTP_STATUS_OK) { + } else if ( + resp->http_status != CH_HTTP_STATUS_OK && + resp->http_status != CH_HTTP_STATUS_NOT_FOUND + ) { + /* 404 means a missing database, which the query itself reports */ elog( WARNING, "pg_clickhouse: SELECT version() failed (HTTP status %d): %s", diff --git a/src/include/http.h b/src/include/http.h index 4d69ed98..b43520b3 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -14,6 +14,7 @@ * and libcurl transport failures through the existing response machinery. */ #define CH_HTTP_STATUS_OK 200L +#define CH_HTTP_STATUS_NOT_FOUND 404L #define CH_HTTP_STATUS_CANCELED 418L #define CH_HTTP_STATUS_TRANSPORT_ERROR 419L diff --git a/test/expected/http.out b/test/expected/http.out index 67e9b01a..80c71c3b 100644 --- a/test/expected/http.out +++ b/test/expected/http.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 diff --git a/test/expected/http_1.out b/test/expected/http_1.out index 3d91bdda..471355c2 100644 --- a/test/expected/http_1.out +++ b/test/expected/http_1.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 diff --git a/test/expected/http_2.out b/test/expected/http_2.out index 6c5acf69..b89fb50b 100644 --- a/test/expected/http_2.out +++ b/test/expected/http_2.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 diff --git a/test/expected/http_3.out b/test/expected/http_3.out index ba1433cc..a7414f94 100644 --- a/test/expected/http_3.out +++ b/test/expected/http_3.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 diff --git a/test/expected/http_4.out b/test/expected/http_4.out index 796927ed..6b40bac9 100644 --- a/test/expected/http_4.out +++ b/test/expected/http_4.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` does not exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404 diff --git a/test/expected/http_5.out b/test/expected/http_5.out index dc5cb8a0..05e88e21 100644 --- a/test/expected/http_5.out +++ b/test/expected/http_5.out @@ -187,7 +187,6 @@ SELECT c3, c4 FROM ft1 ORDER BY c3, c1 LIMIT 1; -- should work ALTER SERVER http_loopback OPTIONS (SET dbname 'no such database'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail -WARNING: pg_clickhouse: SELECT version() failed (HTTP status 404): Code: 81. DB::Exception: Database `no such database` doesn't exist. (UNKNOWN_DATABASE) ERROR: pg_clickhouse: Code: 81. DB::Exception: Database `no such database` doesn't exist. (UNKNOWN_DATABASE) DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 ASC NULLS LAST, c1 ASC NULLS LAST CONTEXT: HTTP status code: 404