diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b87f839..db8803dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,11 @@ All notable changes to this project will be documented in this file. It uses the * Coerce array elements in binary driver. `Array(Int32)` to `bigint[]`, or `quantilesExactLow()` results into `double precision[]`, no longer fails with `could not cast value from integer[] to bigint[]` ([#326]). +* HTTP driver now uses ClickHouse's Native format, sharing encode/decode with + binary driver. `clickhouse_raw_query()` keeps TabSeparated behavior. +* Deprecated `fetch_size`. It is ignored and produces a warning. HTTP driver + always streams, handing Native decoder one chunk at a time. Decoded memory + is set by block size ClickHouse writes, not by client-side byte count. ### 🐞 Bug Fixes diff --git a/Makefile b/Makefile index 55328fed..1be86896 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ CH_C_DIR = $(PGCH_DIR)/clickhouse-c PG_CPPFLAGS = -I./src/include -isystem $(CH_C_DIR) -isystem $(PGCH_DIR) -isystem $(shell $(PG_CONFIG) --includedir-server) -DPGCH_MSG_PREFIX='"pg_clickhouse: "' # Link OpenSSL (for TLS in the binary driver), curl (for the HTTP driver), -# libuuid (for http_streaming.c's query-id generator), and lz4 / zstd +# libuuid (for http.c's query-id generator), and lz4 / zstd # (for the binary driver's compressed-frame codecs). PG_LDFLAGS = -lssl -lcrypto -llz4 -lzstd $(shell $(CURL_CONFIG) --libs) diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index 8d2e560b..2fd83816 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -11,7 +11,7 @@ CREATE EXTENSION pg_clickhouse; This library contains PostgreSQL extension that enables remote query execution on ClickHouse databases, including a [foreign data wrapper]. It supports -PostgreSQL 13 and higher and ClickHouse 23 and higher. +PostgreSQL 13 and higher and ClickHouse 23.3 and higher. ## Getting Started @@ -142,10 +142,6 @@ The supported options are: "none", "lz4", or "zstd". Defaults to "lz4". Ignored by the "http" driver. * `dbname`: The ClickHouse database to use upon connecting. Defaults to "default". -* `fetch_size`: Approximate batch size in bytes for HTTP streaming. Batches - split on row boundaries. Defaults to `50000000` (50 MB). `0` disables - streaming and buffers the full response. Foreign tables can override this - value. * `host`: The host name of the ClickHouse server. Defaults to "localhost"; * `min_tls_version`: Minimum TLS protocol version to negotiate on connections that use TLS. One of `TLSv1`, `TLSv1.1`, `TLSv1.2`, or `TLSv1.3`. Defaults @@ -315,9 +311,6 @@ The supported table options are: * `database`: The name of the remote database. Defaults to the database defined for the foreign server. -* `fetch_size`: Approximate batch size in bytes for HTTP streaming. - Overrides server-level `fetch_size`. Defaults to `50000000` (50 MB). `0` - disables streaming and buffers the full response. * `table_name`: The name of the remote table. Default to the name specified for the foreign table. * `engine`: The [table engine] used by the ClickHouse table. For diff --git a/src/cursor.c b/src/cursor.c new file mode 100644 index 00000000..37ff0a64 --- /dev/null +++ b/src/cursor.c @@ -0,0 +1,338 @@ +/* + * cursor.c + * + * Shared lifecycle for a cursor over a ClickHouse Native block stream: open + * over a driver's response, fetch rows through the decoder, release both on + * teardown. API lives in src/include/cursor.h; the drivers in src/pglink.c + * only supply the response and how to read it. + */ + +#include "postgres.h" + +#include "catalog/pg_type_d.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/memutils.h" +#include "utils/portal.h" + +#include "cursor.h" + +/* Release the reader before the response its blocks came from. */ +static void +cursor_free(void* c) { + ch_cursor* cursor = c; + + pgch_reader_free(&cursor->reader); + cursor->free_response(cursor->response); + cursor->response = NULL; +} + +/* Report decoder error; a driver hook may report cancellation instead. */ +static void +raise_reader_error(ch_cursor* cursor) { + if (cursor->raise_response_error) { + cursor->raise_response_error(cursor); + } + /* Prefer consistent interrupt error message when fetch interrupted */ + CHECK_FOR_INTERRUPTS(); + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", cursor->reader.error), + errdetail_internal("Remote Query: %.64000s", cursor->query) + ); +} + +/* Match the returned columns against the destination the query asked for. */ +static void +configure_columns(ch_cursor* cursor, const ch_query* query) { + pgch_reader* reader = &cursor->reader; + + if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && + (size_t)list_length(query->attr_nums) != cursor->columns_count) { + ereport( + ERROR, + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + (unsigned long)cursor->columns_count, + (unsigned long)list_length(query->attr_nums) + ), + errdetail_internal("Remote Query: %.64000s", query->sql) + ); + } + + /* Preserve JSON text when PostgreSQL destination uses json, not jsonb. */ + if (query->tupdesc && reader->coltypes) { + ListCell* lc; + size_t j = 0; + + foreach (lc, query->attr_nums) { + int i = lfirst_int(lc); + + if (reader->coltypes[j] == JSONBOID && + TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { + reader->coltypes[j] = JSONOID; + } + j++; + } + } +} + +ch_cursor* +chfdw_cursor_open(void* conn, const ch_query* query, const ch_cursor_source* src) { + /* volatile: assigned inside PG_TRY, read after longjmp in PG_CATCH */ + ch_cursor* volatile cursor = NULL; + volatile MemoryContext cxt = NULL; + volatile bool owns_response = false; + MemoryContext oldcxt = CurrentMemoryContext; + + PG_TRY(); + { + cxt = AllocSetContextCreate( + PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES + ); + MemoryContextSwitchTo(cxt); + + cursor = palloc0(sizeof(ch_cursor)); + cursor->memcxt = cxt; + cursor->conn = conn; + cursor->query = pstrdup(query->sql); + cursor->free_response = src->free_response; + cursor->raise_response_error = src->raise_response_error; + + /* Register before taking the response, so unwinding releases it. */ + cursor->callback.func = cursor_free; + cursor->callback.arg = (void*)cursor; + MemoryContextRegisterResetCallback(cxt, &cursor->callback); + cursor->response = src->response; + owns_response = true; + + /* Blocks decode into cxt, outliving the per-row context. */ + src->init_reader(&cursor->reader, cursor->response); + cursor->columns_count = pgch_reader_columns(&cursor->reader); + + if (cursor->reader.error) { + raise_reader_error(cursor); + } + configure_columns(cursor, query); + + MemoryContextSwitchTo(oldcxt); + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + if (!owns_response) { + src->free_response(src->response); + } + if (cxt != NULL) { + MemoryContextDelete(cxt); + } + PG_RE_THROW(); + } + PG_END_TRY(); + + return cursor; +} + +/* Conversion state and target attribute per returned column. */ +static void +build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { + pgch_reader* reader = &cursor->reader; + MemoryContext old = MemoryContextSwitchTo(cursor->memcxt); + size_t ncols = pgch_reader_columns(reader); + ListCell* lc; + size_t j = 0; + + cursor->conversion_states = palloc0(ncols * sizeof(void*)); + cursor->fill_dest = palloc0(ncols * sizeof(int)); + foreach (lc, ctx->retrieved_attrs) { + int attnum = lfirst_int(lc); + Form_pg_attribute att = TupleDescAttr(ctx->tupdesc, attnum - 1); + + cursor->fill_dest[j] = attnum - 1; + cursor->conversion_states[j] = + pgch_reader_convert_init(reader, j, att->atttypid, att->atttypmod); + j++; + } + + MemoryContextSwitchTo(old); +} + +/* Apply PostgreSQL conversions to fetched Native row. */ +static Datum* +apply_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + List* attrs = ctx->retrieved_attrs; + TupleDesc tupdesc = ctx->tupdesc; + Datum* values = ctx->values; + bool* nulls = ctx->nulls; + pgch_reader* reader = &cursor->reader; + size_t attcount = list_length(attrs); + + if (attcount == 0) { + if (pgch_reader_columns(reader) == 1 && reader->nulls[0]) { + nulls[0] = true; + return reader->values; + } + ereport( + ERROR, + errcode(ERRCODE_FDW_ERROR), + errmsg( + "pg_clickhouse: unexpected state: attributes " + "count == 0 and haven't got NULL in the response" + ) + ); + } else if (attcount != pgch_reader_columns(reader)) { + ereport( + ERROR, + errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg_internal( + "pg_clickhouse: returned %lu columns, expected %lu", + pgch_reader_columns(reader), + attcount + ) + ); + } + + if (tupdesc) { + Assert(values && nulls); + + if (cursor->conversion_states == NULL) { + build_conversion(cursor, ctx); + } + pgch_reader_fill_map( + reader, cursor->conversion_states, cursor->fill_dest, values, nulls + ); + } + + return reader->values; +} + +static void +fetch_row_errcb(void* arg) { + const char* sql = (const char*)arg; + + errdetail_internal("Remote Query: %.64000s", sql); +} + +/* + * Fetch a row from the cursor and return its values. + * + * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values + * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts + * values. + * + * Use ctx->tupdesc and ctx->attinmeta to convert the values to the + * appropriate Datums, and store them and the indication of their NULLness in + * ctx->values and ctx->nulls, respectively, then return ctx->values. + * + * If ctx->tupdesc is not set, treat all values as text and return them as + * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, + * which only cares about text. + */ +Datum* +chfdw_cursor_fetch_row(ChFdwScanRowContext* ctx) { + ch_cursor* cursor = ctx->cursor; + ErrorContextCallback errcallback; + bool have_data; + Datum* result; + + errcallback.callback = fetch_row_errcb; + errcallback.arg = (void*)cursor->query; + errcallback.previous = error_context_stack; + error_context_stack = &errcallback; + + have_data = pgch_reader_next(&cursor->reader); + + if (cursor->reader.error) { + error_context_stack = errcallback.previous; + raise_reader_error(cursor); + } + + result = have_data ? apply_row(ctx) : NULL; + + error_context_stack = errcallback.previous; + return result; +} + +/* + * Escape characters that would otherwise corrupt the tab/newline framing or + * collide with the \N null marker. Matches CH's TabSeparated escaping; \0 is + * unreachable since values arrive as cstrings, so it needs no case. + */ +static void +append_tsv_escaped(StringInfo buf, const char* s) { + for (; *s != '\0'; s++) { + switch (*s) { + case '\\': + appendStringInfoString(buf, "\\\\"); + break; + case '\b': + appendStringInfoString(buf, "\\b"); + break; + case '\f': + appendStringInfoString(buf, "\\f"); + break; + case '\n': + appendStringInfoString(buf, "\\n"); + break; + case '\r': + appendStringInfoString(buf, "\\r"); + break; + case '\t': + appendStringInfoString(buf, "\\t"); + break; + default: + appendStringInfoChar(buf, *s); + } + } +} + +text* +chfdw_cursor_render_tsv(ch_cursor* cursor) { + pgch_reader* reader = &cursor->reader; + size_t ncols = pgch_reader_columns(reader); + StringInfoData buf; + + if (ncols == 0) { + return NULL; + } + + initStringInfo(&buf); + + while (pgch_reader_next(reader)) { + for (size_t i = 0; i < ncols; i++) { + if (i > 0) { + appendStringInfoChar(&buf, '\t'); + } + + if (reader->nulls[i]) { + appendStringInfoString(&buf, "\\N"); + } else { + char* val = + pgch_value_to_cstring(reader->coltypes[i], reader->values[i]); + + append_tsv_escaped(&buf, val); + pfree(val); + } + } + appendStringInfoChar(&buf, '\n'); + CHECK_FOR_INTERRUPTS(); + } + + if (reader->error) { + ereport( + ERROR, + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: %s", reader->error) + ); + } + + if (buf.len == 0) { + pfree(buf.data); + return NULL; + } + + return cstring_to_text_with_len(buf.data, buf.len); +} diff --git a/src/fdw.c b/src/fdw.c index 9de9db09..58906e1e 100644 --- a/src/fdw.c +++ b/src/fdw.c @@ -41,6 +41,7 @@ #endif /* extension includes. */ +#include "cursor.h" #include "fdw.h" #include "utils/builtins.h" #include "version.h" @@ -61,9 +62,6 @@ PG_MODULE_MAGIC; /* If no remote estimates, assume a sort costs 20% extra */ #define DEFAULT_FDW_SORT_MULTIPLIER 1.2 -/* Approximate batch size in bytes for HTTP streaming (50 MB). */ -#define DEFAULT_FETCH_SIZE (50 * 1000 * 1000) - /* * Indexes of FDW-private information stored in fdw_private lists. * @@ -76,8 +74,6 @@ enum FdwScanPrivateIndex { FdwScanPrivateSelectSql, /* Integer list of attribute numbers retrieved by the SELECT */ FdwScanPrivateRetrievedAttrs, - /* Approximate batch size in bytes for HTTP streaming */ - FdwScanPrivateFetchSize, /* * String describing join i.e. names of relations being joined and types @@ -132,7 +128,6 @@ typedef struct ChFdwScanState { MemoryContext batch_cxt; /* context holding current batch of tuples */ MemoryContext temp_cxt; /* context for per-tuple temporary data */ - int32 fetch_size; /* approximate batch size in bytes */ bool is_streaming; /* true when using HTTP streaming */ } ChFdwScanState; @@ -357,10 +352,6 @@ merge_fdw_options( const CHFdwRelationInfo* fpinfo_o, const CHFdwRelationInfo* fpinfo_i ); -static int -get_fetch_size_option(DefElem* def); -static DefElem* -ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name); /* Make one query and close the connection */ Datum @@ -389,13 +380,7 @@ clickhouse_raw_query(PG_FUNCTION_ARGS) { } PG_TRY(); - { - ch_cursor* cursor = conn.methods->simple_query(conn.conn, &query); - - res = conn.is_binary ? chfdw_binary_fetch_raw_data(cursor) - : chfdw_http_fetch_raw_data(cursor); - MemoryContextDelete(cursor->memcxt); - } + { res = conn.methods->raw_query(conn.conn, &query); } PG_CATCH(); { conn.methods->disconnect(conn.conn); @@ -549,53 +534,6 @@ time_diff(struct timeval* prior, struct timeval* latter) { return x; } -static int -get_fetch_size_option(DefElem* def) { - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg( - "invalid value for option \"%s\": %s", def->defname, defGetString(def) - ), - errhint("fetch_size must be greater than or equal to 0") - ); - } - - return fetch_size; -} - -/* - * Utility function to fetch a cascading option definition from `fpinfo`. - * Returns the first option that matches in `fpinfo->table->options` or, if - * none, then the first found in `fpinfo->server->options`. Returns `NULL` - * when none found. - */ -static DefElem* -ch_get_table_or_server_option(CHFdwRelationInfo* fpinfo, char* name) { - ListCell* lc; - - foreach (lc, fpinfo->table->options) { - DefElem* def = (DefElem*)lfirst(lc); - - if (strcmp(def->defname, name) == 0) { - return def; - } - } - - foreach (lc, fpinfo->server->options) { - DefElem* def = (DefElem*)lfirst(lc); - - if (strcmp(def->defname, name) == 0) { - return def; - } - } - - return NULL; -} - /* * clickhouseGetForeignRelSize * Estimate # of rows and width of the result of the scan @@ -636,15 +574,6 @@ clickhouseGetForeignRelSize( fpinfo->fdw_tuple_cost = DEFAULT_FDW_TUPLE_COST; fpinfo->shippable_extensions = NIL; - /* - * Extract fetch_size: table option overrides server option, default - * DEFAULT_FETCH_SIZE. Value is approximate batch size in bytes; 0 means - * buffer entire response (disable HTTP streaming). - */ - DefElem* def = ch_get_table_or_server_option(fpinfo, "fetch_size"); - - fpinfo->fetch_size = def ? get_fetch_size_option(def) : DEFAULT_FETCH_SIZE; - chfdw_apply_custom_table_options(fpinfo, foreigntableid); fpinfo->user = NULL; @@ -1096,9 +1025,7 @@ clickhouseGetForeignPlan( * Build the fdw_private list that will be available to the executor. * Items in the list must match order in enum FdwScanPrivateIndex. */ - fdw_private = list_make3( - makeString(sql.data), retrieved_attrs, makeInteger(fpinfo->fetch_size) - ); + fdw_private = list_make2(makeString(sql.data), retrieved_attrs); if (IS_JOIN_REL(foreignrel) || IS_UPPER_REL(foreignrel)) { fdw_private = lappend(fdw_private, makeString(fpinfo->relation_name->data)); } @@ -1198,8 +1125,6 @@ clickhouseBeginForeignScan(ForeignScanState* node, int eflags) { fsstate->query = strVal(list_nth(fsplan->fdw_private, FdwScanPrivateSelectSql)); fsstate->retrieved_attrs = (List*)list_nth(fsplan->fdw_private, FdwScanPrivateRetrievedAttrs); - fsstate->fetch_size = - intVal(list_nth(fsplan->fdw_private, FdwScanPrivateFetchSize)); /* Create contexts for batches of tuples and per-tuple temp workspace. */ fsstate->batch_cxt = AllocSetContextCreate( @@ -1363,12 +1288,11 @@ clickhouseIterateForeignScan(ForeignScanState* node) { } fsstate->is_streaming = - fsstate->fetch_size > 0 && fsstate->scan_conn.gate.methods->streaming_query != NULL; if (fsstate->is_streaming) { fsstate->ch_cursor = fsstate->scan_conn.gate.methods->streaming_query( - fsstate->scan_conn.gate.conn, &query, fsstate->fetch_size + fsstate->scan_conn.gate.conn, &query ); } else { /* Binary still falls back to the simple path for now. */ @@ -2519,7 +2443,6 @@ merge_fdw_options( fpinfo->fdw_tuple_cost = fpinfo_o->fdw_tuple_cost; fpinfo->shippable_extensions = fpinfo_o->shippable_extensions; fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate; - fpinfo->fetch_size = fpinfo_o->fetch_size; /* Merge the table level options from either side of the join. */ if (fpinfo_i) { @@ -2533,12 +2456,6 @@ merge_fdw_options( */ fpinfo->use_remote_estimate = fpinfo_o->use_remote_estimate || fpinfo_i->use_remote_estimate; - - /* - * Set fetch size to maximum of the joining sides, since larger joins - * benefit from bigger batches. - */ - fpinfo->fetch_size = Max(fpinfo_o->fetch_size, fpinfo_i->fetch_size); } } diff --git a/src/http.c b/src/http.c index b5751dec..570f06be 100644 --- a/src/http.c +++ b/src/http.c @@ -1,24 +1,43 @@ +/*------------------------------------------------------------------------- + * + * http.c + * HTTP transport for pg_clickhouse. + * + * Owns the connection, request setup, chunk delivery, status, cancellation + * and cleanup. Uses curl_multi + curl_easy_pause to hand ClickHouse + * responses to the caller one receive chunk at a time, keeping memory + * bounded. + * + * Copyright (c) 2025-2026, ClickHouse, Inc. + * + *------------------------------------------------------------------------- + */ #include -#include #include #include #include -#include -#include -#include +#include + +#include "postgres.h" -static char curl_error_buffer[CURL_ERROR_SIZE]; -static bool curl_error_happened = false; -static long curl_verbose = 0; -static curl_xferinfo_callback curl_progressfunc = NULL; -static bool curl_initialized = false; -static char ch_query_id_prefix[5]; +#include "http.h" +#include "internal.h" +#include "kv_list.h" + +#ifndef CURL_WRITEFUNC_ERROR +#define CURL_WRITEFUNC_ERROR 0xFFFFFFFF +#endif + +#define DATABASE_HEADER "X-ClickHouse-Database" +#define INITIAL_BUF_SIZE (64 * 1024) + +static long curl_verbose = 0; +static bool curl_initialized = false; void -ch_http_init(int verbose, uint32_t query_id_prefix) { +ch_http_init(int verbose) { curl_verbose = verbose; - snprintf(ch_query_id_prefix, 5, "%x", query_id_prefix); if (!curl_initialized) { curl_initialized = true; @@ -26,20 +45,10 @@ ch_http_init(int verbose, uint32_t query_id_prefix) { } } -void -ch_http_set_progress_func(curl_xferinfo_callback progressfunc) { - curl_progressfunc = progressfunc; -} - -curl_xferinfo_callback -ch_http_get_progress_func(void) { - return curl_progressfunc; -} - -long -ch_http_get_verbose(void) { - return curl_verbose; -} +/* ---------------------------------------------------------------- + * Connection + * ---------------------------------------------------------------- + */ #define CLICKHOUSE_PORT 8123 #define CLICKHOUSE_TLS_PORT 8443 @@ -67,24 +76,17 @@ curl_min_tls_version(tls_version v) { } ch_http_connection_t* -ch_http_connection(ch_connection_details* details) { - int n; - char* connstring = NULL; - size_t len = 20; /* all symbols from url string + some extra */ - char *host = details->host, *username = details->username, - *password = details->password; - int port = details->port; +ch_http_connection(ch_connection_details* details, const char** error) { + CURLU* cu = NULL; + char* host = details->host; + int port = details->port; + char port_buf[12]; - curl_error_happened = false; - ch_http_connection_t* conn = calloc(sizeof(ch_http_connection_t), 1); + ch_http_connection_t* conn = calloc(1, sizeof(ch_http_connection_t)); + *error = "out of memory"; if (!conn) { - goto cleanup; - } - - conn->curl = curl_easy_init(); - if (!conn->curl) { - goto cleanup; + return NULL; } conn->ssl_version = curl_min_tls_version(details->min_tls_version); @@ -123,83 +125,550 @@ ch_http_connection(ch_connection_details* details) { break; } - len += strlen(host) + snprintf(NULL, 0, "%d", port); + snprintf(port_buf, sizeof(port_buf), "%d", port); + + cu = curl_url(); + if (cu == NULL) { + goto cleanup; + } + + /* Credentials go in as components so curl escapes them for us. */ + *error = "could not build ClickHouse URL"; + if (curl_url_set(cu, CURLUPART_SCHEME, use_tls ? "https" : "http", 0) != + CURLUE_OK || + curl_url_set(cu, CURLUPART_HOST, host, 0) != CURLUE_OK || + curl_url_set(cu, CURLUPART_PORT, port_buf, 0) != CURLUE_OK || + curl_url_set(cu, CURLUPART_PATH, "/", 0) != CURLUE_OK) { + goto cleanup; + } - if (username) { - username = curl_easy_escape(conn->curl, username, 0); - if (username == NULL) { + if (details->username) { + if (curl_url_set(cu, CURLUPART_USER, details->username, CURLU_URLENCODE) != + CURLUE_OK) { goto cleanup; } - len += strlen(username); - } - if (password) { - password = curl_easy_escape(conn->curl, password, 0); - if (password == NULL) { - curl_free(username); + if (details->password && + curl_url_set(cu, CURLUPART_PASSWORD, details->password, CURLU_URLENCODE) != + CURLUE_OK) { goto cleanup; } - len += strlen(password); } - connstring = calloc(len, 1); - if (!connstring) { + if (curl_url_get(cu, CURLUPART_URL, &conn->base_url, 0) != CURLUE_OK) { goto cleanup; } - char* scheme = use_tls ? "https" : "http"; + curl_url_cleanup(cu); + return conn; + +cleanup: + curl_url_cleanup(cu); + free(conn->dbname); + free(conn); + + return NULL; +} + +void +ch_http_close(ch_http_connection_t* conn) { + curl_free(conn->base_url); + free(conn->dbname); + free(conn); +} + +/* ---------------------------------------------------------------- + * HttpStream — opaque struct. + * ---------------------------------------------------------------- + */ +struct HttpStream { + /* Connection (borrowed, not owned) */ + ch_http_connection_t* conn; + + /* Owned CURL resources */ + CURL* curl; + CURLM* multi; + struct curl_slist* headers; + curl_mime* form; + char* url; /* allocated by curl_url_get, freed with + * curl_free */ + + /* Stream buffer */ + char* buf; + size_t buf_allocated; + size_t write_pos; + bool streaming; /* hand out one chunk at a time, else buffer whole body */ + bool paused; + bool transfer_done; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ + char error_buffer[CURL_ERROR_SIZE]; + + /* Public state readable via C accessors */ + long http_status; + char query_id[CH_HTTP_QUERY_ID_LEN]; + double pretransfer_time; + char* error_msg; /* strdup'd, freed with free() */ +}; + +/* CURLOPT_XFERINFOFUNCTION adapter over the caller's cancellation check. */ +static int +xferinfo_callback( + void* clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow +) { + HttpStream* stream = (HttpStream*)clientp; + + return stream->cancel() ? 1 : 0; +} + +/* True when an override replaces the user setting of this name. */ +static bool +is_overridden(const ch_http_request* req, const char* name) { + for (int i = 0; i < req->num_overrides; i++) { + if (strcmp(req->overrides[i].name, name) == 0) { + return true; + } + } + + return false; +} + +/* ---------------------------------------------------------------- + * write_callback — CURL write callback. Appends data to the stream + * buffer and, when streaming, pauses receipt so the caller drains it. + * ---------------------------------------------------------------- + */ +static size_t +write_callback(void* contents, size_t size, size_t nmemb, void* userp) { + size_t realsize = size * nmemb; + HttpStream* self = (HttpStream*)userp; + size_t needed = self->write_pos + realsize + 1; + + /* Grow buffer if needed */ + if (needed > self->buf_allocated) { + size_t newsize = self->buf_allocated * 2; + char* newbuf; + + if (newsize < needed) { + newsize = needed; + } + + newbuf = (char*)realloc(self->buf, newsize); + if (!newbuf) { + return CURL_WRITEFUNC_ERROR; + } + + self->buf = newbuf; + self->buf_allocated = newsize; + } + + memcpy(self->buf + self->write_pos, contents, realsize); + self->write_pos += realsize; + self->buf[self->write_pos] = '\0'; + + if (self->streaming) { + self->paused = true; + curl_easy_pause(self->curl, CURLPAUSE_RECV); + } + + return realsize; +} + +/* ---------------------------------------------------------------- + * setup_curl — configure the CURL easy handle for this query. + * ---------------------------------------------------------------- + */ +static void +setup_curl(HttpStream* stream, const ch_http_request* req) { + const ch_query* query = req->query; + CURLU* cu = curl_url(); + char temp_buf[512]; + + /* Build URL with query_id and settings */ + curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); + + snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); + curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); + + kv_iter iter = new_kv_iter(query->settings); + while (kv_iter_next(&iter)) { + if (is_overridden(req, iter.name)) { + continue; + } + snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); + curl_url_set( + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE + ); + } - if (username && password) { - n = snprintf( - connstring, len, "%s://%s:%s@%s:%d/", scheme, username, password, host, port + for (int i = 0; i < req->num_overrides; i++) { + snprintf( + temp_buf, + sizeof(temp_buf), + "%s=%s", + req->overrides[i].name, + req->overrides[i].value + ); + curl_url_set( + cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE ); - curl_free(username); - curl_free(password); - } else if (username) { - n = snprintf(connstring, len, "%s://%s@%s:%d/", scheme, username, host, port); - curl_free(username); + } + + curl_url_get(cu, CURLUPART_URL, &stream->url, 0); + curl_url_cleanup(cu); + + /* Configure CURL easy handle */ + curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); + curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); + curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); + curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); + curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); + curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, curl_verbose); + + if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) { + curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); + } + + if (stream->cancel) { + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFOFUNCTION, xferinfo_callback); + curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream); } else { - n = snprintf(connstring, len, "%s://%s:%d/", scheme, host, port); + curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); } - if (n < 0) { - goto cleanup; + if (stream->conn->dbname) { + snprintf( + temp_buf, sizeof(temp_buf), "%s: %s", DATABASE_HEADER, stream->conn->dbname + ); + stream->headers = curl_slist_append(NULL, temp_buf); + curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); + } + + /* POST body or MIME form */ + if (query->body != NULL) { + curl_easy_setopt( + stream->curl, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)query->body_len + ); + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->body); + } else if (query->num_params == 0) { + curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); + } else { + curl_mimepart* part; + + stream->form = curl_mime_init(stream->curl); + part = curl_mime_addpart(stream->form); + curl_mime_name(part, "query"); + curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); + + for (int i = 0; i < query->num_params; i++) { + part = curl_mime_addpart(stream->form); + snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); + curl_mime_name(part, temp_buf); + curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); + } + curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); } +} - conn->base_url = connstring; +static int +pump(HttpStream* stream) { + int running_handles; + CURLMcode mc; + CURLMsg* msg; + int msgs_left; - return conn; + if (stream->paused) { + stream->paused = false; + curl_easy_pause(stream->curl, CURLPAUSE_CONT); + } -cleanup: - snprintf(curl_error_buffer, CURL_ERROR_SIZE, "OOM"); - curl_error_happened = true; - if (connstring) { - free(connstring); + for (;;) { + mc = curl_multi_perform(stream->multi, &running_handles); + if (mc != CURLM_OK) { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup(curl_multi_strerror(mc)); + return -1; + } + + if (running_handles == 0) { + stream->transfer_done = true; + } + + /* Buffered mode waits for complete response. */ + if (stream->paused || stream->transfer_done) { + break; + } + + curl_multi_wait(stream->multi, NULL, 0, 100, NULL); + } + + curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, &stream->http_status); + curl_easy_getinfo( + stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time + ); + + while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { + if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { + if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { + stream->http_status = CH_HTTP_STATUS_CANCELED; + } else { + stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; + free(stream->error_msg); + stream->error_msg = strdup( + stream->error_buffer[0] != '\0' + ? stream->error_buffer + : curl_easy_strerror(msg->data.result) + ); + } + return -1; + } + } + + return 0; +} + +/* Blocking chunk reader; the decoder tracks its position within the chunk. */ +bool +ch_http_stream_next_chunk(void* ud, const void** data, size_t* len, char** error) { + HttpStream* stream = (HttpStream*)ud; + + *data = NULL; + *len = 0; + + /* + * ch_http_stream_begin leaves the first chunk buffered, so pump only when + * the buffer is empty + */ + while (stream->write_pos == 0) { + if (stream->transfer_done) { + return true; /* clean EOF */ + } + if (pump(stream) < 0) { + *error = stream->error_msg; + return false; + } + } + + *data = stream->buf; + *len = stream->write_pos; + /* Bytes stay readable until the next pump refills from offset 0 */ + stream->write_pos = 0; + return true; +} + +/* ---------------------------------------------------------------- + * Public API — lifecycle + * ---------------------------------------------------------------- + */ + +/* True for a status whose body the caller reads as an error message. */ +static bool +error_status(long status) { + /* Synthetic statuses carry no server body. */ + if (status == CH_HTTP_STATUS_CANCELED || status == CH_HTTP_STATUS_TRANSPORT_ERROR) { + return false; + } + + return status > 0 && status != CH_HTTP_STATUS_OK; +} + +/* + * ch_http_stream_end — clean up all owned resources. + */ +void +ch_http_stream_end(HttpStream* stream) { + if (!stream) { + return; } - if (conn) { - if (conn->dbname) { - free(conn->dbname); + if (stream->multi) { + if (stream->curl) { + curl_multi_remove_handle(stream->multi, stream->curl); } - free(conn); + curl_multi_cleanup(stream->multi); + } + + if (stream->curl) { + curl_easy_cleanup(stream->curl); + } + + if (stream->headers) { + curl_slist_free_all(stream->headers); + } + if (stream->form) { + curl_mime_free(stream->form); + } + if (stream->url) { + curl_free(stream->url); + } + if (stream->buf) { + free(stream->buf); + } + if (stream->error_msg) { + free(stream->error_msg); + } + + free(stream); +} + +/* + * ch_http_stream_begin — allocate and initialize a streaming HTTP query. + * Returns NULL on failure. + */ +HttpStream* +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req) { + HttpStream* stream; + uuid_t id; + + stream = calloc(1, sizeof(HttpStream)); + if (!stream) { + return NULL; + } + + stream->conn = conn; + stream->cancel = req->cancel; + stream->streaming = req->stream_chunks; + + /* Generate query ID */ + uuid_generate(id); + uuid_unparse(id, stream->query_id); + + /* + * Each HttpStream owns its easy handle, so concurrent foreign scans in + * subqueries or joins do not fight over one. + */ + stream->curl = curl_easy_init(); + if (!stream->curl) { + goto fail; } + /* Allocate stream buffer */ + stream->buf = (char*)malloc(INITIAL_BUF_SIZE); + if (!stream->buf) { + goto fail; + } + stream->buf_allocated = INITIAL_BUF_SIZE; + stream->buf[0] = '\0'; + + setup_curl(stream, req); + + /* Create multi handle and kick off the transfer */ + stream->multi = curl_multi_init(); + if (!stream->multi) { + goto fail; + } + curl_multi_add_handle(stream->multi, stream->curl); + + pump(stream); + /* Error bodies are reported whole, so stop streaming and buffer the rest. */ + if (stream->streaming && error_status(stream->http_status)) { + stream->streaming = false; + pump(stream); + } + + return stream; + +fail: + ch_http_stream_end(stream); return NULL; } +/* ---------------------------------------------------------------- + * Public API — accessors + * ---------------------------------------------------------------- + */ +char* +ch_http_stream_buffer(HttpStream* stream) { + return stream->buf; +} + +size_t +ch_http_stream_available(HttpStream* stream) { + return stream->write_pos; +} + +long +ch_http_stream_status(HttpStream* stream) { + return stream->http_status; +} + +const char* +ch_http_stream_query_id(HttpStream* stream) { + return stream->query_id; +} + +const char* +ch_http_stream_error(HttpStream* stream) { + return stream->error_msg; +} + +double +ch_http_stream_request_time(HttpStream* stream) { + return stream->pretransfer_time * 1000; +} + /* - * ch_http_simple_query — buffer the full response in memory. + * take_body — transfer ownership of the response body. * - * Built on top of the streaming driver with an effectively-unbounded - * fetch_size, so the whole response lands in one batch that we then hand off - * to the caller as a ch_http_response_t. + * On return, *out_data is a malloc()'d buffer the caller must free(). When + * status is CH_HTTP_STATUS_TRANSPORT_ERROR the body is the strdup'd libcurl + * error message; otherwise it is the accumulated response bytes, NUL + * terminated. *out_size is set to the length in bytes, excluding the NUL. + * Sets *out_data to NULL and *out_size to 0 when there is nothing to hand + * off. Safe to call at most once per stream; the stream itself should still + * be released with ch_http_stream_end(). + */ +static void +take_body(HttpStream* stream, char** out_data, size_t* out_size) { + if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { + *out_data = stream->error_msg; + *out_size = strlen(stream->error_msg); + stream->error_msg = NULL; + return; + } + + if (stream->write_pos == 0 || !stream->buf) { + *out_data = NULL; + *out_size = 0; + return; + } + + *out_data = stream->buf; + *out_size = stream->write_pos; + stream->buf = NULL; +} + +/* + * ch_http_simple_query — buffer the full TabSeparated response in memory. + * + * Server default format is TabSeparated, so only its dialect needs pinning: + * ISO timestamps, \N for NULL and LF line ends, as the text parsers expect. */ ch_http_response_t* -ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { +ch_http_simple_query( + ch_http_connection_t* conn, + const ch_query* query, + ch_cancel_check cancel +) { + static const ch_setting tsv_overrides[] = { + { "date_time_output_format", "iso" }, + { "format_tsv_null_representation", "\\N" }, + { "output_format_tsv_crlf_end_of_line", "0" }, + }; + const ch_http_request req = { .query = query, + .overrides = tsv_overrides, + .num_overrides = lengthof(tsv_overrides), + .cancel = cancel }; HttpStream* stream; ch_http_response_t* resp; - stream = ch_http_stream_begin(conn, query, INT32_MAX); + stream = ch_http_stream_begin(conn, &req); if (stream == NULL) { return NULL; } @@ -210,11 +679,9 @@ ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { return NULL; } - resp->http_status = ch_http_stream_status(stream); - resp->pretransfer_time = ch_http_stream_request_time(stream) / 1000.0; - resp->total_time = ch_http_stream_total_time(stream) / 1000.0; - memcpy(resp->query_id, ch_http_stream_query_id(stream), CH_HTTP_QUERY_ID_LEN); - ch_http_stream_take_body(stream, &resp->data, &resp->datasize); + resp->http_status = stream->http_status; + memcpy(resp->query_id, stream->query_id, CH_HTTP_QUERY_ID_LEN); + take_body(stream, &resp->data, &resp->datasize); if (curl_verbose && resp->http_status != CH_HTTP_STATUS_OK && resp->data) { fprintf(stderr, "%s", resp->data); @@ -224,24 +691,48 @@ ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query) { return resp; } +/* + * Return text before version mentioning + */ +char* +ch_http_format_error(char* errstring) { + size_t n = strlen(errstring); + + for (size_t i = 0; i < n; i++) { + if (strncmp(errstring + i, "version", 7) == 0) { + return pnstrdup(errstring, i - 2); + } + } + + /* + * For some reason ClickHouse 25.12 added a newline to an auth failure + * error. Strip it out. + */ + if (n > 0 && errstring[n - 1] == '\n') { + errstring[--n] = '\0'; + } + + return errstring; +} + /* * Fetches and caches the ClickHouse server version via SELECT version(). - * Writes 0 to all out-params when the version cannot be determined. Caches the - * result on the connection, so only the first call issues a query. + * Returns zeros when the version cannot be determined; a failed lookup counts + * as fetched, so it is not retried and its warning is raised once. */ -void -ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* patch) { - *major = *minor = *patch = 0; +ch_server_version +ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel) { + ch_server_version none = { 0, 0, 0 }; + if (conn == NULL) { - return; + return none; } - /* conn is calloc'd (see ch_http_connect), so version.major == 0 reliably - * means the version has not been fetched and cached yet. */ - if (conn->version.major == 0) { - ch_query query = { "SELECT version()", 0, NULL, NULL, NULL, NULL }; - ch_http_response_t* resp = ch_http_simple_query(conn, &query); + if (!conn->version_fetched) { + ch_query query = { .sql = "SELECT version()" }; + ch_http_response_t* resp = ch_http_simple_query(conn, &query, cancel); + conn->version_fetched = true; if (resp != NULL) { if (resp->http_status == CH_HTTP_STATUS_OK && resp->data != NULL) { int parsed, v_tweak; @@ -270,42 +761,25 @@ ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* } if (parsed < 2) { /* Version string probably trash; zero out. */ - conn->version.major = 0; + conn->version = none; } - } else if (resp->http_status != CH_HTTP_STATUS_OK) { + } else if ( + resp->http_status != CH_HTTP_STATUS_OK && + resp->http_status != CH_HTTP_STATUS_NOT_FOUND + ) { + /* 404 means a missing database, which the query itself reports */ elog( WARNING, - "pg_clickhouse: SELECT version() failed (HTTP status %d): %.*s", + "pg_clickhouse: SELECT version() failed (HTTP status %d): %s", (int)resp->http_status, - resp->data ? (int)resp->datasize : 0, - resp->data ? resp->data : "" + resp->data ? ch_http_format_error(resp->data) : "" ); } ch_http_response_free(resp); } } - *major = conn->version.major; - *minor = conn->version.minor; - *patch = conn->version.patch; -} - -void -ch_http_close(ch_http_connection_t* conn) { - free(conn->base_url); - if (conn->dbname) { - free(conn->dbname); - } - curl_easy_cleanup(conn->curl); -} - -char* -ch_http_last_error(void) { - if (curl_error_happened) { - return curl_error_buffer; - } - - return NULL; + return conn->version; } void diff --git a/src/http_streaming.c b/src/http_streaming.c deleted file mode 100644 index daa8310d..00000000 --- a/src/http_streaming.c +++ /dev/null @@ -1,583 +0,0 @@ -/*------------------------------------------------------------------------- - * - * http_streaming.c - * Streaming HTTP query driver for pg_clickhouse. - * - * Uses curl_multi + CURL_WRITEFUNC_PAUSE to receive ClickHouse HTTP - * responses in row-aligned batches of approximately fetch_size bytes, - * keeping memory proportional to batch size instead of full result set. - * - * Copyright (c) 2025-2026, ClickHouse, Inc. - * - *------------------------------------------------------------------------- - */ -#include -#include -#include -#include - -#include - -#include "postgres.h" - -#include "http.h" -#include "http_streaming.h" -#include "internal.h" -#include "kv_list.h" - -#ifndef CURL_WRITEFUNC_ERROR -#define CURL_WRITEFUNC_ERROR 0xFFFFFFFF -#endif - -#define DATABASE_HEADER "X-ClickHouse-Database" -#define INITIAL_BUF_SIZE (64 * 1024) - -/* ---------------------------------------------------------------- - * HttpStream — opaque struct. - * ---------------------------------------------------------------- - */ -struct HttpStream { - /* Connection (borrowed, not owned) */ - ch_http_connection_t* conn; - - /* Owned CURL resources */ - CURL* curl; - CURLM* multi; - struct curl_slist* headers; - curl_mime* form; - char* url; /* allocated by curl_url_get, freed with - * curl_free */ - - /* Stream buffer */ - char* buf; - size_t buf_allocated; - size_t write_pos; - size_t parse_pos; - size_t batch_end; - int32 fetch_size; /* approximate batch size in bytes */ - bool paused; - bool started; - bool transfer_done; - char error_buffer[CURL_ERROR_SIZE]; - - /* Public state readable via C accessors */ - long http_status; - char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; - char* error_msg; /* strdup'd, freed with free() */ -}; - -/* Forward declarations of static helpers */ -static void -setup_curl(HttpStream* stream, const ch_query* query); -static void -capture_transfer_info(HttpStream* stream); -static void -compact_buffer(HttpStream* stream); -static size_t -find_batch_end(const HttpStream* stream); -static size_t -write_callback(void* contents, size_t size, size_t nmemb, void* userp); - -/* ---------------------------------------------------------------- - * setup_curl — configure the CURL easy handle for this query. - * Mirrors the setup portion of ch_http_simple_query() in http.c. - * ---------------------------------------------------------------- - */ -static void -setup_curl(HttpStream* stream, const ch_query* query) { - CURLU* cu = curl_url(); - char temp_buf[512]; - - /* Build URL with query_id and settings */ - curl_url_set(cu, CURLUPART_URL, stream->conn->base_url, 0); - - snprintf(temp_buf, sizeof(temp_buf), "query_id=%s", stream->query_id); - curl_url_set(cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE); - - kv_iter iter = new_kv_iter(query->settings); - while (kv_iter_next(&iter)) { - if (strcmp(iter.name, "date_time_output_format") == 0 || - strcmp(iter.name, "format_tsv_null_representation") == 0 || - strcmp(iter.name, "output_format_tsv_crlf_end_of_line") == 0) { - continue; - } - snprintf(temp_buf, sizeof(temp_buf), "%s=%s", iter.name, iter.value); - curl_url_set( - cu, CURLUPART_QUERY, temp_buf, CURLU_APPENDQUERY | CURLU_URLENCODE - ); - } - - curl_url_set( - cu, - CURLUPART_QUERY, - "date_time_output_format=iso", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_set( - cu, - CURLUPART_QUERY, - "format_tsv_null_representation=\\N", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_set( - cu, - CURLUPART_QUERY, - "output_format_tsv_crlf_end_of_line=0", - CURLU_APPENDQUERY | CURLU_URLENCODE - ); - curl_url_get(cu, CURLUPART_URL, &stream->url, 0); - curl_url_cleanup(cu); - - /* Configure CURL easy handle */ - curl_easy_setopt(stream->curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(stream->curl, CURLOPT_WRITEDATA, stream); - curl_easy_setopt(stream->curl, CURLOPT_ERRORBUFFER, stream->error_buffer); - curl_easy_setopt(stream->curl, CURLOPT_PATH_AS_IS, 1L); - curl_easy_setopt(stream->curl, CURLOPT_URL, stream->url); - curl_easy_setopt(stream->curl, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(stream->curl, CURLOPT_VERBOSE, ch_http_get_verbose()); - - if (stream->conn->ssl_version != CURL_SSLVERSION_DEFAULT) { - curl_easy_setopt(stream->curl, CURLOPT_SSLVERSION, stream->conn->ssl_version); - } - - if (ch_http_get_progress_func()) { - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt( - stream->curl, CURLOPT_XFERINFOFUNCTION, ch_http_get_progress_func() - ); - curl_easy_setopt(stream->curl, CURLOPT_XFERINFODATA, stream->conn); - } else { - curl_easy_setopt(stream->curl, CURLOPT_NOPROGRESS, 1L); - } - - if (stream->conn->dbname) { - snprintf( - temp_buf, sizeof(temp_buf), "%s: %s", DATABASE_HEADER, stream->conn->dbname - ); - stream->headers = curl_slist_append(NULL, temp_buf); - curl_easy_setopt(stream->curl, CURLOPT_HTTPHEADER, stream->headers); - } - - /* POST body or MIME form */ - if (query->num_params == 0) { - curl_easy_setopt(stream->curl, CURLOPT_POSTFIELDS, query->sql); - } else { - curl_mimepart* part; - - stream->form = curl_mime_init(stream->curl); - part = curl_mime_addpart(stream->form); - curl_mime_name(part, "query"); - curl_mime_data(part, query->sql, CURL_ZERO_TERMINATED); - - for (int i = 0; i < query->num_params; i++) { - part = curl_mime_addpart(stream->form); - snprintf(temp_buf, sizeof(temp_buf), "param_p%d", i + 1); - curl_mime_name(part, temp_buf); - curl_mime_data(part, query->param_values[i], CURL_ZERO_TERMINATED); - } - curl_easy_setopt(stream->curl, CURLOPT_MIMEPOST, stream->form); - } -} - -/* ---------------------------------------------------------------- - * write_callback — CURL write callback. Appends data to the stream - * buffer and asks CURL to pause receipt once a row-aligned batch of - * approximately fetch_size bytes is buffered. - * ---------------------------------------------------------------- - */ -static size_t -write_callback(void* contents, size_t size, size_t nmemb, void* userp) { - size_t realsize = size * nmemb; - HttpStream* self = (HttpStream*)userp; - size_t needed = self->write_pos + realsize + 1; - - /* Grow buffer if needed */ - if (needed > self->buf_allocated) { - size_t newsize = self->buf_allocated * 2; - char* newbuf; - - if (newsize < needed) { - newsize = needed; - } - - newbuf = (char*)realloc(self->buf, newsize); - if (!newbuf) { - return CURL_WRITEFUNC_ERROR; - } - - self->buf = newbuf; - self->buf_allocated = newsize; - } - - memcpy(self->buf + self->write_pos, contents, realsize); - self->write_pos += realsize; - self->buf[self->write_pos] = '\0'; - - /* - * Once we have buffered at least fetch_size bytes AND at least one - * newline (so a row-aligned batch is ready), pause receipt. Pausing on - * byte-count alone can starve the parser of the newline it needs when - * fetch_size is small. We accept this chunk first and pause afterward via - * curl_easy_pause so CURL does not redeliver bytes we already hold. - */ - if (self->write_pos >= (size_t)self->fetch_size && - memchr(self->buf, '\n', self->write_pos) != NULL) { - self->paused = true; - curl_easy_pause(self->curl, CURLPAUSE_RECV); - } - - return realsize; -} - -/* ---------------------------------------------------------------- - * capture_transfer_info — grab HTTP status and timing from CURL. - * ---------------------------------------------------------------- - */ -static void -capture_transfer_info(HttpStream* stream) { - curl_easy_getinfo(stream->curl, CURLINFO_RESPONSE_CODE, &stream->http_status); - curl_easy_getinfo( - stream->curl, CURLINFO_PRETRANSFER_TIME, &stream->pretransfer_time - ); - curl_easy_getinfo(stream->curl, CURLINFO_TOTAL_TIME, &stream->total_time); - stream->started = true; -} - -/* ---------------------------------------------------------------- - * find_batch_end — find a row-aligned split point near fetch_size - * bytes. Looks forward then backward for nearest newline. Never - * returns a partial row: if no newline is buffered, returns 0 so - * the caller keeps ingesting. - * ---------------------------------------------------------------- - */ -static size_t -find_batch_end(const HttpStream* stream) { - const char* base = stream->buf; - const char* found; - - if (stream->fetch_size <= 0) { - return stream->write_pos; - } - - if (stream->write_pos >= (size_t)stream->fetch_size) { - /* Look forward first */ - found = memchr( - base + stream->fetch_size, '\n', stream->write_pos - stream->fetch_size - ); - if (found) { - return (found - base) + 1; - } - - /* Look backward */ - for (size_t i = stream->fetch_size; i > 0; i--) { - if (base[i - 1] == '\n') { - return i; - } - } - } - - if (stream->transfer_done) { - return stream->write_pos; - } - - return 0; -} - -/* ---------------------------------------------------------------- - * compact_buffer — shift unparsed data to the front of the buffer. - * ---------------------------------------------------------------- - */ -static void -compact_buffer(HttpStream* stream) { - if (stream->parse_pos > 0) { - size_t remaining = stream->write_pos - stream->parse_pos; - - memmove(stream->buf, stream->buf + stream->parse_pos, remaining); - stream->write_pos = remaining; - stream->parse_pos = 0; - stream->batch_end = 0; - stream->buf[stream->write_pos] = '\0'; - } -} - -/* ---------------------------------------------------------------- - * http_stream_pump — drive curl_multi until the next batch is ready or the - * transfer completes. Returns 0 on success, -1 on error. - * ---------------------------------------------------------------- - */ -int -ch_http_stream_pump(HttpStream* stream) { - int running_handles; - CURLMcode mc; - CURLMsg* msg; - int msgs_left; - - /* - * Drop the already-consumed batch and see if there is enough buffered - * data for the next one before touching the network again. - */ - if (stream->parse_pos > 0) { - compact_buffer(stream); - } - - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || - (stream->transfer_done && stream->write_pos <= stream->parse_pos)) { - if (!stream->started) { - capture_transfer_info(stream); - } - return 0; - } - - /* Resume if paused from a previous batch */ - if (stream->paused) { - stream->paused = false; - curl_easy_pause(stream->curl, CURLPAUSE_CONT); - } - - /* Drive the transfer */ - for (;;) { - mc = curl_multi_perform(stream->multi, &running_handles); - if (mc != CURLM_OK) { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = strdup(curl_multi_strerror(mc)); - return -1; - } - - if (running_handles == 0) { - stream->transfer_done = true; - } - - stream->batch_end = find_batch_end(stream); - if (stream->batch_end > 0 || stream->paused || stream->transfer_done) { - break; - } - - curl_multi_wait(stream->multi, NULL, 0, 100, NULL); - } - - capture_transfer_info(stream); - stream->batch_end = find_batch_end(stream); - - /* Check for transfer errors */ - while ((msg = curl_multi_info_read(stream->multi, &msgs_left))) { - if (msg->msg == CURLMSG_DONE && msg->data.result != CURLE_OK) { - if (msg->data.result == CURLE_ABORTED_BY_CALLBACK) { - stream->http_status = CH_HTTP_STATUS_CANCELED; - } else { - stream->http_status = CH_HTTP_STATUS_TRANSPORT_ERROR; - free(stream->error_msg); - stream->error_msg = strdup( - stream->error_buffer[0] != '\0' - ? stream->error_buffer - : curl_easy_strerror(msg->data.result) - ); - } - return -1; - } - } - - return 0; -} - -/* ---------------------------------------------------------------- - * Public API — lifecycle - * ---------------------------------------------------------------- - */ - -/* - * ch_http_stream_begin — allocate and initialize a streaming HTTP query. - * Returns NULL on failure. - */ -HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - int32 fetch_size -) { - HttpStream* stream; - uuid_t id; - - stream = calloc(1, sizeof(HttpStream)); - if (!stream) { - return NULL; - } - - stream->conn = conn; - stream->fetch_size = fetch_size; - - /* Generate query ID */ - uuid_generate(id); - uuid_unparse(id, stream->query_id); - - /* - * Create our own CURL easy handle so that multiple HttpStream instances - * (e.g. concurrent foreign scans in subqueries or joins) do not fight - * over the single handle in conn->curl. - */ - stream->curl = curl_easy_init(); - if (!stream->curl) { - goto fail; - } - - /* Allocate stream buffer */ - stream->buf = (char*)malloc(INITIAL_BUF_SIZE); - if (!stream->buf) { - goto fail; - } - stream->buf_allocated = INITIAL_BUF_SIZE; - stream->buf[0] = '\0'; - - setup_curl(stream, query); - - /* Create multi handle and kick off the transfer */ - stream->multi = curl_multi_init(); - if (!stream->multi) { - goto fail; - } - curl_multi_add_handle(stream->multi, stream->curl); - - /* Pump until first batch is ready or transfer completes */ - ch_http_stream_pump(stream); - - return stream; - -fail: - ch_http_stream_end(stream); - return NULL; -} - -/* - * ch_http_stream_end — clean up all owned resources. - */ -void -ch_http_stream_end(HttpStream* stream) { - if (!stream) { - return; - } - - if (stream->multi) { - if (stream->curl) { - curl_multi_remove_handle(stream->multi, stream->curl); - } - curl_multi_cleanup(stream->multi); - } - - if (stream->curl) { - curl_easy_cleanup(stream->curl); - } - - if (stream->headers) { - curl_slist_free_all(stream->headers); - } - if (stream->form) { - curl_mime_free(stream->form); - } - if (stream->url) { - curl_free(stream->url); - } - if (stream->buf) { - free(stream->buf); - } - if (stream->error_msg) { - free(stream->error_msg); - } - - free(stream); -} - -/* ---------------------------------------------------------------- - * Public API — accessors - * ---------------------------------------------------------------- - */ -char* -ch_http_stream_buffer(HttpStream* stream) { - return stream->buf + stream->parse_pos; -} - -size_t -ch_http_stream_available(HttpStream* stream) { - return stream->batch_end > stream->parse_pos ? stream->batch_end - stream->parse_pos - : 0; -} - -void -ch_http_stream_advance(HttpStream* stream, size_t n) { - stream->parse_pos += n; - if (stream->parse_pos > stream->batch_end) { - stream->parse_pos = stream->batch_end; - } -} - -bool -ch_http_stream_transfer_done(HttpStream* stream) { - return stream->transfer_done && (stream->write_pos <= stream->parse_pos); -} - -long -ch_http_stream_status(HttpStream* stream) { - return stream->http_status; -} - -const char* -ch_http_stream_query_id(HttpStream* stream) { - return stream->query_id; -} - -const char* -ch_http_stream_error(HttpStream* stream) { - return stream->error_msg; -} - -double -ch_http_stream_request_time(HttpStream* stream) { - return stream->pretransfer_time * 1000; -} - -double -ch_http_stream_total_time(HttpStream* stream) { - return stream->total_time * 1000; -} - -/* - * ch_http_stream_take_body — transfer ownership of the response body. - * - * On return, *out_data is a malloc()'d buffer the caller must free(). When - * status is CH_HTTP_STATUS_TRANSPORT_ERROR the body is the strdup'd libcurl - * error message; otherwise it is the accumulated response bytes (shifted to - * offset 0 and NUL-terminated). *out_size is set to the length in bytes, - * excluding the NUL. Sets *out_data to NULL and *out_size to 0 when there is - * nothing to hand off. Safe to call at most once per stream; the stream - * itself should still be released with ch_http_stream_end(). - */ -void -ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size) { - size_t avail; - - if (stream->http_status == CH_HTTP_STATUS_TRANSPORT_ERROR && stream->error_msg) { - *out_data = stream->error_msg; - *out_size = strlen(stream->error_msg); - stream->error_msg = NULL; - return; - } - - avail = ch_http_stream_available(stream); - if (avail == 0 || !stream->buf) { - *out_data = NULL; - *out_size = 0; - return; - } - - if (stream->parse_pos > 0) { - memmove(stream->buf, stream->buf + stream->parse_pos, avail); - } - stream->buf[avail] = '\0'; - - *out_data = stream->buf; - *out_size = avail; - stream->buf = NULL; -} diff --git a/src/include/cursor.h b/src/include/cursor.h new file mode 100644 index 00000000..374ef6c7 --- /dev/null +++ b/src/include/cursor.h @@ -0,0 +1,70 @@ +/* + * cursor.h + * + * Cursor for an open ClickHouse Native result + * + * Driver provides response and initializes reader. Cursor validates columns, + * converts rows, and releases resources + */ + +#ifndef CLICKHOUSE_CURSOR_H +#define CLICKHOUSE_CURSOR_H + +#include "postgres.h" + +#include "fdw.h" +#include "pg-clickhouse-decode.h" + +typedef struct ch_cursor { + MemoryContext memcxt; /* Delete this context to close cursor */ + MemoryContextCallback callback; + + pgch_reader reader; + void* response; /* Driver response read by reader */ + void (*free_response)(void* response); + /* + * Report a driver error, such as cancellation, before reporting a decoder + * error. Leave unset if driver needs no special handling + */ + void (*raise_response_error)(ch_cursor* cursor); + + void* conn; + char* query; + double request_time; + size_t columns_count; + /* One conversion state and destination attribute per returned column */ + void** conversion_states; + int* fill_dest; +} ch_cursor; + +/* Describes driver response and how cursor reads it */ +typedef struct ch_cursor_source { + void* response; + /* Initialize reader from blocks or byte chunks */ + void (*init_reader)(pgch_reader* reader, void* response); + /* Accept NULL because error callback may release response first */ + void (*free_response)(void* response); + /* Report driver-specific errors, or NULL if not needed */ + void (*raise_response_error)(ch_cursor* cursor); +} ch_cursor_source; + +/* + * Open cursor and take ownership of src->response. Cursor context releases + * response even if opening fails. Delete cursor->memcxt to close cursor + */ +extern ch_cursor* +chfdw_cursor_open(void* conn, const ch_query* query, const ch_cursor_source* src); + +/* Fetch next row for either driver */ +extern Datum* +chfdw_cursor_fetch_row(ChFdwScanRowContext* ctx); + +/* + * Read remaining rows and return tab-separated text. Use \N for null values. + * Return NULL if there are no rows. Output can differ from HTTP driver because + * PostgreSQL, rather than ClickHouse, formats each value + */ +extern text* +chfdw_cursor_render_tsv(ch_cursor* cursor); + +#endif /* CLICKHOUSE_CURSOR_H */ diff --git a/src/include/engine.h b/src/include/engine.h index eee488fb..e99256d8 100644 --- a/src/include/engine.h +++ b/src/include/engine.h @@ -36,6 +36,12 @@ typedef struct { * when not specified */ } ch_connection_details; +/* + * Polled by a transport while a request is in flight; return true to abort it. + * NULL means the request cannot be cancelled. + */ +typedef bool (*ch_cancel_check)(void); + /* * ch_query an SQL query to execute on ClickHouse. */ @@ -52,9 +58,14 @@ typedef struct { const List* attr_nums; /* List of settings to pass to ClickHouse upon execution. */ const kv_list* settings; + /* Posted verbatim, already prefixed with sql; sql stays set for errors. */ + const void* body; + const size_t body_len; } ch_query; #define new_query(sql, num, vals, tupdesc, attrs) \ { sql, num, vals, tupdesc, attrs, chfdw_get_session_settings() } +#define new_body_query(sql, body, len) \ + { sql, 0, NULL, NULL, NULL, chfdw_get_session_settings(), body, len } #endif /* CLICKHOUSE_ENGINE_H */ diff --git a/src/include/fdw.h b/src/include/fdw.h index 5509acb1..115f8b0c 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -41,23 +41,8 @@ */ #define CH_ESCAPED_NAMEDATALEN NAMEDATALEN * 2 -/* pglink.c */ +/* cursor.h: an open Native result, whichever driver produced it */ typedef struct ch_cursor ch_cursor; -typedef struct ch_cursor { - MemoryContext memcxt; /* used for cleanup */ - MemoryContextCallback callback; - - void* query_response; - void* read_state; - void* conn; - char* query; - double request_time; - double total_time; - size_t columns_count; - /* for binary, per returned column: conversion state, target attribute */ - void** conversion_states; - int* fill_dest; -} ch_cursor; typedef struct ChFdwScanRowContext { TupleDesc tupdesc; /* tuple descriptor for row */ @@ -71,7 +56,7 @@ typedef struct ChFdwScanRowContext { typedef void (*disconnect_method)(void* conn); typedef void (*check_conn_method)(const char* password, UserMapping* user); typedef ch_cursor* (*simple_query_method)(void* conn, const ch_query* query); -typedef void (*simple_insert_method)(void* conn, const ch_query* query); +typedef text* (*raw_query_method)(void* conn, const ch_query* query); typedef Datum* (*cursor_fetch_row_method)(ChFdwScanRowContext* ctx); typedef void* (*prepare_insert_method)( void* conn, @@ -82,11 +67,7 @@ typedef void* (*prepare_insert_method)( ); typedef void (*insert_tuple_method)(void* state, TupleTableSlot* slot); typedef void (*finalize_insert_method)(void* state); -typedef ch_cursor* (*streaming_query_method)( - void* conn, - const ch_query* query, - int32 fetch_size -); +typedef ch_cursor* (*streaming_query_method)(void* conn, const ch_query* query); typedef bool (*is_broken_method)(const void* conn); typedef ch_server_version (*server_version_method)(void* conn); @@ -94,6 +75,7 @@ typedef ch_server_version (*server_version_method)(void* conn); typedef struct { disconnect_method disconnect; simple_query_method simple_query; + raw_query_method raw_query; cursor_fetch_row_method fetch_row; prepare_insert_method prepare_insert; insert_tuple_method insert_tuple; @@ -108,7 +90,6 @@ typedef struct { typedef struct { libclickhouse_methods* methods; void* conn; - bool is_binary; } ch_connection; ch_connection_details* @@ -124,10 +105,6 @@ chfdw_binary_connect(ch_connection_details* details); */ ch_server_version chfdw_get_server_version(UserMapping* user); -text* -chfdw_http_fetch_raw_data(ch_cursor* cursor); -text* -chfdw_binary_fetch_raw_data(ch_cursor* cursor); List* chfdw_construct_create_tables(ImportForeignSchemaStmt* stmt, ForeignServer* server); char* @@ -202,8 +179,6 @@ typedef struct CHFdwRelationInfo { ForeignServer* server; UserMapping* user; /* only set in use_remote_estimate mode */ - int32 fetch_size; /* fetch size for this remote table */ - /* * Name of the relation while EXPLAINing ForeignScan. It is used for join * relations but is set for all relations. For join relation, the name @@ -286,10 +261,6 @@ extern ch_scan_connection chfdw_get_scan_connection(UserMapping* user); extern void chfdw_release_scan_connection(UserMapping* user, ch_scan_connection sconn); -extern void -chfdw_exec_query(ch_connection conn, const char* query); -extern void -chfdw_report_error(int elevel, ch_connection conn, bool clear, const char* sql); /* in option.c */ extern kv_list* diff --git a/src/include/http.h b/src/include/http.h index 6a4089b9..b43520b3 100644 --- a/src/include/http.h +++ b/src/include/http.h @@ -4,8 +4,7 @@ #include "postgres.h" #include "engine.h" -#include "lib/stringinfo.h" -#include "nodes/pg_list.h" +#include "server_version.h" #include #define CH_HTTP_QUERY_ID_LEN 37 @@ -15,63 +14,103 @@ * and libcurl transport failures through the existing response machinery. */ #define CH_HTTP_STATUS_OK 200L +#define CH_HTTP_STATUS_NOT_FOUND 404L #define CH_HTTP_STATUS_CANCELED 418L #define CH_HTTP_STATUS_TRANSPORT_ERROR 419L typedef struct ch_http_connection_t ch_http_connection_t; + +/* + * Opaque handle to one in-flight HTTP query. The real type is the HttpStream + * struct, defined in http.c. + */ +typedef struct HttpStream HttpStream; + +/* A ClickHouse setting the caller sends as a URL parameter. */ +typedef struct ch_setting { + const char* name; + const char* value; +} ch_setting; + +/* + * One HTTP request: the SQL to run plus the response policy its caller needs. + * Overrides win over a query setting of the same name, so a caller pins what + * its decoder requires while user settings fill in the rest. + */ +typedef struct ch_http_request { + const ch_query* query; + const ch_setting* overrides; + int num_overrides; + /* Hand out the body one receive chunk at a time, else buffer it whole */ + bool stream_chunks; + ch_cancel_check cancel; /* NULL leaves the transfer uninterruptible */ +} ch_http_request; + +/* Response body buffered whole, for callers with no incremental decoder. */ typedef struct ch_http_response_t { char* data; size_t datasize; long http_status; char query_id[CH_HTTP_QUERY_ID_LEN]; - double pretransfer_time; - double total_time; } ch_http_response_t; -typedef enum { CH_CONT, CH_EOL, CH_EOF } ch_read_status; - -typedef struct { - char* data; - size_t datalen; - size_t curpos; - StringInfoData val; - bool done; - bool is_null; /* set when the parser saw the wire NULL - * marker `\N` for the field just read */ -} ch_http_read_state; - -typedef struct { - StringInfoData sql; - char* sql_begin; /* beginning part of constructed sql */ - List* target_attrs; /* list of target attribute numbers */ - int p_nums; /* number of parameters to transmit */ - ch_http_connection_t* conn; -} ch_http_insert_state; - -void -ch_http_init(int verbose, uint32_t query_id_prefix); void -ch_http_set_progress_func(curl_xferinfo_callback progressfunc); -curl_xferinfo_callback -ch_http_get_progress_func(void); -long -ch_http_get_verbose(void); +ch_http_init(int verbose); +/* Returns NULL and sets *error to a static message on failure. */ ch_http_connection_t* -ch_http_connection(ch_connection_details* details); +ch_http_connection(ch_connection_details* details, const char** error); void ch_http_close(ch_http_connection_t* conn); -ch_http_response_t* -ch_http_simple_query(ch_http_connection_t* conn, const ch_query* query); + +/* lifecycle */ +HttpStream* +ch_http_stream_begin(ch_http_connection_t* conn, const ch_http_request* req); void -ch_http_server_version(ch_http_connection_t* conn, int* major, int* minor, int* patch); +ch_http_stream_end(HttpStream* stream); + +/* + * pgch_chunk_source next_chunk over the response body. Bytes stay valid until + * the following call. Sets *len 0 at clean EOF; returns false with *error on + * transport failure or cancellation. Takes void* so it can be assigned to the + * callback slot without this header knowing pg-clickhouse-c. + */ +bool +ch_http_stream_next_chunk(void* stream, const void** data, size_t* len, char** error); + +/* accessors — let pglink.c read stream state without seeing the struct */ char* -ch_http_last_error(void); +ch_http_stream_buffer(HttpStream* stream); +size_t +ch_http_stream_available(HttpStream* stream); +long +ch_http_stream_status(HttpStream* stream); +const char* +ch_http_stream_query_id(HttpStream* stream); +const char* +ch_http_stream_error(HttpStream* stream); +double +ch_http_stream_request_time(HttpStream* stream); + +/* + * Trim the ClickHouse build version off an error body so messages stay stable + * across servers. May return errstring itself, modified in place. + */ +char* +ch_http_format_error(char* errstring); + +ch_http_response_t* +ch_http_simple_query( + ch_http_connection_t* conn, + const ch_query* query, + ch_cancel_check cancel +); +/* + * Fetch and cache the server version, returning {0, 0, 0} when it cannot be + * determined. Only the first call issues a query. + */ +ch_server_version +ch_http_server_version(ch_http_connection_t* conn, ch_cancel_check cancel); -/* read */ -void -ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen); -int -ch_http_read_next(ch_http_read_state* state, bool is_array); void ch_http_response_free(ch_http_response_t* resp); diff --git a/src/include/http_streaming.h b/src/include/http_streaming.h deleted file mode 100644 index 135ddaad..00000000 --- a/src/include/http_streaming.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef CLICKHOUSE_HTTP_STREAMING_H -#define CLICKHOUSE_HTTP_STREAMING_H - -#include "engine.h" - -typedef struct ch_http_connection_t ch_http_connection_t; - -/* - * Opaque handle to a streaming HTTP query. The real type is the HttpStream - * struct, defined in http_streaming.c. - */ -typedef struct HttpStream HttpStream; - -/* lifecycle */ -HttpStream* -ch_http_stream_begin( - ch_http_connection_t* conn, - const ch_query* query, - int32 fetch_size -); -int -ch_http_stream_pump(HttpStream* stream); -void -ch_http_stream_end(HttpStream* stream); - -/* accessors — let pglink.c read stream state without seeing the struct */ -char* -ch_http_stream_buffer(HttpStream* stream); -size_t -ch_http_stream_available(HttpStream* stream); -void -ch_http_stream_advance(HttpStream* stream, size_t n); -bool -ch_http_stream_transfer_done(HttpStream* stream); -long -ch_http_stream_status(HttpStream* stream); -const char* -ch_http_stream_query_id(HttpStream* stream); -const char* -ch_http_stream_error(HttpStream* stream); -double -ch_http_stream_request_time(HttpStream* stream); -double -ch_http_stream_total_time(HttpStream* stream); - -/* - * Transfer ownership of the response body to the caller. On return, *out_data - * is a malloc()'d buffer (or the strdup'd transport error message when status - * is CH_HTTP_STATUS_TRANSPORT_ERROR) that the caller must free(). The stream - * itself is unchanged otherwise and should still be released with - * ch_http_stream_end(). - */ -void -ch_http_stream_take_body(HttpStream* stream, char** out_data, size_t* out_size); - -#endif /* CLICKHOUSE_HTTP_STREAMING_H */ diff --git a/src/include/internal.h b/src/include/internal.h index c8e54bb1..d2208717 100644 --- a/src/include/internal.h +++ b/src/include/internal.h @@ -1,17 +1,17 @@ #ifndef CLICKHOUSE_INTERNAL_H #define CLICKHOUSE_INTERNAL_H -#include "curl/curl.h" +#include + #include "server_version.h" typedef struct ch_http_connection_t { - CURL* curl; char* dbname; - char* base_url; + char* base_url; /* built by curl_url_get, freed with curl_free */ long ssl_version; /* CURLOPT_SSLVERSION min; DEFAULT means unset */ - /* Server version, fetched lazily via SELECT version() then cached; a major - * of 0 means not fetched yet. */ + /* Server version, fetched lazily via SELECT version() then cached. */ ch_server_version version; + bool version_fetched; /* version lookup ran, even if it failed */ } ch_http_connection_t; typedef struct ch_binary_connection_t { diff --git a/src/option.c b/src/option.c index 130eb8cd..0a5d2252 100644 --- a/src/option.c +++ b/src/option.c @@ -83,8 +83,6 @@ static bool is_valid_option(const char* keyword, Oid context); static bool is_ch_option(const char* keyword); -static void -validate_fetch_size_option(DefElem* def); static bool parse_min_tls_version(const char* val, tls_version* out); @@ -138,7 +136,11 @@ clickhouse_fdw_validator(PG_FUNCTION_ARGS) { } if (strcmp(def->defname, "fetch_size") == 0) { - validate_fetch_size_option(def); + ereport( + WARNING, + errcode(ERRCODE_WARNING_DEPRECATED_FEATURE), + errmsg("option \"fetch_size\" is deprecated and ignored") + ); } if (strcmp(def->defname, "secure") == 0) { @@ -248,22 +250,6 @@ InitChFdwOptions(void) { popt++; } -static void -validate_fetch_size_option(DefElem* def) { - int fetch_size = pg_strtoint32(defGetString(def)); - - if (fetch_size < 0) { - ereport( - ERROR, - errcode(ERRCODE_FDW_INVALID_ATTRIBUTE_VALUE), - errmsg( - "invalid value for option \"%s\": %s", def->defname, defGetString(def) - ), - errhint("fetch_size must be greater than or equal to 0") - ); - } -} - /* * Map a min_tls_version option value to tls_version. Accepts the same spellings * as Postgres's ssl_min_protocol_version GUC, case-insensitively. Returns false diff --git a/src/parser.c b/src/parser.c deleted file mode 100644 index 45a0b050..00000000 --- a/src/parser.c +++ /dev/null @@ -1,294 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include - -static void -ch_http_read_array_string_literal(ch_http_read_state* state); -static void -ch_http_read_array(ch_http_read_state* state); -static int -ch_http_read_eof(ch_http_read_state* state); -static void -ch_http_parse_error(const char* msg); - -/* - * The streaming path can pass the parser a bounded slice rather than a - * NUL-terminated buffer, so reaching datalen is the equivalent of the old - * '\0' checks. - */ -inline static int -ch_http_read_eof(ch_http_read_state* state) { - state->done = true; - return CH_EOF; -} - -inline static void -ch_http_parse_error(const char* msg) { - ereport( - ERROR, - errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("pg_clickhouse: %s", msg) - ); -} - -void -ch_http_read_state_init(ch_http_read_state* state, char* data, size_t datalen) { - state->data = datalen > 0 ? data : NULL; - state->datalen = datalen; - state->curpos = 0; - state->done = false; - state->is_null = false; - if (state->val.data == NULL) { - initStringInfo(&state->val); - } else { - resetStringInfo(&state->val); - } -} - -/* - * Parse the next tab-separated value from state->data. Parses tab-delimited - * ClickHouse literals including unquoted strings and arrays. Returns CH_CONT - * if there are moe fields on the line to read, CH_EOL if it has reached the - * end of the line, and CH_EOF if it has reached the end of the file. - * - * `is_array` tells the parser whether the destination Postgres column is an - * array. TabSeparated is ambiguous: a `String` value beginning with `[` is not - * escaped on the wire and is indistinguishable byte-for-byte from an array - * literal until you know the column type. The caller knows the type, so it - * makes the call. - */ -int -ch_http_read_next(ch_http_read_state* state, bool is_array) { - char* data = state->data; - - if (state->done) { - return CH_EOF; - } - if (data == NULL) { - return ch_http_read_eof(state); - } - - resetStringInfo(&state->val); - state->is_null = false; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - /* - * Detect the wire NULL marker. ClickHouse's TabSeparated format encodes - * NULL as the bare 2-byte sequence `\N`. A literal backslash in data is - * sent as `\\`, so legitimate non-null output never contains `\N` at the - * start of a field followed by a delimiter. Detect it here, before - * unescaping, so a non-null String value whose unescaped content happens - * to be `\N` is not collapsed with the NULL marker. - */ - if (state->curpos + 1 < state->datalen && data[state->curpos] == '\\' && - data[state->curpos + 1] == 'N' && - (state->curpos + 2 == state->datalen || data[state->curpos + 2] == '\t' || - data[state->curpos + 2] == '\n')) { - state->is_null = true; - state->curpos += 2; - } else if (is_array && data[state->curpos] == '[') { - /* Parse array literal. */ - ch_http_read_array(state); - } else { - while (state->curpos < state->datalen && data[state->curpos] != '\t' && - data[state->curpos] != '\n') { - if (data[state->curpos] == '\\') { - state->curpos++; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - /* unescape some sequences */ - switch (data[state->curpos]) { - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'N': - /* NULL (format_tsv_null_representation) */ - appendStringInfoString(&state->val, "\\N"); - break; - default: - appendStringInfoChar(&state->val, data[state->curpos]); - } - state->curpos++; - } else { - appendStringInfoChar(&state->val, data[state->curpos++]); - } - } - } - - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - if (data[state->curpos] == '\t') { - /* There are more fields. */ - state->curpos++; - return CH_CONT; - } - - /* Should be at the end of the line or the file. */ - if (data[state->curpos] != '\n') { - ereport( - ERROR, - errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), - errmsg("unexpected byte (%d) after array", data[state->curpos]) - ); - } - - state->curpos++; - if (state->curpos >= state->datalen) { - return ch_http_read_eof(state); - } - - return CH_EOL; -} - -/* - * Convert a single-quoted string from a ClickHouse array to a double-quoted - * PostgreSQL array string. Based on `readQuotedStringFieldInto()` from - * `src/IO/ReadHelpers.cpp` in the ClickHouse source. The basic conversion is: - * - * - " => \" - * - \\ => \\ - * - \ followed by b, f, r, n, t, or 0 => literal char - * - \ followed by any other character (including ') => append that character - * - Append any other character - * - * https://clickhouse.com/docs/interfaces/formats/TabSeparated - * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO - */ -void -ch_http_read_array_string_literal(ch_http_read_state* state) { - /* Postgres array string is double-quoted. */ - appendStringInfoChar(&state->val, '"'); - state->curpos++; - - while (state->curpos < state->datalen && state->data[state->curpos] != '\'') { - char ch = state->data[state->curpos]; - - if (ch == '"') { - /* Escape double quotation mark. */ - appendStringInfoChar(&state->val, '\\'); - appendStringInfoChar(&state->val, ch); - state->curpos++; - } else if (ch == '\\') { - /* Emit the escaped character. */ - state->curpos++; - if (state->curpos >= state->datalen) { - ch_http_parse_error("invalid array string"); - } - - switch (state->data[state->curpos]) { - case '\\': - appendStringInfoChar(&state->val, state->data[state->curpos]); - appendStringInfoChar(&state->val, state->data[state->curpos]); - break; - case 'b': - appendStringInfoChar(&state->val, '\b'); - break; - case 'f': - appendStringInfoChar(&state->val, '\f'); - break; - case 'r': - appendStringInfoChar(&state->val, '\r'); - break; - case 'n': - appendStringInfoChar(&state->val, '\n'); - break; - case 't': - appendStringInfoChar(&state->val, '\t'); - break; - case '0': - appendStringInfoChar(&state->val, '\0'); - break; - default: - /* Includes ' and probably no other character. */ - appendStringInfoChar(&state->val, state->data[state->curpos]); - } - state->curpos++; - } else { - /* Append any other character. */ - appendStringInfoChar(&state->val, ch); - state->curpos++; - } - } - - if (state->curpos >= state->datalen || state->data[state->curpos] != '\'') { - ch_http_parse_error("invalid array string"); - } - - appendStringInfoChar(&state->val, '"'); - state->curpos++; -} - -/* - * Convert a ClickHouse array literal to a Postgres array literal. Supports - * nested array. The conversions are: - * - * - [ => { - * - ] => } - * - ' => Start of string, convert to double-quoted string - * - Append any other character - * - * Based on `readQuotedFieldInBracketsInto()` from `src/IO/ReadHelpers.cpp` in - * the ClickHouse source. Additional References: - * - * https://clickhouse.com/docs/interfaces/formats/TabSeparated - * https://www.postgresql.org/docs/current/arrays.html#ARRAYS-IO - */ -void -ch_http_read_array(ch_http_read_state* state) { - size_t balance = 1; - - /* Postgres arrays are wrapped in { and }. */ - appendStringInfoChar(&state->val, '{'); - state->curpos++; - - while (state->curpos < state->datalen && balance) { - switch (state->data[state->curpos]) { - case '\'': - ch_http_read_array_string_literal(state); - break; - case '[': - ++balance; - appendStringInfoChar(&state->val, '{'); - state->curpos++; - break; - case ']': - --balance; - appendStringInfoChar(&state->val, '}'); - state->curpos++; - break; - default: - appendStringInfoChar(&state->val, state->data[state->curpos]); - state->curpos++; - } - } - - if (balance != 0) { - ch_http_parse_error("malformed array literal"); - } -} diff --git a/src/pglink.c b/src/pglink.c index 504143ef..c04eced0 100644 --- a/src/pglink.c +++ b/src/pglink.c @@ -9,6 +9,7 @@ #include "parser/parse_coerce.h" #include "parser/parse_type.h" #include "utils/builtins.h" +#include "utils/date.h" #include "utils/fmgroids.h" #include "utils/lsyscache.h" #include "utils/syscache.h" @@ -17,9 +18,11 @@ #include "utils/uuid.h" #include "binary.h" +#include "cursor.h" #include "fdw.h" #include "http.h" -#include "http_streaming.h" +#include "pg-clickhouse-decode.h" +#include "pg-clickhouse-encode.h" #include #include @@ -27,43 +30,45 @@ static bool initialized = false; +/* Rows buffered for one Native INSERT over HTTP. */ +typedef struct { + char* sql; /* INSERT statement, for error reporting */ + char* sql_begin; /* sql plus the FORMAT clause the body follows */ + pgch_writer* writer; + AttrNumber* attnums; /* slot attribute feeding each column */ + Oid* atttypids; + size_t ncols; + ch_http_connection_t* conn; +} ch_http_insert_state; + static void http_disconnect(void* conn); -static ch_cursor* -http_simple_query(void* conn, const ch_query* query); -static ch_cursor* -http_streaming_query(void* conn, const ch_query* query, int32 fetch_size); +static text* +http_raw_query(void* conn, const ch_query* query); static void http_simple_insert(void* conn, const ch_query* query); +static ch_cursor* +http_native_cursor(void* conn, const ch_query* query); static void -http_cursor_free(void*); -static void -http_streaming_cursor_free(void*); -static Datum* -http_fetch_row(ChFdwScanRowContext* ctx); -static Datum* -http_streaming_fetch_row(ChFdwScanRowContext* ctx); -static Datum* -http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state); +http_native_read_error(ch_cursor* cursor); static void* http_prepare_insert(void*, ResultRelInfo*, List*, const ch_query*, char*); static void http_insert_tuple(void*, TupleTableSlot*); static void -char_to_datum(ChFdwScanRowContext* ctx, int attnum, char* data, size_t len); -static void report_http_stream_query_failure(void* conn, const ch_query* query, HttpStream* stream); static ch_server_version http_server_version(void* conn); static libclickhouse_methods http_methods = { .disconnect = http_disconnect, - .simple_query = http_simple_query, - .fetch_row = http_fetch_row, + .simple_query = http_native_cursor, + .raw_query = http_raw_query, + .fetch_row = chfdw_cursor_fetch_row, .prepare_insert = http_prepare_insert, .insert_tuple = http_insert_tuple, - .streaming_query = http_streaming_query, - .streaming_fetch_row = http_streaming_fetch_row, + .streaming_query = http_native_cursor, + .streaming_fetch_row = chfdw_cursor_fetch_row, .server_version = http_server_version, }; @@ -71,14 +76,12 @@ static void binary_disconnect(void* conn); static ch_cursor* binary_simple_query(void* conn, const ch_query* query); -static void -binary_cursor_free(void* cursor); +static text* +binary_raw_query(void* conn, const ch_query* query); static bool binary_is_broken(const void* conn); /* static void binary_simple_insert(void *conn, const char *query); */ -static Datum* -binary_fetch_row(ChFdwScanRowContext* ctx); static void binary_insert_tuple(void*, TupleTableSlot* slot); static void @@ -105,7 +108,8 @@ binary_server_version(void* conn); static libclickhouse_methods binary_methods = { .disconnect = binary_disconnect, .simple_query = binary_simple_query, - .fetch_row = binary_fetch_row, + .raw_query = binary_raw_query, + .fetch_row = chfdw_cursor_fetch_row, .prepare_insert = binary_prepare_insert, .insert_tuple = binary_insert_tuple, .finalize_insert = binary_finalize_insert, @@ -115,19 +119,10 @@ static libclickhouse_methods binary_methods = { .server_version = binary_server_version, }; -static int -http_progress_callback( - void* clientp, - curl_off_t dltotal, - curl_off_t dlnow, - curl_off_t ultotal, - curl_off_t ulnow -) { - if (ProcDiePending || QueryCancelPending) { - return 1; - } - - return 0; +/* ch_cancel_check for the HTTP transport, polled while a request is in flight. */ +static bool +http_canceled(void) { + return QueryCancelPending || ProcDiePending; } static bool @@ -144,10 +139,11 @@ ch_connection chfdw_http_connect(ch_connection_details* details) { ch_connection res; ch_http_connection_t* conn; + const char* error; if (!initialized) { initialized = true; - ch_http_init(0, (uint32_t)MyProcPid); + ch_http_init(0); } /* @@ -172,14 +168,8 @@ chfdw_http_connect(ch_connection_details* details) { } } - conn = ch_http_connection(details); + conn = ch_http_connection(details, &error); if (conn == NULL) { - char* error = ch_http_last_error(); - - if (error == NULL) { - error = "undefined"; - } - ereport( ERROR, errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), @@ -187,9 +177,8 @@ chfdw_http_connect(ch_connection_details* details) { ); } - res.conn = conn; - res.methods = &http_methods; - res.is_binary = false; + res.conn = conn; + res.methods = &http_methods; return res; } @@ -205,34 +194,7 @@ http_disconnect(void* conn) { static ch_server_version http_server_version(void* conn) { - ch_server_version v = { 0, 0, 0 }; - - ch_http_server_version((ch_http_connection_t*)conn, &v.major, &v.minor, &v.patch); - return v; -} - -/* - * Return text before version mentioning - */ -static char* -format_error(char* errstring) { - size_t n = strlen(errstring); - - for (size_t i = 0; i < n; i++) { - if (strncmp(errstring + i, "version", 7) == 0) { - return pnstrdup(errstring, i - 2); - } - } - - /* - * For some reason ClickHouse 25.12 added a newline to an auth failure - * error. Strip it out. - */ - if (n > 0 && errstring[n - 1] == '\n') { - errstring[--n] = '\0'; - } - - return errstring; + return ch_http_server_version((ch_http_connection_t*)conn, http_canceled); } static void @@ -246,8 +208,8 @@ kill_query(void* conn, const char* query_id) { NULL ); - ch_http_set_progress_func(NULL); - resp = ch_http_simple_query(conn, &query); + /* Not cancellable: it is the cleanup for a query already cancelled. */ + resp = ch_http_simple_query(conn, &query, NULL); if (resp != NULL) { ch_http_response_free(resp); } @@ -292,7 +254,7 @@ report_http_stream_query_failure( ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) @@ -304,23 +266,16 @@ report_http_stream_query_failure( PG_END_TRY(); } -static ch_cursor* -http_simple_query(void* conn, const ch_query* query) { +/* Whole response body as one text value, for clickhouse_raw_query() */ +static text* +http_raw_query(void* conn, const ch_query* query) { int attempts = 0; - /* - * volatile: changed after setjmp (PG_TRY) and read after longjmp - * (PG_CATCH); longjmp needn't restore register-cached locals, so a - * non-volatile such local has an indeterminate value per C setjmp rules. - */ - volatile MemoryContext tempcxt = NULL; - MemoryContext oldcxt; - ch_cursor* cursor; ch_http_response_t* resp; - - ch_http_set_progress_func(http_progress_callback); + /* volatile: assigned inside PG_TRY, so longjmp may leave it in a register */ + text* volatile result; again: - resp = ch_http_simple_query(conn, query); + resp = ch_http_simple_query(conn, query, http_canceled); if (resp == NULL) { ereport(ERROR, errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory")); } @@ -358,7 +313,7 @@ http_simple_query(void* conn, const ch_query* query) { ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) ); @@ -366,59 +321,32 @@ http_simple_query(void* conn, const ch_query* query) { PG_TRY(); { - /* - * If any palloc below throws, use PG_CATCH to free the Curl response. - */ - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES - ); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); - cursor->query = pstrdup(query->sql); - cursor->request_time = resp->pretransfer_time * 1000; - cursor->total_time = resp->total_time * 1000; - ch_http_read_state_init(cursor->read_state, resp->data, resp->datasize); - - cursor->memcxt = tempcxt; - cursor->callback.func = http_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - MemoryContextSwitchTo(oldcxt); - } - PG_CATCH(); - { - if (resp) { - ch_http_response_free(resp); - } - if (tempcxt) { - MemoryContextDelete(tempcxt); - } - PG_RE_THROW(); + result = + resp->data ? cstring_to_text_with_len(resp->data, resp->datasize) : NULL; } + PG_FINALLY(); + { ch_http_response_free(resp); } PG_END_TRY(); - return cursor; + return result; } static void http_simple_insert(void* conn, const ch_query* query) { - ch_http_response_t* resp = ch_http_simple_query(conn, query); + ch_http_response_t* resp = ch_http_simple_query(conn, query, http_canceled); if (resp == NULL) { - char* error = ch_http_last_error(); + ereport(ERROR, errcode(ERRCODE_FDW_OUT_OF_MEMORY), errmsg("out of memory")); + } - if (error == NULL) { - error = "undefined"; - } + if (resp->http_status == CH_HTTP_STATUS_CANCELED) { + kill_query(conn, resp->query_id); + ch_http_response_free(resp); ereport( ERROR, - errcode(ERRCODE_SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION), - errmsg("pg_clickhouse: communication error: %s", error) + errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), + errmsg("pg_clickhouse: query was aborted") ); } @@ -431,7 +359,7 @@ http_simple_insert(void* conn, const ch_query* query) { ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", format_error(error)), + errmsg("pg_clickhouse: %s", ch_http_format_error(error)), status < 404 ? 0 : errdetail_internal("Remote Query: %.64000s", query->sql), errcontext("HTTP status code: %li", status) ); @@ -440,39 +368,69 @@ http_simple_insert(void* conn, const ch_query* query) { ch_http_response_free(resp); } -inline static void -http_cursor_free(void* c) { - ch_http_response_free(((ch_cursor*)c)->query_response); +/* pgch_chunk_source cancellation poll, checked between reads. */ +static bool +native_chunks_cancelled(void* ud pg_attribute_unused()) { + return http_canceled(); } -inline static void -http_streaming_cursor_free(void* c) { - if (((ch_cursor*)c)->query_response) { - ch_http_stream_end(((ch_cursor*)c)->query_response); - } -} +/* Room for every setting native_overrides writes. */ +#define NATIVE_OVERRIDES_MAX 3 /* - * Create a streaming cursor with row-aligned batches of ~fetch_size bytes - * via CURL pause/resume, keeping memory proportional to batch size. + * Settings the shared decoder needs from a Native response: the pair + * PGCH_NATIVE_SETTINGS joins, plus the format itself. Listed one by one + * because each needs its own server version gate; an unknown HTTP setting + * fails the query. */ +static int +native_overrides(void* conn, ch_setting out[NATIVE_OVERRIDES_MAX]) { + ch_server_version version = + ch_http_server_version((ch_http_connection_t*)conn, http_canceled); + int n = 0; + + /* Format as a setting keeps SQL unchanged, so query parameters work. */ + out[n++] = (ch_setting){ "default_format", "Native" }; + if (chfdw_version_ge(version, 24, 7)) { + out[n++] = + (ch_setting){ "output_format_native_encode_types_in_binary_format", "0" }; + } + if (chfdw_version_ge(version, 24, 10)) { + out[n++] = (ch_setting){ "output_format_native_write_json_as_string", "1" }; + } + + return n; +} + +static void +http_stream_reader_init(pgch_reader* reader, void* response) { + pgch_chunk_source src = { .ud = response, + .next_chunk = ch_http_stream_next_chunk, + .cancelled = native_chunks_cancelled }; + + pgch_reader_init_chunks(reader, &src, NULL); +} + +static void +http_stream_free(void* response) { + ch_http_stream_end(response); +} + +/* Create shared-decoder cursor over HTTP Native response. */ static ch_cursor* -http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { +http_native_cursor(void* conn, const ch_query* query) { int attempts = 0; - /* - * volatile: changed after setjmp (PG_TRY) and read after longjmp - * (PG_CATCH); longjmp needn't restore register-cached locals, so a - * non-volatile such local has an indeterminate value per C setjmp rules. - */ - volatile MemoryContext tempcxt = NULL; - MemoryContext oldcxt; - ch_cursor* cursor; HttpStream* stream; - - ch_http_set_progress_func(http_progress_callback); + ch_cursor* cursor; + ch_setting overrides[NATIVE_OVERRIDES_MAX]; + ch_http_request req = { .query = query, + .overrides = overrides, + .num_overrides = native_overrides(conn, overrides), + .stream_chunks = true, + .cancel = http_canceled }; again: - stream = ch_http_stream_begin(conn, query, fetch_size); + stream = ch_http_stream_begin(conn, &req); if (stream == NULL) { ereport( ERROR, @@ -482,266 +440,26 @@ http_streaming_query(void* conn, const ch_query* query, int32 fetch_size) { } attempts++; - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR) { - if (attempts < 3) { - ch_http_stream_end(stream); - goto again; - } + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_TRANSPORT_ERROR && + attempts < 3) { + ch_http_stream_end(stream); + goto again; } if (ch_http_stream_status(stream) != CH_HTTP_STATUS_OK) { report_http_stream_query_failure(conn, query, stream); } - PG_TRY(); - { - /* - * If any palloc below throws, clean up the stream which is not - * tracked by a memory context yet. - */ - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse streaming cursor", ALLOCSET_DEFAULT_SIZES - ); - oldcxt = MemoryContextSwitchTo(tempcxt); - - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = stream; - cursor->read_state = palloc0(sizeof(ch_http_read_state)); - cursor->query = pstrdup(query->sql); - cursor->request_time = ch_http_stream_request_time(stream); - cursor->total_time = ch_http_stream_total_time(stream); - - ch_http_read_state_init( - cursor->read_state, - ch_http_stream_buffer(stream), - ch_http_stream_available(stream) - ); - - cursor->memcxt = tempcxt; - cursor->callback.func = http_streaming_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); + ch_cursor_source src = { .response = stream, + .init_reader = http_stream_reader_init, + .free_response = http_stream_free, + .raise_response_error = http_native_read_error }; - MemoryContextSwitchTo(oldcxt); - - /* Ownership transferred to the cursor callback */ - stream = NULL; - } - PG_CATCH(); - { - if (stream) { - ch_http_stream_end(stream); - } - if (tempcxt) { - MemoryContextDelete(tempcxt); - } - PG_RE_THROW(); - } - PG_END_TRY(); + cursor = chfdw_cursor_open(conn, query, &src); + cursor->request_time = ch_http_stream_request_time(stream); return cursor; } -/* - * Streaming variant of http_fetch_row. When the parser exhausts the current - * buffer and the transfer isn't done, pump more data from curl and - * reinitialize the parser on the refilled buffer. - */ -static Datum* -http_streaming_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - ch_http_read_state* state = cursor->read_state; - HttpStream* stream = cursor->query_response; - - /* Pump the next batch when the current one has been exhausted. */ - if (state->done || state->data == NULL) { - /* Sync parse position: tell stream how far the parser advanced */ - ch_http_stream_advance(stream, state->curpos); - - if (ch_http_stream_pump(stream) < 0) { - if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED) { - char qid[CH_HTTP_QUERY_ID_LEN]; - - memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); - ch_http_stream_end(stream); - cursor->query_response = NULL; - kill_query(cursor->conn, qid); - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: query was aborted") - ); - } - ereport( - ERROR, - errcode(ERRCODE_CONNECTION_FAILURE), - errmsg( - "pg_clickhouse: streaming error - %s", - ch_http_stream_error(stream) ? ch_http_stream_error(stream) - : "unknown" - ) - ); - } - - /* Reinitialize parser on the (possibly compacted) buffer */ - ch_http_read_state_init( - state, ch_http_stream_buffer(stream), ch_http_stream_available(stream) - ); - } - - return http_fetch_row_from_state(ctx, state); -} - -static Datum* -http_fetch_row_from_state(ChFdwScanRowContext* ctx, ch_http_read_state* state) { - int rc = CH_CONT; - size_t attcount = list_length(ctx->retrieved_attrs); - Datum* values; - - /* All rows or empty table. */ - if (state->done || state->data == NULL) { - return NULL; - } - - /* Special case: SELECT NULL. */ - if (attcount == 0) { - Assert(ctx->values && ctx->nulls); - rc = ch_http_read_next(state, false); - if (rc != CH_CONT && state->is_null) { - ctx->nulls[0] = true; - ctx->values[0] = (Datum)0; - return ctx->values; - } - - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg("pg_clickhouse: unexpected response for a zero-column result"), - errdetail("Expected a NULL marker (\\N) in the TabSeparated response.") - ); - } - - /* - * Create Datums based on the retrieved_attrs for the TupleDesc. - * ctx->values and ctx->nulls must already be initialized with memory for - * ctx->tupdesc->natts Datums. - */ - if (ctx->tupdesc) { - values = ctx->values; - ListCell* lc; - int i; - - Assert(ctx->values && ctx->nulls && ctx->attinmeta); - foreach (lc, ctx->retrieved_attrs) { - Oid pgtype; - - i = lfirst_int(lc) - 1; - pgtype = TupleDescAttr(ctx->tupdesc, i)->atttypid; - rc = ch_http_read_next(state, type_is_array(pgtype)); - char_to_datum( - ctx, i, state->is_null ? NULL : state->val.data, state->val.len - ); - } - } - /* No TupleDesc, everything is text. */ - else { - values = palloc(attcount * sizeof(Datum)); - for (size_t idx = 0; idx < attcount; idx++) { - rc = ch_http_read_next(state, false); - if (state->is_null) { - values[idx] = (Datum)0; - } else { - values[idx] = PointerGetDatum(cstring_to_text(state->val.data)); - } - } - } - - if (attcount > 0 && rc != CH_EOL && rc != CH_EOF) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal("pg_clickhouse: columns mismatch"), - errdetail( - "Number of returned columns does not match " - "expected column count (%lu).", - attcount - ) - ); - } - - return values; -} - -/* - * Fetch a row from the http response and return its values. - * - * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values - * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts - * values. - * - * Use ctx->tupdesc and ctx->attinmeta to convert the values to the - * appropriate Datums, and store them and the indication of their NULLness in - * ctx->values and ctx->nulls, respectively, then return ctx->values. - * - * If ctx->tupdesc is not set, treat all values as text and return them as - * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, - * which only cares about text. - */ -static Datum* -http_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - ch_http_read_state* state = cursor->read_state; - - return http_fetch_row_from_state(ctx, state); -} - -/* - * Convert the raw data of length len to a Datum identified by attidx. - * Determines the Postgres type and input function from the attidx values in - * ctx->tupdesc and ctx->attinmeta. - */ -static void -char_to_datum(ChFdwScanRowContext* ctx, int attidx, char* data, size_t len) { - static const char time_prefix[] = "1970-01-01T"; - Oid pgtype = TupleDescAttr(ctx->tupdesc, attidx)->atttypid; - - if (data && len > sizeof(time_prefix) - 1 && - (pgtype == TIMEOID || pgtype == TIMETZOID) && data[len - 1] == 'Z') { - /* - * date_time_output_format=iso formats times as ISO timestamps. Remove - * the leading `YYYY-mm-ddT`. - */ - data += sizeof(time_prefix) - 1; - } else if (pgtype == BYTEAOID) { - /* Postgres input function won't work, we have raw data. */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = - data == NULL ? (Datum)0 - : PointerGetDatum((bytea*)cstring_to_text_with_len(data, len)); - return; - } - - /* Apply the input function even to nulls, to support domains */ - ctx->nulls[attidx] = data == NULL; - ctx->values[attidx] = InputFunctionCall( - &ctx->attinmeta->attinfuncs[attidx], - data, - ctx->attinmeta->attioparams[attidx], - ctx->attinmeta->atttypmods[attidx] - ); -} - -text* -chfdw_http_fetch_raw_data(ch_cursor* cursor) { - ch_http_read_state* state = cursor->read_state; - - if (state->data == NULL) { - return NULL; - } - - return cstring_to_text(state->data); -} - /* * Convert a Datum to a ClickHouse literal string. Returns NULL if the value * cannot be converted to a literal. @@ -814,58 +532,28 @@ chfdw_datum_to_ch_literal(Datum value, Oid type) { } /* - * extend_insert_query - * Construct values part of INSERT query + * Serialize buffered rows as a Native block and POST them. + * + * Column types come from PostgreSQL, so they rarely match the destination + * exactly. ClickHouse casts them per column name under + * input_format_native_allow_types_conversion, on by default since 23.3. */ static void -extend_insert_query(ch_http_insert_state* state, TupleTableSlot* slot) { -#ifdef USE_ASSERT_CHECKING - int pindex = 0; -#endif - bool first = true; - - if (state->sql.len == 0) { - appendStringInfoString(&state->sql, state->sql_begin); - } +http_flush_insert(ch_http_insert_state* state) { + pgch_buf body = {}; - /* get following parameters from slot */ - if (slot != NULL && state->target_attrs != NIL) { - ListCell* lc; + if (pgch_writer_rows(state->writer) == 0) { + return; + } - foreach (lc, state->target_attrs) { - int attnum = lfirst_int(lc); - Datum value; - Oid type; - bool isnull; - char* string; + pgch_buf_append(&body, state->sql_begin, strlen(state->sql_begin)); + /* NULL opts: no block info or custom serialization, matching the reader. */ + pgch_writer_flush(state->writer, &body, NULL); - value = slot_getattr(slot, attnum, &isnull); - type = TupleDescAttr(slot->tts_tupleDescriptor, attnum - 1)->atttypid; + ch_query query = new_body_query(state->sql, body.data, body.len); - if (!first) { - appendStringInfoChar(&state->sql, '\t'); - } - first = false; - - if (isnull) { - appendStringInfoString(&state->sql, "\\N"); -#ifdef USE_ASSERT_CHECKING - pindex++; -#endif - continue; - } - - string = chfdw_datum_to_ch_literal(value, type); - appendStringInfoString(&state->sql, string); - pfree(string); -#ifdef USE_ASSERT_CHECKING - pindex++; -#endif - } - appendStringInfoChar(&state->sql, '\n'); - - Assert(pindex == state->p_nums); - } + http_simple_insert(state->conn, &query); + pgch_buf_reset(&body); } static void* @@ -877,12 +565,75 @@ http_prepare_insert( char* table_name ) { ch_http_insert_state* state = palloc0(sizeof(ch_http_insert_state)); + Relation rel = rri->ri_RelationDesc; + TupleDesc tupdesc = RelationGetDescr(rel); + Oid relid = RelationGetRelid(rel); + size_t ncols = list_length(target_attrs); + pgch_col* cols = palloc0(ncols * sizeof(pgch_col)); + ListCell* lc; + size_t i = 0; + + state->ncols = ncols; + state->attnums = palloc0(ncols * sizeof(AttrNumber)); + state->atttypids = palloc0(ncols * sizeof(Oid)); + + foreach (lc, target_attrs) { + AttrNumber attnum = lfirst_int(lc); + Form_pg_attribute attr = TupleDescAttr(tupdesc, attnum - 1); + CustomColumnInfo* cinfo = chfdw_get_custom_column_info(relid, attnum); + /* Name must match the INSERT column list chfdw_deparse_insert_sql built */ + const char* colname = + (cinfo && cinfo->colname[0]) ? cinfo->colname : NameStr(attr->attname); + const char* chtype; + chc_err err = {}; + chc_type* coltype; + + /* + * ClickHouse gained Time64 in 25.6 and casts it to none of the types + * a table holds a time of day in, so send a timestamp on the epoch + * date, as the TabSeparated payload did. + */ + if (attr->atttypid == TIMEOID) { + chtype = attr->attnotnull ? "DateTime64(6, 'UTC')" + : "Nullable(DateTime64(6, 'UTC'))"; + } else { + chtype = pgch_ch_type_for( + attr->atttypid, attr->atttypmod, attr->attnotnull, NULL + ); + } - initStringInfo(&state->sql); - state->sql_begin = psprintf("%s FORMAT TSV\n", query->sql); - state->target_attrs = target_attrs; - state->p_nums = list_length(state->target_attrs); - state->conn = conn; + /* A PostgreSQL array type carries no dimension count, only the + * declared attndims does, and ClickHouse nests one Array per + * dimension */ + for (int dim = 1; dim < attr->attndims; dim++) { + chtype = psprintf("Array(%s)", chtype); + } + + if (chc_type_parse(chtype, strlen(chtype), &pgch_alloc, &coltype, &err) != + CHC_OK) { + ereport( + ERROR, + errcode(ERRCODE_FDW_INVALID_DATA_TYPE), + errmsg( + "pg_clickhouse: could not build ClickHouse type for column \"%s\"", + colname + ), + errdetail_internal("%s: %s", chtype, err.msg) + ); + } + + state->attnums[i] = attnum; + state->atttypids[i] = attr->atttypid; + cols[i].name = colname; + cols[i].name_len = strlen(colname); + cols[i].type = coltype; + i++; + } + + state->writer = pgch_writer_new(CurrentMemoryContext, cols, ncols); + state->sql = pstrdup(query->sql); + state->sql_begin = psprintf("%s FORMAT Native\n", query->sql); + state->conn = conn; return state; } @@ -891,15 +642,37 @@ static void http_insert_tuple(void* istate, TupleTableSlot* slot) { ch_http_insert_state* state = istate; - extend_insert_query(state, slot); + if (slot != NULL) { + for (size_t i = 0; i < state->ncols; i++) { + bool isnull; + Datum value = slot_getattr(slot, state->attnums[i], &isnull); + Oid valtype = state->atttypids[i]; + + /* PostgreSQL casts inet to text through network_show, which + * appends a netmask ClickHouse rejects for IPv4 and IPv6. The + * output function omits it for single hosts. */ + if (valtype == INETOID && !isnull) { + value = CStringGetTextDatum(OidOutputFunctionCall(F_INET_OUT, value)); + valtype = TEXTOID; + } else if (valtype == TIMEOID && !isnull) { + /* Pair with the DateTime64 column http_prepare_insert declares */ + value = TimestampTzGetDatum( + DatumGetTimeADT(value) - + (TimestampTz)(POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * + USECS_PER_DAY + ); + valtype = TIMESTAMPTZOID; + } - if ((slot == NULL && state->sql.len > 0) || - (size_t)state->sql.len > (MaxAllocSize / 2 /* 512MB */)) { - ch_query query = new_query(state->sql.data, 0, NULL, NULL, NULL); + pgch_append_datum(state->writer, i, value, valtype, isnull); + } - http_simple_insert(state->conn, &query); - resetStringInfo(&state->sql); + /* Flush at 64MiB so bulk loads stream instead of buffering every row */ + if (pgch_writer_bytes(state->writer) < 64 * 1024 * 1024) { + return; + } } + http_flush_insert(state); } /*** BINARY PROTOCOL ***/ @@ -908,9 +681,8 @@ ch_connection chfdw_binary_connect(ch_connection_details* details) { ch_connection res; - res.conn = ch_binary_connect(details); - res.methods = &binary_methods; - res.is_binary = true; + res.conn = ch_binary_connect(details); + res.methods = &binary_methods; return res; } @@ -936,12 +708,20 @@ binary_server_version(void* conn) { return v; } +static void +binary_reader_init(pgch_reader* reader, void* response) { + pgch_block_source src = ch_binary_response_block_source(response); + + pgch_reader_init(reader, &src); +} + +static void +binary_response_free(void* response) { + ch_binary_response_free(response); +} + static ch_cursor* binary_simple_query(void* conn, const ch_query* query) { - MemoryContext tempcxt, oldcxt; - ch_cursor* cursor; - pgch_reader* state; - ch_binary_response_t* resp = ch_binary_simple_query(conn, query, &is_canceled); if (!ch_binary_response_success(resp)) { @@ -959,301 +739,52 @@ binary_simple_query(void* conn, const ch_query* query) { ); } - tempcxt = AllocSetContextCreate( - PortalContext, "pg_clickhouse cursor", ALLOCSET_DEFAULT_SIZES - ); - - oldcxt = MemoryContextSwitchTo(tempcxt); - cursor = palloc0(sizeof(ch_cursor)); - cursor->conn = conn; - cursor->query_response = resp; - state = (pgch_reader*)palloc0(sizeof(pgch_reader)); - cursor->query = pstrdup(query->sql); - cursor->read_state = state; - pgch_block_source src = ch_binary_response_block_source(resp); - pgch_reader_init(cursor->read_state, &src); - cursor->columns_count = pgch_reader_columns(state); - cursor->memcxt = tempcxt; - cursor->callback.func = binary_cursor_free; - cursor->callback.arg = cursor; - MemoryContextRegisterResetCallback(tempcxt, &cursor->callback); - - /* - * Validate declared shape before any per-column access. Empty attr_nums - * keeps the zero-attribute NULL sentinel handled at fetch time. Ignore - * columns_count == 0 (DDL) to support callers passing a placeholder - * column list since clickhouse_query() requires one syntactically. - */ - if (query->tupdesc && query->attr_nums && cursor->columns_count > 0 && - (size_t)list_length(query->attr_nums) != cursor->columns_count) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal( - "pg_clickhouse: returned %lu columns, expected %lu", - (unsigned long)cursor->columns_count, - (unsigned long)list_length(query->attr_nums) - ), - errdetail_internal("Remote Query: %.64000s", query->sql) - ); - } - - /* - * CH JSON columns default to JSONBOID in state->coltypes. When foreign - * table column is declared `json` (JSONOID), override so - * binary_make_datum returns json Datum from CH's STRING bytes, skipping - * jsonb_in / jsonb_out round-trip that would reformat CH's emit and break - * expected outputs that pin CH's exact formatting. - */ - if (query->tupdesc && state->coltypes) { - ListCell* lc; - size_t j = 0; + ch_cursor_source src = { .response = resp, + .init_reader = binary_reader_init, + .free_response = binary_response_free }; - foreach (lc, query->attr_nums) { - int i = lfirst_int(lc); - - if (state->coltypes[j] == JSONBOID && - TupleDescAttr(query->tupdesc, i - 1)->atttypid == JSONOID) { - state->coltypes[j] = JSONOID; - } - j++; - } - } - - MemoryContextSwitchTo(oldcxt); - - if (state->error) { - /* Prefer consistent interrupt error message when query interrupted */ - CHECK_FOR_INTERRUPTS(); - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", query->sql) - ); - } - - return cursor; + return chfdw_cursor_open(conn, query, &src); } -/* - * Escape characters that would otherwise corrupt the tab/newline framing or - * collide with the \N null marker. Matches CH's TabSeparated escaping; \0 is - * unreachable since values arrive as cstrings, so it needs no case. - */ -static void -append_tsv_escaped(StringInfo buf, const char* s) { - for (; *s != '\0'; s++) { - switch (*s) { - case '\\': - appendStringInfoString(buf, "\\\\"); - break; - case '\b': - appendStringInfoString(buf, "\\b"); - break; - case '\f': - appendStringInfoString(buf, "\\f"); - break; - case '\n': - appendStringInfoString(buf, "\\n"); - break; - case '\r': - appendStringInfoString(buf, "\\r"); - break; - case '\t': - appendStringInfoString(buf, "\\t"); - break; - default: - appendStringInfoChar(buf, *s); - } - } -} - -/* - * Drain a binary cursor into tab-separated rows, mirroring the single-text - * result of the http path. Nulls render as \N, other values escape the - * control characters CH's TabSeparated format does so they stay unambiguous. - * Output formatting otherwise differs from the http driver since values pass - * through PG output functions rather than ClickHouse's wire formatting. - */ -text* -chfdw_binary_fetch_raw_data(ch_cursor* cursor) { - pgch_reader* state = cursor->read_state; - size_t ncols = pgch_reader_columns(state); - StringInfoData buf; - - if (ncols == 0) { - return NULL; - } - - initStringInfo(&buf); +static text* +binary_raw_query(void* conn, const ch_query* query) { + ch_cursor* cursor = binary_simple_query(conn, query); + /* volatile: assigned inside PG_TRY, so longjmp may leave it in a register */ + text* volatile result; - while (pgch_reader_next(state)) { - for (size_t i = 0; i < ncols; i++) { - if (i > 0) { - appendStringInfoChar(&buf, '\t'); - } - - if (state->nulls[i]) { - appendStringInfoString(&buf, "\\N"); - } else { - char* val = pgch_value_to_cstring(state->coltypes[i], state->values[i]); - - append_tsv_escaped(&buf, val); - pfree(val); - } - } - appendStringInfoChar(&buf, '\n'); - CHECK_FOR_INTERRUPTS(); - } - - if (state->error) { - ereport( - ERROR, - errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error) - ); - } - - if (buf.len == 0) { - pfree(buf.data); - return NULL; - } + PG_TRY(); + { result = chfdw_cursor_render_tsv(cursor); } + PG_FINALLY(); + { MemoryContextDelete(cursor->memcxt); } + PG_END_TRY(); - return cstring_to_text_with_len(buf.data, buf.len); + return result; } -/* - * Fetch a row from the binary cursor and return its values. - * - * If ctx->tupdesc is set, ctx->attinmeta must also be set, and ctx->values - * and ctx->nulls must already be palloc'd with space for ctx->tupdesc->natts - * values. - * - * Use ctx->tupdesc and ctx->attinmeta to convert the values to the - * appropriate Datums, and store them and the indication of their NULLness in - * ctx->values and ctx->nulls, respectively, then return ctx->values. - * - * If ctx->tupdesc is not set, treat all values as text and return them as - * text `Datum`s. This is the use case for `chfdw_construct_create_tables()`, - * which only cares about text. - */ +/* Report a truncated response as cancellation when that is what caused it. */ static void -binary_fetch_row_errcb(void* arg) { - const char* sql = (const char*)arg; - - errdetail_internal("Remote Query: %.64000s", sql); -} +http_native_read_error(ch_cursor* cursor) { + HttpStream* stream = cursor->response; -/* Conversion state and target attribute per returned column. */ -static void -build_conversion(ch_cursor* cursor, const ChFdwScanRowContext* ctx) { - pgch_reader* state = cursor->read_state; - MemoryContext old = MemoryContextSwitchTo(cursor->memcxt); - size_t ncols = pgch_reader_columns(state); - ListCell* lc; - size_t j = 0; - - cursor->conversion_states = palloc0(ncols * sizeof(void*)); - cursor->fill_dest = palloc0(ncols * sizeof(int)); - foreach (lc, ctx->retrieved_attrs) { - int attnum = lfirst_int(lc); - Form_pg_attribute att = TupleDescAttr(ctx->tupdesc, attnum - 1); - - cursor->fill_dest[j] = attnum - 1; - cursor->conversion_states[j] = - pgch_reader_convert_init(state, j, att->atttypid, att->atttypmod); - j++; + if (stream == NULL) { + return; } - MemoryContextSwitchTo(old); -} + if (ch_http_stream_status(stream) == CH_HTTP_STATUS_CANCELED || + QueryCancelPending || ProcDiePending) { + char qid[CH_HTTP_QUERY_ID_LEN]; -static Datum* -binary_fetch_row(ChFdwScanRowContext* ctx) { - ch_cursor* cursor = ctx->cursor; - List* attrs = ctx->retrieved_attrs; - TupleDesc tupdesc = ctx->tupdesc; - Datum* values = ctx->values; - bool* nulls = ctx->nulls; - pgch_reader* state = cursor->read_state; - ErrorContextCallback errcallback; - - errcallback.callback = binary_fetch_row_errcb; - errcallback.arg = (void*)cursor->query; - errcallback.previous = error_context_stack; - error_context_stack = &errcallback; - - bool have_data = pgch_reader_next(state); - size_t attcount = list_length(attrs); - - if (state->error) { - error_context_stack = errcallback.previous; - - /* Prefer consistent interrupt error message when fetch interrupted */ - CHECK_FOR_INTERRUPTS(); + memcpy(qid, ch_http_stream_query_id(stream), sizeof(qid)); + /* Drop the transfer before asking the server to kill the query. */ + ch_http_stream_end(stream); + cursor->response = NULL; + kill_query(cursor->conn, qid); ereport( ERROR, errcode(ERRCODE_SQL_ROUTINE_EXCEPTION), - errmsg("pg_clickhouse: %s", state->error), - errdetail_internal("Remote Query: %.64000s", cursor->query) - ); - } - - if (!have_data) { - error_context_stack = errcallback.previous; - return NULL; - } - - if (attcount == 0) { - if (pgch_reader_columns(state) == 1 && state->nulls[0]) { - nulls[0] = true; - goto ok; - } else { - ereport( - ERROR, - errcode(ERRCODE_FDW_ERROR), - errmsg( - "pg_clickhouse: unexpected state: attributes " - "count == 0 and haven't got NULL in the response" - ) - ); - } - } else if (attcount != pgch_reader_columns(state)) { - ereport( - ERROR, - errcode(ERRCODE_DATATYPE_MISMATCH), - errmsg_internal( - "pg_clickhouse: returned %lu columns, expected %lu", - pgch_reader_columns(state), - attcount - ) - ); - } - - if (tupdesc) { - Assert(values && nulls); - - if (cursor->conversion_states == NULL) { - build_conversion(cursor, ctx); - } - pgch_reader_fill_map( - state, cursor->conversion_states, cursor->fill_dest, values, nulls + errmsg("pg_clickhouse: query was aborted") ); } - -ok: - error_context_stack = errcallback.previous; - return state->values; -} - -static void -binary_cursor_free(void* c) { - ch_cursor* cursor = c; - - /* Conversion states live in the context this callback fires for. */ - pgch_reader_free(cursor->read_state); - ch_binary_response_free(cursor->query_response); } static void* diff --git a/test/expected/aggregates.out b/test/expected/aggregates.out index 6e53a4c0..085f9508 100644 --- a/test/expected/aggregates.out +++ b/test/expected/aggregates.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1498,9 +1498,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1525,9 +1525,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1552,9 +1552,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1579,9 +1579,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1633,9 +1633,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1660,9 +1660,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1687,9 +1687,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1714,9 +1714,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1741,9 +1741,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_1.out b/test/expected/aggregates_1.out index 09018fd2..f83aa63b 100644 --- a/test/expected/aggregates_1.out +++ b/test/expected/aggregates_1.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1498,9 +1498,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1525,9 +1525,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1552,9 +1552,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1579,9 +1579,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1633,9 +1633,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1660,9 +1660,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1687,9 +1687,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1714,9 +1714,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1741,9 +1741,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_2.out b/test/expected/aggregates_2.out index a12ae333..12b77545 100644 --- a/test/expected/aggregates_2.out +++ b/test/expected/aggregates_2.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1484,9 +1484,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1511,9 +1511,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1538,9 +1538,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1565,9 +1565,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1619,9 +1619,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1646,9 +1646,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1673,9 +1673,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1700,9 +1700,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1727,9 +1727,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_3.out b/test/expected/aggregates_3.out index 795d7480..a4e67d78 100644 --- a/test/expected/aggregates_3.out +++ b/test/expected/aggregates_3.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1488,9 +1488,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1515,9 +1515,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1542,9 +1542,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1569,9 +1569,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1623,9 +1623,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_samp(cast(b, 'Nullable(Decimal)')), stddev_pop(cast(b, 'Nullable(Decimal)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 151.2183851256189 | 151.2183851256189 | 130.95896303804486 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 151.218385125619 | 151.218385125619 | 130.958963038045 (1 row) -- var_pop(int2), var_samp(int2), variance(int2) @@ -1650,9 +1650,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1677,9 +1677,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1704,9 +1704,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1731,9 +1731,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_4.out b/test/expected/aggregates_4.out index a2b231e2..90b22c49 100644 --- a/test/expected/aggregates_4.out +++ b/test/expected/aggregates_4.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1488,9 +1488,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1515,9 +1515,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1542,9 +1542,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1569,9 +1569,9 @@ DETAIL: Remote Query: SELECT datestamp, groupConcat(', ')(path) FROM agg_test.h Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1645,9 +1645,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1672,9 +1672,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1699,9 +1699,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1726,9 +1726,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/aggregates_5.out b/test/expected/aggregates_5.out index c2a7522c..fb3aeffc 100644 --- a/test/expected/aggregates_5.out +++ b/test/expected/aggregates_5.out @@ -570,9 +570,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits (4 rows) - array_agg ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- - {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.6,7.01,4.26,7.4,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.1,1.99,1.81,6.8,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.2,3.55,8.17,8.24,2.36,4.79,6.75,8.9,7.69,8.09,5.9} + array_agg +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + {1.29,8.38,4.28,6.51,2.15,3.02,5.94,2.98,5.60,7.01,4.26,7.40,7.55,2.34,7.57,5.66,2.61,3.51,3.52,4.43,5.55,5.24,5.10,1.99,1.81,6.80,3.54,8.14,8.21,4.55,1.33,7.42,2.23,5.24,6.04,8.53,4.43,3.57,2.47,1.20,3.55,8.17,8.24,2.36,4.79,6.75,8.90,7.69,8.09,5.90} (1 row) QUERY PLAN @@ -596,9 +596,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT groupArray(cost) FROM agg_test.hits WHERE ((cost < 1000000000)) (4 rows) - array_agg ----------------------------------------------------- - {5.1,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} + array_agg +----------------------------------------------------- + {5.10,1.99,8.14,8.53,3.57,2.47,8.24,4.79,6.75,7.69} (1 row) -- min(UInt64) @@ -888,9 +888,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT min(cost) FROM agg_test.hits (4 rows) - min ------ - 1.2 + min +------ + 1.20 (1 row) QUERY PLAN @@ -1206,9 +1206,9 @@ FDW options: (database 'agg_test', table_name 'hits', engine 'MergeTree') Remote SQL: SELECT max(cost) FROM agg_test.hits (4 rows) - max ------ - 8.9 + max +------ + 8.90 (1 row) QUERY PLAN @@ -1479,9 +1479,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(a), stddev_samp(a), stddev_pop(a) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int4), stddev_samp(int4), stddev_pop(int4) @@ -1506,9 +1506,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int32)')), stddev_samp(cast(a, 'Nullable(Int32)')), stddev_pop(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(int8), stddev_samp(int8), stddev_pop(int8) @@ -1533,9 +1533,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(cast(a, 'Nullable(Int64)')), stddev_samp(cast(a, 'Nullable(Int64)')), stddev_pop(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop --------------------+-------------------+-------------------- - 41.22701379758988 | 41.22701379758988 | 35.703641270884404 + stddev | stddev_samp | stddev_pop +------------------+------------------+------------------ + 41.2270137975899 | 41.2270137975899 | 35.7036412708844 (1 row) -- stddev(float), stddev_samp(float), stddev_pop(float) @@ -1560,9 +1560,9 @@ DETAIL: Remote Query: SELECT groupBitXor(cast(duration, 'Nullable(Int32)')) FRO Remote SQL: SELECT stddev_samp(b), stddev_samp(b), stddev_pop(b) FROM agg_test.agg_numbers (4 rows) - stddev | stddev_samp | stddev_pop ------------+-------------+------------ - 151.38937 | 151.38937 | 131.10704 + stddev | stddev_samp | stddev_pop +--------------------+--------------------+-------------------- + 151.38937377929688 | 151.38937377929688 | 131.10704040527344 (1 row) -- stddev(float8), stddev_samp(float8), stddev_pop(float8) @@ -1636,9 +1636,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(a), var_samp(a), var_samp(a) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int4), var_samp(int4), variance(int4) @@ -1663,9 +1663,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')), var_samp(cast(a, 'Nullable(Int32)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(int8), var_samp(int8), variance(int8) @@ -1690,9 +1690,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')), var_samp(cast(a, 'Nullable(Int64)')) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ----------+--------------------+-------------------- - 1274.75 | 1699.6666666666667 | 1699.6666666666667 + var_pop | var_samp | variance +---------+------------------+------------------ + 1274.75 | 1699.66666666667 | 1699.66666666667 (1 row) -- var_pop(float), var_samp(float), variance(float) @@ -1717,9 +1717,9 @@ CONTEXT: HTTP status code: 500 Remote SQL: SELECT var_pop(b), var_samp(b), var_samp(b) FROM agg_test.agg_numbers (4 rows) - var_pop | var_samp | variance ------------+-----------+----------- - 17189.057 | 22918.742 | 22918.742 + var_pop | var_samp | variance +-----------------+---------------+--------------- + 17189.056640625 | 22918.7421875 | 22918.7421875 (1 row) -- var_pop(float8), var_samp(float8), variance(float8) diff --git a/test/expected/gucs.out b/test/expected/gucs.out index 623cd0e0..b8bbd36f 100644 --- a/test/expected/gucs.out +++ b/test/expected/gucs.out @@ -80,8 +80,8 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i ------------------------------------+---------------------- connect_timeout | 2 count_distinct_implementation | uniq - date_time_output_format | iso - format_tsv_null_representation | \N + date_time_output_format | unix_timestamp + format_tsv_null_representation | NOPE join_algorithm | prefer_partial_merge join_use_nulls | 1 log_queries_min_type | QUERY_FINISH @@ -90,7 +90,7 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i max_result_rows | 1024 metrics_perf_events_list | this,that network_compression_method | ZSTD - output_format_tsv_crlf_end_of_line | 0 + output_format_tsv_crlf_end_of_line | 1 poll_interval | 5 totals_mode | after_having_auto (15 rows) @@ -118,7 +118,7 @@ NOTICE: ERR 42601 - pg_clickhouse: missing comma after "join_use_nulls" value i ------------------------------------+---------- connect_timeout | t count_distinct_implementation | t - date_time_output_format | f + date_time_output_format | t format_tsv_null_representation | t join_algorithm | t join_use_nulls | t diff --git a/test/expected/http.out b/test/expected/http.out index d97798b3..80c71c3b 100644 --- a/test/expected/http.out +++ b/test/expected/http.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -858,64 +828,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -944,20 +870,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1049,8 +961,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1071,18 +983,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1133,8 +1043,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1170,19 +1080,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1239,16 +1145,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_1.out b/test/expected/http_1.out index aa1769e6..471355c2 100644 --- a/test/expected/http_1.out +++ b/test/expected/http_1.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -856,64 +826,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1047,8 +959,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1069,18 +981,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1131,8 +1041,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1168,19 +1078,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1143,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_2.out b/test/expected/http_2.out index 9d7cf836..b89fb50b 100644 --- a/test/expected/http_2.out +++ b/test/expected/http_2.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -856,64 +826,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1047,8 +959,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1069,18 +981,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1131,8 +1041,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1168,19 +1078,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1143,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_3.out b/test/expected/http_3.out index 463ad947..a7414f94 100644 --- a/test/expected/http_3.out +++ b/test/expected/http_3.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -856,64 +826,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1047,8 +959,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1069,18 +981,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1131,8 +1041,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1168,19 +1078,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1143,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_4.out b/test/expected/http_4.out index 85e671f2..6b40bac9 100644 --- a/test/expected/http_4.out +++ b/test/expected/http_4.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -856,64 +826,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -942,20 +868,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1047,8 +959,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1069,18 +981,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1131,8 +1041,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1168,19 +1078,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1237,16 +1143,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_5.out b/test/expected/http_5.out index ab3dfd5c..05e88e21 100644 --- a/test/expected/http_5.out +++ b/test/expected/http_5.out @@ -68,16 +68,6 @@ CREATE FOREIGN TABLE ft1 ( c7 char(10) default 'ft1', c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( c1 int NOT NULL, @@ -155,27 +145,6 @@ INSERT INTO ft4 'AAA' || to_char(id, 'FM000'), (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - c1 | c3 -----+------- - 1 | 00001 - 2 | 00002 - 3 | 00003 - 4 | 00004 - 5 | 00005 - 6 | 00006 - 7 | 00007 - 8 | 00008 - 9 | 00009 - 10 | 00010 - 11 | 00011 - 12 | 00012 - 13 | 00013 - 14 | 00014 - 15 | 00015 -(15 rows) - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; c1 | c2 | c3 | c4 ----+----+--------+---- @@ -223,6 +192,7 @@ DETAIL: Remote Query: SELECT c1, c3, c4 FROM "no such database".t1 ORDER BY c3 CONTEXT: HTTP status code: 404 ALTER USER MAPPING FOR CURRENT_USER SERVER http_loopback OPTIONS (ADD user 'no such user'); SELECT c3, c4 FROM ft1 ORDER BY c3, c1; -- should fail +WARNING: pg_clickhouse: SELECT version() failed (HTTP status 403): Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) ERROR: pg_clickhouse: Code: 516. DB::Exception: no such user: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) CONTEXT: HTTP status code: 403 ALTER SERVER http_loopback OPTIONS (SET dbname 'http_test'); @@ -858,64 +828,20 @@ SELECT * FROM bad_name; ERROR: pg_clickhouse: unsupported line ending character in database name DETAIL: Invalid database name: 'http_test X-My-Header: 123' -/* ===== fetch_size option tests ===== */ -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - c3 -------- - 00001 - 00002 - 00003 -(3 rows) - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); -ERROR: invalid value for option "fetch_size": -1 -HINT: fetch_size must be greater than or equal to 0 +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); +WARNING: option "fetch_size" is deprecated and ignored +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -944,20 +870,6 @@ SELECT id, term FROM ft_brk ORDER BY id; 6 | leading text [bracketed] trailing (6 rows) -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - id | term -----+----------------------------------- - 1 | [abc_def]_ghi.jkl_mno - 2 | plainstring - 3 | [just_brackets] - 4 | [] - 5 | [a,b,c]_trailing - 6 | leading text [bracketed] trailing -(6 rows) - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -1049,8 +961,8 @@ SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; (4 rows) /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -1071,18 +983,16 @@ SELECT clickhouse_raw_query('INSERT INTO http_test.t_timestr VALUES CREATE FOREIGN TABLE ft_timestr (id text, v time) SERVER http_loopback OPTIONS (table_name 't_timestr'); SELECT v FROM ft_timestr WHERE id = '3'; - v ----------- - 00:00:00 -(1 row) - +ERROR: invalid input syntax for type time: "1970-01-01T00:00:00Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '3')) SELECT v FROM ft_timestr WHERE id = '1'; ERROR: invalid input syntax for type time: "Z" +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '1')) SELECT v FROM ft_timestr WHERE id = '2'; ERROR: invalid input syntax for type time: "xZ" -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +DETAIL: Remote Query: SELECT v FROM http_test.t_timestr WHERE ((id = '2')) +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -1133,8 +1043,8 @@ SELECT * FROM ft_nested_arrays ORDER BY c1; (2 rows) SELECT * FROM ft_ragged_arrays ORDER BY c1; -ERROR: malformed array literal: "{{1,2,3},{4}}" -DETAIL: Multidimensional arrays must have sub-arrays with matching dimensions. +ERROR: pg_clickhouse: nested arrays must have sub-arrays with matching dimensions +DETAIL: Remote Query: SELECT c1, c2 FROM http_test.ragged_arrays ORDER BY c1 ASC NULLS LAST -- clickhouse_query: server-based typed rowset over the http driver SELECT * FROM clickhouse_query( 'http_loopback', 'SELECT c1, c3 FROM t1 ORDER BY c1 LIMIT 3' @@ -1170,19 +1080,15 @@ LINE 1: SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1'); ^ -- fewer columns declared than returned is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT 1, 2') AS t(x int); -ERROR: pg_clickhouse: columns mismatch -DETAIL: Number of returned columns does not match expected column count (1). +ERROR: pg_clickhouse: returned 2 columns, expected 1 +DETAIL: Remote Query: SELECT 1, 2 -- value not coercible to the declared type is rejected SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); ERROR: invalid input syntax for type integer: "abc" +DETAIL: Remote Query: SELECT 'abc' -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); ERROR: server "no_such_server" does not exist -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to foreign table ft_no_stream -drop cascades to foreign table ft_override_stream /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ CREATE SERVER http_secure_on FOREIGN DATA WRAPPER clickhouse_fdw @@ -1239,16 +1145,14 @@ NOTICE: drop cascades to foreign table bad_name DROP SERVER http_loopback2 CASCADE; NOTICE: drop cascades to foreign table ft6 DROP SERVER http_loopback CASCADE; -NOTICE: drop cascades to 15 other objects +NOTICE: drop cascades to 13 other objects DETAIL: drop cascades to foreign table ft1 -drop cascades to foreign table ft1_stream drop cascades to foreign table ft2 drop cascades to foreign table ft3 drop cascades to foreign table ft4 drop cascades to foreign table ft5 drop cascades to foreign table ftcopy drop cascades to foreign table ft_brk -drop cascades to foreign table ft_brk_stream drop cascades to foreign table ft_brk_mix drop cascades to foreign table ft_nullmark drop cascades to foreign table ft_bytea diff --git a/test/expected/http_inserts.out b/test/expected/http_inserts.out index 21605fd4..9462e64d 100644 --- a/test/expected/http_inserts.out +++ b/test/expected/http_inserts.out @@ -235,9 +235,9 @@ INSERT INTO complex VALUES SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- - 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 - 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 - 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 + 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 17:01:02.123 + 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 18:01:02.234 + 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 19:01:02 4 | 1970-01-01 | 1970-01-01 00:00:00+00 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 5 | 2000-01-01 | 2000-01-01 00:00:00+00 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 (5 rows) @@ -249,8 +249,8 @@ SELECT * FROM complex ORDER BY c1; 1 | 2020-06-01 | 2020-06-02 10:01:02-07 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 2 | 2020-06-02 | 2020-06-03 10:01:02-07 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 3 | 2020-06-03 | 2020-06-04 10:01:02-07 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 - 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 - 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 + 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1969-12-31 16:00:00 + 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 1999-12-31 16:00:00 (5 rows) /* check arrays */ @@ -475,11 +475,11 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); +ERROR: pg_clickhouse: cannot append NULL to NOT NULL Int16 column "c2" SELECT * FROM default_vals; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 -----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------------+------------+------------------------+------------------------+-----+--------------------------------------+---------+----- - 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 0.0 | 0 | | | | 1970-01-01 | 1970-01-01 | 1969-12-31 16:00:00-08 | 1969-12-31 16:00:00-08 | {0} | 00000000-0000-0000-0000-000000000000 | 0.0.0.0 | :: -(1 row) + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 +----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) /* COPY FROM bulk-loads rows into the remote table. */ COPY ints (c1, c2, c3, c4) FROM stdin; diff --git a/test/expected/http_inserts_1.out b/test/expected/http_inserts_1.out index 9db6ce79..e231ec5e 100644 --- a/test/expected/http_inserts_1.out +++ b/test/expected/http_inserts_1.out @@ -226,9 +226,9 @@ INSERT INTO complex VALUES SELECT * FROM complex ORDER BY c1; c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 ----+------------+------------------------+----+--------+------+------------+------------------------- - 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 - 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 - 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 + 1 | 2020-06-01 | 2020-06-02 17:01:02+00 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 17:01:02.123 + 2 | 2020-06-02 | 2020-06-03 17:01:02+00 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 18:01:02.234 + 3 | 2020-06-03 | 2020-06-04 17:01:02+00 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 19:01:02 4 | 1970-01-01 | 1970-01-01 00:00:00+00 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 5 | 2000-01-01 | 2000-01-01 00:00:00+00 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 (5 rows) @@ -240,8 +240,8 @@ SELECT * FROM complex ORDER BY c1; 1 | 2020-06-01 | 2020-06-02 10:01:02-07 | t1 | fix_t1 | low1 | 2020-06-01 | 2020-06-02 10:01:02.123 2 | 2020-06-02 | 2020-06-03 10:01:02-07 | 5 | fix_t2 | low2 | 2020-06-02 | 2020-06-03 11:01:02.234 3 | 2020-06-03 | 2020-06-04 10:01:02-07 | 5 | fix_t3 | low3 | 2020-06-03 | 2020-06-04 12:01:02 - 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1970-01-01 00:00:00 - 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 2000-01-01 00:00:00 + 4 | 1970-01-01 | 1969-12-31 16:00:00-08 | 5 | fix_t4 | low4 | 1970-01-01 | 1969-12-31 16:00:00 + 5 | 2000-01-01 | 1999-12-31 16:00:00-08 | 5 | fix_t5 | low5 | 2000-01-01 | 1999-12-31 16:00:00 (5 rows) /* check arrays */ @@ -466,11 +466,11 @@ INSERT INTO default_vals VALUES( NULL, NULL, NULL, NULL, ARRAY[NULL]::int[], NULL, NULL, NULL ); +ERROR: pg_clickhouse: cannot append NULL to NOT NULL Int16 column "c2" SELECT * FROM default_vals; - c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 -----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+------------+------------+------------------------+------------------------+-----+--------------------------------------+---------+----- - 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0.0 | 0.0 | 0.0 | 0 | | | | 1970-01-01 | 1970-01-01 | 1969-12-31 16:00:00-08 | 1969-12-31 16:00:00-08 | {0} | 00000000-0000-0000-0000-000000000000 | 0.0.0.0 | :: -(1 row) + c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | c10 | c11 | c12 | c13 | c14 | c15 | c16 | c17 | c20 | c21 | c22 | c23 | c24 | c25 | c26 | c27 | c28 +----+----+----+----+----+----+----+----+----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+-----+----- +(0 rows) /* COPY FROM bulk-loads rows into the remote table. */ COPY ints (c1, c2, c3, c4) FROM stdin; diff --git a/test/expected/import_schema.out b/test/expected/import_schema.out index 5774f7c6..eda5ce6a 100644 --- a/test/expected/import_schema.out +++ b/test/expected/import_schema.out @@ -364,10 +364,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -713,7 +713,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/import_schema_1.out b/test/expected/import_schema_1.out index 79d2a416..5c626875 100644 --- a/test/expected/import_schema_1.out +++ b/test/expected/import_schema_1.out @@ -328,10 +328,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -613,7 +613,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/import_schema_2.out b/test/expected/import_schema_2.out index 143cd76a..2298d336 100644 --- a/test/expected/import_schema_2.out +++ b/test/expected/import_schema_2.out @@ -364,10 +364,10 @@ SELECT * FROM clickhouse.arrays ORDER BY c1 LIMIT 2; (2 rows) SELECT * FROM clickhouse.tuples ORDER BY c1 LIMIT 2; - c1 | c2 | c3.a | c3.b | c4 -----+-----------+---------+---------+---- - 0 | (0,'0',1) | {0,1,1} | {0,2,2} | 0 - 1 | (1,'1',2) | {1,1,1} | {1,2,2} | 1 + c1 | c2 | c3.a | c3.b | c4 +----+---------+---------+---------+---- + 0 | (0,0,1) | {0,1,1} | {0,2,2} | 0 + 1 | (1,1,2) | {1,1,1} | {1,2,2} | 1 (2 rows) SELECT * FROM clickhouse.timezones ORDER BY t1 LIMIT 2; @@ -713,7 +713,8 @@ DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM impor SELECT * FROM clickhouse.ints WHERE c1 IN (127, -128) ORDER BY c1; -ERROR: value "18446744073709551615" is out of range for type bigint +ERROR: pg_clickhouse: value 18446744073709551615 is out of range of bigint +DETAIL: Remote Query: SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, c10 FROM import_test.ints WHERE ((c1 IN (127,(-128)))) ORDER BY c1 ASC NULLS LAST -- Ignore 18446744073709551615 SELECT * FROM clickhouse_bin.ints WHERE c1 = -128 UNION diff --git a/test/expected/json_3.out b/test/expected/json_3.out index ee946eb3..a8224f53 100644 --- a/test/expected/json_3.out +++ b/test/expected/json_3.out @@ -61,12 +61,8 @@ SELECT * FROM json_bin.things ORDER BY id; ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.things ORDER BY id; - id | data -----+------------------------------------------------------------------ - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} - 4 | {"id": "4", "name": "doodad", "size": "large", "stocked": false} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Should also support JSON mapping. CREATE FOREIGN TABLE json_bin.json_things ( id integer NOT NULL, @@ -85,13 +81,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; - id | data -----+----------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} - 4 | {"id":"4","name":"doodad","size":"large","stocked":false} - 6 | {"id":"6","name":"curio","size":"medium","stocked":false} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -106,13 +97,8 @@ SELECT data['name'] FROM json_http.things; (3 rows) SELECT data['name'] FROM json_http.things ORDER BY id; - data ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data['name'] FROM json_bin.things; QUERY PLAN @@ -139,12 +125,8 @@ SELECT DISTINCT data['size'] FROM json_http.things; (6 rows) SELECT DISTINCT data['size'] FROM json_http.things; - data ----------- - "medium" - "large" -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT data FROM json_test.things EXPLAIN (VERBOSE, COSTS OFF) SELECT DISTINCT data['size'] FROM json_bin.things; QUERY PLAN @@ -174,12 +156,8 @@ SELECT data['size'], count(*) FROM json_http.things GROUP BY data['size']; (6 rows) SELECT data['size'], count(*) FROM json_http.things GROUP BY data['size']; - data | count -----------+------- - "medium" | 2 - "large" | 1 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT data FROM json_test.things EXPLAIN (VERBOSE, COSTS OFF) SELECT data['size'], count(*) FROM json_bin.things GROUP BY data['size']; QUERY PLAN @@ -206,13 +184,8 @@ SELECT data ->> 'name' FROM json_http.things; (3 rows) SELECT data ->> 'name' FROM json_http.things ORDER BY id; - ?column? ----------- - gizmo - doodad - curio -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.things; QUERY PLAN @@ -236,13 +209,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; - ?column? ----------- - gizmo - doodad - curio -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -446,11 +414,8 @@ SELECT * FROM json_http.things SELECT * FROM json_http.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; - id | data -----+----------------------------------------------------------------- - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; @@ -475,11 +440,8 @@ SELECT * FROM json_http.json_things SELECT * FROM json_http.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; - id | data -----+---------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE (((data.name = 'widget') OR (data.name = 'gizmo'))) ORDER BY id ASC NULLS LAST SELECT * FROM json_bin.json_things WHERE data ->> 'name' = 'widget' OR data ->> 'name' = 'gizmo' ORDER BY id; @@ -501,13 +463,8 @@ SELECT * FROM json_http.things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.things ORDER BY data ->> 'name'; - id | data -----+------------------------------------------------------------------ - 6 | {"id": "6", "name": "curio", "size": "medium", "stocked": false} - 4 | {"id": "4", "name": "doodad", "size": "large", "stocked": false} - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.things ORDER BY data ->> 'name'; QUERY PLAN @@ -540,13 +497,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; - id | data -----+----------------------------------------------------------- - 6 | {"id":"6","name":"curio","size":"medium","stocked":false} - 4 | {"id":"4","name":"doodad","size":"large","stocked":false} - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -579,13 +531,8 @@ SELECT data -> 'name' FROM json_http.things; (3 rows) SELECT data -> 'name' FROM json_http.things ORDER BY id; - ?column? ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.things; QUERY PLAN @@ -608,13 +555,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; - ?column? ----------- - "gizmo" - "doodad" - "curio" -(3 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -696,11 +638,8 @@ SELECT * FROM json_http.things WHERE data -> 'stocked' = 'true'::jsonb; (3 rows) SELECT * FROM json_http.things WHERE data -> 'stocked' = 'true'::jsonb ORDER BY id; - id | data -----+----------------------------------------------------------------- - 3 | {"id": "3", "name": "gizmo", "size": "medium", "stocked": true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((toJSONString(data.stocked) = 'true')) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.things WHERE data -> 'stocked' = 'true'::jsonb; QUERY PLAN @@ -724,11 +663,8 @@ SELECT * FROM json_http.json_things WHERE (data -> 'stocked')::text = 'true'; (3 rows) SELECT * FROM json_http.json_things WHERE (data -> 'stocked')::text = 'true' ORDER BY id; - id | data -----+---------------------------------------------------------- - 3 | {"id":"3","name":"gizmo","size":"medium","stocked":true} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.things WHERE ((CAST(toJSONString(data.stocked) AS String) = 'true')) ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things WHERE (data -> 'stocked')::text = 'true'; QUERY PLAN @@ -837,11 +773,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -864,11 +797,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -892,11 +822,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -919,11 +846,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -947,11 +871,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; - id | data -----+------------------------------------------------------------------- - 1 | {"select": "reserved", "my field": "hello", "CamelCase": "world"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -974,11 +895,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; - id | data -----+-------------------------------------------------------------- - 1 | {"CamelCase":"world","my field":"hello","select":"reserved"} -(1 row) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1380,12 +1298,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; - jsonb_extract_path_text -------------------------- - C100 - C200 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1409,12 +1323,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; - jsonb_extract_path_text -------------------------- - Paris - London -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1438,12 +1348,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; - jsonb_extract_path ------------------------------------ - {"zip": "75001", "city": "Paris"} - {"zip": "SW1A", "city": "London"} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1573,12 +1479,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; - json_extract_path_text ------------------------- - C100 - C200 -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1602,12 +1504,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; - json_extract_path_text ------------------------- - Paris - London -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1631,12 +1529,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; - json_extract_path --------------------------------- - {"city":"Paris","zip":"75001"} - {"city":"London","zip":"SW1A"} -(2 rows) - +ERROR: pg_clickhouse: unsupported JSON serialization version 0 (set output_format_native_write_json_as_string=1) +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_4.out b/test/expected/json_4.out index 3143cfa1..be4b616f 100644 --- a/test/expected/json_4.out +++ b/test/expected/json_4.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_5.out b/test/expected/json_5.out index 668c6979..a09da14a 100644 --- a/test/expected/json_5.out +++ b/test/expected/json_5.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/json_6.out b/test/expected/json_6.out index 8c5d03c1..1a6a7845 100644 --- a/test/expected/json_6.out +++ b/test/expected/json_6.out @@ -74,9 +74,8 @@ SELECT * FROM json_bin.json_things ORDER BY id; ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST SELECT * FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST -- Subscript access on JSON columns must not be pushed down to ClickHouse. -- ClickHouse JSON does not support the jsonb `column['key']` syntax (it -- requires dot notation), so subscripts must be evaluated locally by @@ -167,9 +166,8 @@ SELECT data ->> 'name' FROM json_http.json_things; (3 rows) SELECT data ->> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data ->> 'name' FROM json_bin.json_things; QUERY PLAN @@ -428,9 +426,8 @@ SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; (3 rows) SELECT * FROM json_http.json_things ORDER BY data ->> 'name'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY data.name ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_things ORDER BY data ->> 'name'; QUERY PLAN @@ -481,9 +478,8 @@ SELECT data -> 'name' FROM json_http.json_things; (3 rows) SELECT data -> 'name' FROM json_http.json_things ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(id Int8, name String, size String, stocked UInt8) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.things ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT data -> 'name' FROM json_bin.json_things; QUERY PLAN @@ -668,9 +664,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -693,9 +688,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'my field' = 'hello'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."my field" = 'hello')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'my field' = 'hello'; QUERY PLAN @@ -719,9 +713,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -744,9 +737,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'CamelCase' = 'world'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."CamelCase" = 'world')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'CamelCase' = 'world'; QUERY PLAN @@ -770,9 +762,8 @@ SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as jsonb +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -795,9 +786,8 @@ SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; (3 rows) SELECT * FROM json_http.json_special_keys WHERE data ->> 'select' = 'reserved'; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(CamelCase String, `my field` String, select String) as json +DETAIL: Remote Query: SELECT id, data FROM json_test.special_keys WHERE ((data."select" = 'reserved')) EXPLAIN (VERBOSE, COSTS OFF) SELECT * FROM json_bin.json_special_keys WHERE data ->> 'select' = 'reserved'; QUERY PLAN @@ -1163,9 +1153,8 @@ SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'customerId') FROM json_bin.events; QUERY PLAN @@ -1189,9 +1178,8 @@ SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events; (3 rows) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_http.events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path_text(props, 'address', 'city') FROM json_bin.events; QUERY PLAN @@ -1215,9 +1203,8 @@ SELECT jsonb_extract_path(props, 'address') FROM json_http.events; (3 rows) SELECT jsonb_extract_path(props, 'address') FROM json_http.events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as jsonb +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT jsonb_extract_path(props, 'address') FROM json_bin.events; QUERY PLAN @@ -1347,9 +1334,8 @@ SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events; (3 rows) SELECT json_extract_path_text(props, 'customerId') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'customerId') FROM json_bin.json_events; QUERY PLAN @@ -1373,9 +1359,8 @@ SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_even (3 rows) SELECT json_extract_path_text(props, 'address', 'city') FROM json_http.json_events ORDER BY id; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT id, props FROM json_test.events ORDER BY id ASC NULLS LAST EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path_text(props, 'address', 'city') FROM json_bin.json_events; QUERY PLAN @@ -1399,9 +1384,8 @@ SELECT json_extract_path(props, 'address') FROM json_http.json_events; (3 rows) SELECT json_extract_path(props, 'address') FROM json_http.json_events; -ERROR: invalid input syntax for type json -DETAIL: Token "(" is invalid. -CONTEXT: JSON data, line 1: (... +ERROR: pg_clickhouse: cannot return Tuple(address Tuple(city String, zip String), customerId String) as json +DETAIL: Remote Query: SELECT props FROM json_test.events EXPLAIN (VERBOSE, COSTS OFF) SELECT json_extract_path(props, 'address') FROM json_bin.json_events; QUERY PLAN diff --git a/test/expected/query_cancel.out b/test/expected/query_cancel.out index 4b78aa45..6cbc332b 100644 --- a/test/expected/query_cancel.out +++ b/test/expected/query_cancel.out @@ -1,13 +1,9 @@ -- Test that query cancellation (e.g. Ctrl+C / statement_timeout) works for -- both the HTTP and binary drivers during remote query execution. --- HTTP driver (streaming path) +-- HTTP driver CREATE SERVER cancel_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'cancel_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http; --- HTTP driver (buffered path) -CREATE SERVER cancel_http_buf FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; -- Binary driver CREATE SERVER cancel_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'cancel_test', driver 'binary'); @@ -40,8 +36,6 @@ SELECT clickhouse_raw_query('INSERT INTO cancel_test.t1 CREATE FOREIGN TABLE cancel_http_ft (c1 int) SERVER cancel_http OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE cancel_http_buf_ft (c1 int) - SERVER cancel_http_buf OPTIONS (table_name 't1'); CREATE FOREIGN TABLE cancel_binary_ft (c1 int) SERVER cancel_binary OPTIONS (table_name 't1'); -- Warm up connections so the cancel test only covers query execution. @@ -51,12 +45,6 @@ SELECT count(*) FROM cancel_http_ft; 100 (1 row) -SELECT count(*) FROM cancel_http_buf_ft; - count -------- - 100 -(1 row) - SELECT count(*) FROM cancel_binary_ft; count ------- @@ -73,14 +61,6 @@ SELECT count(*) FROM cancel_http_ft a CROSS JOIN cancel_http_ft b CROSS JOIN cancel_http_ft c CROSS JOIN cancel_http_ft d; ERROR: pg_clickhouse: query was aborted COMMIT; --- HTTP buffered: exercises the curl progress-callback cancel path via --- the legacy non-streaming (simple_query) implementation. -BEGIN; -SET LOCAL statement_timeout = '10ms'; -SELECT count(*) FROM cancel_http_buf_ft a CROSS JOIN cancel_http_buf_ft b - CROSS JOIN cancel_http_buf_ft c CROSS JOIN cancel_http_buf_ft d; -ERROR: pg_clickhouse: query was aborted -COMMIT; -- Binary: same test, exercising the OnProgress cancel path. BEGIN; SET LOCAL statement_timeout = '10ms'; @@ -96,12 +76,6 @@ SELECT count(*) FROM cancel_http_ft; 100 (1 row) -SELECT count(*) FROM cancel_http_buf_ft; - count -------- - 100 -(1 row) - SELECT count(*) FROM cancel_binary_ft; count ------- @@ -110,13 +84,10 @@ SELECT count(*) FROM cancel_binary_ft; -- Cleanup DROP FOREIGN TABLE cancel_http_ft; -DROP FOREIGN TABLE cancel_http_buf_ft; DROP FOREIGN TABLE cancel_binary_ft; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http; -DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_binary; DROP SERVER cancel_http; -DROP SERVER cancel_http_buf; DROP SERVER cancel_binary; SELECT clickhouse_raw_query('DROP DATABASE cancel_test'); clickhouse_raw_query diff --git a/test/expected/stream_out.out b/test/expected/stream_out.out index c9a46397..3a6510aa 100644 --- a/test/expected/stream_out.out +++ b/test/expected/stream_out.out @@ -1,5 +1,5 @@ CREATE SERVER try_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'try_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'try_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER try_http; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS try_test'); clickhouse_raw_query @@ -3534,7 +3534,6 @@ SELECT * FROM try_http_ft; 3499 (3500 rows) -ALTER SERVER try_http OPTIONS (SET fetch_size '3'); SELECT count(*), min(c1), max(c1), sum(c1::bigint) FROM try_http_ft; count | min | max | sum -------+-----+------+--------- diff --git a/test/sql/http.sql b/test/sql/http.sql index 18bfa73e..f5d03e73 100644 --- a/test/sql/http.sql +++ b/test/sql/http.sql @@ -37,17 +37,6 @@ CREATE FOREIGN TABLE ft1 ( c8 text ) SERVER http_loopback OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE ft1_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text, - c4 date, - c5 date, - c6 varchar(10), - c7 char(10) default 'ft1', - c8 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); - ALTER FOREIGN TABLE ft1 DROP COLUMN c0; CREATE FOREIGN TABLE ft2 ( @@ -120,9 +109,6 @@ INSERT INTO ft4 (id % 2)::bool FROM generate_series(1, 100) id; --- 15 rows with fetch_size 100 bytes forces multiple streaming batches. -SELECT c1, c3 FROM ft1_stream WHERE c1 <= 15 ORDER BY c1; - SELECT * FROM ft5 ORDER BY c1 LIMIT 5; COPY ftcopy FROM stdin; @@ -263,48 +249,21 @@ CREATE FOREIGN TABLE bad_name ( ) SERVER http_loopback_bad OPTIONS ( table_name 't3' ); SELECT * FROM bad_name; -/* ===== fetch_size option tests ===== */ - -/* Server-level fetch_size: set to 0 to disable streaming. */ -CREATE SERVER http_no_stream FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER http_no_stream; - -CREATE FOREIGN TABLE ft_no_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1'); - -/* Query with streaming disabled (fetch_size = 0). */ -SELECT c3 FROM ft_no_stream ORDER BY c1 LIMIT 3; - -/* Table-level fetch_size overrides server-level. */ -CREATE FOREIGN TABLE ft_override_stream ( - c1 int NOT NULL, - c2 int NOT NULL, - c3 text -) SERVER http_no_stream OPTIONS (table_name 't1', fetch_size '100'); - -/* Query with table-level streaming override (fetch_size = 100). */ -SELECT c3 FROM ft_override_stream ORDER BY c1 LIMIT 3; - -/* Default (no fetch_size set) uses streaming — already tested via ft1. */ -SELECT c3 FROM ft1 ORDER BY c1 LIMIT 3; - -/* Negative fetch_size should be rejected. */ +/* Deprecated fetch_size options warn but remain accepted. */ CREATE SERVER http_bad_fetch FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'http_test', driver 'http', fetch_size '-1'); + OPTIONS(dbname 'http_test', driver 'http', fetch_size '100'); CREATE FOREIGN TABLE ft_bad_fetch ( c1 int NOT NULL, c3 text -) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '-1'); +) SERVER http_loopback OPTIONS (table_name 't1', fetch_size '100'); + +DROP FOREIGN TABLE ft_bad_fetch; +DROP SERVER http_bad_fetch; /* - * TabSeparated does not escape `[` or `]` in String values, so a value like - * `[foo]bar` is wire-indistinguishable from a CH array literal. The parser - * uses the destination Postgres column type to decide. + * Native names the column type on the wire, so a String value like `[foo]bar` + * stays text instead of reading as a CH array literal. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_brk (id String, term String) ENGINE = MergeTree ORDER BY id'); @@ -322,11 +281,6 @@ INSERT INTO ft_brk VALUES SELECT id, term FROM ft_brk ORDER BY id; -/* Streaming path with a small fetch_size forces multi-batch parsing. */ -CREATE FOREIGN TABLE ft_brk_stream (id text, term text) - SERVER http_loopback OPTIONS (table_name 't_brk', fetch_size '32'); -SELECT id, term FROM ft_brk_stream ORDER BY id; - /* * Mixed row: real Array(String) column alongside a String column whose value * starts with `[`. The parser must stay in sync across columns. @@ -381,8 +335,8 @@ CREATE FOREIGN TABLE ft_bytea (id text, v bytea) SELECT id, v, v IS NULL AS is_null, octet_length(v) FROM ft_bytea ORDER BY id; /* - * time columns strip the exact ISO epoch date prefix. Shorter values or ones - * with another prefix route through the input function unchanged. + * time columns route String values through the input function unchanged, so + * an ISO timestamp is rejected rather than trimmed to its time of day. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.t_timestr (id String, v String) ENGINE = MergeTree ORDER BY id'); @@ -398,9 +352,8 @@ SELECT v FROM ft_timestr WHERE id = '3'; SELECT v FROM ft_timestr WHERE id = '1'; SELECT v FROM ft_timestr WHERE id = '2'; -/* nested arrays via http (TabSeparated): rectangular maps to multi-dim, - * jagged shapes route through array_in and surface its malformed-literal - * error -- matching the binary path. */ +/* nested arrays via http: rectangular maps to multi-dim, jagged shapes are + * rejected -- matching the binary path. */ SELECT clickhouse_raw_query('CREATE TABLE http_test.nested_arrays ( c1 Int8, c2 Array(Array(Int32)), c3 Array(Array(String)) ) ENGINE = MergeTree PARTITION BY c1 ORDER BY (c1); @@ -448,9 +401,6 @@ SELECT * FROM clickhouse_query('http_loopback', 'SELECT ''abc''') AS t(x int); -- unknown server is rejected SELECT * FROM clickhouse_query('no_such_server', 'SELECT 1') AS t(x int); -DROP USER MAPPING FOR CURRENT_USER SERVER http_no_stream; -DROP SERVER http_no_stream CASCADE; - /* ===== secure option tests ===== */ /* All accepted values for the secure option should pass validation. */ diff --git a/test/sql/query_cancel.sql b/test/sql/query_cancel.sql index 22b0af9f..62e4e488 100644 --- a/test/sql/query_cancel.sql +++ b/test/sql/query_cancel.sql @@ -1,16 +1,11 @@ -- Test that query cancellation (e.g. Ctrl+C / statement_timeout) works for -- both the HTTP and binary drivers during remote query execution. --- HTTP driver (streaming path) +-- HTTP driver CREATE SERVER cancel_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'cancel_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http; --- HTTP driver (buffered path) -CREATE SERVER cancel_http_buf FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'cancel_test', driver 'http', fetch_size '0'); -CREATE USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; - -- Binary driver CREATE SERVER cancel_binary FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'cancel_test', driver 'binary'); @@ -25,14 +20,11 @@ SELECT clickhouse_raw_query('INSERT INTO cancel_test.t1 CREATE FOREIGN TABLE cancel_http_ft (c1 int) SERVER cancel_http OPTIONS (table_name 't1'); -CREATE FOREIGN TABLE cancel_http_buf_ft (c1 int) - SERVER cancel_http_buf OPTIONS (table_name 't1'); CREATE FOREIGN TABLE cancel_binary_ft (c1 int) SERVER cancel_binary OPTIONS (table_name 't1'); -- Warm up connections so the cancel test only covers query execution. SELECT count(*) FROM cancel_http_ft; -SELECT count(*) FROM cancel_http_buf_ft; SELECT count(*) FROM cancel_binary_ft; -- The repeated CROSS JOINs intentionally create a large remote result set, @@ -45,14 +37,6 @@ SELECT count(*) FROM cancel_http_ft a CROSS JOIN cancel_http_ft b CROSS JOIN cancel_http_ft c CROSS JOIN cancel_http_ft d; COMMIT; --- HTTP buffered: exercises the curl progress-callback cancel path via --- the legacy non-streaming (simple_query) implementation. -BEGIN; -SET LOCAL statement_timeout = '10ms'; -SELECT count(*) FROM cancel_http_buf_ft a CROSS JOIN cancel_http_buf_ft b - CROSS JOIN cancel_http_buf_ft c CROSS JOIN cancel_http_buf_ft d; -COMMIT; - -- Binary: same test, exercising the OnProgress cancel path. BEGIN; SET LOCAL statement_timeout = '10ms'; @@ -63,17 +47,13 @@ COMMIT; -- Verify each connection is still usable after the canceled query tears down -- any remote state. SELECT count(*) FROM cancel_http_ft; -SELECT count(*) FROM cancel_http_buf_ft; SELECT count(*) FROM cancel_binary_ft; -- Cleanup DROP FOREIGN TABLE cancel_http_ft; -DROP FOREIGN TABLE cancel_http_buf_ft; DROP FOREIGN TABLE cancel_binary_ft; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http; -DROP USER MAPPING FOR CURRENT_USER SERVER cancel_http_buf; DROP USER MAPPING FOR CURRENT_USER SERVER cancel_binary; DROP SERVER cancel_http; -DROP SERVER cancel_http_buf; DROP SERVER cancel_binary; SELECT clickhouse_raw_query('DROP DATABASE cancel_test'); diff --git a/test/sql/stream_out.sql b/test/sql/stream_out.sql index 9c30982e..7bedbd3c 100644 --- a/test/sql/stream_out.sql +++ b/test/sql/stream_out.sql @@ -1,5 +1,5 @@ CREATE SERVER try_http FOREIGN DATA WRAPPER clickhouse_fdw - OPTIONS(dbname 'try_test', driver 'http', fetch_size '1'); + OPTIONS(dbname 'try_test', driver 'http'); CREATE USER MAPPING FOR CURRENT_USER SERVER try_http; SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS try_test'); @@ -14,8 +14,6 @@ CREATE FOREIGN TABLE try_http_ft (c1 int) SELECT * FROM try_http_ft; -ALTER SERVER try_http OPTIONS (SET fetch_size '3'); - SELECT count(*), min(c1), max(c1), sum(c1::bigint) FROM try_http_ft; DROP USER MAPPING FOR CURRENT_USER SERVER try_http;