diff --git a/doc/pg_clickhouse.md b/doc/pg_clickhouse.md index 8d2e560b..4861b2a2 100644 --- a/doc/pg_clickhouse.md +++ b/doc/pg_clickhouse.md @@ -1313,6 +1313,7 @@ equivalents as follows: * `!~*` (case insensitive regexp not match): [match](https://clickhouse.com/docs/sql-reference/functions/string-search-functions#match) * `->>` (JSON/JSONB extract element as text): [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) * `->` (JSON/JSONB extract): [toJSONString](https://clickhouse.com/docs/sql-reference/functions/json-functions#toJSONString) + [sub-column syntax](https://clickhouse.com/docs/sql-reference/data-types/newjson#reading-json-paths-as-sub-columns) +* `?` (JSONB top-level key or array string existence): [JSONHas](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonhas) for objects and [JSONExtractArrayRaw](https://clickhouse.com/docs/sql-reference/functions/json-functions#jsonextractarrayraw) for arrays. This also supports compatibility views that parse a JSON document stored in a ClickHouse `String`. ### IN and NULL Semantics diff --git a/src/custom_types.c b/src/custom_types.c index 19171d71..c3a93fdb 100644 --- a/src/custom_types.c +++ b/src/custom_types.c @@ -946,6 +946,7 @@ chfdw_check_for_custom_type(Oid typeoid) { #define OID_TEXT_REGEX_NE_OP 642 #define OID_TEXT_IREGEX_NE_OP 1229 #define OID_JSONB_FETCHVAL_OP 3211 +#define OID_JSONB_EXISTS_OP 3247 #define OID_JSONB_FETCHVAL_TEXT_OP 3477 #define OID_JSON_FETCHVAL_OP 3962 #define OID_JSON_FETCHVAL_TEXT_OP 3963 @@ -977,6 +978,8 @@ classify_builtin_operator(Oid opoid) { case OID_JSONB_FETCHVAL_TEXT_OP: case OID_JSON_FETCHVAL_TEXT_OP: return CF_JSON_FETCHVAL_TEXT; + case OID_JSONB_EXISTS_OP: + return CF_JSON_EXISTS; case OID_ARRAY_CONTAINS_OP: return CF_ARRAY_CONTAINS; case OID_ARRAY_CONTAINED_OP: diff --git a/src/deparse.c b/src/deparse.c index 66d50d0b..2287f0cb 100644 --- a/src/deparse.c +++ b/src/deparse.c @@ -198,6 +198,12 @@ typedef enum ExprTruthCtx { EXPR_CTX_EXACT, } ExprTruthCtx; +typedef enum JsonbDocumentKind { + JSONB_DOCUMENT_UNSUPPORTED, + JSONB_DOCUMENT_NATIVE, + JSONB_DOCUMENT_STRING, +} JsonbDocumentKind; + /* * Functions to determine whether an expression can be evaluated safely on * remote server. @@ -258,6 +264,13 @@ deparseSQLValueFunction(SQLValueFunction* node, deparse_expr_cxt* context); static void deparseOpExpr(OpExpr* node, deparse_expr_cxt* context); static void +deparseJsonbExists( + Expr* document, + Expr* key, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +); +static void deparseOperatorName(StringInfo buf, Form_pg_operator opform); static void deparseDistinctExpr(DistinctExpr* node, deparse_expr_cxt* context); @@ -382,6 +395,8 @@ get_relation_column_alias_ids( int* relno, int* colno ); +static JsonbDocumentKind +classifyJsonbDocument(Expr* expr, Expr** document); /* * Examine each qual clause in input_conds, and classify them into two groups, @@ -894,6 +909,55 @@ classify_notin_subplan(SubPlan* subplan, PlannerInfo* root, Relids relids) { return NOTIN_SHIP_NONE; } +/* + * Recognize either a native ClickHouse JSON column imported as jsonb or the + * compatibility-view shape used for a String containing a JSON document: + * + * jsonb_in(foreign_text_column::cstring) + * + * Arbitrary jsonb expressions stay local because their remote representation + * is not known. + */ +static JsonbDocumentKind +classifyJsonbDocument(Expr* expr, Expr** document) { + while (IsA(expr, RelabelType)) { + expr = ((RelabelType*)expr)->arg; + } + + if (IsA(expr, Var) && exprType((Node*)expr) == JSONBOID) { + *document = expr; + return JSONB_DOCUMENT_NATIVE; + } + + if (IsA(expr, FuncExpr)) { + FuncExpr* func = (FuncExpr*)expr; + + if (func->funcid == F_JSONB_IN && list_length(func->args) == 1) { + Expr* input = (Expr*)linitial(func->args); + + while (IsA(input, RelabelType)) { + input = ((RelabelType*)input)->arg; + } + if (IsA(input, CoerceViaIO)) { + CoerceViaIO* cast = (CoerceViaIO*)input; + Expr* raw = cast->arg; + + while (IsA(raw, RelabelType)) { + raw = ((RelabelType*)raw)->arg; + } + if (cast->resulttype == CSTRINGOID && + exprType((Node*)raw) == TEXTOID) { + *document = raw; + return JSONB_DOCUMENT_STRING; + } + } + } + } + + *document = NULL; + return JSONB_DOCUMENT_UNSUPPORTED; +} + /* * Check if expression is safe to execute remotely, and return true if so. * @@ -1055,6 +1119,7 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { case T_DistinctExpr: /* struct-equivalent to OpExpr */ { OpExpr* oe = (OpExpr*)node; + CustomObjectDef* cdef; /* * Similarly, only shippable operators can be sent to remote. @@ -1065,6 +1130,24 @@ foreign_expr_walker(Node* node, foreign_glob_cxt* glob_cxt, ExprTruthCtx ctx) { return false; } + cdef = chfdw_check_for_custom_operator(oe->opno, NULL); + if (cdef && cdef->cf_type == CF_JSON_EXISTS) { + Expr* document; + JsonbDocumentKind document_kind = + classifyJsonbDocument((Expr*)linitial(oe->args), &document); + + if (document_kind == JSONB_DOCUMENT_UNSUPPORTED || + !foreign_expr_walker( + (Node*)document, glob_cxt, EXPR_CTX_EXACT + ) || + !foreign_expr_walker( + (Node*)lsecond(oe->args), glob_cxt, EXPR_CTX_EXACT + )) { + return false; + } + break; + } + /* * Recurse to input subexpressions. */ @@ -4869,6 +4952,81 @@ findFunction(Oid typoid, char* name) { return result; } +static void +deparseJsonbDocument( + Expr* document, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + if (document_kind == JSONB_DOCUMENT_NATIVE) { + appendStringInfoString(context->buf, "toJSONString("); + } + deparseExpr(document, context); + if (document_kind == JSONB_DOCUMENT_NATIVE) { + appendStringInfoChar(context->buf, ')'); + } +} + +static void +deparseJsonbNonnullDocument( + Expr* document, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + appendStringInfoString(context->buf, "ifNull("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(context->buf, ", 'null')"); +} + +static void +deparseJsonbExists( + Expr* document, + Expr* key, + JsonbDocumentKind document_kind, + deparse_expr_cxt* context +) { + StringInfo buf = context->buf; + + appendStringInfoString(buf, "(if(isNull("); + deparseJsonbDocument(document, document_kind, context); + appendStringInfoString(buf, ") OR isNull("); + deparseExpr(key, context); + appendStringInfoString(buf, "), NULL, "); + + if (document_kind == JSONB_DOCUMENT_STRING) { + appendStringInfoString(buf, "if(NOT isValidJSON("); + deparseJsonbNonnullDocument(document, document_kind, context); + appendStringInfoString( + buf, + "), throwIf(1, 'invalid input syntax for type json'), " + ); + } + + appendStringInfoString(buf, "multiIf(JSONType("); + deparseJsonbNonnullDocument(document, document_kind, context); + appendStringInfoString(buf, ") = 'Object', JSONHas("); + deparseJsonbNonnullDocument(document, document_kind, context); + appendStringInfoString(buf, ", "); + deparseExpr(key, context); + appendStringInfoString(buf, "), JSONType("); + deparseJsonbNonnullDocument(document, document_kind, context); + appendStringInfoString( + buf, + ") = 'Array', arrayExists(jsonb_exists_element -> " + "JSONType(jsonb_exists_element) = 'String' AND " + "JSONExtractString(jsonb_exists_element) = " + ); + deparseExpr(key, context); + appendStringInfoString(buf, ", JSONExtractArrayRaw("); + deparseJsonbNonnullDocument(document, document_kind, context); + appendStringInfoString(buf, ")), 0)"); + + if (document_kind == JSONB_DOCUMENT_STRING) { + appendStringInfoChar(buf, ')'); + } + appendStringInfoString(buf, "))"); +} + /* * Deparse given operator expression. To avoid problems around priority of * operations, we always parenthesize the arguments. @@ -4999,6 +5157,17 @@ deparseOpExpr(OpExpr* node, deparse_expr_cxt* context) { appendStringInfoChar(buf, ')'); goto cleanup; } break; + case CF_JSON_EXISTS: { + Expr* document; + JsonbDocumentKind document_kind = + classifyJsonbDocument(linitial(node->args), &document); + + Assert(document_kind != JSONB_DOCUMENT_UNSUPPORTED); + deparseJsonbExists( + document, lsecond(node->args), document_kind, context + ); + goto cleanup; + } break; case CF_JSON_FETCHVAL: case CF_JSON_FETCHVAL_TEXT: { Expr* arg1 = linitial(node->args); diff --git a/src/include/fdw.h b/src/include/fdw.h index 5509acb1..2f071803 100644 --- a/src/include/fdw.h +++ b/src/include/fdw.h @@ -417,6 +417,7 @@ typedef enum { CF_REGEX_ICASE_NO_MATCH, /* !~* case-insensitive regex operator */ CF_JSON_FETCHVAL, /* -> operator on json & jsonb */ CF_JSON_FETCHVAL_TEXT, /* ->> operator on json & jsonb */ + CF_JSON_EXISTS, /* ? operator on jsonb */ CF_JSON_EXTRACT_PATH_TEXT, /* jsonb?_extract_path_text() → * col."k1"."k2" */ CF_JSON_EXTRACT_PATH, /* json?_extract_path() → diff --git a/test/expected/jsonb_exists.out b/test/expected/jsonb_exists.out new file mode 100644 index 00000000..b711c601 --- /dev/null +++ b/test/expected/jsonb_exists.out @@ -0,0 +1,171 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE SERVER +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +CREATE USER MAPPING +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String) + ) ENGINE = MergeTree ORDER BY id +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '["key","second",3]'), + (3, '{"other":2}'), + (4, '["other",3]'), + (5, NULL), + (6, '"key"') +$$); + clickhouse_raw_query +---------------------- + +(1 row) + +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); +CREATE FOREIGN TABLE +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); +CREATE FOREIGN TABLE +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +FROM jsonb_exists_string; +CREATE VIEW +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_native + Output: id + Remote SQL: SELECT id FROM jsonb_exists.native_documents WHERE ((if(isNull(toJSONString(document)) OR isNull('key'), NULL, multiIf(JSONType(ifNull(toJSONString(document), 'null')) = 'Object', JSONHas(ifNull(toJSONString(document), 'null'), 'key'), JSONType(ifNull(toJSONString(document), 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(toJSONString(document), 'null'))), 0)))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + id +---- + 1 +(1 row) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE ((if(isNull(document) OR isNull('key'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'key'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'key', JSONExtractArrayRaw(ifNull(document, 'null'))), 0))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Remote SQL: SELECT id FROM jsonb_exists.string_documents WHERE (((if(isNull(document) OR isNull('first'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'first'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'first', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))) OR (if(isNull(document) OR isNull('second'), NULL, if(NOT isValidJSON(ifNull(document, 'null')), throwIf(1, 'invalid input syntax for type json'), multiIf(JSONType(ifNull(document, 'null')) = 'Object', JSONHas(ifNull(document, 'null'), 'second'), JSONType(ifNull(document, 'null')) = 'Array', arrayExists(jsonb_exists_element -> JSONType(jsonb_exists_element) = 'String' AND JSONExtractString(jsonb_exists_element) = 'second', JSONExtractArrayRaw(ifNull(document, 'null'))), 0)))))) ORDER BY id ASC NULLS LAST +(3 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + id +---- + 1 + 2 +(2 rows) + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + QUERY PLAN +------------------------------------------------------------------------------------------------- + Foreign Scan on public.jsonb_exists_string + Output: jsonb_exists_string.id + Filter: ((jsonb_in((jsonb_exists_string.document)::cstring) || '{}'::jsonb) ? 'key'::text) + Remote SQL: SELECT id, document FROM jsonb_exists.string_documents ORDER BY id ASC NULLS LAST +(4 rows) + +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + id +---- + 1 + 2 + 6 +(3 rows) + +DROP VIEW jsonb_exists_compatibility; +DROP VIEW +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE +DROP FOREIGN TABLE jsonb_exists_native; +DROP FOREIGN TABLE +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +DROP USER MAPPING +SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); + clickhouse_raw_query +---------------------- + +(1 row) + +DROP SERVER jsonb_exists_binary; +DROP SERVER diff --git a/test/sql/jsonb_exists.sql b/test/sql/jsonb_exists.sql new file mode 100644 index 00000000..4ed6eeef --- /dev/null +++ b/test/sql/jsonb_exists.sql @@ -0,0 +1,82 @@ +CREATE SERVER jsonb_exists_binary FOREIGN DATA WRAPPER clickhouse_fdw + OPTIONS (dbname 'jsonb_exists', driver 'binary'); +CREATE USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; + +SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS jsonb_exists'); +SELECT clickhouse_raw_query('CREATE DATABASE jsonb_exists'); +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.native_documents ( + id Int32, + document JSON + ) ENGINE = MergeTree ORDER BY id +$$); +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.native_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '{"other":2}'), + (3, '{"other":null}') +$$); +SELECT clickhouse_raw_query($$ + CREATE TABLE jsonb_exists.string_documents ( + id Int32, + document Nullable(String) + ) ENGINE = MergeTree ORDER BY id +$$); +SELECT clickhouse_raw_query($$ + INSERT INTO jsonb_exists.string_documents VALUES + (1, '{"key":1,"first":true}'), + (2, '["key","second",3]'), + (3, '{"other":2}'), + (4, '["other",3]'), + (5, NULL), + (6, '"key"') +$$); + +CREATE FOREIGN TABLE jsonb_exists_native ( + id integer, + document jsonb +) SERVER jsonb_exists_binary OPTIONS (table_name 'native_documents'); + +CREATE FOREIGN TABLE jsonb_exists_string ( + id integer, + document text +) SERVER jsonb_exists_binary OPTIONS (table_name 'string_documents'); + +CREATE VIEW jsonb_exists_compatibility AS +SELECT id, pg_catalog.jsonb_in(document::pg_catalog.cstring) AS document +FROM jsonb_exists_string; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; +SELECT id FROM jsonb_exists_native WHERE document ? 'key' ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'key' +ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE document ? 'first' OR document ? 'second' +ORDER BY id; + +EXPLAIN (VERBOSE, COSTS OFF) +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; +SELECT id FROM jsonb_exists_compatibility +WHERE (document || '{}'::jsonb) ? 'key' +ORDER BY id; + +DROP VIEW jsonb_exists_compatibility; +DROP FOREIGN TABLE jsonb_exists_string; +DROP FOREIGN TABLE jsonb_exists_native; +DROP USER MAPPING FOR CURRENT_USER SERVER jsonb_exists_binary; +SELECT clickhouse_raw_query('DROP DATABASE jsonb_exists'); +DROP SERVER jsonb_exists_binary;