feat: portable text search with contains, startsWith and endsWith

.changeset/text-search.md

@@ -0,0 +1,5 @@
+---
+"@maple-dev/effect-orm": minor
+---
+
+String expressions add `.contains(value)`, `.startsWith(value)` and `.endsWith(value)`: text search that treats the value as text, not as a LIKE pattern. `%`, `_` and `\` in the value are escaped with a backslash. A string is escaped before it is bound; a param or expression is escaped in SQL with `replace()`. `{ insensitive: true }` writes `ILIKE`. Postgres gets an explicit `ESCAPE '\'`; ClickHouse already escapes with a backslash. `escapeLike(text)` is the escape step on its own, for a `.like()` pattern. Postgres and ClickHouse.

docs/expressions.md

@@ -81,6 +81,31 @@ list is written as the constant it means, `1 = 0` for `in_()` and `1 = 1` for `n
 `.like(pattern)`, `.notLike(pattern)`, `.ilike(pattern)` are constrained by `this` to
 `Expr<string>`, so calling them on a numeric column is a type error.
 
+### Text search
+
+`.contains(value)`, `.startsWith(value)` and `.endsWith(value)` match text, not a pattern. `%`,
+`_` and `\` in the value match themselves: the helper escapes them with a backslash and adds the
+wildcards. Use them for a search box; use `.like()` only for a pattern you wrote.
+
+```ts
+.where(($) => [$.OrgId.eq("org_123"), $.Name.contains("50%_off", { insensitive: true })])
+// ClickHouse: … AND events.Name ILIKE '%50\\%\\_off%'
+// Postgres:   … AND "events"."Name" ILIKE $1 ESCAPE '\'   -- $1 = '%50\%\_off%'
+```
+
+- The value is a string, or a param or expression of one. A string is escaped before it is bound.
+  A param or expression is escaped in SQL (`replace(…)` three times), because its value is known
+  only when the query runs.
+- Postgres gets an explicit `ESCAPE '\'`. ClickHouse uses the backslash as its escape character
+  already, so it gets none.
+- `{ insensitive: true }` writes `ILIKE`. Which characters fold is the database's choice.
+  ClickHouse folds UTF-8 (`'ÉCOLE' ILIKE 'é%'` is true). Postgres folds by the database's
+  `LC_CTYPE`: a `C` locale (PGlite's default) folds ASCII only.
+- An empty value matches every non-NULL row. A NULL column never matches.
+- `escapeLike(text)` is the escape step on its own, for a pattern you build for `.like()`.
+
+_(Backed by `docs/expressions.md > Text search escapes LIKE wildcards`.)_
+
 ## Combining conditions
 
 ```ts
@@ -139,7 +164,7 @@ unbounded read. The [optional-filter recipe](./recipes.md#optional-filters-and-e
 an explicit choice.
 
 Escaping SQL strings does not escape LIKE wildcards. `%` and `_` in a `.like()` argument are
-patterns; use `.eq()` for exact text. Keep table names, aliases, and raw SQL under application
+patterns; use `.eq()` for exact text and [`.contains()`](#text-search) to search for it. Keep table names, aliases, and raw SQL under application
 control: value escaping is not an allowlist for identifiers.
 
 ## Arithmetic

docs/reference.md

@@ -113,6 +113,7 @@ time; see [Params and compilation](./params-and-compilation.md#what-each-kind-ac
 | `undecoded(column)`       | the column as the driver sends it, typed as its wire form  |
 | `inExprList(expr, exprs)` | Same for expression lists                                  |
 | `notInList(expr, values)` | `expr NOT IN ('a', 'b')`                                   |
+| `escapeLike(text)`        | `text` with `%`, `_` and `\` escaped, for a `.like()` pattern |
 | `not(condition)`          | `NOT (…)`                                                  |
 | `and(...conds)` / `or(...conds)` | One flat `(… AND …)` / `(… OR …)`; skips `undefined`, returns `undefined` when none are left |
 | `dynamicColumn(name, t?)` | An `Expr` from a runtime column name — a `GROUP BY` alias  |
@@ -123,7 +124,9 @@ time; see [Params and compilation](./params-and-compilation.md#what-each-kind-ac
 
 `Expr<T>` methods: `eq`, `neq`, `gt`, `gte`, `lt`, `lte`, `in_`, `notIn`, `between`, `notBetween`,
 `isNull`, `isNotNull`, `like`, `notLike`,
-`ilike` (string-only), and `add`, `sub`, `mul`, `div`, `mod` (number-only, **no parentheses**). `div` and `mod` decode
+`ilike`, `contains`, `startsWith`, `endsWith` (string-only; the last three escape `%`, `_` and `\`
+in the value and take `TextMatchOptions`, `{ insensitive?: boolean }`, see
+[Text search](./expressions.md#text-search)), and `add`, `sub`, `mul`, `div`, `mod` (number-only, **no parentheses**). `div` and `mod` decode
 as `number | null` — ClickHouse sends `inf`/`nan` as JSON `null` — except by a numeric literal of
 magnitude ≥ 1 (`Quotient<L, R>`), which keeps the dividend's nullability; use
 `ifNull(ifNotFinite(expr, 0), lit(0))` for a guaranteed number otherwise.

src/ch/expr.test-d.ts

@@ -135,6 +135,19 @@ numExpr.ilike("%test%")
 // @ts-expect-error — .notLike() requires Expr<string>
 numExpr.notLike("%test%")
 
+expectTypeOf(strExpr.contains("50%")).toMatchTypeOf<Condition>()
+expectTypeOf(strExpr.startsWith("a", { insensitive: true })).toMatchTypeOf<Condition>()
+expectTypeOf(strExpr.endsWith(strExpr)).toMatchTypeOf<Condition>()
+
+// @ts-expect-error — .contains() requires Expr<string>
+numExpr.contains("1")
+
+// @ts-expect-error — the value is a string, not a number
+strExpr.startsWith(1)
+
+// @ts-expect-error — insensitive is a boolean
+strExpr.endsWith("a", { insensitive: "yes" })
+
 // ColumnRef .get() — only valid for Map columns
 
 const mapRef = makeColumnRef<"Attrs", CHMap<CHString, CHString>>("Attrs")

src/ch/expr.ts

@@ -193,6 +193,14 @@ export interface Expr<TSType, P = never> {
 	like<Q = never>(this: Expr<string | null>, pattern: string | Expr<string, Q>): Condition<P | Q>
 	notLike<Q = never>(this: Expr<string | null>, pattern: string | Expr<string, Q>): Condition<P | Q>
 	ilike<Q = never>(this: Expr<string | null>, pattern: string | Expr<string, Q>): Condition<P | Q>
+	/**
+	 * Text search. The value is text, not a pattern: `%`, `_` and `\` in it
+	 * match themselves. `{ insensitive: true }` ignores case (`ILIKE`). A value
+	 * is a string or a param or expression of one.
+	 */
+	contains<Q = never>(this: Expr<string | null>, value: string | Expr<string, Q>, options?: TextMatchOptions): Condition<P | Q>
+	startsWith<Q = never>(this: Expr<string | null>, value: string | Expr<string, Q>, options?: TextMatchOptions): Condition<P | Q>
+	endsWith<Q = never>(this: Expr<string | null>, value: string | Expr<string, Q>, options?: TextMatchOptions): Condition<P | Q>
 
 	// NULL and ranges
 	/** `expr IS NULL`. */
@@ -227,6 +235,12 @@ export interface Expr<TSType, P = never> {
 	mod<R extends number | null, Q = never>(this: Expr<number | null>, n: R | Expr<R, Q>): Expr<Quotient<TSType, R>, P | Q>
 }
 
+/** Options of `contains`, `startsWith` and `endsWith`. */
+export interface TextMatchOptions {
+	/** Ignore case: `ILIKE` instead of `LIKE`. Which characters fold is the database's choice; see docs/expressions.md. */
+	readonly insensitive?: boolean | undefined
+}
+
 /**
  * What a comparison takes on its right: a value of the column's type (never
  * `null`), or an expression of it.
@@ -387,6 +401,54 @@ const likePattern = (pattern: string | Expr<string, any>): string => {
 	)
 }
 
+/** The LIKE escape character of every text match. */
+const LIKE_ESCAPE = "\\"
+
+/**
+ * `value` with `%`, `_` and `\` escaped by a backslash, so a LIKE pattern
+ * built from it matches the text itself. ClickHouse and Postgres escape with a
+ * backslash by default; `contains` and friends also write `ESCAPE '\'` where
+ * the dialect is not ClickHouse.
+ */
+export const escapeLike = (value: string): string => value.replace(/[\\%_]/g, (char) => `${LIKE_ESCAPE}${char}`)
+
+type TextMatch = "contains" | "startsWith" | "endsWith"
+
+/** `value` escaped and wrapped in the wildcards of `match`, as SQL. */
+const textPattern = (value: string | Expr<string, any>, match: TextMatch): string => {
+	const before = match === "contains" || match === "endsWith" ? "%" : ""
+	const after = match === "contains" || match === "startsWith" ? "%" : ""
+	if (typeof value === "string") return compile(boundString(`${before}${escapeLike(value)}${after}`))
+	if (isExprLike(value)) {
+		// The value is only known at run time: escape it in SQL, the same three
+		// characters, escape character first.
+		const q = (text: string) => compile(str(text))
+		let escaped = compile(value.toFragment())
+		for (const char of [LIKE_ESCAPE, "%", "_"]) escaped = `replace(${escaped}, ${q(char)}, ${q(LIKE_ESCAPE + char)})`
+		return `(${[before && q(before), escaped, after && q(after)].filter(Boolean).join(" || ")})`
+	}
+	return fail(
+		new QueryBuilderError({
+			code: "InvalidLiteral",
+			message: `a ${match} value must be a string or a string expression, not ${describeUntyped(value)}`,
+		}),
+		"NULL",
+	)
+}
+
+/** `expr LIKE pattern`, or `ILIKE` when insensitive, with the escape clause where the dialect needs one. */
+const textMatch = (fragment: SqlFragment, value: string | Expr<string, any>, match: TextMatch, options?: TextMatchOptions): Condition<any> =>
+	makeCond(
+		known(() => {
+			const op = options?.insensitive === true ? "ILIKE" : "LIKE"
+			// Outside a compile, ClickHouse is the default dialect. Its backslash
+			// escape is built in; standard SQL has no default escape character.
+			const dialect = activeSqlSyntax()?.dialect ?? "clickhouse"
+			const escape = dialect === "clickhouse" ? "" : ` ESCAPE ${compile(str(LIKE_ESCAPE))}`
+			return `${compile(fragment)} ${op} ${textPattern(value, match)}${escape}`
+		}),
+	)
+
 const dateTimeLiteral = (value: DateTime.Utc): string =>
 	activeSqlSyntax()?.dateTimeLiteral(value) ?? compile(str(chDateTimeLiteral(value)))
 
@@ -534,6 +596,9 @@ export function makeExpr<T>(
 		like: (pattern) => makeCond(known(() => `${compile(fragment)} LIKE ${likePattern(pattern)}`)),
 		notLike: (pattern) => makeCond(known(() => `${compile(fragment)} NOT LIKE ${likePattern(pattern)}`)),
 		ilike: (pattern) => makeCond(known(() => `${compile(fragment)} ILIKE ${likePattern(pattern)}`)),
+		contains: (value, options) => textMatch(fragment, value, "contains", options),
+		startsWith: (value, options) => textMatch(fragment, value, "startsWith", options),
+		endsWith: (value, options) => textMatch(fragment, value, "endsWith", options),
 
 		in_: (...values) => inCond(fragment, "IN", values.map((v) => () => operand(v))),
 		notIn: (...values) => inCond(fragment, "NOT IN", values.map((v) => () => operand(v))),

src/ch/param.ts

@@ -115,6 +115,9 @@ function makeParamMarker<N extends string, T>(
 		like: raise,
 		notLike: raise,
 		ilike: raise,
+		contains: raise,
+		startsWith: raise,
+		endsWith: raise,
 		isNull: raise,
 		isNotNull: raise,
 		between: raise,

src/ch/text-match.test.ts

@@ -0,0 +1,62 @@
+// `contains`, `startsWith` and `endsWith` match text, not a pattern: `%`, `_`
+// and the escape character in the value are escaped, on every dialect. Live
+// runs on ClickHouse and Postgres are the `text-match` core cases.
+
+import { describe, expect, it } from "@effect/vitest"
+import { Cause, Effect, Exit } from "effect"
+import * as CH from "../clickhouse"
+import * as PG from "../postgres"
+
+const Events = CH.table("events", {
+	columns: { Id: CH.string, Name: CH.string, Note: CH.nullable(CH.string) },
+	engine: CH.engine.mergeTree(),
+	orderBy: ["Id"],
+})
+const Users = PG.table("users", { columns: { id: PG.text, name: PG.text }, primaryKey: ["id"] })
+
+describe("text match", () => {
+	it("contains escapes LIKE wildcards in a ClickHouse value", () => {
+		const compiled = CH.compileUnsafe(CH.from(Events).select("Id").where(($) => [$.Name.contains("50%_off\\")]))
+		expect(compiled.sql).toContain(String.raw`Name LIKE '%50\\%\\_off\\\\%'`)
+	})
+
+	it("startsWith and endsWith put the wildcard on one side, also on a nullable column", () => {
+		const compiled = CH.compileUnsafe(
+			CH.from(Events).select("Id").where(($) => [$.Name.startsWith("a_"), $.Note.endsWith("%z")]),
+		)
+		expect(compiled.sql).toContain(String.raw`Name LIKE 'a\\_%'`)
+		expect(compiled.sql).toContain(String.raw`Note LIKE '%\\%z'`)
+	})
+
+	it("insensitive is ILIKE", () => {
+		const compiled = CH.compileUnsafe(CH.from(Events).select("Id").where(($) => [$.Name.contains("Ab", { insensitive: true })]))
+		expect(compiled.sql).toContain(`Name ILIKE '%Ab%'`)
+	})
+
+	it("binds the escaped value on Postgres and writes the escape character", () => {
+		const compiled = PG.compileUnsafe(
+			PG.from(Users).select("id").where(($) => [$.name.contains("50%_", { insensitive: true }), $.name.startsWith("x")]),
+		)
+		expect(compiled.sql).toContain(String.raw`"name" ILIKE $1 ESCAPE '\'`)
+		expect(compiled.sql).toContain(String.raw`"name" LIKE $2 ESCAPE '\'`)
+		expect(compiled.parameters).toEqual([String.raw`%50\%\_%`, "x%"])
+	})
+
+	it("escapes a param in SQL, since its value is known only at run time", () => {
+		const query = PG.from(Users).select("id").where(($) => [$.name.endsWith(PG.param.string("q"))])
+		const compiled = PG.compileUnsafe(query, { q: "a%" })
+		expect(compiled.sql).toContain(
+			String.raw`"name" LIKE ('%' || replace(replace(replace($1, '\', '\\'), '%', '\%'), '_', '\_')) ESCAPE '\'`,
+		)
+		expect(compiled.parameters).toEqual(["a%"])
+	})
+
+	it("escapeLike escapes the escape character first", () => {
+		expect(CH.escapeLike(String.raw`\%_x`)).toBe(String.raw`\\\%\_x`)
+	})
+
+	it.each([[1], [{}], [null]])("a value of %o fails as QueryBuilderError", (value) => {
+		const exit = Effect.runSyncExit(CH.compile(CH.from(Events).select("Id").where(($) => [$.Name.contains(value as never)])))
+		expect(Exit.isFailure(exit) && String(Cause.squash(exit.cause))).toMatch(/contains value must be a string/)
+	})
+})

src/core.ts

@@ -25,6 +25,8 @@ export {
 	type Condition,
 	type Comparable,
 	type MapValueOf,
+	type TextMatchOptions,
+	escapeLike,
 	lit,
 	rawExpr,
 	untypedExpr,

src/docs-examples.test.ts

@@ -18,6 +18,7 @@ import { Effect, Exit, Option, Schema } from "effect"
 import * as CH from "./ch/index"
 import * as T from "./ch/types"
 import * as CHE from "./clickhouse"
+import * as PG from "./postgres"
 import { parseStatement, renderStatement, withSettings } from "./sql/statement"
 import { compileCHUnsafe, compileUnionUnsafe } from "./ch/compile"
 import { raw as rawFragment, compile as compileFragment } from "./sql/sql-fragment"
@@ -278,6 +279,24 @@ describe("docs/expressions.md", () => {
 		}),
 	)
 
+	it("Text search escapes LIKE wildcards", () => {
+		const search = (text: string) =>
+			CH.from(Events)
+				.select(($) => ({ name: $.Name }))
+				.where(($) => [$.OrgId.eq("org_123"), $.Name.contains(text, { insensitive: true })])
+		expect(oneLine(compileCHUnsafe(search("50%_off"), {}).sql)).toBe(
+			String.raw`SELECT events.Name AS name FROM events WHERE events.OrgId = 'org_123' AND events.Name ILIKE '%50\\%\\_off%'`,
+		)
+
+		const Users = PG.table("users", { columns: { id: PG.text, name: PG.text }, primaryKey: ["id"] })
+		const compiled = PG.compileUnsafe(PG.from(Users).select("id").where(($) => [$.name.startsWith("50%")]))
+		expect(oneLine(compiled.sql)).toBe(String.raw`SELECT "users"."id" AS "id" FROM "users" WHERE "users"."name" LIKE $1 ESCAPE '\'`)
+		expect(compiled.parameters).toEqual([String.raw`50\%%`])
+
+		// `.like()` takes a pattern; escapeLike makes text safe inside one.
+		expect(CHE.escapeLike("50%_off")).toBe(String.raw`50\%\_off`)
+	})
+
 	it("Optional predicates with when", () => {
 		const build = (nameFilter?: string) =>
 			CH.from(Events)

src/expr.ts

@@ -6,7 +6,9 @@ export {
 	type Condition,
 	type Expr,
 	type MapValueOf,
+	type TextMatchOptions,
 	and,
+	escapeLike,
 	dynamicColumn,
 	inExprList,
 	inList,