From 459f90b1e9edec7859d5a8d5a531f749a826d183 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 1/7] feat(database): defineRelations and findMany, one statement per level of with --- src/database.ts | 15 ++ src/database/relations.test-d.ts | 82 ++++++++ src/database/relations.test.ts | 192 +++++++++++++++++++ src/database/relations.ts | 293 +++++++++++++++++++++++++++++ tests/relations.clickhouse.test.ts | 72 +++++++ 5 files changed, 654 insertions(+) create mode 100644 src/database/relations.test-d.ts create mode 100644 src/database/relations.test.ts create mode 100644 src/database/relations.ts create mode 100644 tests/relations.clickhouse.test.ts diff --git a/src/database.ts b/src/database.ts index ba66164..7b9b94a 100644 --- a/src/database.ts +++ b/src/database.ts @@ -38,6 +38,21 @@ export { TransactionUnsupported, type TransactionError, } from "./database/errors" +export { + defineRelations, + findMany, + type FindManyOptions, + type FindManyRow, + type KeyColumn, + type Relation, + type RelationalSchema, + type RelationalTable, + type RelationBuilder, + type RelationOptions, + type RelationsConfig, + type RelationTables, + type WithOf, +} from "./database/relations" export { QuerySummary, withQuerySummary } from "./database/tracing" export { sql, type SqlIdentifier, type SqlRawText, type SqlTemplate } from "./database/sql" export type { DialectTransactions, IsolationLevel, TransactionSettings } from "./ch/dialect" diff --git a/src/database/relations.test-d.ts b/src/database/relations.test-d.ts new file mode 100644 index 0000000..05aa0eb --- /dev/null +++ b/src/database/relations.test-d.ts @@ -0,0 +1,82 @@ +// Type-level tests: relation declarations are checked against the tables, and +// `findMany` rows follow the `with` tree. + +import type { DateTime, Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as Db from "../database" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.int4, title: PG.text, at: PG.timestamptz }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +type User = { readonly id: number; readonly name: string } +type FindError = Db.DatabaseError | import("../postgres").QueryBuilderError | import("../postgres").CompiledQueryDecodeError + +// No `with`: the table's own row. +expectTypeOf(Db.findMany(schema.users)).toEqualTypeOf, FindError, Db.Database>>() +expectTypeOf(Db.findMany(schema.users, { where: ($) => [$.name.eq("ada")] })).toEqualTypeOf< + Effect.Effect, FindError, Db.Database> +>() + +// `many` is an array, `one` is a row or null, nested to any depth. +expectTypeOf( + Db.findMany(schema.users, { + with: { posts: { where: ($) => [$.title.eq("a")], with: { author: true } } }, + }), +).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ + readonly id: number + readonly name: string + readonly posts: ReadonlyArray<{ + readonly id: number + readonly authorId: number + readonly title: string + readonly at: DateTime.Utc + readonly author: User | null + }> + }>, + FindError, + Db.Database + > +>() + +// `where` and `orderBy` on a nested level see that level's columns. +Db.findMany(schema.users, { + // @ts-expect-error -- `title` is a column of posts, not users + where: ($) => [$.title.eq("a")], +}) + +// A relation name `with` does not declare. +// @ts-expect-error -- users has no relation `author` +Db.findMany(schema.users, { with: { author: true } }) + +// Declarations are checked: target table, `to` column, `from` column. +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- no table `comments` + users: { posts: r.many("comments", { from: "id", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- posts has no column `userId` + users: { posts: r.many("posts", { from: "id", to: "userId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- users has no column `uid` + users: { posts: r.many("posts", { from: "uid", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- a timestamptz decodes to DateTime, which is not a key + posts: { author: r.one("users", { from: "at", to: "id" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- `name` is already a column of users + users: { name: r.many("posts", { from: "id", to: "authorId" }) }, +})) diff --git a/src/database/relations.test.ts b/src/database/relations.test.ts new file mode 100644 index 0000000..59358ac --- /dev/null +++ b/src/database/relations.test.ts @@ -0,0 +1,192 @@ +// Relational reads (`defineRelations`, `findMany`) on PGlite: one statement per +// level of `with`, rows decoded through each level's own row codec. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, expect, layer } from "@effect/vitest" +import { Cause, DateTime, Effect, Exit, Layer } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as PG from "../postgres" +import { postgresDialect } from "../pg/dialect" +import * as Db from "../database" + +const statements: Array = [] + +const Live = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const setup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob'), (3, 'cy')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'a1'), (11, 1, 'a2'), (12, 2, 'b1')`) +}) + +layer(Live, { excludeTestServices: true })("findMany on PGlite", (it) => { + it.effect("loads a many relation and a nested one, one statement per level", () => + Effect.gen(function* () { + yield* setup + statements.length = 0 + const users = yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { orderBy: ($) => [[$.id, "asc"]], with: { author: true } } }, + }) + expect(users).toEqual([ + { + id: 1, + name: "ada", + posts: [ + { id: 10, authorId: 1, title: "a1", author: { id: 1, name: "ada" } }, + { id: 11, authorId: 1, title: "a2", author: { id: 1, name: "ada" } }, + ], + }, + { id: 2, name: "bob", posts: [{ id: 12, authorId: 2, title: "b1", author: { id: 2, name: "bob" } }] }, + { id: 3, name: "cy", posts: [] }, + ]) + expect(statements).toHaveLength(3) + expect(statements[1]).toMatch(/IN \(/) + }), + ) + + it.effect("a one relation is null when no row matches, and an empty level sends nothing", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (13, 99, 'orphan')`) + const orphan = yield* Db.findMany(schema.posts, { where: ($) => [$.id.eq(13)], with: { author: true } }) + expect(orphan).toEqual([{ id: 13, authorId: 99, title: "orphan", author: null }]) + statements.length = 0 + const none = yield* Db.findMany(schema.users, { where: ($) => [$.id.eq(404)], with: { posts: true } }) + expect(none).toEqual([]) + expect(statements).toHaveLength(1) + }), + ) + + it.effect("a nested where filters that level only", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.in_(1, 2)], + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { where: ($) => [$.title.eq("a2")] } }, + }) + expect(users.map((user) => [user.name, user.posts.map((post) => post.title)])).toEqual([ + ["ada", ["a2"]], + ["bob", []], + ]) + }), + ) + + it.effect("nested rows decode through their column codecs", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE events (id int4 PRIMARY KEY, user_id int4 NOT NULL, at timestamptz NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO events VALUES (1, 1, '2026-01-02T03:04:05Z')`) + const Events = PG.table("events", { + columns: { id: PG.int4, userId: PG.column(PG.int4, { name: "user_id" }), at: PG.timestamptz }, + primaryKey: ["id"], + }) + const withEvents = Db.defineRelations({ users: Users, events: Events }, (r) => ({ + users: { events: r.many("events", { from: "id", to: "userId" }) }, + })) + const [ada] = yield* Db.findMany(withEvents.users, { where: ($) => [$.id.eq(1)], with: { events: true } }) + const at = ada!.events[0]!.at + assert(DateTime.isDateTime(at)) + expect(DateTime.formatIso(at)).toBe("2026-01-02T03:04:05.000Z") + }), + ) +}) + +// Tenant scope on every level: a separate database that refuses unscoped statements. +const Orgs = PG.table("orgs", { columns: { id: PG.text }, primaryKey: ["id"] }) +const Units = PG.table("units", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Locks = PG.table("locks", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }), unitId: PG.column(PG.int4, { name: "unit_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const tenancy = Db.defineRelations({ orgs: Orgs, units: Units, locks: Locks }, (r) => ({ + orgs: { units: r.many("units", { from: "id", to: "orgId" }) }, + units: { locks: r.many("locks", { from: "id", to: "unitId" }) }, +})) + +const Scoped = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const tenantSetup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE orgs (id text PRIMARY KEY)`) + yield* Db.execute(Db.sql`CREATE TABLE units (id int4 PRIMARY KEY, org_id text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE locks (id int4 PRIMARY KEY, org_id text NOT NULL, unit_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO orgs VALUES ('a'), ('b')`) + yield* Db.execute(Db.sql`INSERT INTO units VALUES (1, 'a'), (2, 'b')`) + // Lock 21 points at unit 1 but belongs to org b: bad data a tenant pin must not leak. + yield* Db.execute(Db.sql`INSERT INTO locks VALUES (10, 'a', 1), (20, 'b', 2), (21, 'b', 1)`) +}) + +layer(Scoped, { excludeTestServices: true })("findMany tenant scope", (it) => { + it.effect("a level under one tenant's rows is pinned to that tenant", () => + Effect.gen(function* () { + yield* tenantSetup + statements.length = 0 + const units = yield* Db.findMany(tenancy.units, { where: ($) => [$.orgId.eq("a")], with: { locks: true } }) + expect(units).toEqual([{ id: 1, orgId: "a", locks: [{ id: 10, orgId: "a", unitId: 1 }] }]) + expect(statements[1]).toMatch(/org_id" = /) + }), + ) + + it.effect("an unscoped top level is refused before any nested level runs", () => + Effect.gen(function* () { + statements.length = 0 + const exit = yield* Effect.exit(Db.findMany(tenancy.units, { with: { locks: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + expect(statements).toEqual([]) + }), + ) + + it.effect("an untenanted parent gives a tenanted level nothing to pin, so it is refused", () => + Effect.gen(function* () { + const exit = yield* Effect.exit(Db.findMany(tenancy.orgs, { with: { units: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + }), + ) + + it.effect("crossTenant declares every level", () => + Effect.gen(function* () { + const units = yield* Db.findMany(tenancy.units, { + orderBy: ($) => [[$.id, "asc"]], + with: { locks: { orderBy: ($) => [[$.id, "asc"]] } }, + crossTenant: true, + }) + expect(units.map((unit) => unit.locks.map((lock) => lock.id))).toEqual([[10, 21], [20]]) + }), + ) +}) diff --git a/src/database/relations.ts b/src/database/relations.ts new file mode 100644 index 0000000..d5fae05 --- /dev/null +++ b/src/database/relations.ts @@ -0,0 +1,293 @@ +// Relational reads: relations declared over tables, and `findMany` with a +// nested `with`, as Drizzle's relational queries and Prisma's `include`. +// +// Each level of `with` is one ordinary builder query, `WHERE IN (...)` +// over the keys the level above returned. So every level compiles, decodes and +// derives its tenant scope the way any other query does, and `run` checks it. +// See design/relations.md for why this and not JSON aggregation. + +import { Effect } from "effect" +import type { CompiledQueryDecodeError } from "../ch/compile" +import type { QueryBuilderError } from "../ch/errors" +import type { Condition, Expr } from "../ch/expr" +import { from, type ColumnAccessor, type CHQuery } from "../ch/query" +import type { SelectRowOf, Table } from "../ch/table" +import type { ColumnDefs, InferTS } from "../ch/types" +import { Database } from "./database" +import type { DatabaseError } from "./errors" + +type AnyTable = Table + +/** The tables a relation schema covers, by the key relations use to name them. */ +export type RelationTables = Record + +type ColumnsOf = T extends Table ? Cols : never + +/** + * Columns a relation can join on: their decoded value is a string, number or + * bigint, which the loader matches by value. A `DateTime` or bytes column is + * not a key. + */ +export type KeyColumn = { + [K in keyof ColumnsOf]: NonNullable[K]>> extends string | number | bigint ? K : never +}[keyof ColumnsOf] & + string + +/** + * A relation from one table to another: rows of `target` whose `to` column + * equals this row's `from` column. `"one"` loads the first such row or `null`, + * `"many"` loads all of them. + */ +export interface Relation< + Kind extends "one" | "many" = "one" | "many", + Target extends string = string, + From extends string = string, + To extends string = string, +> { + readonly _tag: "Relation" + readonly kind: Kind + readonly target: Target + readonly from: From + readonly to: To +} + +/** What the `defineRelations` callback gets: `r.one` and `r.many`, checked against the tables. */ +export interface RelationBuilder { + readonly one: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"one", Target, From, To> + readonly many: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"many", Target, From, To> +} + +/** + * The relations of each table, by relation name. `from` must be a key column + * of the source table, and a relation name must not be one of its columns: the + * loaded rows carry both. + */ +export type RelationsConfig = { + readonly [S in keyof Tables]?: { readonly [name: string]: Relation<"one" | "many", keyof Tables & string, KeyColumn> } & { + readonly [C in keyof ColumnsOf]?: never + } +} + +type RelationsOf = K extends keyof R ? NonNullable : {} + +/** One table of a relation schema, as `findMany` takes it: `schema.users`. */ +export interface RelationalTable { + readonly _tag: "RelationalTable" + readonly key: K + readonly table: Tables[K] + /** @internal */ + readonly tables: Tables + /** @internal */ + readonly relations: R +} + +/** What `defineRelations` returns: one `RelationalTable` per table key. */ +export type RelationalSchema = { + readonly [K in keyof Tables & string]: RelationalTable +} + +/** + * Relations over a set of tables, Drizzle's `defineRelations`. Relations name + * their target by its key in `tables`, so two tables can point at each other + * and a table at itself. + * + * ```ts + * const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + * users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + * posts: { author: r.one("users", { from: "authorId", to: "id" }) }, + * })) + * ``` + */ +export const defineRelations = >( + tables: Tables, + build: (r: RelationBuilder) => R, +): RelationalSchema => { + const relation = + (kind: "one" | "many") => + (target: string, on: { readonly from: string; readonly to: string }): Relation => ({ + _tag: "Relation", + kind, + target, + from: on.from, + to: on.to, + }) + const relations = build({ one: relation("one"), many: relation("many") } as unknown as RelationBuilder) + return Object.fromEntries( + Object.keys(tables).map((key) => [key, { _tag: "RelationalTable", key, table: tables[key], tables, relations }]), + ) as unknown as RelationalSchema +} + +/** Options for one nested level of `with`. */ +export interface RelationOptions { + /** More conditions on this level, ANDed with the relation's key match. */ + readonly where?: ($: ColumnAccessor>) => ReadonlyArray + /** The order of this level's rows, within each parent. */ + readonly orderBy?: ($: ColumnAccessor>) => ReadonlyArray, "asc" | "desc"]> + /** Relations to load under each row of this level. */ + readonly with?: WithOf +} + +/** `with`: each relation to load, as `true` or with its own options. */ +export type WithOf = { + readonly [N in keyof RelationsOf]?: + | true + | (RelationsOf[N] extends Relation ? RelationOptions : never) +} + +/** Options for `findMany`: the top level also takes `limit`, `offset` and `crossTenant`. */ +export interface FindManyOptions + extends RelationOptions { + readonly limit?: number + readonly offset?: number + /** Declare every level `.crossTenant()`; see docs/tenant-scoping.md. */ + readonly crossTenant?: boolean +} + +type WithIn = O extends { readonly with?: infer W } ? (W extends object ? W : {}) : {} + +type Simplify = { readonly [K in keyof A]: A[K] } & {} + +/** A row `findMany` returns: every column of the table, plus each relation `with` asked for. */ +export type FindManyRow = Simplify< + SelectRowOf & { + readonly [N in keyof W & keyof RelationsOf as W[N] extends undefined ? never : N]: RelationsOf[N] extends Relation< + infer Kind, + infer T extends keyof Tables & string + > + ? Kind extends "many" + ? ReadonlyArray>> + : FindManyRow> | null + : never + } +> + +type FindError = DatabaseError | QueryBuilderError | CompiledQueryDecodeError +type Row = Record +interface Level { + readonly where?: ($: any) => ReadonlyArray + readonly orderBy?: ($: any) => ReadonlyArray, "asc" | "desc"]> + readonly with?: Record + readonly limit?: number + readonly offset?: number +} + +const distinct = (values: ReadonlyArray): Array => [ + ...new Set(values.filter((value) => value !== null && value !== undefined)), +] + +const levelQuery = ( + table: AnyTable, + level: Level, + match: ($: any) => ReadonlyArray, + crossTenant: boolean, +): CHQuery => { + let query: CHQuery = from(table as Table) + .select() + .where(($: any) => [...match($), ...(level.where?.($) ?? [])]) + if (level.orderBy !== undefined) query = query.orderBy(level.orderBy as any) + if (level.limit !== undefined) query = query.limit(level.limit) + if (level.offset !== undefined) query = query.offset(level.offset) + return crossTenant ? query.crossTenant() : query +} + +/** + * Load the relations `level.with` asks for under `rows` of `table`: one query + * per relation, then the next level under its rows. + */ +const attach = ( + tables: RelationTables, + relations: Record | undefined>, + key: string, + rows: ReadonlyArray, + level: Level, + crossTenant: boolean, +): Effect.Effect, FindError, Database> => + Effect.gen(function* () { + const names = Object.entries(level.with ?? {}).filter(([, spec]) => spec !== undefined) + if (names.length === 0) return rows + const db = yield* Database + const parent = tables[key]! + const loaded: Record>> = {} + for (const [name, spec] of names) { + const relation = relations[key]?.[name] + if (relation === undefined) { + return yield* Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) + } + const target = tables[relation.target]! + const nested: Level = spec === true ? {} : (spec as Level) + const keys = distinct(rows.map((row) => row[relation.from])) + let children: ReadonlyArray = [] + if (keys.length > 0) { + // The child level keeps the tenant its parents were read for: when the + // parent rows all carry one tenant, the child query pins its own tenant + // column to it, so its scope is proved like any other query's. + const tenants = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined + ? distinct(rows.map((row) => row[parent.tenantColumn!])) + : [] + const tenantColumn = target.tenantColumn + const query = levelQuery( + target, + nested, + ($) => [ + $[relation.to].in_(...keys), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ], + crossTenant, + ) + const decoded = (yield* db.run(query as any)) as ReadonlyArray + children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) + } + const byKey = new Map>() + for (const child of children) { + const value = child[relation.to] + const group = byKey.get(value) + if (group === undefined) byKey.set(value, [child]) + else group.push(child) + } + loaded[name] = byKey + } + return rows.map((row) => { + const out: Row = { ...row } + for (const [name] of names) { + const relation = relations[key]![name]! + const group = loaded[name]!.get(row[relation.from]) ?? [] + out[name] = relation.kind === "many" ? group : (group[0] ?? null) + } + return out + }) + }) + +/** + * Read rows of one table with the relations `with` names, nested to any depth. + * Each level is one statement, `WHERE IN (...)` over the keys of the + * level above, decoded through its own row codec. A level whose parents carry + * one tenant is pinned to it. Levels are separate statements: run `findMany` + * inside `transaction` for one snapshot across them. + * + * ```ts + * Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) + * ``` + */ +export const findMany: { + ( + table: RelationalTable, + ): Effect.Effect>, FindError, Database> + >( + table: RelationalTable, + options: O, + ): Effect.Effect>>, FindError, Database> +} = (table: RelationalTable, options?: FindManyOptions) => + Effect.gen(function* () { + const level = (options ?? {}) as Level & { readonly crossTenant?: boolean } + const crossTenant = level.crossTenant === true + const db = yield* Database + const rows = (yield* db.run(levelQuery(table.table, level, () => [], crossTenant) as any)) as ReadonlyArray + return yield* attach(table.tables, table.relations as any, table.key, rows, level, crossTenant) + }) as any diff --git a/tests/relations.clickhouse.test.ts b/tests/relations.clickhouse.test.ts new file mode 100644 index 0000000..98cd43c --- /dev/null +++ b/tests/relations.clickhouse.test.ts @@ -0,0 +1,72 @@ +// `findMany` on a live ClickHouse: relations need no foreign keys, so the same +// loader runs here. Creates two tables in a throwaway database. + +import { ClickhouseClient } from "@effect/sql-clickhouse" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as Db from "@maple-dev/effect-orm/database" +import { endpoint } from "./clickhouse-support" + +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +const Services = CH.table("services", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, Name: CH.string }, +}) +const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, ServiceId: CH.uint32, At: CH.dateTime64 }, +}) +const schema = Db.defineRelations({ services: Services, events: Events }, (r) => ({ + services: { events: r.many("events", { from: "Id", to: "ServiceId" }) }, + events: { service: r.one("services", { from: "ServiceId", to: "Id" }) }, +})) + +describe.skipIf(!endpoint)("findMany on live ClickHouse", () => { + it("loads nested relations, pinned to the tenant, one statement per level", async () => { + const { rows, sent } = await Effect.runPromise( + Effect.gen(function* () { + const database = `eo_relations_${Date.now()}_${Math.floor(Math.random() * 1e6)}` + const admin = yield* ClickhouseClient.ClickhouseClient + yield* admin.asCommand(admin.unsafe(`CREATE DATABASE ${database}`)) + return yield* Effect.gen(function* () { + const client = yield* ClickhouseClient.ClickhouseClient + const sent: Array = [] + const db = Db.fromSqlClient(client, { + dialect: CH.clickhouseDialect, + command: client.asCommand, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void sent.push(statement.sql)), + }) + yield* db.execute(Db.sql`CREATE TABLE services (OrgId String, Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) + yield* db.execute( + Db.sql`CREATE TABLE events (OrgId String, ServiceId UInt32, At DateTime64(3)) ENGINE = MergeTree ORDER BY At`, + ) + yield* db.execute(Db.sql`INSERT INTO services VALUES ('a', 1, 'api'), ('b', 2, 'web')`) + yield* db.execute( + Db.sql`INSERT INTO events VALUES ('a', 1, '2026-01-01 00:00:00.000'), ('b', 1, '2026-01-01 00:00:01.000')`, + ) + sent.length = 0 + const rows = yield* Db.findMany(schema.services, { + where: ($) => [$.OrgId.eq("a")], + with: { events: { with: { service: true } } }, + }).pipe(Effect.provideService(Db.Database, db)) + return { rows, sent } + }).pipe( + Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database })), + Effect.ensuring(Effect.orDie(admin.asCommand(admin.unsafe(`DROP DATABASE IF EXISTS ${database} SYNC`)))), + ) + }).pipe(Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database: "default" }))), + ) + expect(sent).toHaveLength(3) + expect(rows).toHaveLength(1) + const [api] = rows + expect(api!.Name).toBe("api") + expect(api!.events).toHaveLength(1) + expect(api!.events[0]!.service?.Name).toBe("api") + }) +}) From d08bef864e1b2c92528ba00bfca9c4111b316a2c Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 2/7] docs: relational reads with defineRelations and findMany --- README.md | 1 + docs/README.md | 1 + docs/reference.md | 1 + docs/relations.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 docs/relations.md diff --git a/README.md b/README.md index 4863efe..afa3ed5 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ Full guides live in [`docs/`](./docs/README.md): | [Decoding results](./docs/decoding-results.md) | `rowSchema`, `decodeRows`, decode errors | | [Running a query](./docs/running-queries.md) | Executing the SQL with a real client, wire settings, `SETTINGS` | | [Tenant scoping](./docs/tenant-scoping.md) | `tenantColumn`, what marks a query scoped, `crossTenant()` | +| [Relational reads](./docs/relations.md) | `defineRelations`, `findMany` with nested `with` | | [Postgres](./docs/postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./docs/migrations.md) | DDL from `table`, `effect-orm generate`, applying migrations | | [Extending the DSL](./docs/extending.md) | `defineFn`, raw escape hatches, handwritten SQL | diff --git a/docs/README.md b/docs/README.md index 7a61d11..a150170 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ Roughly in reading order. | [Postgres](./postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./migrations.md) | DDL from `CH.table` / `PG.table`, `materializedView`, `effect-orm generate`, applying migrations, adopting drizzle-kit | | [Statements and transactions](./database.md) | `Database` over your `SqlClient`: `run`, `execute`, `transaction`, retry | +| [Relational reads](./relations.md) | `defineRelations`, `findMany` with nested `with`, tenant scope per level | ## Reference diff --git a/docs/reference.md b/docs/reference.md index abc2d46..f34bc3c 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -513,6 +513,7 @@ placeholders, double-quoted identifiers. | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | +| `defineRelations`, `findMany`, `Relation`, `RelationBuilder`, `RelationsConfig`, `RelationTables`, `RelationalSchema`, `RelationalTable`, `RelationOptions`, `FindManyOptions`, `FindManyRow`, `WithOf`, `KeyColumn` — relational reads, one statement per level ([Relational reads](./relations.md)) | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | diff --git a/docs/relations.md b/docs/relations.md new file mode 100644 index 0000000..173046e --- /dev/null +++ b/docs/relations.md @@ -0,0 +1,129 @@ +# Relational reads + +`Db.defineRelations` declares how tables relate. `Db.findMany` reads rows of one table with the +related rows under them, nested to any depth, as Drizzle's relational queries and Prisma's +`include` do. Both are in `@maple-dev/effect-orm/database`. + +```ts title="relations-find-many.ts" +import { PgliteClient } from "@effect/sql-pglite" +import { Effect, Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +export const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const DatabaseLive = Db.layerSqlClient({ dialect: PG.postgresDialect }).pipe( + Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" })), +) + +export const users = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'hello')`) + return yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { with: { author: true } } }, + }) +}).pipe(Effect.provide(DatabaseLive)) +// [ +// { id: 1, name: "ada", posts: [{ id: 10, authorId: 1, title: "hello", author: { id: 1, name: "ada" } }] }, +// { id: 2, name: "bob", posts: [] }, +// ] +``` + +## Declare relations + +`defineRelations(tables, build)` takes the tables by key, and a callback that returns the +relations of each table by name. A relation names its target by that key, so two tables can point +at each other, and a table can point at itself. + +| Builder | Loads | Result | +| --- | --- | --- | +| `r.many(target, { from, to })` | Every `target` row whose `to` column equals this row's `from` column | `ReadonlyArray` | +| `r.one(target, { from, to })` | The first such row | `Row \| null` | + +The types check each declaration. The target must be a key of `tables`. `from` must be a column of +the source table and `to` a column of the target. Both must decode to a `string`, `number` or +`bigint`, because the loader matches keys by value. A relation name must not be a column name of +its table, because the loaded row carries both. + +A `one` relation is always `| null`. A row can be missing even when a foreign key says it cannot: +another statement can delete it between two levels. + +Relations are metadata only. They do not need a foreign key in the database, so they work on +ClickHouse too. + +_(Backed by `src/database/relations.test-d.ts`, and `tests/relations.clickhouse.test.ts` on ClickHouse.)_ + +## Read with `findMany` + +`findMany(schema.table, options?)` returns every column of the table, plus each relation that +`with` names. `with` takes `true`, or the options of that level: + +| Option | Levels | Meaning | +| --- | --- | --- | +| `where` | All | More conditions, ANDed with the relation's key match | +| `orderBy` | All | The order of the rows; nested rows keep it within each parent | +| `with` | All | Relations to load under each row of this level | +| `limit`, `offset` | Top only | Paging of the top level | +| `crossTenant` | Top only | Declare every level `.crossTenant()` | + +Rows come back decoded through the codecs of their columns at every level. A `timestamptz` in a +nested row is a `DateTime.Utc`, not a string. + +## One statement per level + +Each level of `with` is one ordinary builder query: every column of the target table, +`WHERE IN ()`, plus the level's own `where` and +`orderBy`. The loader groups the rows by key and puts them under their parents. A level with no +keys sends no statement. A tree with three levels sends three statements, whatever the number of +rows. + +The levels are separate statements. Outside a transaction, a write from another connection can +land between two of them. Run `findMany` inside [`Db.transaction`](./database.md#transactions) +with `isolationLevel: "repeatable read"` for one snapshot across all levels. + +All keys of a level go into one `IN` list. Postgres allows at most 65,535 bound values in one +statement, so keep the top level paged with `limit`. + +## Tenant scope on every level + +Each level runs through `run`, so each level has its own +[`tenantScope`](./tenant-scoping.md), and `requireTenantScope` checks each one. + +When the parent rows all carry one tenant, and both tables declare a `tenantColumn`, the nested +query also gets ` = `. That pins the level the same way a hand-written +filter does. It also hides a child row whose tenant differs from its parent's. + +A level gets no pin when: + +- the parent table has no `tenantColumn`; +- the parent rows carry more than one tenant. + +That level is then `"cross-tenant"`, and `requireTenantScope` refuses it. There is one exception: +if the relation's `to` column is the target's tenant column and the level has one key, the +one-value `IN` scopes it already. With `crossTenant: true`, every level is declared +cross-tenant and runs. + +_(Backed by `src/database/relations.test.ts`.)_ + +## Not yet + +- Choosing columns per level (`columns: { id: true }`). Every level reads every column. +- A `limit` per parent on a nested level. That needs a window function or a lateral join. +- Composite keys (`from` and `to` with more than one column). +- `findFirst`. Use `findMany` with `limit: 1`. +- Filters and counts over relations (`_count`, `some`, `every`, `none`). +- `param.*` markers in a `where`: `findMany` takes no params, so write values in. + +See [design/relations.md](../design/relations.md) for the design and the alternatives. From 1311ae81acd3c1145c41c3704a9dd6050cc22496 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 3/7] docs(design): relational reads, one statement per level --- .changeset/relational-reads.md | 5 ++ design/relations.md | 108 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .changeset/relational-reads.md create mode 100644 design/relations.md diff --git a/.changeset/relational-reads.md b/.changeset/relational-reads.md new file mode 100644 index 0000000..a2858dd --- /dev/null +++ b/.changeset/relational-reads.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. See docs/relations.md and design/relations.md. diff --git a/design/relations.md b/design/relations.md new file mode 100644 index 0000000..a7f5377 --- /dev/null +++ b/design/relations.md @@ -0,0 +1,108 @@ +# Relational reads: `defineRelations` and `findMany` + +Status: first slice built (issue #79; `docs/relations.md`). `one` and `many` relations over +single-column keys, `with` to any depth, `where` and `orderBy` per level, `limit` and `offset` on +the top level. Section 5 lists what is not built. + +## 1. Goal + +Read a row with its related rows in one call, as Drizzle's relational queries +(`db.query.users.findMany({ with: { posts: true } })`) and Prisma's `include` do. Keep what the +rest of the library gives a query: + +- rows decoded through the column codecs, at every level; +- a tenant scope derived for every statement, which `requireTenantScope` checks; +- a result type derived from the `with` tree. + +## 2. The API + +```ts +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) +``` + +- **One callback over all tables**, as Drizzle v2 `defineRelations`. A relation names its target + by key, not by value, so tables that point at each other need no thunks. +- **Relations live beside the tables, not in them.** `table()` stays as it is, and the DDL, the + diff and the importers do not change. A Prisma importer can write the `defineRelations` call. +- **`findMany` lives in `/database`.** It runs more than one statement, so it needs the + `Database` service. `defineRelations` sits next to it, so the dialect entries do not change. +- **Checked in the types.** The target must be a table key, `to` a column of the target, `from` a + column of the source, and both must decode to `string | number | bigint`. A relation name must + not be a column name. + +## 3. One statement per level, not JSON aggregation + +The issue proposed one statement with JSON aggregation (`jsonb_agg` / `json_object`), as Drizzle +RQB v2 does. This slice runs **one statement per level** instead: +`SELECT FROM WHERE IN () [AND ] +[AND ] [ORDER BY ...]`. The loader then groups the rows by key in memory. Prisma does the +same by default. + +Why: + +1. **Decoding stays correct for free.** Each level is an ordinary builder query, decoded by its + own derived row codec. With JSON aggregation, every column comes back in its JSON form, not + its wire form. An `int8` becomes a JSON number and loses precision past 2^53, a `timestamptz` + becomes an ISO string in a different format, `bytea` becomes hex, and a `numeric` can change. + Each column type would need a second "from JSON" codec per dialect, or a cast to text in the + SQL. That is a large change to the type system for one feature. +2. **Tenant scope needs no new analysis.** Each level is a query the existing analysis already + understands. A nested JSON subquery would need correlated-subquery evidence through + `jsonb_agg`, which the analysis does not have yet. +3. **Every dialect.** The SQL is only `IN (...)`, so the same code runs on Postgres, on + ClickHouse, and later on SQLite / D1 and MySQL. JSON aggregation needs a different function + set per dialect, and ClickHouse has no `jsonb_agg`. +4. **The statement count is fixed.** It is the depth of the `with` tree, not the row count, so + there is no N+1. + +The costs: + +- **More round trips.** Three levels are three statements. On a server close to the app this is + small. JSON aggregation is better on a slow link. +- **No single snapshot.** Outside a transaction, a write can land between two levels. The docs + say to use `Db.transaction({ isolationLevel: "repeatable read" })`. `findMany` does not open a + transaction itself: ClickHouse has none, and the caller may already be in one. +- **Long `IN` lists.** All keys of a level are in one list. Postgres has a limit of 65,535 bound + values. Chunking belongs with #57 (parameter limits). + +JSON aggregation can come later as a second strategy behind the same API, per dialect, once the +column types have JSON codecs. The result type and the tenant rule would not change. + +## 4. Tenant scope on nested levels + +Each level goes through `Database.run`, so it is checked like any other statement. A nested +level filters on the parent keys, which says nothing about the tenant. So the loader adds a pin: + +> When the parent table and the target table both declare a `tenantColumn`, and every parent row +> carries the same tenant value `V`, the nested query also gets ` = V`. + +This is sound. The pin is an ordinary `eq` on a literal, so the existing analysis proves +`"single-tenant"`. `V` comes from rows the parent query read, so a level never reads a tenant its +parent did not read. The pin also hides bad data: a child row whose tenant differs from its +parent's. + +No pin when the parent table is untenanted, or the parent rows carry more than one tenant. The +level is then `"cross-tenant"`, and `requireTenantScope` refuses it, which is the safe default. A +level whose `to` column is the target's tenant column, with one key, is scoped by the one-value +`IN` already. `crossTenant: true` declares every level `.crossTenant()`. + +The pin does not pass through an untenanted level: a grandparent's tenant says nothing about a +grandchild reached through a shared table. + +## 5. Not built + +| What | Why not now | +| --- | --- | +| JSON aggregation strategy | Needs JSON codecs per column type (section 3) | +| `columns` per level | Small, but changes the row type of every level; next slice | +| `limit` per parent on a nested level | Needs `row_number()` over a partition, or a lateral join | +| Composite keys | The loader would match tuples; no issue asks for it yet | +| `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | +| `_count`, `some` / `every` / `none` | Issue #80 | +| Params in nested `where` | `findMany` takes no params; values go in as literals | +| SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | From 098e3b8b10279cc05e501a18d42026cba1a8a12a Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:39:24 +0200 Subject: [PATCH 4/7] feat(database): relation filters some/none/every and relation counts Correlated EXISTS / NOT EXISTS conditions and a correlated count(*) over a declared relation, on the builder (Db.related) and in findMany's where. findMany's _count counts a many relation with one grouped statement per level. A subquery that correlates on the outer tenant column takes that column's tenant binding. --- src/ch/compile.ts | 22 +++- src/ch/tenant.ts | 16 +++ src/database.ts | 5 + src/database/relations.ts | 270 ++++++++++++++++++++++++++++++++------ 4 files changed, 270 insertions(+), 43 deletions(-) diff --git a/src/ch/compile.ts b/src/ch/compile.ts index 0aaaac1..8ebd846 100644 --- a/src/ch/compile.ts +++ b/src/ch/compile.ts @@ -28,7 +28,7 @@ import { checkedLiteral, clickhouseDialect, currentDialect, withDialect, type Di import { Effect, Option, Schema } from "effect" import { QueryBuilderDefect, QueryBuilderError } from "./errors" import { withSubqueryCompiler } from "./subquery-context" -import { tenantBoundOf, tenantColumnOf, tenantPredicatesOf, withTenantBound, type TenantPredicate } from "./tenant" +import { correlatedBound, correlatedTenantOf, tenantBoundOf, tenantColumnOf, tenantPredicatesOf, withTenantBound, type TenantPredicate } from "./tenant" // `QueryBuilderError` moved to ./errors so `expr.ts` can raise it too; still // exported from here, which is where every caller imports it from. @@ -741,6 +741,12 @@ function compileInner< * There is no reason to pass it by hand. */ enclosingCtes?: ReadonlyArray + /** + * Set for a correlated subquery that `markCorrelatedTenant` marked: its + * main table's tenant column is bound to this enclosing column, and the + * enclosing compile resolves that binding (see `tenant.ts`). + */ + correlatedTenant?: string }, ): CompiledQuery { // A nested query's columns are its own: nothing it renders reaches an @@ -843,6 +849,9 @@ function compileInner< fromFragment = raw(`(\n${body}\n) AS ${quoteIdent(state.fromQueryAlias ?? "")}`) } else { fromSource = sourceForTable(state.tableName, mainColumn) + if (options?.correlatedTenant !== undefined && fromSource.column !== undefined) { + fromSource = { ...fromSource, bound: correlatedBound(options.correlatedTenant) } + } fromFragment = mainAlias !== state.tableName ? raw(`${quoteIdentPath(state.tableName)} AS ${quoteIdentPath(mainAlias)}`) : identPath(state.tableName) @@ -856,13 +865,22 @@ function compileInner< sources.push({ scope: "cross-tenant" }) return subquery } + const correlatedTenant = correlatedTenantOf(subquery) const compiled = compileInner(subquery, params, { skipFormat: true, deferParams, nested: true, enclosingCtes: visibleCtes, + correlatedTenant, }) - sources.push(sourceOf(compiled)) + // A correlated subquery bound only through the enclosing tenant column + // reads the tenant that column is bound to: it is that column's source. + sources.push( + correlatedTenant !== undefined && compiled.tenantScope === "single-tenant" && + tenantBoundOf(compiled) === correlatedBound(correlatedTenant) + ? { column: correlatedTenant, scope: "cross-tenant" } + : sourceOf(compiled), + ) return compiled.sql }, () => { const joins = state.typedJoins.map((j) => { diff --git a/src/ch/tenant.ts b/src/ch/tenant.ts index b3a68aa..60a5f03 100644 --- a/src/ch/tenant.ts +++ b/src/ch/tenant.ts @@ -32,3 +32,19 @@ export const withTenantBound = (query: A, bound: string | if (bound !== undefined) bounds.set(query, bound) return query } + +/** + * A correlated subquery whose main table's tenant column equals a tenant column + * of the enclosing query (`inner.OrgId = outer.OrgId` in its own WHERE). Only + * the relation filters mark one, and they write that equality themselves. + * Compiled as a subquery, it takes the enclosing column's binding instead of + * needing one of its own; compiled alone, the mark means nothing. + */ +const correlated = new WeakMap() +export const correlatedTenantOf = (query: object): string | undefined => correlated.get(query) +export const markCorrelatedTenant = (query: A, outerColumn: string): A => { + correlated.set(query, outerColumn) + return query +} +/** The bound a correlated subquery reports: the outer column, not a value. */ +export const correlatedBound = (outerColumn: string): string => `\u0000correlated:${outerColumn}` diff --git a/src/database.ts b/src/database.ts index 7b9b94a..dca1bfc 100644 --- a/src/database.ts +++ b/src/database.ts @@ -41,6 +41,8 @@ export { export { defineRelations, findMany, + related, + type CountOf, type FindManyOptions, type FindManyRow, type KeyColumn, @@ -48,9 +50,12 @@ export { type RelationalSchema, type RelationalTable, type RelationBuilder, + type RelationFilter, type RelationOptions, type RelationsConfig, type RelationTables, + type RelationWhere, + type Related, type WithOf, } from "./database/relations" export { QuerySummary, withQuerySummary } from "./database/tracing" diff --git a/src/database/relations.ts b/src/database/relations.ts index d5fae05..f10d5b8 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -9,10 +9,15 @@ import { Effect } from "effect" import type { CompiledQueryDecodeError } from "../ch/compile" import type { QueryBuilderError } from "../ch/errors" -import type { Condition, Expr } from "../ch/expr" +import { and, lit, makeCond, makeExpr, not, type Condition, type Expr, type ParamsIn } from "../ch/expr" +import { builtins } from "../ch/functions/builtin" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" +import { exists, subqueryExpr } from "../ch/subquery" import type { SelectRowOf, Table } from "../ch/table" -import type { ColumnDefs, InferTS } from "../ch/types" +import { markCorrelatedTenant, tenantColumnOf } from "../ch/tenant" +import { custom, type ColumnDefs, type InferTS } from "../ch/types" +import { PgNumber } from "../pg/types" +import { raw } from "../sql/sql-fragment" import { Database } from "./database" import type { DatabaseError } from "./errors" @@ -123,14 +128,138 @@ export const defineRelations = } +// Relation filters and counts +// +// `some`, `none` and `every` are correlated `EXISTS` / `NOT EXISTS` conditions, +// and `count` is a correlated `count(*)` subquery. The subquery reads the target +// table under its own alias, matches ` = `, and, when both +// tables declare a tenant column, ` = `. That last +// equality is what keeps the subquery inside the outer query's tenant: the +// compiler gives the subquery the outer column's binding (see `tenant.ts`). + +type TargetOf = N extends keyof RelationsOf + ? RelationsOf[N] extends Relation + ? T + : never + : never + +/** The conditions a relation filter adds to its subquery. */ +export type RelationWhere = ( + $: ColumnAccessor>, + r: Related, +) => ReadonlyArray + +type Conditions = ReadonlyArray + +/** The filters and the count of one relation, built on the outer query's `$`. */ +export interface RelationFilter { + /** At least one related row matches: `EXISTS (…)`. With no `where`, any related row. */ + readonly some: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** No related row matches: `NOT EXISTS (…)`. With no `where`, no related row at all. */ + readonly none: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** Every related row matches: `NOT EXISTS (… AND NOT (where))`. True when there is none. */ + readonly every: ( + where: ($: ColumnAccessor>, r: Related) => C, + ) => Condition> + /** The number of related rows that match, as a correlated `count(*)` subquery. */ + readonly count: ( + where?: ($: ColumnAccessor>, r: Related) => C, + ) => Expr> +} + +/** What `related` returns: one `RelationFilter` per relation of the table. */ +export type Related = { + readonly [N in keyof RelationsOf & string]: RelationFilter & keyof Tables & string> +} + +const aggregate = builtins("portable", "aggregate") +/** `count(*)` is the same in every dialect; a driver sends it as a number, a string or a bigint. */ +const countType = custom("Int64", PgNumber) +const countAll = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), countType.schema) +const always = (): Condition => makeCond(raw("1 = 1")) + +const relatedOf = ( + tables: RelationTables, + relations: Record | undefined>, + key: string, + $: any, +): Record> => + Object.fromEntries( + Object.entries(relations[key] ?? {}).map(([name, relation]) => { + const parent = tables[key]! + const target = tables[relation.target]! + const outerFrom = $[relation.from] + // `$` is qualified (`alias.column`): the subquery's alias extends it, so + // it never hides the outer source, even for a table related to itself. + const outerName: string = outerFrom.columnName + const alias = `${outerName.slice(0, outerName.length - relation.from.length - 1)}__${name}` + const outerTenant = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined ? $[parent.tenantColumn] : undefined + const outerTenantColumn = outerTenant === undefined ? undefined : tenantColumnOf(outerTenant) + const inner = ( + where: (($: any, r: any) => ReadonlyArray) | undefined, + select: () => Record>, + negate = false, + ): CHQuery => { + const query = from(target as Table, alias) + .select(select as any) + .where((c: any) => { + const conditions = where?.(c, relatedOf(tables, relations, relation.target, c)) ?? [] + const own = negate ? [not(and(...conditions) ?? always())] : conditions + return [ + c[relation.to].eq(outerFrom), + outerTenant === undefined ? undefined : c[target.tenantColumn!].eq(outerTenant), + ...own, + ] + }) + return outerTenantColumn === undefined ? query : markCorrelatedTenant(query, outerTenantColumn) + } + const one = () => ({ one: lit(1) }) + const filter: RelationFilter = { + some: (where?: any) => exists(inner(where, one) as any), + none: (where?: any) => not(exists(inner(where, one) as any)), + every: (where: any) => not(exists(inner(where, one, true) as any)), + count: (where?: any) => + subqueryExpr(inner(where, () => ({ n: countAll() })) as any, countType, (sql) => `coalesce((${sql}), 0)`) as any, + } + return [name, filter] + }), + ) + +/** + * The relation filters and counts of a table, on the `$` of a builder query + * over that table: + * + * ```ts + * PG.from(Users) + * .select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) + * .where(($) => [Db.related(schema.users, $).posts.some(($) => [$.published.eq(true)])]) + * ``` + * + * `findMany`'s `where` gets the same object as its second argument. + */ +export const related = ( + table: RelationalTable, + $: ColumnAccessor>, +): Related => relatedOf(table.tables, table.relations as any, table.key, $) as unknown as Related + /** Options for one nested level of `with`. */ export interface RelationOptions { - /** More conditions on this level, ANDed with the relation's key match. */ - readonly where?: ($: ColumnAccessor>) => ReadonlyArray + /** + * More conditions on this level, ANDed with the relation's key match. The + * second argument holds the relation filters of this level's table. + */ + readonly where?: RelationWhere /** The order of this level's rows, within each parent. */ readonly orderBy?: ($: ColumnAccessor>) => ReadonlyArray, "asc" | "desc"]> /** Relations to load under each row of this level. */ readonly with?: WithOf + /** Relations to count under each row of this level, Prisma's `_count`. */ + readonly _count?: CountOf } /** `with`: each relation to load, as `true` or with its own options. */ @@ -140,6 +269,13 @@ export type WithOf[N] extends Relation ? RelationOptions : never) } +/** `_count`: each `many` relation to count, as `true` or with a `where`. A `one` relation is refused. */ +export type CountOf = { + readonly [N in keyof RelationsOf]?: RelationsOf[N] extends Relation<"many", infer T extends keyof Tables & string> + ? true | { readonly where?: RelationWhere } + : never +} + /** Options for `findMany`: the top level also takes `limit`, `offset` and `crossTenant`. */ export interface FindManyOptions extends RelationOptions { @@ -150,59 +286,94 @@ export interface FindManyOptions = O extends { readonly with?: infer W } ? (W extends object ? W : {}) : {} +type CountIn = O extends { readonly _count?: infer C } ? (C extends object ? C : {}) : {} type Simplify = { readonly [K in keyof A]: A[K] } & {} -/** A row `findMany` returns: every column of the table, plus each relation `with` asked for. */ -export type FindManyRow = Simplify< +/** + * A row `findMany` returns: every column of the table, plus each relation + * `with` asked for, plus `_count` when the level counts relations. + */ +export type FindManyRow = Simplify< SelectRowOf & { readonly [N in keyof W & keyof RelationsOf as W[N] extends undefined ? never : N]: RelationsOf[N] extends Relation< infer Kind, infer T extends keyof Tables & string > ? Kind extends "many" - ? ReadonlyArray>> - : FindManyRow> | null + ? ReadonlyArray, CountIn>> + : FindManyRow, CountIn> | null : never - } + } & ([keyof C] extends [never] + ? {} + : { readonly _count: { readonly [N in keyof C as C[N] extends undefined ? never : N]: number } }) > type FindError = DatabaseError | QueryBuilderError | CompiledQueryDecodeError type Row = Record +type LevelWhere = ($: any, r: any) => ReadonlyArray interface Level { - readonly where?: ($: any) => ReadonlyArray + readonly where?: LevelWhere readonly orderBy?: ($: any) => ReadonlyArray, "asc" | "desc"]> readonly with?: Record + readonly _count?: Record readonly limit?: number readonly offset?: number } +type Relations = Record | undefined> const distinct = (values: ReadonlyArray): Array => [ ...new Set(values.filter((value) => value !== null && value !== undefined)), ] const levelQuery = ( - table: AnyTable, + tables: RelationTables, + relations: Relations, + key: string, level: Level, match: ($: any) => ReadonlyArray, crossTenant: boolean, ): CHQuery => { - let query: CHQuery = from(table as Table) + let query: CHQuery = from(tables[key] as Table) .select() - .where(($: any) => [...match($), ...(level.where?.($) ?? [])]) + .where(($: any) => [...match($), ...(level.where?.($, relatedOf(tables, relations, key, $)) ?? [])]) if (level.orderBy !== undefined) query = query.orderBy(level.orderBy as any) if (level.limit !== undefined) query = query.limit(level.limit) if (level.offset !== undefined) query = query.offset(level.offset) return crossTenant ? query.crossTenant() : query } +/** + * The key match of a nested level, ` IN ()`. The child level keeps + * the tenant its parents were read for: when the parent rows all carry one + * tenant, the child query pins its own tenant column to it, so its scope is + * proved like any other query's. + */ +const keyMatch = (parent: AnyTable, target: AnyTable, relation: Relation, rows: ReadonlyArray, keys: ReadonlyArray) => { + const tenants = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined + ? distinct(rows.map((row) => row[parent.tenantColumn!])) + : [] + const tenantColumn = target.tenantColumn + return ($: any): ReadonlyArray => [ + $[relation.to].in_(...keys), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ] +} + +const relationOf = (relations: Relations, key: string, name: string): Effect.Effect => { + const relation = relations[key]?.[name] + return relation === undefined ? Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) : Effect.succeed(relation) +} + /** * Load the relations `level.with` asks for under `rows` of `table`: one query - * per relation, then the next level under its rows. + * per relation, then the next level under its rows. Then count the relations + * `level._count` asks for: one grouped query per relation. */ const attach = ( tables: RelationTables, - relations: Record | undefined>, + relations: Relations, key: string, rows: ReadonlyArray, level: Level, @@ -210,37 +381,19 @@ const attach = ( ): Effect.Effect, FindError, Database> => Effect.gen(function* () { const names = Object.entries(level.with ?? {}).filter(([, spec]) => spec !== undefined) - if (names.length === 0) return rows + const counted = Object.entries(level._count ?? {}).filter(([, spec]) => spec !== undefined) + if (names.length === 0 && counted.length === 0) return rows const db = yield* Database const parent = tables[key]! const loaded: Record>> = {} for (const [name, spec] of names) { - const relation = relations[key]?.[name] - if (relation === undefined) { - return yield* Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) - } + const relation = yield* relationOf(relations, key, name) const target = tables[relation.target]! const nested: Level = spec === true ? {} : (spec as Level) const keys = distinct(rows.map((row) => row[relation.from])) let children: ReadonlyArray = [] if (keys.length > 0) { - // The child level keeps the tenant its parents were read for: when the - // parent rows all carry one tenant, the child query pins its own tenant - // column to it, so its scope is proved like any other query's. - const tenants = - parent.tenantColumn !== undefined && target.tenantColumn !== undefined - ? distinct(rows.map((row) => row[parent.tenantColumn!])) - : [] - const tenantColumn = target.tenantColumn - const query = levelQuery( - target, - nested, - ($) => [ - $[relation.to].in_(...keys), - tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, - ], - crossTenant, - ) + const query = levelQuery(tables, relations, relation.target, nested, keyMatch(parent, target, relation, rows, keys), crossTenant) const decoded = (yield* db.run(query as any)) as ReadonlyArray children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) } @@ -253,6 +406,28 @@ const attach = ( } loaded[name] = byKey } + const counts: Record> = {} + for (const [name, spec] of counted) { + const relation = yield* relationOf(relations, key, name) + const target = tables[relation.target]! + const where = spec === true ? undefined : spec!.where + const keys = distinct(rows.map((row) => row[relation.from])) + const byKey = new Map() + if (keys.length > 0) { + // `SELECT , count(*) … WHERE IN () GROUP BY `: the + // same key match and tenant pin as a loaded level. + const match = keyMatch(parent, target, relation, rows, keys) + let query: CHQuery = from(target as Table) + .select(($: any) => ({ key: $[relation.to], n: countAll() })) + .where(($: any) => [...match($), ...(where?.($, relatedOf(tables, relations, relation.target, $)) ?? [])]) + .groupBy("key" as never) + if (crossTenant) query = query.crossTenant() + for (const row of (yield* db.run(query as any)) as ReadonlyArray<{ key: unknown; n: number }>) { + byKey.set(row.key, row.n) + } + } + counts[name] = byKey + } return rows.map((row) => { const out: Row = { ...row } for (const [name] of names) { @@ -260,6 +435,11 @@ const attach = ( const group = loaded[name]!.get(row[relation.from]) ?? [] out[name] = relation.kind === "many" ? group : (group[0] ?? null) } + if (counted.length > 0) { + out._count = Object.fromEntries( + counted.map(([name]) => [name, counts[name]!.get(row[relations[key]![name]!.from]) ?? 0]), + ) + } return out }) }) @@ -271,8 +451,15 @@ const attach = ( * one tenant is pinned to it. Levels are separate statements: run `findMany` * inside `transaction` for one snapshot across them. * + * `where` gets the level's relation filters as its second argument, and + * `_count` counts a `many` relation with one grouped statement per level. + * * ```ts - * Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) + * Db.findMany(schema.users, { + * where: ($, r) => [$.orgId.eq(orgId), r.posts.some()], + * with: { posts: { with: { author: true } } }, + * _count: { posts: true }, + * }) * ``` */ export const findMany: { @@ -282,12 +469,13 @@ export const findMany: { >( table: RelationalTable, options: O, - ): Effect.Effect>>, FindError, Database> + ): Effect.Effect, CountIn>>, FindError, Database> } = (table: RelationalTable, options?: FindManyOptions) => Effect.gen(function* () { const level = (options ?? {}) as Level & { readonly crossTenant?: boolean } const crossTenant = level.crossTenant === true + const relations = table.relations as Relations const db = yield* Database - const rows = (yield* db.run(levelQuery(table.table, level, () => [], crossTenant) as any)) as ReadonlyArray - return yield* attach(table.tables, table.relations as any, table.key, rows, level, crossTenant) + const rows = (yield* db.run(levelQuery(table.tables, relations, table.key, level, () => [], crossTenant) as any)) as ReadonlyArray + return yield* attach(table.tables, relations, table.key, rows, level, crossTenant) }) as any From 9af3aabc27b3bd6516fcf7f8ef929ae9c8238abf Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:39:24 +0200 Subject: [PATCH 5/7] test(database): relation filters and counts on PGlite and ClickHouse --- src/database/relation-filters.test-d.ts | 80 ++++++ src/database/relation-filters.test.ts | 330 ++++++++++++++++++++++ tests/relation-filters.clickhouse.test.ts | 149 ++++++++++ 3 files changed, 559 insertions(+) create mode 100644 src/database/relation-filters.test-d.ts create mode 100644 src/database/relation-filters.test.ts create mode 100644 tests/relation-filters.clickhouse.test.ts diff --git a/src/database/relation-filters.test-d.ts b/src/database/relation-filters.test-d.ts new file mode 100644 index 0000000..68c84f9 --- /dev/null +++ b/src/database/relation-filters.test-d.ts @@ -0,0 +1,80 @@ +// Type-level tests: relation filters and counts follow the relation schema, and +// `_count` adds a typed `_count` to the rows of its level. + +import type { Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as Db from "../database" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.int4, title: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +type FindError = Db.DatabaseError | PG.QueryBuilderError | PG.CompiledQueryDecodeError + +// `_count` adds `_count` with one number per counted relation, at any level. +expectTypeOf( + Db.findMany(schema.users, { + _count: { posts: true }, + with: { posts: { with: { author: { _count: { posts: { where: ($) => [$.title.eq("a")] } } } } } }, + }), +).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ + readonly id: number + readonly name: string + readonly _count: { readonly posts: number } + readonly posts: ReadonlyArray<{ + readonly id: number + readonly authorId: number + readonly title: string + readonly author: { + readonly id: number + readonly name: string + readonly _count: { readonly posts: number } + } | null + }> + }>, + FindError, + Db.Database + > +>() + +// `_count` takes `many` relations only, and only relation names. +// @ts-expect-error `author` is a `one` relation +Db.findMany(schema.posts, { _count: { author: true } }) +// @ts-expect-error no relation named `comments` +Db.findMany(schema.users, { _count: { comments: true } }) + +// The filters see the target's columns, and the next level's relations. +Db.findMany(schema.users, { + where: (_, r) => [ + r.posts.some(($) => [$.title.eq("a")]), + r.posts.every(($, r) => [r.author.none(($) => [$.name.eq("b")])]), + // @ts-expect-error `name` is not a column of posts + r.posts.none(($) => [$.name.eq("a")]), + ], +}) +// @ts-expect-error no relation named `comments` +Db.findMany(schema.users, { where: (_, r) => [r.comments.some()] }) +// @ts-expect-error `every` needs its condition +Db.findMany(schema.users, { where: (_, r) => [r.posts.every()] }) + +// On the builder: a count is a number, and params inside a filter reach the query. +PG.from(Users).select(($) => { + expectTypeOf(Db.related(schema.users, $).posts.count()).toEqualTypeOf>() + return { id: $.id } +}) +const withParam = PG.from(Users) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(schema.users, $).posts.some(($) => [$.title.eq(PG.param.string("title"))])]) +expectTypeOf(PG.compileUnsafe(withParam, { title: "a" }).sql).toEqualTypeOf() +// @ts-expect-error the `title` param is required +PG.compileUnsafe(withParam, {}) diff --git a/src/database/relation-filters.test.ts b/src/database/relation-filters.test.ts new file mode 100644 index 0000000..925404a --- /dev/null +++ b/src/database/relation-filters.test.ts @@ -0,0 +1,330 @@ +// Relation filters (`some`, `none`, `every`), relation counts, and `_count` in +// `findMany`, on PGlite. The filters are correlated EXISTS subqueries; the +// tenant tests check that the subquery stays inside the outer query's tenant. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, describe, expect, it as plain, layer } from "@effect/vitest" +import { Cause, Effect, Exit, Layer } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as PG from "../postgres" +import { postgresDialect } from "../pg/dialect" +import * as Db from "../database" + +const statements: Array = [] + +const live = (requireTenantScope: boolean) => + Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + requireTenantScope, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), + ).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { + id: PG.int4, + authorId: PG.column(PG.int4, { name: "author_id" }), + title: PG.text, + published: PG.bool, + }, + primaryKey: ["id"], +}) +const Comments = PG.table("comments", { + columns: { id: PG.int4, postId: PG.column(PG.int4, { name: "post_id" }), body: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts, comments: Comments }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { + author: r.one("users", { from: "authorId", to: "id" }), + comments: r.many("comments", { from: "id", to: "postId" }), + }, +})) + +const setup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute( + Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL, published bool NOT NULL)`, + ) + yield* Db.execute(Db.sql`CREATE TABLE comments (id int4 PRIMARY KEY, post_id int4 NOT NULL, body text NOT NULL)`) + // ada: one published and one draft post; bob: one published post; cy: none. + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob'), (3, 'cy')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'a1', true), (11, 1, 'a2', false), (12, 2, 'b1', true)`) + yield* Db.execute(Db.sql`INSERT INTO comments VALUES (100, 10, 'nice'), (101, 12, 'ok')`) +}) + +const names = (rows: ReadonlyArray<{ readonly name: string }>) => rows.map((row) => row.name) + +layer(live(false), { excludeTestServices: true })("relation filters on PGlite", (it) => { + it.effect("some: at least one related row matches", () => + Effect.gen(function* () { + yield* setup + const any = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some()], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(any)).toEqual(["ada", "bob"]) + const drafts = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some(($) => [$.published.eq(false)])], + }) + expect(names(drafts)).toEqual(["ada"]) + }), + ) + + it.effect("none: no related row matches", () => + Effect.gen(function* () { + const noPosts = yield* Db.findMany(schema.users, { where: (_, r) => [r.posts.none()] }) + expect(names(noPosts)).toEqual(["cy"]) + const noDrafts = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.none(($) => [$.published.eq(false)])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(noDrafts)).toEqual(["bob", "cy"]) + }), + ) + + it.effect("every: all related rows match, and a row with none passes", () => + Effect.gen(function* () { + const allPublished = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.every(($) => [$.published.eq(true)])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(allPublished)).toEqual(["bob", "cy"]) + }), + ) + + it.effect("filters nest, through a one relation too", () => + Effect.gen(function* () { + // Users with a post that has a comment. + const commented = yield* Db.findMany(schema.users, { + where: (_, r) => [r.posts.some((_, r) => [r.comments.some()])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(names(commented)).toEqual(["ada", "bob"]) + // Posts whose author has a draft. + const posts = yield* Db.findMany(schema.posts, { + where: (_, r) => [r.author.some((_, r) => [r.posts.some(($) => [$.published.eq(false)])])], + orderBy: ($) => [[$.id, "asc"]], + }) + expect(posts.map((post) => post.id)).toEqual([10, 11]) + }), + ) + + it.effect("a filter applies on a nested level of with", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.eq(1)], + with: { posts: { where: (_, r) => [r.comments.none()] } }, + }) + expect(users[0]!.posts.map((post) => post.id)).toEqual([11]) + }), + ) + + it.effect("on the builder: related() filters a where and counts in a select", () => + Effect.gen(function* () { + const query = PG.from(Users) + .select(($) => ({ + name: $.name, + posts: Db.related(schema.users, $).posts.count(), + published: Db.related(schema.users, $).posts.count(($) => [$.published.eq(true)]), + })) + .where(($) => [Db.related(schema.users, $).posts.none(($) => [$.title.eq("b1")])]) + .orderBy(["name", "asc"]) + const rows = yield* Db.run(query) + expect(rows).toEqual([ + { name: "ada", posts: 2, published: 1 }, + { name: "cy", posts: 0, published: 0 }, + ]) + }), + ) + + it.effect("a relation count works in a where", () => + Effect.gen(function* () { + const query = PG.from(Users) + .select(($) => ({ name: $.name })) + .where(($) => [Db.related(schema.users, $).posts.count().gte(2)]) + expect(yield* Db.run(query)).toEqual([{ name: "ada" }]) + }), + ) + + it.effect("_count: one grouped statement per counted relation, 0 for none", () => + Effect.gen(function* () { + statements.length = 0 + const users = yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + _count: { posts: true }, + }) + expect(users.map((user) => [user.name, user._count.posts])).toEqual([ + ["ada", 2], + ["bob", 1], + ["cy", 0], + ]) + expect(statements).toHaveLength(2) + expect(statements[1]).toMatch(/count\(\*\)/) + expect(statements[1]).toMatch(/GROUP BY/) + expect(statements[1]).toMatch(/IN \(/) + }), + ) + + it.effect("_count takes a where, and works on a nested level", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.eq(1)], + _count: { posts: { where: ($) => [$.published.eq(true)] } }, + with: { posts: { orderBy: ($) => [[$.id, "asc"]], _count: { comments: true } } }, + }) + expect(users).toEqual([ + { + id: 1, + name: "ada", + _count: { posts: 1 }, + posts: [ + { id: 10, authorId: 1, title: "a1", published: true, _count: { comments: 1 } }, + { id: 11, authorId: 1, title: "a2", published: false, _count: { comments: 0 } }, + ], + }, + ]) + }), + ) +}) + +describe("relation filters and the subquery alias", () => { + const Nodes = PG.table("nodes", { + columns: { id: PG.int4, parentId: PG.column(PG.nullable(PG.int4), { name: "parent_id" }) }, + primaryKey: ["id"], + }) + const tree = Db.defineRelations({ nodes: Nodes }, (r) => ({ + nodes: { children: r.many("nodes", { from: "id", to: "parentId" }) }, + })) + + plain("a table related to itself reads the subquery under its own alias", () => { + const compiled = PG.compileUnsafe( + PG.from(Nodes) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(tree.nodes, $).children.some()]), + {}, + ) + expect(compiled.sql).toMatch(/FROM "nodes" AS "nodes__children"\s+WHERE "nodes__children"."parent_id" = "nodes"."id"/) + }) +}) + +// Tenant scope: the subquery is pinned to the outer row's tenant. +const Orgs = PG.table("orgs", { columns: { id: PG.text }, primaryKey: ["id"] }) +const Units = PG.table("units", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Locks = PG.table("locks", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }), unitId: PG.column(PG.int4, { name: "unit_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Gateways = PG.table("gateways", { + columns: { id: PG.int4, lockId: PG.column(PG.int4, { name: "lock_id" }) }, + primaryKey: ["id"], +}) +const tenancy = Db.defineRelations({ orgs: Orgs, units: Units, locks: Locks, gateways: Gateways }, (r) => ({ + orgs: { units: r.many("units", { from: "id", to: "orgId" }) }, + units: { locks: r.many("locks", { from: "id", to: "unitId" }) }, + locks: { gateways: r.many("gateways", { from: "id", to: "lockId" }) }, +})) + +describe("relation filters keep the tenant scope", () => { + const scope = (query: PG.CHQuery) => PG.compileUnsafe(query, {}).tenantScope + + plain("a filter under a scoped outer query is single-tenant, with the tenant equality in the subquery", () => { + const query = PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some()]) + const compiled = PG.compileUnsafe(query, {}) + expect(compiled.tenantScope).toBe("single-tenant") + expect(compiled.sql).toContain(`"units__locks"."org_id" = "units"."org_id"`) + }) + + plain("nested filters and counts stay single-tenant", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id, n: Db.related(tenancy.units, $).locks.count() })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some((_, r) => [r.gateways.none()])]), + ), + ).toBe("single-tenant") + }) + + plain("a filter does not scope an unscoped outer query", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [Db.related(tenancy.units, $).locks.some(($) => [$.orgId.eq("a")])]), + ), + ).toBe("cross-tenant") + }) + + plain("a filter from an untenanted table into a tenanted one needs its own tenant equality", () => { + const orgs = (where: (r: Db.Related) => PG.Condition) => + scope(PG.from(Orgs).select(($) => ({ id: $.id })).where(($) => [where(Db.related(tenancy.orgs, $))])) + expect(orgs((r) => r.units.some())).toBe("cross-tenant") + expect(orgs((r) => r.units.some(($) => [$.orgId.eq("a")]))).toBe("single-tenant") + }) + + plain("a filter whose own tenant equality names another tenant is cross-tenant", () => { + expect( + scope( + PG.from(Units) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq("a"), Db.related(tenancy.units, $).locks.some(($) => [$.orgId.eq("b")])]), + ), + ).toBe("cross-tenant") + }) +}) + +layer(live(true), { excludeTestServices: true })("relation filters under requireTenantScope", (it) => { + const tenantSetup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE orgs (id text PRIMARY KEY)`) + yield* Db.execute(Db.sql`CREATE TABLE units (id int4 PRIMARY KEY, org_id text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE locks (id int4 PRIMARY KEY, org_id text NOT NULL, unit_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE gateways (id int4 PRIMARY KEY, lock_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO orgs VALUES ('a'), ('b')`) + yield* Db.execute(Db.sql`INSERT INTO units VALUES (1, 'a'), (2, 'a'), (3, 'b')`) + // Lock 21 points at unit 2 but belongs to org b: bad data the tenant equality must not count. + yield* Db.execute(Db.sql`INSERT INTO locks VALUES (10, 'a', 1), (11, 'a', 1), (21, 'b', 2), (30, 'b', 3)`) + }) + + it.effect("filters and _count run scoped, and a row of another tenant does not match", () => + Effect.gen(function* () { + yield* tenantSetup + const units = yield* Db.findMany(tenancy.units, { + where: ($, r) => [$.orgId.eq("a"), r.locks.none()], + }) + expect(units.map((unit) => unit.id)).toEqual([2]) + const counted = yield* Db.findMany(tenancy.units, { + where: ($) => [$.orgId.eq("a")], + orderBy: ($) => [[$.id, "asc"]], + _count: { locks: true }, + }) + expect(counted.map((unit) => [unit.id, unit._count.locks])).toEqual([ + [1, 2], + [2, 0], + ]) + }), + ) + + it.effect("an unscoped filter is refused", () => + Effect.gen(function* () { + const exit = yield* Effect.exit( + Db.findMany(tenancy.orgs, { where: ($, r) => [$.id.eq("a"), r.units.some()] }), + ) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + }), + ) +}) diff --git a/tests/relation-filters.clickhouse.test.ts b/tests/relation-filters.clickhouse.test.ts new file mode 100644 index 0000000..6d4cb01 --- /dev/null +++ b/tests/relation-filters.clickhouse.test.ts @@ -0,0 +1,149 @@ +// Relation filters and counts on a live ClickHouse. `some` / `none` / `every` +// and `count()` are correlated subqueries, which ClickHouse runs since 25.x +// (`allow_experimental_correlated_subqueries`, on by default in 26.8). +// `_count` in `findMany` is a grouped `IN` query, which needs no correlation. + +import { ClickhouseClient } from "@effect/sql-clickhouse" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as Db from "@maple-dev/effect-orm/database" +import { endpoint } from "./clickhouse-support" + +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +const Services = CH.table("services", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, Name: CH.string }, +}) +const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, ServiceId: CH.uint32, Level: CH.string }, +}) +const Notes = CH.table("notes", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, EventId: CH.uint32 }, +}) +const schema = Db.defineRelations({ services: Services, events: Events, notes: Notes }, (r) => ({ + services: { events: r.many("events", { from: "Id", to: "ServiceId" }) }, + events: { notes: r.many("notes", { from: "Id", to: "EventId" }) }, +})) + +const withDatabase = (body: (db: Db.Database["Service"]) => Effect.Effect) => + Effect.runPromise( + Effect.gen(function* () { + const database = `eo_relation_filters_${Date.now()}_${Math.floor(Math.random() * 1e6)}` + const admin = yield* ClickhouseClient.ClickhouseClient + yield* admin.asCommand(admin.unsafe(`CREATE DATABASE ${database}`)) + return yield* Effect.gen(function* () { + const client = yield* ClickhouseClient.ClickhouseClient + const db = Db.fromSqlClient(client, { + dialect: CH.clickhouseDialect, + command: client.asCommand, + requireTenantScope: true, + }) + yield* db.execute(Db.sql`CREATE TABLE services (OrgId String, Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) + yield* db.execute( + Db.sql`CREATE TABLE events (OrgId String, Id UInt32, ServiceId UInt32, Level String) ENGINE = MergeTree ORDER BY Id`, + ) + yield* db.execute(Db.sql`CREATE TABLE notes (OrgId String, EventId UInt32) ENGINE = MergeTree ORDER BY EventId`) + // api: an error and an info event; web: one info event; db: none. + // Event 3 points at api but belongs to org b: the tenant equality must skip it. + yield* db.execute(Db.sql`INSERT INTO services VALUES ('a', 1, 'api'), ('a', 2, 'web'), ('a', 3, 'db'), ('b', 4, 'other')`) + yield* db.execute( + Db.sql`INSERT INTO events VALUES ('a', 1, 1, 'error'), ('a', 2, 1, 'info'), ('a', 4, 2, 'info'), ('b', 3, 1, 'error')`, + ) + yield* db.execute(Db.sql`INSERT INTO notes VALUES ('a', 1)`) + return yield* body(db).pipe(Effect.provideService(Db.Database, db)) + }).pipe( + Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database })), + Effect.ensuring(Effect.orDie(admin.asCommand(admin.unsafe(`DROP DATABASE IF EXISTS ${database} SYNC`)))), + ) + }).pipe(Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database: "default" }))), + ) + +const names = (rows: ReadonlyArray<{ readonly Name: string }>) => rows.map((row) => row.Name) + +describe.skipIf(!endpoint)("relation filters on live ClickHouse", () => { + it("some, none and every run as correlated subqueries, scoped to the tenant", async () => { + const result = await withDatabase(() => + Effect.gen(function* () { + const find = (where: Db.RelationWhere) => + Db.findMany(schema.services, { + where: ($, r) => [$.OrgId.eq("a"), ...where($, r)], + orderBy: ($) => [[$.Id, "asc"]], + }).pipe(Effect.map(names)) + return { + some: yield* find((_, r) => [r.events.some()]), + errors: yield* find((_, r) => [r.events.some(($) => [$.Level.eq("error")])]), + none: yield* find((_, r) => [r.events.none()]), + every: yield* find((_, r) => [r.events.every(($) => [$.Level.eq("info")])]), + nested: yield* find((_, r) => [r.events.some((_, r) => [r.notes.some()])]), + } + }), + ) + expect(result).toEqual({ + some: ["api", "web"], + errors: ["api"], + none: ["db"], + every: ["web", "db"], + nested: ["api"], + }) + }) + + // ClickHouse 26.8 runs a correlated subquery in SELECT next to a WHERE only + // when the outer columns it correlates on are selected too ("can't find + // correlated column … in current header", NOT_IMPLEMENTED). So this selects + // the key (`Id`) and the tenant column (`OrgId`) beside the counts. + it("count() in a select and a where, 0 when nothing matches", async () => { + const rows = await withDatabase(() => + Db.run( + CH.from(Services) + .select(($) => ({ + org: $.OrgId, + id: $.Id, + name: $.Name, + events: Db.related(schema.services, $).events.count(), + errors: Db.related(schema.services, $).events.count(($) => [$.Level.eq("error")]), + })) + .where(($) => [$.OrgId.eq("a"), Db.related(schema.services, $).events.count().lt(2)]) + .orderBy(["name", "asc"]), + ), + ) + expect(rows).toEqual([ + { org: "a", id: 3, name: "db", events: 0, errors: 0 }, + { org: "a", id: 2, name: "web", events: 1, errors: 0 }, + ]) + }) + + it("count() in a where needs no selected key", async () => { + const rows = await withDatabase(() => + Db.run( + CH.from(Services) + .select(($) => ({ name: $.Name })) + .where(($) => [$.OrgId.eq("a"), Db.related(schema.services, $).events.count().gte(2)]), + ), + ) + expect(rows).toEqual([{ name: "api" }]) + }) + + it("_count in findMany is one grouped statement per level", async () => { + const rows = await withDatabase(() => + Db.findMany(schema.services, { + where: ($) => [$.OrgId.eq("a")], + orderBy: ($) => [[$.Id, "asc"]], + _count: { events: true }, + with: { events: { orderBy: ($) => [[$.Id, "asc"]], _count: { notes: true } } }, + }), + ) + expect(rows.map((row) => [row.Name, row._count.events, row.events.map((event) => event._count.notes)])).toEqual([ + ["api", 2, [1, 0]], + ["web", 1, [0]], + ["db", 0, []], + ]) + }) +}) From c1717c9a438741492f0bd481e7c39ba5e5673b83 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:41:36 +0200 Subject: [PATCH 6/7] fix(database): refuse relation filters on an unqualified accessor --- src/database/relation-filters.test.ts | 5 +++++ src/database/relations.ts | 25 +++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/database/relation-filters.test.ts b/src/database/relation-filters.test.ts index 925404a..0882308 100644 --- a/src/database/relation-filters.test.ts +++ b/src/database/relation-filters.test.ts @@ -196,6 +196,11 @@ layer(live(false), { excludeTestServices: true })("relation filters on PGlite", }) describe("relation filters and the subquery alias", () => { + plain("a filter on an update's or a delete's unqualified $ is refused, not correlated to itself", () => { + const query = PG.deleteFrom(Users).where(($) => [Db.related(schema.users, $ as any).posts.none()]) + expect(() => PG.compileUnsafe(query, {})).toThrow(/an update or delete has no alias/) + }) + const Nodes = PG.table("nodes", { columns: { id: PG.int4, parentId: PG.column(PG.nullable(PG.int4), { name: "parent_id" }) }, primaryKey: ["id"], diff --git a/src/database/relations.ts b/src/database/relations.ts index f10d5b8..ccdb1ec 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -8,7 +8,7 @@ import { Effect } from "effect" import type { CompiledQueryDecodeError } from "../ch/compile" -import type { QueryBuilderError } from "../ch/errors" +import { QueryBuilderDefect, type QueryBuilderError } from "../ch/errors" import { and, lit, makeCond, makeExpr, not, type Condition, type Expr, type ParamsIn } from "../ch/expr" import { builtins } from "../ch/functions/builtin" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" @@ -17,7 +17,8 @@ import type { SelectRowOf, Table } from "../ch/table" import { markCorrelatedTenant, tenantColumnOf } from "../ch/tenant" import { custom, type ColumnDefs, type InferTS } from "../ch/types" import { PgNumber } from "../pg/types" -import { raw } from "../sql/sql-fragment" +import { fail } from "../ch/failure" +import { known, raw } from "../sql/sql-fragment" import { Database } from "./database" import type { DatabaseError } from "./errors" @@ -182,6 +183,25 @@ const countType = custom("Int64", PgNumber) const countAll = (): Expr => makeExpr(aggregate.lazy(() => "count(*)", "count"), countType.schema) const always = (): Condition => makeCond(raw("1 = 1")) +/** + * The filters of a relation on an accessor whose columns are not qualified, as + * an update's or a delete's. The correlation would name an inner column, so + * each filter fails the compile instead. + */ +const unqualified = (key: string, name: string): RelationFilter => { + const failure = (placeholder: string) => + known(() => + fail( + new QueryBuilderDefect({ + message: `related(${key}).${name}: $ must be the accessor of a select over ${key}; an update or delete has no alias to correlate on`, + }), + placeholder, + ), + ) + const condition = () => makeCond(failure("FALSE")) + return { some: condition, none: condition, every: condition, count: () => makeExpr(failure("0"), countType.schema) as any } +} + const relatedOf = ( tables: RelationTables, relations: Record | undefined>, @@ -196,6 +216,7 @@ const relatedOf = ( // `$` is qualified (`alias.column`): the subquery's alias extends it, so // it never hides the outer source, even for a table related to itself. const outerName: string = outerFrom.columnName + if (!outerName.endsWith(`.${relation.from}`)) return [name, unqualified(key, name)] const alias = `${outerName.slice(0, outerName.length - relation.from.length - 1)}__${name}` const outerTenant = parent.tenantColumn !== undefined && target.tenantColumn !== undefined ? $[parent.tenantColumn] : undefined From 8be9e667c84743639e49c5accd91ca4448478e86 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:41:36 +0200 Subject: [PATCH 7/7] docs: relation filters, relation counts and _count --- .changeset/relation-filters.md | 5 ++ design/relations.md | 59 +++++++++++++++++- docs/reference.md | 1 + docs/relations.md | 109 ++++++++++++++++++++++++++++++++- 4 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 .changeset/relation-filters.md diff --git a/.changeset/relation-filters.md b/.changeset/relation-filters.md new file mode 100644 index 0000000..cec33db --- /dev/null +++ b/.changeset/relation-filters.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +Relation filters and counts in `/database`, as Prisma's `some` / `none` / `every` and `_count`. `findMany`'s `where` gets the relations of its level as a second argument: `where: ($, r) => [r.posts.some(($) => [$.published.eq(false)])]`. `Db.related(schema.users, $)` gives the same filters on a builder query, and `r.posts.count(where?)`, an `Expr` for a `select` or a `where`. The filters are correlated `EXISTS` / `NOT EXISTS` subqueries. `_count: { posts: true }` in `findMany` adds `_count: { posts: number }` to each row, with one grouped statement per level. When both tables declare a tenant column, the subquery matches the outer row's tenant, and a filter under a scoped query stays `"single-tenant"`. See docs/relations.md. diff --git a/design/relations.md b/design/relations.md index a7f5377..a7a51a0 100644 --- a/design/relations.md +++ b/design/relations.md @@ -2,7 +2,8 @@ Status: first slice built (issue #79; `docs/relations.md`). `one` and `many` relations over single-column keys, `with` to any depth, `where` and `orderBy` per level, `limit` and `offset` on -the top level. Section 5 lists what is not built. +the top level. Relation filters and counts (issue #80) are in section 6. Section 5 lists what is +not built. ## 1. Goal @@ -103,6 +104,60 @@ grandchild reached through a shared table. | `limit` per parent on a nested level | Needs `row_number()` over a partition, or a lateral join | | Composite keys | The loader would match tuples; no issue asks for it yet | | `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | -| `_count`, `some` / `every` / `none` | Issue #80 | +| `_count` of a `one` relation | `r.author.some()` says the same; no call site needs a 0/1 count | +| Relation filters in `update` / `delete` | Their accessor is not qualified, so there is no alias to correlate on; refused | +| `orderBy` a relation count | Select `count()` on a builder query and order by its alias | | Params in nested `where` | `findMany` takes no params; values go in as literals | | SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | + +## 6. Relation filters and counts + +```ts +Db.findMany(schema.users, { + where: ($, r) => [$.orgId.eq(orgId), r.posts.some(($) => [$.published.eq(false)])], + _count: { posts: true }, +}) +PG.from(Users).select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) +``` + +- **Filters are correlated subqueries.** `some` is `EXISTS`, `none` is `NOT EXISTS`, `every` is + `NOT EXISTS (… AND NOT (where))`, as Prisma compiles them. A filter is a condition inside one + statement, so it cannot be a separate level: the outer rows it filters do not exist yet. The + subquery reads the target under the alias `__`, which differs from every + enclosing alias, so a self-relation needs nothing special. +- **`Db.related(schema.table, $)` is the one entry point.** It builds the filters from the `$` of + any select over the table, so the builder and `findMany` share one implementation. `findMany` + passes it as the second argument of every `where`. +- **`_count` in `findMany` is a grouped level, not a subquery.** `SELECT , count(*) … + WHERE IN () GROUP BY `, with the same tenant pin as a loaded level. This keeps + #79's model: no correlation, so it runs on every dialect and every ClickHouse version, and the + count decodes through one codec. `r.posts.count()` is the correlated form, for builder queries, + wrapped in `coalesce(…, 0)` because ClickHouse returns NULL for a correlated count over no rows. + +### Tenant scope inside the subquery + +A subquery's scope counts as one more source of the outer query (docs/tenant-scoping.md), so a +subquery that pins no tenant made every filter `"cross-tenant"`. Writing the tenant value into +the subquery is not possible: the filter is built before the outer `where` is known, and the +value can be a param. + +So when both tables declare a tenant column, the filter writes +` = ` into the subquery and marks the subquery with the outer column +(`markCorrelatedTenant`, internal). Compiled as a subquery of the outer query, the marked query's +main source starts bound to a token for that outer column. If the subquery is single-tenant on +that token, the outer compile counts it as a source on the outer tenant column, which the outer +`where` binds like any other. Nested filters chain: the inner filter's token resolves against the +middle subquery's own main column. + +This is sound because only the filters set the mark, and they always write the equality the mark +claims. A subquery whose own `where` pins a literal tenant is bound to that literal instead, and +the outer query must pin the same one. A marked query compiled alone ignores the mark. The write +paths (insert, update, delete) do not read the mark, so a marked subquery there stays +`"cross-tenant"`. + +### ClickHouse + +Correlated subqueries run on ClickHouse 25 and later. On 26.8 (CI): `EXISTS` / `NOT EXISTS`, +nested, and a correlated count in `WHERE` work. A correlated scalar subquery in `SELECT` next to a +`WHERE` fails with `NOT_IMPLEMENTED` unless the correlated outer columns are also selected. The +docs say to select them, or to use `_count`, which needs no correlation. diff --git a/docs/reference.md b/docs/reference.md index f34bc3c..64f5535 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -514,6 +514,7 @@ placeholders, double-quoted identifiers. | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | | `defineRelations`, `findMany`, `Relation`, `RelationBuilder`, `RelationsConfig`, `RelationTables`, `RelationalSchema`, `RelationalTable`, `RelationOptions`, `FindManyOptions`, `FindManyRow`, `WithOf`, `KeyColumn` — relational reads, one statement per level ([Relational reads](./relations.md)) | `/database` | +| `related`, `Related`, `RelationFilter`, `RelationWhere`, `CountOf` — relation filters `some` / `none` / `every`, relation counts, and `_count` in `findMany` ([Filter by relations](./relations.md#filter-by-relations)) | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | diff --git a/docs/relations.md b/docs/relations.md index 173046e..941a3ba 100644 --- a/docs/relations.md +++ b/docs/relations.md @@ -68,13 +68,15 @@ _(Backed by `src/database/relations.test-d.ts`, and `tests/relations.clickhouse. ## Read with `findMany` `findMany(schema.table, options?)` returns every column of the table, plus each relation that -`with` names. `with` takes `true`, or the options of that level: +`with` names, plus `_count` when the level counts relations. `with` takes `true`, or the options +of that level: | Option | Levels | Meaning | | --- | --- | --- | | `where` | All | More conditions, ANDed with the relation's key match | | `orderBy` | All | The order of the rows; nested rows keep it within each parent | | `with` | All | Relations to load under each row of this level | +| `_count` | All | `many` relations to count under each row of this level | | `limit`, `offset` | Top only | Paging of the top level | | `crossTenant` | Top only | Declare every level `.crossTenant()` | @@ -117,13 +119,116 @@ cross-tenant and runs. _(Backed by `src/database/relations.test.ts`.)_ +## Filter by relations + +A `where` gets a second argument, `r`, with the relations of its table. Each relation has three +filters, as Prisma's `some`, `none` and `every`: + +| Filter | True when | SQL | +| --- | --- | --- | +| `r.posts.some(where?)` | At least one related row matches (with no `where`: there is one) | `EXISTS (…)` | +| `r.posts.none(where?)` | No related row matches (with no `where`: there is none) | `NOT EXISTS (…)` | +| `r.posts.every(where)` | Every related row matches; also true when there is none | `NOT EXISTS (… AND NOT (where))` | + +```ts title="relations-filters.ts" +import { Effect } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), published: PG.bool }, + primaryKey: ["id"], +}) +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +// Users with a draft, and users whose posts are all published. +export const withDrafts = Db.findMany(schema.users, { where: (_, r) => [r.posts.some(($) => [$.published.eq(false)])] }) +export const allPublished = Db.findMany(schema.users, { where: (_, r) => [r.posts.every(($) => [$.published.eq(true)])] }) + +// The same filters on a builder query: `Db.related` takes the query's `$`. +export const query = PG.from(Users) + .select(($) => ({ id: $.id, posts: Db.related(schema.users, $).posts.count() })) + .where(($) => [Db.related(schema.users, $).posts.none()]) +export const rows: Effect.Effect, unknown, Db.Database> = + Db.run(query) +``` + +Each filter is one correlated subquery in the statement that holds it. It reads the target +table under its own alias, `__`, so a table related to itself works. The +`where` of a filter gets its own `r`, so filters nest: `r.posts.some((_, r) => [r.comments.some()])`. +A `one` relation has the same filters: `some` is "the related row exists and matches", `none` is +"it does not". + +`every` negates the condition, so a related row where the condition is NULL passes it. Use a +condition that cannot be NULL, or say what NULL means with `isNull` / `isNotNull`. + +_(Backed by `src/database/relation-filters.test.ts`, and `tests/relation-filters.clickhouse.test.ts` on ClickHouse.)_ + +## Count relations + +Two ways, for two shapes of read: + +- **`_count` in `findMany`**, as Prisma's `_count`. `_count: { posts: true }` adds + `_count: { posts: number }` to each row of that level. `_count: { posts: { where } }` counts + only the rows that match. Only a `many` relation can be counted. It runs one statement per + counted relation, the same way as a level of `with`: + `SELECT , count(*) FROM WHERE IN () [AND ] GROUP BY `. + A row with no related rows gets `0`. +- **`r.posts.count(where?)` on a query**, an `Expr`: a correlated + `coalesce((SELECT count(*) …), 0)`. Use it in a builder `select`, or in a `where` such as + `Db.related(schema.users, $).posts.count().gte(2)`. + +_(Backed by `src/database/relation-filters.test.ts`.)_ + +## Tenant scope in filters and counts + +When both tables declare a `tenantColumn`, a filter's subquery also gets +` = `. The compiler then gives the subquery the +tenant the outer query is bound to: a filter under `$.orgId.eq("a")` is `"single-tenant"`. The +equality also hides a related row whose tenant differs from the outer row's. + +A filter does not scope the query that holds it. Without a tenant filter of its own, the outer +query stays `"cross-tenant"`. A filter from an untenanted table into a tenanted one has no outer +tenant to match, so it is `"cross-tenant"` too, unless its own `where` pins the tenant +(`r.units.some(($) => [$.orgId.eq("a")])`). A filter whose `where` pins a different tenant from the +outer query makes the query `"cross-tenant"`. + +`_count` uses the same pin as a level of `with` (see above). + +_(Backed by `src/database/relation-filters.test.ts`.)_ + +## On ClickHouse + +ClickHouse 25 and later run correlated subqueries (`allow_experimental_correlated_subqueries`, +on by default in 26.8). On 26.8: + +- `some`, `none`, `every`, and filters nested in filters work. +- `count()` in a `where` works. +- `count()` in a `select` works when the outer columns it correlates on are selected too, or the + query has no `where`. Otherwise ClickHouse refuses the query with `NOT_IMPLEMENTED` + ("can't find correlated column"). Select the relation's `from` column and the tenant column + next to the count, or use `_count` in `findMany`. +- A correlated `count()` over no rows returns NULL on ClickHouse, not 0. The `coalesce` in + `count()` turns it into 0. +- `_count` needs no correlation, so it works on every version. + +_(Backed by `tests/relation-filters.clickhouse.test.ts`.)_ + ## Not yet - Choosing columns per level (`columns: { id: true }`). Every level reads every column. - A `limit` per parent on a nested level. That needs a window function or a lateral join. - Composite keys (`from` and `to` with more than one column). - `findFirst`. Use `findMany` with `limit: 1`. -- Filters and counts over relations (`_count`, `some`, `every`, `none`). +- `_count` of a `one` relation. Test for the row with `r.author.some()`. +- Relation filters in `update` and `delete`. Their `$` has no alias to correlate on, so the + compile dies with a `QueryBuilderDefect`. Select the keys first, then write by key. +- Ordering by a relation count (`orderBy: { posts: { _count: "desc" } }`). On a builder query, + select the count and order by its alias. - `param.*` markers in a `where`: `findMany` takes no params, so write values in. See [design/relations.md](../design/relations.md) for the design and the alternatives.