Skip to content

Commit 8bf0e2f

Browse files
dmealingclaude
andcommitted
fix(codegen-ts): SQLite collect views open on SQLite 3.35, not only 3.44
A projection view using origin.collect ordered its elements with an in-aggregate ORDER BY (json_group_array(x ORDER BY ...)), which needs SQLite 3.44. SQLite parses every view when it opens a database, so such a database could not be opened at all by Ubuntu 22.04's 3.37 or Python 3.10's stdlib. A collect column is now built in two steps. The grouped query gathers each element as [sort keys..., value] with no ORDER BY, and an outer query over it sorts the elements through json_each and re-aggregates them. The sort cannot sit in a subquery of the grouped query itself: a subquery may not read an aggregate of the query around it ("misuse of aggregate function"). Element order, the empty-set [] result and whole-object elements (still JSON objects, re-wrapped with json()) are unchanged. Postgres output is unchanged. Verified on real engines: the emitted migration applies and returns the same rows on SQLite 3.35.5 (built from the amalgamation), 3.37.2 (system) and libsql, and the re-diff is empty. The old form fails on 3.37 with `near "order": syntax error`. The integration probe now seeds rows whose label order and id order are opposite, so a collect that returned scan order would fail it. Next migrate against an existing SQLite/D1 database replaces such views once. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01M1mRRB1K2WdYy82kntMQdx
1 parent ec1874b commit 8bf0e2f

4 files changed

Lines changed: 170 additions & 54 deletions

File tree

‎server/typescript/packages/codegen-ts/src/projection/view-ddl-emit.ts‎

Lines changed: 113 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@ function renderFilterCond(clause: ViewFilterClause, dialect: EmitOptions["dialec
6868
return `${lhs} ${op} ${sqlLiteral(clause.value, dialect)}`;
6969
}
7070

71-
/** Subquery alias for the json_each re-wrap in the SQLite whole-object collect. Scoped to
72-
* its own scalar subquery, so it cannot collide with a JOIN-tree alias; named distinctly
73-
* anyway so it is obvious in emitted DDL where it came from. */
71+
/** Alias of the json_each table that a SQLite collect column sorts its elements over.
72+
* Scoped to its own scalar subquery, so it cannot collide with a JOIN-tree alias; named
73+
* distinctly anyway so it is obvious in emitted DDL where it came from. */
7474
const JSON_EACH_ALIAS = "mo_je";
7575

7676
/**
@@ -146,6 +146,95 @@ function renderFirst(
146146
);
147147
}
148148

149+
type CollectColumn = Extract<SelectColumn, { kind: "collectAgg" | "collectObjectAgg" }>;
150+
151+
/** Alias of the grouped inner query in a two-level SQLite view. */
152+
const GROUPED_ALIAS = "mo_g";
153+
/** Alias of the sorted element subquery in a SQLite collect column. */
154+
const SORTED_ALIAS = "mo_s";
155+
156+
/**
157+
* How a SQLite collect column orders its elements. Each element is stored as a JSON array
158+
* `[key1, …, keyN, payload]`, or as the bare value when `bare` is set (a `@distinct`
159+
* collect orders by the value itself).
160+
*/
161+
interface SqliteCollectLayout {
162+
readonly keys: readonly { readonly expr: string; readonly order: string }[];
163+
readonly payload: string;
164+
readonly bare: boolean;
165+
/** Wrap each element in json() when re-aggregating — true for a JSON object payload. */
166+
readonly json: boolean;
167+
}
168+
169+
function sqliteCollectLayout(c: CollectColumn): SqliteCollectLayout {
170+
const col = (name: string): string => `${c.sourceAlias}.${quoteIfNeeded(name)}`;
171+
const orderKeys = c.orderBy.map((k) => ({ expr: col(k.column), order: `${k.dir.toUpperCase()} NULLS LAST` }));
172+
if (c.kind === "collectObjectAgg") {
173+
const pairs = c.members.map((m) => `'${m.memberName}', ${col(m.sourceColumn)}`).join(", ");
174+
// Default element order is the related entity's PK; an explicit @orderBy leads with the
175+
// PK appended as a tie-break, so equal-order rows stay byte-deterministic.
176+
return {
177+
keys: [...orderKeys, { expr: col(c.joinedPkColumn), order: "ASC" }],
178+
payload: `json_object(${pairs})`,
179+
bare: false,
180+
json: true,
181+
};
182+
}
183+
const src = col(c.sourceColumn);
184+
// @distinct always orders by the value; otherwise an explicit @orderBy, else value ascending.
185+
if (c.distinct) return { keys: [], payload: src, bare: true, json: false };
186+
return {
187+
keys: orderKeys.length > 0 ? orderKeys : [{ expr: src, order: "ASC" }],
188+
payload: src,
189+
bare: false,
190+
json: false,
191+
};
192+
}
193+
194+
/**
195+
* SQLite collect, inner half: the grouped query aggregates the related rows UNSORTED, each
196+
* element carrying its sort keys. Ordering happens in the outer query
197+
* ({@link sqliteCollectOuter}).
198+
*
199+
* Why two levels. The ordered aggregate `json_group_array(x ORDER BY …)` needs SQLite 3.44,
200+
* and a database holding such a view will not open on anything older (Ubuntu 22.04 ships
201+
* 3.37). Before 3.44 the only way to order an aggregate's input is to aggregate over a
202+
* sorted subquery — and a subquery cannot read an aggregate of the query around it
203+
* ("misuse of aggregate function"), so the sort has to sit one query further out, reading
204+
* the grouped result as a plain column. Measured on SQLite 3.35.5, 3.37.2, 3.44 and 3.53.
205+
*
206+
* This also keeps whole-object elements as JSON objects: without an in-aggregate ORDER BY
207+
* `json_group_array(json_array(…, json_object(…)))` nests correctly, where on 3.44
208+
* `json_group_array(json_object(…) ORDER BY …)` returned an array of quoted strings.
209+
*/
210+
function sqliteCollectInner(c: CollectColumn): string {
211+
const layout = sqliteCollectLayout(c);
212+
const guard = `${c.sourceAlias}.${quoteIfNeeded(c.joinedPkColumn)} IS NOT NULL`;
213+
const element = layout.bare
214+
? `DISTINCT ${layout.payload}`
215+
: `json_array(${[...layout.keys.map((k) => k.expr), layout.payload].join(", ")})`;
216+
return `COALESCE(json_group_array(${element}) FILTER (WHERE ${guard}), json_array())`;
217+
}
218+
219+
/** SQLite collect, outer half: sort the grouped column's elements and re-aggregate them. */
220+
function sqliteCollectOuter(c: CollectColumn): string {
221+
const layout = sqliteCollectLayout(c);
222+
const grouped = `${GROUPED_ALIAS}.${quoteIfNeeded(c.dbColAlias)}`;
223+
const at = (i: number): string => `json_extract(${JSON_EACH_ALIAS}.value, '$[${i}]')`;
224+
const value = layout.bare ? `${JSON_EACH_ALIAS}.value` : at(layout.keys.length);
225+
const orderBy = layout.bare
226+
? `${JSON_EACH_ALIAS}.value ASC`
227+
: layout.keys.map((k, i) => `${at(i)} ${k.order}`).join(", ");
228+
const element = layout.json ? `json(${SORTED_ALIAS}.v)` : `${SORTED_ALIAS}.v`;
229+
return (
230+
`(SELECT json_group_array(${element}) FROM (SELECT ${value} AS v FROM json_each(${grouped}) ${JSON_EACH_ALIAS}` +
231+
` ORDER BY ${orderBy}) ${SORTED_ALIAS}) AS ${quoteIfNeeded(c.dbColAlias)}`
232+
);
233+
}
234+
235+
const isCollect = (c: SelectColumn): c is CollectColumn =>
236+
c.kind === "collectAgg" || c.kind === "collectObjectAgg";
237+
149238
function renderColumn(c: SelectColumn, options: EmitOptions, baseAlias: string): string {
150239
const dialect = options.dialect;
151240
const alias = quoteIfNeeded(c.dbColAlias);
@@ -200,12 +289,18 @@ function renderColumn(c: SelectColumn, options: EmitOptions, baseAlias: string):
200289
? `ORDER BY ${renderOrderKeys(c.orderBy, c.sourceAlias)}`
201290
: `ORDER BY ${src} ASC`;
202291
if (dialect === "sqlite") {
203-
return `COALESCE(json_group_array(${distinctKw}${src} ${orderClause}) FILTER (WHERE ${guard}), json_array()) AS ${alias}`;
292+
// Unsorted here; the outer query of the two-level SQLite view sorts it (see
293+
// sqliteCollectInner / sqliteCollectOuter).
294+
return `${sqliteCollectInner(c)} AS ${alias}`;
204295
}
205296
return `COALESCE(array_agg(${distinctKw}${src} ${orderClause}) FILTER (WHERE ${guard}), '{}') AS ${alias}`;
206297
}
207298

208299
if (c.kind === "collectObjectAgg") {
300+
if (dialect === "sqlite") {
301+
// SQLite: unsorted here; the outer query of the two-level view sorts it.
302+
return `${sqliteCollectInner(c)} AS ${alias}`;
303+
}
209304
// #335 whole-object rollup. jsonb, not json: PG's `json` has neither an equality
210305
// nor an ordering operator, so `json_agg(json_build_object(…) ORDER BY …)` does not
211306
// run — verified against a real PG 15.
@@ -224,31 +319,6 @@ function renderColumn(c: SelectColumn, options: EmitOptions, baseAlias: string):
224319
const pairs = c.members
225320
.map((m) => `'${m.memberName}', ${c.sourceAlias}.${quoteIfNeeded(m.sourceColumn)}`)
226321
.join(", ");
227-
// In-aggregate ORDER BY needs SQLite >= 3.44 — not a new constraint: the scalar
228-
// collect above already emits it, and D1's baseline is pinned at 3.44.0.
229-
if (dialect === "sqlite") {
230-
// SQLite cannot do BOTH in-aggregate ORDER BY and JSON nesting in one call.
231-
// Measured on SQLite 3.44.0, D1's pinned baseline:
232-
//
233-
// json_group_array(json_object(…)) -> nests correctly
234-
// json_group_array(json_object(…) ORDER BY …) -> array of QUOTED STRINGS
235-
// json_group_array(json(json_object(…)) ORDER BY …) -> array of QUOTED STRINGS
236-
//
237-
// The ORDER BY clause itself destroys the JSON subtype, and a json() wrapper on the
238-
// argument does not survive it. Dropping ORDER BY is not an option: element order
239-
// would stop being deterministic and an author's @orderBy would silently do nothing.
240-
//
241-
// So build the ordered array first (elements quoted), then re-wrap element by
242-
// element through json_each — which iterates in ARRAY ORDER, so the ordering
243-
// survives while json(value) restores each element's JSON subtype. Still a grouped
244-
// LEFT JOIN: unlike origin.first this needs no correlation info, so a multi-hop
245-
// @via lowers here exactly as a single-hop one does.
246-
//
247-
// Found by the real-engine probe in integration-tests. Emitted SQL text cannot show
248-
// this, which is exactly why golden SQL is not evidence for new DDL.
249-
const ordered = `COALESCE(json_group_array(json_object(${pairs}) ${orderClause}) FILTER (WHERE ${guard}), json_array())`;
250-
return `(SELECT json_group_array(json(${JSON_EACH_ALIAS}.value)) FROM json_each(${ordered}) ${JSON_EACH_ALIAS}) AS ${alias}`;
251-
}
252322
// PG's jsonb_build_object already yields real jsonb, so jsonb_agg nests it correctly
253323
// with no wrapper — verified against a real engine, not assumed by symmetry.
254324
return `COALESCE(jsonb_agg(jsonb_build_object(${pairs}) ${orderClause}) FILTER (WHERE ${guard}), '[]'::jsonb) AS ${alias}`;
@@ -314,10 +384,23 @@ export function emitViewDdl(spec: ViewSpec, options: EmitOptions): string {
314384
? `\n GROUP BY ${spec.groupBy.map(quoteRef).join(", ")}`
315385
: "";
316386

317-
const body = ` SELECT
387+
const grouped = ` SELECT
318388
${cols}
319389
${fromClause}${joinsClause ? "\n" + joinsClause : ""}${whereClause}${groupByClause}`;
320390

391+
// SQLite orders collect elements in an outer query over the grouped one — see
392+
// sqliteCollectInner for why. Every other column passes straight through.
393+
const collects = options.dialect === "sqlite" && spec.selectSpec.columns.some(isCollect);
394+
const body = !collects ? grouped : ` SELECT
395+
${spec.selectSpec.columns
396+
.map((c) => " " + (isCollect(c)
397+
? sqliteCollectOuter(c)
398+
: `${GROUPED_ALIAS}.${quoteIfNeeded(c.dbColAlias)} AS ${quoteIfNeeded(c.dbColAlias)}`))
399+
.join(",\n")}
400+
FROM (
401+
${grouped}
402+
) ${GROUPED_ALIAS}`;
403+
321404
if (options.bodyOnly) return body;
322405
return `CREATE VIEW ${quoteIfNeeded(spec.viewName)} AS
323406
${body};`;

‎server/typescript/packages/codegen-ts/test/projection/collect-whole-object-ddl.test.ts‎

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -56,17 +56,18 @@ describe("emitViewDdl — #335 whole-object collect", () => {
5656
);
5757
});
5858

59-
// SQLite needs the json_each re-wrap. Measured on 3.44.0: the in-aggregate ORDER BY
60-
// destroys the JSON subtype, so json_group_array(json_object(...) ORDER BY ...) returns
61-
// an array of QUOTED STRINGS — and a json() wrapper on the argument does not survive it
62-
// either. json_each iterates in array order, so re-wrapping element-by-element restores
63-
// the objects while keeping the ordering. Found by the real-engine probe.
64-
test("sqlite: the ordered array is re-wrapped through json_each so elements stay OBJECTS", () => {
59+
// SQLite collects each row as [sort keys…, object] UNSORTED in the grouped query, then
60+
// sorts in an outer query and re-wraps each object with json(). No in-aggregate ORDER BY:
61+
// it needs SQLite 3.44, and on 3.44 it also turned the objects into quoted strings.
62+
test("sqlite: objects are collected unsorted with their PK key, then sorted and re-wrapped", () => {
6563
const sql = emitViewDdl(wholeObject(), { dialect: "sqlite", ...OPTS });
6664
expect(sql).toContain(
67-
"(SELECT json_group_array(json(mo_je.value)) FROM json_each(" +
68-
"COALESCE(json_group_array(json_object('id', s.id, 'name', s.supplier_name) ORDER BY s.id ASC) " +
69-
"FILTER (WHERE s.id IS NOT NULL), json_array())) mo_je) AS supplier_briefs",
65+
"COALESCE(json_group_array(json_array(s.id, json_object('id', s.id, 'name', s.supplier_name))) " +
66+
"FILTER (WHERE s.id IS NOT NULL), json_array()) AS supplier_briefs",
67+
);
68+
expect(sql).toContain(
69+
"(SELECT json_group_array(json(mo_s.v)) FROM (SELECT json_extract(mo_je.value, '$[1]') AS v " +
70+
"FROM json_each(mo_g.supplier_briefs) mo_je ORDER BY json_extract(mo_je.value, '$[0]') ASC) mo_s) AS supplier_briefs",
7071
);
7172
});
7273

@@ -124,9 +125,11 @@ describe("emitViewDdl — #335 no-churn pin on the SCALAR collect arm", () => {
124125
);
125126
});
126127

127-
test("sqlite: json_group_array / json_array() — unchanged", () => {
128-
expect(emitViewDdl(scalar(false, []), { dialect: "sqlite", ...opts })).toContain(
129-
"COALESCE(json_group_array(i.category ORDER BY i.category ASC) FILTER (WHERE i.id IS NOT NULL), json_array()) AS categories",
128+
test("sqlite: the scalar arm orders by the value, with no PK tie-break", () => {
129+
const sql = emitViewDdl(scalar(false, []), { dialect: "sqlite", ...opts });
130+
expect(sql).toContain(
131+
"COALESCE(json_group_array(json_array(i.category, i.category)) FILTER (WHERE i.id IS NOT NULL), json_array()) AS categories",
130132
);
133+
expect(sql).toContain("ORDER BY json_extract(mo_je.value, '$[0]') ASC) mo_s) AS categories");
131134
});
132135
});

‎server/typescript/packages/codegen-ts/test/projection/view-ddl-emit.test.ts‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -230,11 +230,36 @@ describe("emitViewDdl — #195 array rollup (collect)", () => {
230230
);
231231
});
232232

233-
test("sqlite → json_group_array coalesced to json_array()", () => {
233+
// SQLite before 3.44 has no in-aggregate ORDER BY, so the grouped query collects the
234+
// elements unsorted and an outer query over it sorts them (the floor is 3.35).
235+
test("sqlite → unsorted json_group_array in the grouped query, sorted in the outer one", () => {
234236
const sql = emitViewDdl(collect(true, []), { dialect: "sqlite", baseTableName: "orders", joinTables: { Item: "items" } });
235237
expect(sql).toContain(
236-
"COALESCE(json_group_array(DISTINCT i.category ORDER BY i.category ASC) FILTER (WHERE i.id IS NOT NULL), json_array()) AS categories",
238+
"COALESCE(json_group_array(DISTINCT i.category) FILTER (WHERE i.id IS NOT NULL), json_array()) AS categories",
237239
);
240+
expect(sql).toContain(
241+
"(SELECT json_group_array(mo_s.v) FROM (SELECT mo_je.value AS v FROM json_each(mo_g.categories) mo_je" +
242+
" ORDER BY mo_je.value ASC) mo_s) AS categories",
243+
);
244+
});
245+
246+
test("sqlite: an explicit @orderBy key travels with the element and sorts in the outer query", () => {
247+
const sql = emitViewDdl(collect(false, [{ column: "created_at", dir: "desc" }]),
248+
{ dialect: "sqlite", baseTableName: "orders", joinTables: { Item: "items" } });
249+
expect(sql).toContain(
250+
"COALESCE(json_group_array(json_array(i.created_at, i.category)) FILTER (WHERE i.id IS NOT NULL), json_array()) AS categories",
251+
);
252+
expect(sql).toContain(
253+
"(SELECT json_group_array(mo_s.v) FROM (SELECT json_extract(mo_je.value, '$[1]') AS v FROM json_each(mo_g.categories) mo_je" +
254+
" ORDER BY json_extract(mo_je.value, '$[0]') DESC NULLS LAST) mo_s) AS categories",
255+
);
256+
});
257+
258+
test("sqlite: no aggregate carries an in-aggregate ORDER BY (it needs SQLite 3.44)", () => {
259+
for (const spec of [collect(true, []), collect(false, []), collect(false, [{ column: "created_at", dir: "desc" }])]) {
260+
const sql = emitViewDdl(spec, { dialect: "sqlite", baseTableName: "orders", joinTables: { Item: "items" } });
261+
expect(sql).not.toMatch(/json_group_array\([^()]*(\([^()]*\)[^()]*)*ORDER BY/);
262+
}
238263
});
239264
});
240265

‎server/typescript/packages/integration-tests/test/view-lifecycle-sqlite.test.ts‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,14 @@ describe("view value-probe — real SQLite (#195 origins)", () => {
112112
// SQLite lowers collect to json_group_array (a JSON string), any/all to MAX/MIN over 1/0.
113113
expect(up).toContain(`CREATE VIEW "v_program_summary" AS`);
114114
expect(up).toContain("json_group_array");
115-
// #335 whole-object rollup: json_object per row, related-PK ascending, [] on empty.
116-
// #335 whole-object rollup: an ordered json_object array re-wrapped through json_each
117-
// (SQLite's in-aggregate ORDER BY destroys the JSON subtype — see view-ddl-emit).
118-
expect(up).toContain("json_group_array(json_object('id', w.id, 'label', w.label) ORDER BY w.id ASC)");
119-
expect(up).toContain("SELECT json_group_array(json(mo_je.value)) FROM json_each(");
115+
// Collect is two-level on SQLite: the grouped query gathers [sort keys…, value] unsorted,
116+
// an outer query sorts. No in-aggregate ORDER BY, which needs SQLite 3.44 — the floor
117+
// for every view is 3.35 (see view-ddl-emit).
118+
expect(up).toContain("json_group_array(json_array(w.id, json_object('id', w.id, 'label', w.label)))");
119+
expect(up).toContain(") mo_g;");
120+
for (const line of up.split("\n").filter((l) => l.includes("FILTER (WHERE"))) {
121+
expect(line).not.toContain("ORDER BY");
122+
}
120123
expect(up).toMatch(/COALESCE\(MAX\(CASE WHEN[^)]*\)[^,]*, 0\) AS "anyLongWeek"/);
121124
expect(up).toMatch(/COALESCE\(MIN\(CASE WHEN[^)]*\)[^,]*, 1\) AS "allLongWeeks"/);
122125
await applyRaw(up);
@@ -133,8 +136,8 @@ describe("view value-probe — real SQLite (#195 origins)", () => {
133136
await sql.raw(`INSERT INTO "programs" ("id","status") VALUES (1,'PUBLISHED'),(2,'DRAFT')`).execute(k);
134137
await sql.raw(
135138
`INSERT INTO "weeks" ("id","programId","label","durationMinutes","createdAt") VALUES
136-
(1, 1, 'A', 30, '2026-01-01T00:00:00Z'),
137-
(2, 1, 'B', 90, '2026-02-01T00:00:00Z')`,
139+
(1, 1, 'B', 30, '2026-01-01T00:00:00Z'),
140+
(2, 1, 'A', 90, '2026-02-01T00:00:00Z')`,
138141
).execute(k);
139142

140143
const rows = await sql.raw(
@@ -152,15 +155,17 @@ describe("view value-probe — real SQLite (#195 origins)", () => {
152155
const full = byId.get("1")!;
153156
expect(full.anyLongWeek).toBe(1); // 90 > 60
154157
expect(full.allLongWeeks).toBe(0); // 30 is NOT > 60
158+
// Labels sort by label, briefs by id — deliberately OPPOSITE orders over the same rows,
159+
// so neither can pass by returning the rows in scan order.
155160
expect(JSON.parse(full.weekLabels)).toEqual(["A", "B"]);
156161
// #335 — the DECLARED members only, in related-PK order. Week also has programId /
157162
// durationMinutes / createdAt; if any of those appear the "declared VO IS the
158163
// exposure" guarantee (#270) has broken at the SQL tier.
159164
expect(JSON.parse(full.weekBriefs)).toEqual([
160-
{ id: 1, label: "A" },
161-
{ id: 2, label: "B" },
165+
{ id: 1, label: "B" },
166+
{ id: 2, label: "A" },
162167
]);
163-
expect(full.latestWeekLabel).toBe("B"); // most recent by createdAt
168+
expect(full.latestWeekLabel).toBe("A"); // most recent by createdAt
164169
expect(full.isPublished).toBe(1); // status = 'PUBLISHED'
165170

166171
// Empty (id 2): ZERO weeks — the empty-set pins.

0 commit comments

Comments
 (0)