fix(pull): capture index-only projections and render them correctly

.changeset/index-only-projections.md

@@ -0,0 +1,12 @@
+---
+"@chkit/clickhouse": patch
+"@chkit/plugin-pull": patch
+"@chkit/core": patch
+"chkit": patch
+---
+
+Support index-only projections (`PROJECTION p INDEX (a, b) TYPE basic`) end to end. `ProjectionDefinition` is now a union of the existing `{ name, query }` SELECT form and a new `{ name, index, type }` index-only form, which renders without the wrapping parens that made the SELECT form invalid for it. `chkit pull` previously parsed only the SELECT form and dropped index-only projections on the floor, so a pulled schema silently recreated the table without them; they now round-trip through pull, generate, migrate, and drift.
+
+Index expressions are normalized to the exact form ClickHouse stores — a single expression bare (`INDEX a`), several as a tuple (`INDEX (a, b)`), redundant parens peeled at every level, and a space after each argument separator — so `'(a)'` and `'a'`, or `'concat(x,y)'` and `'concat(x, y)'`, describe the same table and no longer read as drift.
+
+Two new validation errors guard the new form: `projection_ambiguous_kind` when an entry sets both `query` and `index` (which would otherwise silently discard the SELECT body), and `projection_empty_index` when the index expression is empty (which would otherwise emit invalid DDL).

apps/docs/src/content/docs/schema/dsl-reference.md

@@ -273,19 +273,34 @@ indexes: [
 
 ## Projections
 
-Each entry in the `projections` array is a `ProjectionDefinition`.
+Each entry in the `projections` array is a `ProjectionDefinition`, which takes one of two forms.
+
+A **SELECT projection** stores a rewritten copy of the data.
 
 | Field | Type | Description |
 |-------|------|-------------|
 | `name` | `string` | Projection name |
 | `query` | `string` | Projection SELECT query |
 
+An **index-only projection** stores no SELECT body. It reorders parts by a secondary key so lookups on that key prune instead of scanning.
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `name` | `string` | Projection name |
+| `index` | `string` | Expression list to order by, e.g. `receiver, sender` |
+| `type` | `string` | Projection index type. ClickHouse currently accepts `basic` |
+
 ```ts
 projections: [
   { name: 'p_recent', query: 'SELECT id ORDER BY received_at DESC LIMIT 10' },
+  { name: 'by_receiver', index: 'receiver, sender', type: 'basic' },
 ]
 ```
 
+The `index` expression is rendered the way ClickHouse normalizes it: a single expression is emitted bare (`INDEX receiver`), several are emitted as a tuple (`INDEX (receiver, sender)`), redundant parentheses are dropped, and a space follows every argument separator. Writing `'(receiver)'` and `'receiver'` therefore produce the same table, and neither reads as drift.
+
+A projection must be exactly one of the two kinds. Setting both `query` and `index` on the same entry is a `projection_ambiguous_kind` validation error, and an empty `index` is a `projection_empty_index` error.
+
 ## `view()`
 
 Creates a view definition.
@@ -443,6 +458,8 @@ chkit validates schema definitions and throws a `ChxValidationError` if any issu
 - **Duplicate column names** -- repeated column name within a table
 - **Duplicate index names** -- repeated index name within a table
 - **Duplicate projection names** -- repeated projection name within a table
+- **Ambiguous projection kind** (`projection_ambiguous_kind`) -- a projection sets both `query` and `index`; use one or the other (see [Projections](#projections))
+- **Empty projection index** (`projection_empty_index`) -- an index-only projection whose `index` expression is empty
 - **Primary key references missing column** -- `primaryKey` includes a bare column name not in `columns` (function expressions like `toDate(ts)` are passed through to ClickHouse unchecked)
 - **Order by references missing column** -- `orderBy` includes a bare column name not in `columns` (function expressions like `toStartOfHour(ts)` are passed through to ClickHouse unchecked)
 - **Empty codec chain** (`codec_chain_empty`) -- a `codec` array with no steps; provide at least one codec or omit the field

packages/cli/src/commands/drift/compare.ts

@@ -1,5 +1,7 @@
 import {
   normalizeEngine as coreNormalizeEngine,
+  isIndexProjection,
+  normalizeProjectionIndex,
   normalizeSQLFragment,
   type ColumnDefinition,
   type ProjectionDefinition,
@@ -227,6 +229,12 @@ function normalizeIndexShape(index: SkipIndexDefinition): string {
 }
 
 function normalizeProjectionShape(projection: ProjectionDefinition): string {
+  if (isIndexProjection(projection)) {
+    return [
+      `index=${normalizeProjectionIndex(projection.index)}`,
+      `type=${projection.type.trim()}`,
+    ].join('|')
+  }
   return `query=${normalizeSQLFragment(projection.query)}`
 }
 

packages/cli/src/test/drift.test.ts

@@ -123,6 +123,101 @@ describe('@chkit/cli drift comparer', () => {
     expect(result).toBeNull()
   })
 
+  // ClickHouse rewrites a single-column `INDEX (id)` to `INDEX id`, so a schema
+  // written with parens must not read as drift against the live table.
+  test('treats an index-only projection as clean regardless of index parens', () => {
+    const expected = table({
+      database: 'app',
+      name: 'events',
+      engine: 'MergeTree()',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      primaryKey: ['id'],
+      orderBy: ['id'],
+      projections: [{ name: 'by_receiver', index: '(receiver)', type: 'basic' }],
+    })
+
+    const result = compareTableShape(expected, {
+      engine: 'MergeTree()',
+      primaryKey: '(id)',
+      orderBy: '(id)',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      settings: {},
+      indexes: [],
+      projections: [{ name: 'by_receiver', index: 'receiver', type: 'basic' }],
+    })
+
+    expect(result).toBeNull()
+  })
+
+  test('reports projection_mismatch when an index projection changes type', () => {
+    const expected = table({
+      database: 'app',
+      name: 'events',
+      engine: 'MergeTree()',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      primaryKey: ['id'],
+      orderBy: ['id'],
+      projections: [{ name: 'by_receiver', index: 'receiver, id', type: 'basic' }],
+    })
+
+    const result = compareTableShape(expected, {
+      engine: 'MergeTree()',
+      primaryKey: '(id)',
+      orderBy: '(id)',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      settings: {},
+      indexes: [],
+      projections: [{ name: 'by_receiver', index: 'receiver', type: 'basic' }],
+    })
+
+    expect(result?.reasonCodes).toContain('projection_mismatch')
+    expect(result?.projectionDiffs).toEqual(['by_receiver'])
+  })
+
+  // A SELECT projection and an index-only projection sharing a name are
+  // different objects; the fingerprint must not collapse them.
+  test('reports projection_mismatch when a select projection becomes index-only', () => {
+    const expected = table({
+      database: 'app',
+      name: 'events',
+      engine: 'MergeTree()',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      primaryKey: ['id'],
+      orderBy: ['id'],
+      projections: [{ name: 'p', index: 'receiver', type: 'basic' }],
+    })
+
+    const result = compareTableShape(expected, {
+      engine: 'MergeTree()',
+      primaryKey: '(id)',
+      orderBy: '(id)',
+      columns: [
+        { name: 'id', type: 'UInt64' },
+        { name: 'receiver', type: 'String' },
+      ],
+      settings: {},
+      indexes: [],
+      projections: [{ name: 'p', query: 'SELECT receiver' }],
+    })
+
+    expect(result?.reasonCodes).toContain('projection_mismatch')
+  })
+
   test('treats quoted string defaults and implicit engine settings as equivalent', () => {
     const expected = table({
       database: 'app',

packages/clickhouse/src/create-table-parser.ts

@@ -1,9 +1,8 @@
-import { normalizeSQLFragment, splitTopLevelComma } from '@chkit/core'
+import { normalizeProjectionIndex, normalizeSQLFragment, splitTopLevelComma } from '@chkit/core'
 
-interface ProjectionDefinitionShape {
-  name: string
-  query: string
-}
+type ProjectionDefinitionShape =
+  | { name: string; query: string }
+  | { name: string; index: string; type: string }
 
 function parseClauseFromCreateTableQuery(
   createTableQuery: string | undefined,
@@ -134,7 +133,7 @@
   )
 }
 
 export function parseProjectionsFromCreateTableQuery(
   createTableQuery: string | undefined
 ): ProjectionDefinitionShape[] {
   const body = extractCreateTableBody(createTableQuery)
@@ -142,12 +141,26 @@
   const parts = splitTopLevelComma(body)
   const projections: ProjectionDefinitionShape[] = []
   for (const part of parts) {
+    // Index-only projections have no SELECT body, so they must be matched
+    // before the parenthesized SELECT form.
+    const indexMatch = part.match(
+      /^\s*PROJECTION\s+(?:`([^`]+)`|([A-Za-z_][A-Za-z0-9_]*))\s+INDEX\s+([\s\S]+?)\s+TYPE\s+([A-Za-z_][A-Za-z0-9_]*)\s*$/i
+    )
+    if (indexMatch) {
+      const name = (indexMatch[1] ?? indexMatch[2] ?? '').trim()
+      const index = normalizeProjectionIndex(indexMatch[3] ?? '')
+      const type = (indexMatch[4] ?? '').trim()
+      if (!name || !index || !type) continue
+      projections.push({ name, index, type })
+      continue
+    }
+
     const match = part.match(
-      /^\s*PROJECTION\s+(`([^`]+)`|([A-Za-z_][A-Za-z0-9_]*))\s*\(([\s\S]*)\)\s*$/i
+      /^\s*PROJECTION\s+(?:`([^`]+)`|([A-Za-z_][A-Za-z0-9_]*))\s*\(([\s\S]*)\)\s*$/i
     )
     if (!match) continue
-    const name = (match[2] ?? match[3] ?? '').trim()
-    const query = normalizeSQLFragment((match[4] ?? '').trim())
+    const name = (match[1] ?? match[2] ?? '').trim()
+    const query = normalizeSQLFragment((match[3] ?? '').trim())
     if (!name || !query) continue
     projections.push({ name, query })
   }

packages/clickhouse/src/index.test.ts

@@ -426,6 +426,51 @@ ORDER BY id;`
       },
     ])
   })
+
+  // Verbatim SHOW CREATE TABLE output from ClickHouse 26.3, including its own
+  // pretty-printing of the SELECT body. Index-only projections used to be
+  // dropped here, which is what made `chkit pull` lose them entirely.
+  test('parses index-only projections alongside select projections', () => {
+    const query = `CREATE TABLE default.address_counterparts
+(
+    \`sender\` String,
+    \`receiver\` String,
+    \`cnt\` UInt64,
+    PROJECTION by_receiver INDEX (receiver, sender) TYPE basic,
+    PROJECTION agg_by_sender
+    (
+        SELECT
+            sender,
+            sum(cnt)
+        GROUP BY sender
+    )
+)
+ENGINE = SharedMergeTree
+ORDER BY (sender, receiver)
+SETTINGS storage_policy = 's3', index_granularity = 8192`
+
+    expect(parseProjectionsFromCreateTableQuery(query)).toEqual([
+      { name: 'by_receiver', index: '(receiver, sender)', type: 'basic' },
+      { name: 'agg_by_sender', query: 'SELECT sender, sum(cnt) GROUP BY sender' },
+    ])
+  })
+
+  // ClickHouse drops the parens around a single-column index expression, so the
+  // parser sees the bare form even when the table was created with `INDEX (b)`.
+  test('parses a single-column index projection with a backticked name', () => {
+    const query = `CREATE TABLE app.events
+(
+  \`a\` String,
+  \`b\` String,
+  PROJECTION \`p_one\` INDEX b TYPE basic
+)
+ENGINE = MergeTree
+ORDER BY a`
+
+    expect(parseProjectionsFromCreateTableQuery(query)).toEqual([
+      { name: 'p_one', index: 'b', type: 'basic' },
+    ])
+  })
 })
 
 describe('formatConnectionError', () => {

packages/core/src/canonical.ts

@@ -2,7 +2,6 @@ import type {
   ColumnDefinition,
   MaterializedViewDefinition,
   MaterializedViewRefresh,
-  ProjectionDefinition,
   SchemaDefinition,
   SkipIndexDefinition,
   TableDefinition,
@@ -11,6 +10,7 @@ import type {
 import { normalizeKeyColumns } from './key-clause.js'
 import { isSchemaDefinition } from './model.js'
 import { canonicalizeCodec } from './codec.js'
+import { canonicalizeProjection } from './projection.js'
 import { normalizeEngine, normalizeSQLFragment } from './sql-normalizer.js'
 
 function sortByName<T extends { name: string }>(items: T[]): T[] {
@@ -41,13 +41,6 @@ function canonicalizeIndex(index: SkipIndexDefinition): SkipIndexDefinition {
   }
 }
 
-function canonicalizeProjection(projection: ProjectionDefinition): ProjectionDefinition {
-  return {
-    ...projection,
-    query: normalizeSQLFragment(projection.query),
-  }
-}
-
 function canonicalizeTable(def: TableDefinition): TableDefinition {
   const settings = def.settings
     ? Object.fromEntries(