Skip to content

Commit cc294f5

Browse files
fix(snowflake): drop a clustering key before any column it names
Snowflake still rejected DROP COLUMN when a key column was inside an expression, when a quoted lowercase name was uppercased again, or when a type change kept the same key. Cover the inverse expression key and issue 5813. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 23f6e3b commit cc294f5

3 files changed

Lines changed: 277 additions & 72 deletions

File tree

‎sqlmesh/core/engine_adapter/mixins.py‎

Lines changed: 28 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
if t.TYPE_CHECKING:
2020
from sqlmesh.core._typing import TableName
21+
from sqlmesh.core.engine_adapter.shared import DataObject
2122
from sqlmesh.core.engine_adapter._typing import (
2223
DCL,
2324
DF,
@@ -385,15 +386,15 @@ def get_alter_operations(
385386

386387
clustering_operation: t.Optional[TableAlterClusterByOperation] = None
387388
if current_table_info and target_table_info:
388-
if target_table_info.is_clustered:
389-
if target_table_info.clustering_key and (
390-
current_table_info.clustering_key != target_table_info.clustering_key
391-
):
392-
clustering_operation = TableAlterChangeClusterKeyOperation(
393-
target_table=current_table,
394-
clustering_key=target_table_info.clustering_key,
395-
dialect=self.dialect,
396-
)
389+
if target_table_info.is_clustered and target_table_info.clustering_key:
390+
# Include an unchanged key. Snowflake may have to drop it and set it
391+
# again when a key column is dropped and re-added. Other engines ignore
392+
# that operation in _apply_clustering_operation.
393+
clustering_operation = TableAlterChangeClusterKeyOperation(
394+
target_table=current_table,
395+
clustering_key=target_table_info.clustering_key,
396+
dialect=self.dialect,
397+
)
397398
elif current_table_info.is_clustered:
398399
clustering_operation = TableAlterDropClusterKeyOperation(target_table=current_table)
399400

@@ -402,19 +403,32 @@ def get_alter_operations(
402403

403404
return operations
404405

406+
def _clustering_key_is_unchanged(
407+
self,
408+
clustering_operation: "TableAlterClusterByOperation",
409+
current_table_info: t.Optional["DataObject"],
410+
) -> bool:
411+
return (
412+
isinstance(clustering_operation, TableAlterChangeClusterKeyOperation)
413+
and current_table_info is not None
414+
and current_table_info.clustering_key == clustering_operation.clustering_key
415+
)
416+
405417
def _apply_clustering_operation(
406418
self,
407419
operations: t.List[TableAlterOperation],
408420
clustering_operation: "TableAlterClusterByOperation",
409-
current_table_info: t.Any,
421+
current_table_info: t.Optional["DataObject"],
410422
) -> None:
411423
"""Insert the clustering operation into the alter operations list.
412424
413-
The default implementation appends the clustering change after all column
414-
changes (historical order). Snowflake overrides this to handle the case
415-
where the old key must be dropped before a column it names is removed,
416-
and the new key must be set after new columns are added.
425+
The default implementation appends a real clustering change after all column
426+
changes. An unchanged key is ignored here. Snowflake overrides this so it can
427+
drop that key before removing a column the key names, then set the same key
428+
after the column comes back.
417429
"""
430+
if self._clustering_key_is_unchanged(clustering_operation, current_table_info):
431+
return
418432
operations.append(clustering_operation)
419433

420434

‎sqlmesh/core/engine_adapter/snowflake.py‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import contextlib
4+
import itertools
45
import logging
56
import typing as t
67

@@ -699,22 +700,26 @@ def _apply_clustering_operation(
699700
self,
700701
operations: t.List[TableAlterOperation],
701702
clustering_operation: TableAlterClusterByOperation,
702-
current_table_info: t.Any,
703+
current_table_info: t.Optional[DataObject],
703704
) -> None:
704705
"""Apply clustering DDL in the order Snowflake requires.
705706
706707
Snowflake rejects DROP COLUMN while the column is still named in the clustering
707-
key, and also rejects CLUSTER BY on a column that has not yet been added. The
708+
key, including when the name sits inside an expression such as TO_DATE("TS").
709+
It also rejects CLUSTER BY on a column that has not yet been added. The
708710
required sequence when the old key names a column being dropped is:
709711
710-
1. DROP CLUSTERING KEY — before any drop of a column the old key still names.
711-
2. Column changes (drops and adds) — in their existing schema-diff order.
712-
3. CLUSTER BY — after all column changes, if the target table is clustered.
712+
1. DROP CLUSTERING KEY, before any drop of a column the old key still names.
713+
2. Column changes (drops and adds), in their existing schema-diff order.
714+
3. CLUSTER BY, after all column changes, when the target table is clustered.
713715
714-
When no column being dropped belongs to the current key the historical order is
715-
kept: column changes first, then the single clustering change at the end.
716+
A type change that schema diff renders as DROP plus ADD uses the same sequence
717+
even when the key text does not change. When no dropped column belongs to the
718+
current key, an unchanged key emits nothing and a changed key stays at the end.
719+
Column names from DESCRIBE are already stored as Snowflake resolved them, so
720+
they are compared as returned.
716721
"""
717-
clustering_key = getattr(current_table_info, "clustering_key", None)
722+
clustering_key = current_table_info.clustering_key if current_table_info else None
718723
if clustering_key:
719724
referenced = self._clustering_key_column_names(clustering_key)
720725
drop_index = next(
@@ -723,10 +728,7 @@ def _apply_clustering_operation(
723728
for i, op in enumerate(operations)
724729
if isinstance(op, TableAlterDropColumnOperation)
725730
and op.column_parts
726-
and normalize_identifiers(
727-
exp.to_identifier(op.column_parts[0].name), dialect=self.dialect
728-
).name
729-
in referenced
731+
and op.column_parts[0].name in referenced
730732
),
731733
None,
732734
)
@@ -738,7 +740,8 @@ def _apply_clustering_operation(
738740
if isinstance(clustering_operation, TableAlterChangeClusterKeyOperation):
739741
operations.append(clustering_operation)
740742
return
741-
# No key column is being dropped: append at end (historical order).
743+
if self._clustering_key_is_unchanged(clustering_operation, current_table_info):
744+
return
742745
operations.append(clustering_operation)
743746

744747
def _clustering_key_column_names(self, clustering_key: str) -> t.Set[str]:
@@ -747,9 +750,11 @@ def _clustering_key_column_names(self, clustering_key: str) -> t.Set[str]:
747750
clustering_key=clustering_key,
748751
dialect=self.dialect,
749752
).cluster_key_expressions
753+
cluster_columns = itertools.chain.from_iterable(
754+
expression.find_all(exp.Column) for expression in expressions
755+
)
750756
return {
751-
normalize_identifiers(expression, dialect=self.dialect).name
752-
for expression in expressions
757+
normalize_identifiers(column, dialect=self.dialect).name for column in cluster_columns
753758
}
754759

755760
def alter_table(

0 commit comments

Comments
 (0)